From a2c3c3edc5b54dcfddbcd8fb15202ed3f31addef Mon Sep 17 00:00:00 2001
From: Kaidan Theron <kaidan13th@gmail.com>
Date: Fri, 27 Jun 2025 10:08:37 +0200
Subject: [PATCH 001/391] add datalevin

---
 README.md                                    | 11 +++++
 deps.edn                                     |  3 +-
 src/source/datastore/datalevin.clj           | 43 ++++++++++++++++++++
 src/source/datastore/output_schema.clj       | 24 +++++++++++
 src/source/datastore/selection_schema.clj    | 24 +++++++++++
 src/source/datastore/util.clj                | 23 +++++++++++
 src/source/db/master.clj                     |  8 ++++
 src/source/migrations/001_init_master_db.clj |  3 +-
 src/source/routes/reitit.clj                 |  7 +++-
 src/source/routes/rss.clj                    | 20 +++++++++
 src/source/services/interface.clj            | 12 +++++-
 src/source/services/xml.clj                  | 16 ++++++++
 12 files changed, 189 insertions(+), 5 deletions(-)
 create mode 100644 src/source/datastore/datalevin.clj
 create mode 100644 src/source/datastore/output_schema.clj
 create mode 100644 src/source/datastore/selection_schema.clj
 create mode 100644 src/source/datastore/util.clj
 create mode 100644 src/source/routes/rss.clj
 create mode 100644 src/source/services/xml.clj

diff --git a/README.md b/README.md
index 6c1f189d..5ea62489 100644
--- a/README.md
+++ b/README.md
@@ -2,6 +2,17 @@
 
 This is the backend for the Source platform. You can find documentation on setup for development below.
 
+## Dependencies
+
+
+- >= openjdk version 17.
+- libc, libomp, libmvec (Refer to the Datalevin [installation docs](https://github.com/juji-io/datalevin/blob/master/doc/install.md#native-dependencies) for more info)
+
+If you are on MacOS you can run:
+```
+brew install libomp llvm openjdk@17
+```
+
 ## Development setup
 
 - Pull the project from GitHub.
diff --git a/deps.edn b/deps.edn
index 3b450cf2..46434127 100644
--- a/deps.edn
+++ b/deps.edn
@@ -25,4 +25,5 @@
         com.kepler16/mallard {:mvn/version "3.2.1"}
         com.kepler16/mallard-sqlite-store {:mvn/version "3.2.1"}
         com.github.seancorfield/honeysql {:mvn/version "2.7.1310"}
-        metosin/reitit {:mvn/version "0.9.1"}}}
+        metosin/reitit {:mvn/version "0.9.1"}
+        datalevin/datalevin {:mvn/version "0.9.22"}}}
diff --git a/src/source/datastore/datalevin.clj b/src/source/datastore/datalevin.clj
new file mode 100644
index 00000000..3b2bbaa8
--- /dev/null
+++ b/src/source/datastore/datalevin.clj
@@ -0,0 +1,43 @@
+(ns source.datastore.datalevin
+  (:require [datalevin.core :as d]))
+
+;; Something to consider (async transactions) https://cljdoc.org/d/datalevin/datalevin/0.9.22/doc/transaction#asynchronous-transaction
+
+(defn get-value [ds {:keys [tname key]}]
+  (d/get-value ds (name tname) key))
+
+(defn delete! [ds {:keys [tname keys]}]
+  (->> (mapv (fn [key] [:del (name tname) key]) keys)
+       (d/transact-kv ds)))
+
+(defn put! [ds {:keys [tname data]}]
+  (->> (mapv (fn [item] [:put (name tname) (first item) (last item)]) data)
+       (d/transact-kv ds)))
+
+(defn get-all
+  [ds {:keys [tname]}]
+  (let [tn (name tname)]
+    (mapv (fn [[k v]] [k v])
+          (d/entries ds tn))))
+
+(comment
+  (require '[source.datastore.util :as ds.util])
+  (let [ds (ds.util/conn "store")
+        key "somekey"
+        value "somestringvalue"
+        tname :some-table]
+    (println "Putting a value")
+    (d/open-dbi ds (name tname))
+    (put! ds {:tname tname
+              :data [[key value]]})
+    (assert (= value
+               (get-value ds {:tname tname
+                              :key key})))
+    (println "Test passed")
+    (println "Deleting a value")
+    (delete! ds {:tname tname
+                 :keys [key]})
+    (assert (= nil
+               (get-value ds {:tname tname
+                              :key key})))
+    (ds.util/close ds)))
diff --git a/src/source/datastore/output_schema.clj b/src/source/datastore/output_schema.clj
new file mode 100644
index 00000000..b1388cc7
--- /dev/null
+++ b/src/source/datastore/output_schema.clj
@@ -0,0 +1,24 @@
+(ns source.datastore.output-schema
+  (:require [source.datastore.datalevin :as dl]))
+
+(defn get-value
+  "Finds a value in the `output-schema` table by key."
+  [ds key]
+  (dl/get-value ds {:tname   :output-schema
+                    :key     key}))
+
+(defn delete!
+  "Deletes one or more keys from the `output-schema` table."
+  [ds keys]
+  (dl/delete! ds {:tname :output-schema
+                  :data   keys}))
+
+(defn put!
+  "Puts one or more key-value pairs into `output-schema`."
+  [ds items]
+  (dl/put! ds {:tname :output-schema
+               :data  items}))
+
+(defn get-all
+  [ds]
+  (dl/get-all ds {:tname :output-schema}))
diff --git a/src/source/datastore/selection_schema.clj b/src/source/datastore/selection_schema.clj
new file mode 100644
index 00000000..80df8c91
--- /dev/null
+++ b/src/source/datastore/selection_schema.clj
@@ -0,0 +1,24 @@
+(ns source.datastore.selection-schema
+  (:require [source.datastore.datalevin :as dl]))
+
+(defn get-value
+  "Finds a value in the `selection-schema` table by key."
+  [ds key]
+  (dl/get-value ds {:tname   :selection-schema
+                    :key     key}))
+
+(defn delete!
+  "Deletes one or more keys from the `selection-schema` table."
+  [ds keys]
+  (dl/delete! ds {:tname :selection-schema
+                  :data   keys}))
+
+(defn put!
+  "Puts one or more key-value pairs into `selection-schema`."
+  [ds items]
+  (dl/put! ds {:tname :selection-schema
+               :data  items}))
+
+(defn get-all
+  [ds]
+  (dl/get-all ds {:tname :selection-schema}))
diff --git a/src/source/datastore/util.clj b/src/source/datastore/util.clj
new file mode 100644
index 00000000..61f6f6e4
--- /dev/null
+++ b/src/source/datastore/util.clj
@@ -0,0 +1,23 @@
+(ns source.datastore.util
+  (:require [datalevin.core :as d]
+            [source.config :as conf]
+            [clojure.string :as string]))
+
+(defn absolute [path]
+  (-> (java.io.File. ".")
+      .getAbsolutePath
+      (clojure.string/replace "/." "/")
+      (str path)))
+
+(defn store-path [store-name]
+  (-> (str (conf/read-value :database :dir) store-name)
+      (absolute)))
+
+(defn conn
+  "Open a connection to a datalevin kv store"
+  [store-name] (d/open-kv (store-path store-name)))
+
+(defn close
+  "Close a connection to a datalevin store"
+  [conn]
+  (d/close-kv conn))
diff --git a/src/source/db/master.clj b/src/source/db/master.clj
index 11127d77..8678d16d 100644
--- a/src/source/db/master.clj
+++ b/src/source/db/master.clj
@@ -130,6 +130,14 @@
    (tables/foreign-key :feed-id :feeds :id)
    (tables/foreign-key :sector-id :sectors :id)))
 
+(def selection-schemas
+  (tables/create-table-sql
+   :selection-schemas
+   (tables/table-id)
+   [:output-schema-id :integer :not nil]
+   [:provider-id :integer :not nil]
+   (tables/foreign-key :provider-id :providers :id)))
+
 (comment
   (require '[honey.sql :as sql])
 
diff --git a/src/source/migrations/001_init_master_db.clj b/src/source/migrations/001_init_master_db.clj
index ec0b753c..2b10997f 100644
--- a/src/source/migrations/001_init_master_db.clj
+++ b/src/source/migrations/001_init_master_db.clj
@@ -84,7 +84,8 @@
       :providers
       :businesses
       :user-sectors
-      :feed-sectors])
+      :feed-sectors
+      :selection-schemas])
 
     (db/insert! ds-master baselines-seed)
     (db/insert! ds-master cadences-seed)
diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index 47ace2c5..b52c63db 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -10,7 +10,8 @@
             [source.routes.google-launch :as google-launch]
             [source.routes.google-redirect :as google-redirect]
             [source.routes.admin :as admin]
-            [source.routes.authorized :as authorized]))
+            [source.routes.authorized :as authorized]
+            [source.routes.rss :as rss]))
 
 (defn create-app []
   (let [ds (db/ds :master)]
@@ -31,7 +32,9 @@
         ["protected" {:middleware [[mw/apply-auth]]}
          ["/authorized" {:get authorized/get}]]
         ["admin" {:middleware [[mw/apply-auth {:required-type :admin}]]}
-         ["/add-admin" {:post admin/post}]]]]))))
+         ["/add-admin" {:post admin/post}]
+         ["/selection-schemas" {:get rss/selection-schemas}]
+         ["/output-schemas" {:get rss/output-schemas}]]]]))))
 
 (comment
   (require '[source.middleware.auth.util :as auth.util])
diff --git a/src/source/routes/rss.clj b/src/source/routes/rss.clj
new file mode 100644
index 00000000..5008e446
--- /dev/null
+++ b/src/source/routes/rss.clj
@@ -0,0 +1,20 @@
+(ns source.routes.rss
+  (:require [source.services.interface :as services]
+            [ring.util.response :as res]))
+
+(defn post-output-schema [{:keys [ds body] :as _request}]
+  (let [{:keys [key value]} body]
+    (services/insert-output-schema! ds {:key key
+                                        :value value})))
+
+(defn selection-schemas [{:keys [store] :as _request}]
+  (services/selection-schemas store))
+
+(defn output-schemas [{:keys [store] :as _request}]
+  (services/output-schemas store))
+
+(defn output-schemas [])
+
+(defn get-ast [])
+
+(defn get-data [])
diff --git a/src/source/services/interface.clj b/src/source/services/interface.clj
index 508864ab..cb1ade9c 100644
--- a/src/source/services/interface.clj
+++ b/src/source/services/interface.clj
@@ -1,7 +1,8 @@
 (ns source.services.interface
   (:require  [source.services.users :as users]
              [source.db.interface :as db]
-             [source.services.auth :as auth]))
+             [source.services.auth :as auth]
+             [source.services.xml :as xml]))
 
 (defn users
   [& args]
@@ -22,6 +23,15 @@
 (defn register [ds user]
   (auth/register ds user))
 
+(defn selection-schemas [ds]
+  (xml/get-all ds {:tname :selection-schema}))
+
+(defn output-schemas [ds]
+  (xml/get-all ds {:tname :output-schema}))
+
+(defn insert-output-schema! [ds {:keys [key value]}]
+  (xml/add-output-schema! ds {:data [[key value]]}))
+
 (comment
   (users (db/ds :master))
   (user (db/ds :master) {:id 2})
diff --git a/src/source/services/xml.clj b/src/source/services/xml.clj
new file mode 100644
index 00000000..fe02f0b0
--- /dev/null
+++ b/src/source/services/xml.clj
@@ -0,0 +1,16 @@
+(ns source.services.xml
+  (:require [source.db.interface :as db]
+            [source.datastore.datalevin :as dl]))
+
+(defn get-all
+  [ds {:keys [tname]}]
+  (dl/get-all ds {:tname tname}))
+
+(defn add-output-schema!
+  [ds {:keys [data]}]
+  (dl/put! ds {:tname :output-schema
+               :data data}))
+
+(defn insert-selection-schema!
+  [ds]
+  (db/insert! ds {:tname :selection-schemas}))

From 97cfbe7aaa9bb2471dcd920a9445bae787115a96 Mon Sep 17 00:00:00 2001
From: Kaidan Theron <kaidan13th@gmail.com>
Date: Sun, 29 Jun 2025 20:22:38 +0200
Subject: [PATCH 002/391] add kv-store service use to routes

---
 src/source/datastore/datalevin.clj        | 81 ++++++++++++++++++-----
 src/source/datastore/interface.clj        | 27 ++++++++
 src/source/datastore/output_schema.clj    | 24 -------
 src/source/datastore/selection_schema.clj | 24 -------
 src/source/datastore/util.clj             |  6 ++
 src/source/db/honey.clj                   | 12 ++--
 src/source/routes/reitit.clj              | 28 ++++++--
 src/source/routes/rss.clj                 | 20 ------
 src/source/routes/selection_schema.clj    |  8 +++
 src/source/services/interface.clj         | 12 +---
 src/source/services/xml.clj               | 16 -----
 src/source/services/xml_schemas.clj       | 15 +++++
 src/source/util.clj                       |  6 ++
 13 files changed, 158 insertions(+), 121 deletions(-)
 create mode 100644 src/source/datastore/interface.clj
 delete mode 100644 src/source/datastore/output_schema.clj
 delete mode 100644 src/source/datastore/selection_schema.clj
 delete mode 100644 src/source/routes/rss.clj
 create mode 100644 src/source/routes/selection_schema.clj
 delete mode 100644 src/source/services/xml.clj
 create mode 100644 src/source/services/xml_schemas.clj

diff --git a/src/source/datastore/datalevin.clj b/src/source/datastore/datalevin.clj
index 3b2bbaa8..1f182014 100644
--- a/src/source/datastore/datalevin.clj
+++ b/src/source/datastore/datalevin.clj
@@ -1,24 +1,66 @@
 (ns source.datastore.datalevin
-  (:require [datalevin.core :as d]))
+  (:require [datalevin.core :as d]
+            [source.util :as util]))
 
-;; Something to consider (async transactions) https://cljdoc.org/d/datalevin/datalevin/0.9.22/doc/transaction#asynchronous-transaction
+;; If we want to have higher write speed in future we can use transact-kv-async
 
-(defn get-value [ds {:keys [tname key]}]
+;; TODO:
+;; - when operations on kv store are done the relevant tables need to be opened beforehand
+;;   using (datalevin.core/open-dbi "table-name")
+;; - we need a nice way of opening and closing a connection to the datastore before and after
+;;  an operation is performed. Closing a connection to the datastore will also close the tables
+;;  in that datastore. Although it isn't documented as far as I could see in the datalevin or
+;;  LMDB docs, I think we should avoid opening the same table multiple times. So we need to figure
+;;  out a way to open a connection once (both on a datastore level and a datastore level), reuse it
+;;  where required, and then close it.
+;;
+;;  A possible solution here is to open up the kv-store and the tables we use on server start and
+;;  pass the kv-store connection around. As you might notice from the implementation, we don't need
+;;  to "def" a table connection and pass it around, only the store connection. As long as you call
+;;  (datalevin.core/open-dbi "table-name"). Refer to the comment block below.
+
+(defn find [ds {:keys [tname key]}]
   (d/get-value ds (name tname) key))
 
-(defn delete! [ds {:keys [tname keys]}]
-  (->> (mapv (fn [key] [:del (name tname) key]) keys)
-       (d/transact-kv ds)))
+(defn exists? [ds opts]
+  (-> (find ds opts)
+      (some?)))
+
+(defn delete!
+  "Removes one or multiple keys from kv store."
+  [ds {:keys [tname keys]}]
+  (->> (mapv (fn [key] [:del key]) keys)
+       (d/transact-kv ds (name tname))))
 
-(defn put! [ds {:keys [tname data]}]
-  (->> (mapv (fn [item] [:put (name tname) (first item) (last item)]) data)
-       (d/transact-kv ds)))
+(defn insert!
+  "Inserts kv's into store. Skips keys that already exist.
+  Returns the key-value pairs that were inserted."
+  [ds {:keys [tname data]}]
+  (let [multi? (util/vectors? data)
+        input-kvs (if multi? data [data])
+        kvs-to-insert (->> input-kvs
+                           (filterv (fn [[k _]]
+                                      (not (exists? ds {:tname tname :key k})))))]
+    (->> kvs-to-insert
+         (mapv (fn [[k v]] [:put k v]))
+         (d/transact-kv ds (name tname)))
+    kvs-to-insert))
+
+(defn update!
+  "Replaces values for keys in store. Skips keys that don't exist"
+  [ds {:keys [tname data]}]
+  (let [transducer (comp
+                    (filter (fn [[k _]]
+                              (exists? ds {:tname tname :key k})))
+                    (map (fn [[k v]]
+                           [:put k v])))]
+    (->> data
+         (into [] transducer)
+         (d/transact-kv ds (name tname)))))
 
 (defn get-all
   [ds {:keys [tname]}]
-  (let [tn (name tname)]
-    (mapv (fn [[k v]] [k v])
-          (d/entries ds tn))))
+  (d/get-range ds (name tname) [:all]))
 
 (comment
   (require '[source.datastore.util :as ds.util])
@@ -28,16 +70,19 @@
         tname :some-table]
     (println "Putting a value")
     (d/open-dbi ds (name tname))
-    (put! ds {:tname tname
-              :data [[key value]]})
+    (insert! ds {:tname tname
+                 :data [key value]})
     (assert (= value
-               (get-value ds {:tname tname
-                              :key key})))
+               (find ds {:tname tname
+                         :key key})))
+    (insert! ds {:tname tname
+                 :data [key value]})
     (println "Test passed")
     (println "Deleting a value")
     (delete! ds {:tname tname
                  :keys [key]})
     (assert (= nil
-               (get-value ds {:tname tname
-                              :key key})))
+               (find ds {:tname tname
+                         :key key})))
+    (println "Test passed")
     (ds.util/close ds)))
diff --git a/src/source/datastore/interface.clj b/src/source/datastore/interface.clj
new file mode 100644
index 00000000..ad66abad
--- /dev/null
+++ b/src/source/datastore/interface.clj
@@ -0,0 +1,27 @@
+(ns source.datastore.interface
+  (:require [source.datastore.datalevin :as dl]
+            [source.datastore.util :as store.util]))
+
+(defn ds [store-name]
+  (store.util/conn store-name))
+
+(defn store-name [& args]
+  (apply store.util/store-name args))
+
+(defn find [ds opts]
+  (dl/find ds opts))
+
+(defn exists? [ds opts]
+  (dl/exists? ds opts))
+
+(defn insert! [ds opts]
+  (dl/insert! ds opts))
+
+(defn update! [ds opts]
+  (dl/update! ds opts))
+
+(defn get-all [ds opts]
+  (dl/get-all ds opts))
+
+(defn delete! [ds opts]
+  (dl/delete! ds opts))
diff --git a/src/source/datastore/output_schema.clj b/src/source/datastore/output_schema.clj
deleted file mode 100644
index b1388cc7..00000000
--- a/src/source/datastore/output_schema.clj
+++ /dev/null
@@ -1,24 +0,0 @@
-(ns source.datastore.output-schema
-  (:require [source.datastore.datalevin :as dl]))
-
-(defn get-value
-  "Finds a value in the `output-schema` table by key."
-  [ds key]
-  (dl/get-value ds {:tname   :output-schema
-                    :key     key}))
-
-(defn delete!
-  "Deletes one or more keys from the `output-schema` table."
-  [ds keys]
-  (dl/delete! ds {:tname :output-schema
-                  :data   keys}))
-
-(defn put!
-  "Puts one or more key-value pairs into `output-schema`."
-  [ds items]
-  (dl/put! ds {:tname :output-schema
-               :data  items}))
-
-(defn get-all
-  [ds]
-  (dl/get-all ds {:tname :output-schema}))
diff --git a/src/source/datastore/selection_schema.clj b/src/source/datastore/selection_schema.clj
deleted file mode 100644
index 80df8c91..00000000
--- a/src/source/datastore/selection_schema.clj
+++ /dev/null
@@ -1,24 +0,0 @@
-(ns source.datastore.selection-schema
-  (:require [source.datastore.datalevin :as dl]))
-
-(defn get-value
-  "Finds a value in the `selection-schema` table by key."
-  [ds key]
-  (dl/get-value ds {:tname   :selection-schema
-                    :key     key}))
-
-(defn delete!
-  "Deletes one or more keys from the `selection-schema` table."
-  [ds keys]
-  (dl/delete! ds {:tname :selection-schema
-                  :data   keys}))
-
-(defn put!
-  "Puts one or more key-value pairs into `selection-schema`."
-  [ds items]
-  (dl/put! ds {:tname :selection-schema
-               :data  items}))
-
-(defn get-all
-  [ds]
-  (dl/get-all ds {:tname :selection-schema}))
diff --git a/src/source/datastore/util.clj b/src/source/datastore/util.clj
index 61f6f6e4..47efc9d3 100644
--- a/src/source/datastore/util.clj
+++ b/src/source/datastore/util.clj
@@ -21,3 +21,9 @@
   "Close a connection to a datalevin store"
   [conn]
   (d/close-kv conn))
+
+(defn store-name
+  ([type]
+   (name type))
+  ([type id]
+   (str (name type) "_" id)))
diff --git a/src/source/db/honey.clj b/src/source/db/honey.clj
index ed4d05c6..79101b88 100644
--- a/src/source/db/honey.clj
+++ b/src/source/db/honey.clj
@@ -42,10 +42,14 @@
   "inserts a single record or a set of records into a table. records passed in map form where the keys can be snake-case keywords. all keys are converted to snake_case strings before executing prepared statements."
   [ds {:keys [tname data]}]
   (let [multi? (vector? data)
-        values (if multi? data [data])]
-    (execute! ds
-              (-> (hsql/insert-into (csk/->snake_case_keyword tname))
-                  (hsql/values values)))))
+        values (if multi? data [data])
+        result (execute! ds
+                         (-> (hsql/insert-into (csk/->snake_case_keyword tname))
+                             (hsql/values values)
+                             (hsql/returning :*)))]
+    (if multi?
+      result
+      (first result))))
 
 (defn delete!
   "deletes a record or set of records that match a predicate where clause. the where
diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index b52c63db..9b37ec57 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -11,7 +11,7 @@
             [source.routes.google-redirect :as google-redirect]
             [source.routes.admin :as admin]
             [source.routes.authorized :as authorized]
-            [source.routes.rss :as rss]))
+            [source.routes.selection-schema :as selection-schema]))
 
 (defn create-app []
   (let [ds (db/ds :master)]
@@ -33,11 +33,12 @@
          ["/authorized" {:get authorized/get}]]
         ["admin" {:middleware [[mw/apply-auth {:required-type :admin}]]}
          ["/add-admin" {:post admin/post}]
-         ["/selection-schemas" {:get rss/selection-schemas}]
-         ["/output-schemas" {:get rss/output-schemas}]]]]))))
+         ["/selection-schema" {:post selection-schema/post}]]]]))))
 
 (comment
-  (require '[source.middleware.auth.util :as auth.util])
+  (require '[source.middleware.auth.util :as auth.util]
+           '[source.datastore.util :as store.util]
+           '[datalevin.core :as dl])
 
   (let [app (create-app)
         request {:uri "/users" :request-method :get}]
@@ -88,8 +89,23 @@
         request {:uri "/oauth2/google"
                  :request-method :get}]
     (-> request
-        app 
-        :body 
+        app
+        :body
         (json/read-json {:key-fn keyword})))
 
+  (let [app (create-app)
+        store (store.util/conn "datalevin")
+        ;; open table before operation
+        thing (dl/open-dbi store "selection-schemas")
+        request {:uri "/admin/selection-schema"
+                 :request-method :post
+                 :store store
+                 :body {:record {:provider-id 1
+                                 :output-schema-id 1}
+                        :schema {:hi "hello"}}
+                 :headers {"authorization" (str "Bearer " (auth.util/sign-jwt {:id 1 :type "admin"}))}}]
+    (-> request
+        app
+        :body
+        (json/read-json {:key-fn keyword})))
   ())
diff --git a/src/source/routes/rss.clj b/src/source/routes/rss.clj
deleted file mode 100644
index 5008e446..00000000
--- a/src/source/routes/rss.clj
+++ /dev/null
@@ -1,20 +0,0 @@
-(ns source.routes.rss
-  (:require [source.services.interface :as services]
-            [ring.util.response :as res]))
-
-(defn post-output-schema [{:keys [ds body] :as _request}]
-  (let [{:keys [key value]} body]
-    (services/insert-output-schema! ds {:key key
-                                        :value value})))
-
-(defn selection-schemas [{:keys [store] :as _request}]
-  (services/selection-schemas store))
-
-(defn output-schemas [{:keys [store] :as _request}]
-  (services/output-schemas store))
-
-(defn output-schemas [])
-
-(defn get-ast [])
-
-(defn get-data [])
diff --git a/src/source/routes/selection_schema.clj b/src/source/routes/selection_schema.clj
new file mode 100644
index 00000000..c5bc7bf7
--- /dev/null
+++ b/src/source/routes/selection_schema.clj
@@ -0,0 +1,8 @@
+(ns source.routes.selection-schema
+  (:require [source.services.interface :as services]
+            [ring.util.response :as res]))
+
+(defn post [{:keys [store ds body] :as _request}]
+  (let [{:keys [schema record] :as opts} body]
+    (-> (services/add-selection-schema! store ds opts)
+        (res/response))))
diff --git a/src/source/services/interface.clj b/src/source/services/interface.clj
index cb1ade9c..e0f33479 100644
--- a/src/source/services/interface.clj
+++ b/src/source/services/interface.clj
@@ -2,7 +2,7 @@
   (:require  [source.services.users :as users]
              [source.db.interface :as db]
              [source.services.auth :as auth]
-             [source.services.xml :as xml]))
+             [source.services.xml-schemas :as xml]))
 
 (defn users
   [& args]
@@ -23,14 +23,8 @@
 (defn register [ds user]
   (auth/register ds user))
 
-(defn selection-schemas [ds]
-  (xml/get-all ds {:tname :selection-schema}))
-
-(defn output-schemas [ds]
-  (xml/get-all ds {:tname :output-schema}))
-
-(defn insert-output-schema! [ds {:keys [key value]}]
-  (xml/add-output-schema! ds {:data [[key value]]}))
+(defn add-selection-schema! [store db {:keys [schema record] :as opts}]
+  (xml/add-selection-schema! store db opts))
 
 (comment
   (users (db/ds :master))
diff --git a/src/source/services/xml.clj b/src/source/services/xml.clj
deleted file mode 100644
index fe02f0b0..00000000
--- a/src/source/services/xml.clj
+++ /dev/null
@@ -1,16 +0,0 @@
-(ns source.services.xml
-  (:require [source.db.interface :as db]
-            [source.datastore.datalevin :as dl]))
-
-(defn get-all
-  [ds {:keys [tname]}]
-  (dl/get-all ds {:tname tname}))
-
-(defn add-output-schema!
-  [ds {:keys [data]}]
-  (dl/put! ds {:tname :output-schema
-               :data data}))
-
-(defn insert-selection-schema!
-  [ds]
-  (db/insert! ds {:tname :selection-schemas}))
diff --git a/src/source/services/xml_schemas.clj b/src/source/services/xml_schemas.clj
new file mode 100644
index 00000000..77742888
--- /dev/null
+++ b/src/source/services/xml_schemas.clj
@@ -0,0 +1,15 @@
+(ns source.services.xml-schemas
+  (:require [source.db.interface :as db]
+            [source.datastore.interface :as store]))
+
+(defn get-all
+  [ds {:keys [tname]}]
+  (store/get-all ds {:tname tname}))
+
+(defn add-selection-schema!
+  [store db {:keys [schema record]}]
+  (let [db-result (db/insert! db {:tname :selection-schemas
+                                  :data record})]
+    (println db-result)
+    (store/insert! store {:tname :selection-schemas
+                          :data [(:id db-result) schema]})))
diff --git a/src/source/util.clj b/src/source/util.clj
index 504d046d..4b86b8ac 100644
--- a/src/source/util.clj
+++ b/src/source/util.clj
@@ -2,6 +2,12 @@
   (:require [buddy.core.codecs :as codecs]
             [buddy.core.nonce :as nonce]))
 
+(defn vectors?
+  "Returns true if coll is a 2d vector"
+  [coll]
+  (and (vector? coll)
+       (vector? (first coll))))
+
 (defn content-type [request]
   (or (get-in request [:headers "Content-Type"])
       (get-in request [:headers :content-type])

From 39823ff7c8fbe8d4b1b750ba19ce7a3cacd8c2ed Mon Sep 17 00:00:00 2001
From: Kaidan Theron <kaidan13th@gmail.com>
Date: Sun, 29 Jun 2025 21:59:51 +0200
Subject: [PATCH 003/391] add selection schema util

---
 deps.edn                |   3 +-
 src/source/rss/core.clj | 131 +++++++++++++++++++++++++++++++++++++++-
 2 files changed, 132 insertions(+), 2 deletions(-)

diff --git a/deps.edn b/deps.edn
index 46434127..ec63e157 100644
--- a/deps.edn
+++ b/deps.edn
@@ -26,4 +26,5 @@
         com.kepler16/mallard-sqlite-store {:mvn/version "3.2.1"}
         com.github.seancorfield/honeysql {:mvn/version "2.7.1310"}
         metosin/reitit {:mvn/version "0.9.1"}
-        datalevin/datalevin {:mvn/version "0.9.22"}}}
+        datalevin/datalevin {:mvn/version "0.9.22"}
+        hickory/hickory {:mvn/version "0.7.1"}}}
diff --git a/src/source/rss/core.clj b/src/source/rss/core.clj
index e6a1aa1a..e311d3e8 100644
--- a/src/source/rss/core.clj
+++ b/src/source/rss/core.clj
@@ -1 +1,130 @@
-(ns source.rss.core)
+(ns source.rss.core
+  (:require [hickory.core :as h]
+            [hickory.select :as s]))
+
+(defn get-ast
+  "Constructs a hickory tree representation from an xml string."
+  [xml]
+  (-> xml h/parse h/as-hickory))
+
+(defn collect-leaf-paths
+  "This function does a DFS on a hickory tree and assigns paths to each node relative to the root and
+  then returns the ast with paths.
+  
+  Paths are made up of a sequence of tag names."
+  ([root-node]
+   (collect-leaf-paths root-node []))
+  ([node current-path]
+   (if (= (type node) java.lang.String)
+     node
+
+     (let [{:keys [tag content]} node
+           new-path (if tag
+                      (conj current-path tag)
+                      current-path)
+           content-with-paths (when content
+                                (let [nodes (if (> (count content) 1) ;; filter out erroneous string children
+                                              (filterv #(not= (type %) java.lang.String) content)
+                                              content)]
+                                  (mapv #(collect-leaf-paths % new-path) nodes)))]
+       (-> node
+           (assoc :path new-path)
+           (dissoc :content)
+           (assoc :content content-with-paths))))))
+
+(defn build-child-selector
+  "given a sequence of tag keywords this will return a hickory selector
+  that hickory.select/select can use to get the node you want"
+  [tags]
+  (let [ts (mapv #(last %) tags)
+        root (first ts)]
+    (reduce (fn [sel tag]
+              (s/child sel (s/tag tag)))
+            (s/tag root)
+            (rest ts))))
+
+(defn split
+  "splits path segment into [namespace name]
+  i.e. \"attr/attr-name\" becomes [:attr :attr-name]"
+  [path-segment]
+  (let [path-keyword (keyword path-segment)]
+    (mapv
+     #(keyword %)
+     [(namespace path-keyword)
+      (name path-keyword)])))
+
+(defn leaf?
+  "Returns true if :seg-type from namespaced keyword path segment's type is content or attribute."
+  [kw-path-seg]
+  (let [seg-type (first kw-path-seg)]
+    (or (= seg-type :content) (= seg-type :attr))))
+
+(defn extract-leaf
+  "Get's the content from a leaf node using the leaf path segment."
+  [node kw-path-seg]
+  (let [[seg-type seg-val] kw-path-seg]
+    (if (= seg-type :attr)
+      (-> (:attrs node)
+          (get seg-val))
+
+      (-> (:content node)
+          (get (Integer/parseInt (name seg-val)))))))
+
+(defn extract-data
+  "Recursively extracts data from a hickory xml tree according to the input selection schema.
+  The input selection schema contains paths to each field in the schema. These paths are relative to their parent node
+  with any number of tag/tag-name segments and end with a attr/attribute-name or content/n-th segment if the field is a string."
+  [schema ast]
+  (reduce-kv
+   (fn [result field {:keys [type path schema]}]
+     (let [keyword-path (mapv #(split %) path)
+           is-leaf? (leaf? (last keyword-path))
+           selector-path (if is-leaf? (butlast keyword-path) keyword-path)
+           selector (build-child-selector selector-path)]
+
+       (cond
+         (= type "map")
+         (let [child-node (first (s/select selector ast))]
+           (assoc result field
+                  (extract-data schema child-node)))
+
+         (= type "vector")
+         (let [child-nodes (s/select selector ast)]
+           (assoc result field
+                  (mapv #(extract-data schema %) child-nodes)))
+
+         is-leaf?
+         (assoc result field
+                (-> (s/select selector ast)
+                    first
+                    (extract-leaf (last keyword-path))))
+
+         :else
+         result)))
+   {}
+   schema))
+
+(comment
+  (let [ast (get-ast (slurp "https://www.youtube.com/feeds/videos.xml?channel_id=UCWI-ohtRu8eEeDj93hmUsUQ"))
+        schema {:title
+                {:type "string",
+                 :required true,
+                 :path ["html" "body" "feed" "title" "content/0"]},
+                :url
+                {:type "string",
+                 :required true,
+                 :path ["html" "body" "feed" "author" "uri" "content/0"]},
+                :posts
+                {:type "vector",
+                 :required true,
+                 :schema
+                 {:title {:type "string", :required true, :path ["title" "content/0"]},
+                  :stream-url
+                  {:type "string", :required true, :path ["link" "attr/href"]},
+                  :description
+                  {:type "string",
+                   :required false,
+                   :path ["media:group" "media:description" "content/0"]},
+                  :posted-at {:type "string", :required false, :path ["published" "content/0"]}},
+                 :path ["html" "body" "feed" "entry"]}}]
+    (extract-data schema ast)))

From bc2dc64a5051b5a90519dd3fb986b5e041e4a166 Mon Sep 17 00:00:00 2001
From: Kaidan Theron <kaidan13th@gmail.com>
Date: Mon, 30 Jun 2025 10:35:22 +0200
Subject: [PATCH 004/391] add services for xml schemas

---
 src/source/datastore/datalevin.clj  | 17 ++++++++++++---
 src/source/datastore/interface.clj  |  3 +++
 src/source/datastore/tables.clj     | 22 +++++++++++++++++++
 src/source/services/providers.clj   | 25 +++++++++++++++++++++
 src/source/services/xml_schemas.clj | 34 ++++++++++++++++++++++++++++-
 5 files changed, 97 insertions(+), 4 deletions(-)
 create mode 100644 src/source/datastore/tables.clj
 create mode 100644 src/source/services/providers.clj

diff --git a/src/source/datastore/datalevin.clj b/src/source/datastore/datalevin.clj
index 1f182014..9ef934fc 100644
--- a/src/source/datastore/datalevin.clj
+++ b/src/source/datastore/datalevin.clj
@@ -19,13 +19,22 @@
 ;;  to "def" a table connection and pass it around, only the store connection. As long as you call
 ;;  (datalevin.core/open-dbi "table-name"). Refer to the comment block below.
 
-(defn find [ds {:keys [tname key]}]
+(defn find
+  "Returns the value for a key in the kv-store"
+  [ds {:keys [tname key]}]
   (d/get-value ds (name tname) key))
 
-(defn exists? [ds opts]
+(defn exists?
+  "Returns true if value exists for key in kv-store"
+  [ds opts]
   (-> (find ds opts)
       (some?)))
 
+(defn entries
+  "Get the number of entries in a table"
+  [ds {:keys [tname]}]
+  (d/entries ds tname))
+
 (defn delete!
   "Removes one or multiple keys from kv store."
   [ds {:keys [tname keys]}]
@@ -59,12 +68,13 @@
          (d/transact-kv ds (name tname)))))
 
 (defn get-all
+  "Returns all key value pairs in table in kv-store."
   [ds {:keys [tname]}]
   (d/get-range ds (name tname) [:all]))
 
 (comment
   (require '[source.datastore.util :as ds.util])
-  (let [ds (ds.util/conn "store")
+  (let [ds (ds.util/conn "datalevin")
         key "somekey"
         value "somestringvalue"
         tname :some-table]
@@ -85,4 +95,5 @@
                (find ds {:tname tname
                          :key key})))
     (println "Test passed")
+    ()
     (ds.util/close ds)))
diff --git a/src/source/datastore/interface.clj b/src/source/datastore/interface.clj
index ad66abad..a955478c 100644
--- a/src/source/datastore/interface.clj
+++ b/src/source/datastore/interface.clj
@@ -25,3 +25,6 @@
 
 (defn delete! [ds opts]
   (dl/delete! ds opts))
+
+(defn entries [ds opts]
+  (dl/entries ds opts))
diff --git a/src/source/datastore/tables.clj b/src/source/datastore/tables.clj
new file mode 100644
index 00000000..d17c411c
--- /dev/null
+++ b/src/source/datastore/tables.clj
@@ -0,0 +1,22 @@
+(ns source.datastore.tables
+  (:require [datalevin.core :as d]))
+
+(defn open-table!
+  "Opens table in the store."
+  [store tname]
+  (d/open-dbi store (name tname)))
+
+(defn drop-table!
+  "Clears data from and deleted table"
+  [store tname]
+  (d/drop-dbi store (name tname)))
+
+(defn clear-table!
+  "Clear data from table"
+  [store tname]
+  (d/clear-dbi store (name tname)))
+
+(defn tables
+  "Returns a vector of all tables in store"
+  [store]
+  (d/list-dbis store))
diff --git a/src/source/services/providers.clj b/src/source/services/providers.clj
new file mode 100644
index 00000000..7bafe60c
--- /dev/null
+++ b/src/source/services/providers.clj
@@ -0,0 +1,25 @@
+(ns source.services.providers
+  (:require [source.db.interface :as db]
+            [source.datastore.interface :as store]))
+
+(defn insert-provider! [ds provider]
+  (->> {:tname :providers
+        :data provider}
+       (db/insert! ds)))
+
+(defn providers
+  ([ds] (providers ds {}))
+  ([ds opts]
+   (->> {:tname :providers}
+        (merge opts)
+        (db/find ds))))
+
+(defn provider [ds {:keys [id where] :as opts}]
+  (->> {:tname :providers
+        :where (if (some? id)
+                 [:= :id id]
+                 where)
+        :ret :1}
+       (merge opts)
+       (db/find ds)))
+
diff --git a/src/source/services/xml_schemas.clj b/src/source/services/xml_schemas.clj
index 77742888..fcb450b3 100644
--- a/src/source/services/xml_schemas.clj
+++ b/src/source/services/xml_schemas.clj
@@ -6,10 +6,42 @@
   [ds {:keys [tname]}]
   (store/get-all ds {:tname tname}))
 
+(defn add-output-schema!
+  [store {:keys [schema]}]
+  (let [id (-> (store/entries store {:tname :output-schemas})
+               inc)]
+    (store/insert! store {:tname :selection-schemas
+                          :data [id schema]})))
+
 (defn add-selection-schema!
   [store db {:keys [schema record]}]
   (let [db-result (db/insert! db {:tname :selection-schemas
                                   :data record})]
-    (println db-result)
     (store/insert! store {:tname :selection-schemas
                           :data [(:id db-result) schema]})))
+
+(defn selection-schemas
+  ([ds] (selection-schemas ds {}))
+  ([ds opts]
+   (->> {:tname :selection-schemas}
+        (merge opts)
+        (db/find ds))))
+
+(defn selection-schema [ds {:keys [id where] :as opts}]
+  (->> {:tname :selection-schemas
+        :where (if (some? id)
+                 [:= :id id]
+                 where)
+        :ret :1}
+       (merge opts)
+       (db/find ds)))
+
+(defn output-schemas
+  [store]
+  (store/get-all store {:tname :output-schemas}))
+
+(defn output-schema
+  [store id]
+  (store/find store {:tname :output-schemas
+                     :key id}))
+

From e94eef33a9d1d7eeafea2a25b3c9c8e0de21dcf4 Mon Sep 17 00:00:00 2001
From: Kaidan Theron <kaidan13th@gmail.com>
Date: Mon, 30 Jun 2025 10:52:52 +0200
Subject: [PATCH 005/391] add services for getting selection-schema(s)

---
 src/source/routes/selection_schema.clj  | 7 ++++++-
 src/source/routes/selection_schemas.clj | 7 +++++++
 src/source/services/interface.clj       | 5 ++++-
 3 files changed, 17 insertions(+), 2 deletions(-)
 create mode 100644 src/source/routes/selection_schemas.clj

diff --git a/src/source/routes/selection_schema.clj b/src/source/routes/selection_schema.clj
index c5bc7bf7..10dfbc9d 100644
--- a/src/source/routes/selection_schema.clj
+++ b/src/source/routes/selection_schema.clj
@@ -3,6 +3,11 @@
             [ring.util.response :as res]))
 
 (defn post [{:keys [store ds body] :as _request}]
-  (let [{:keys [schema record] :as opts} body]
+  (let [{:keys [_schema _record] :as opts} body]
     (-> (services/add-selection-schema! store ds opts)
         (res/response))))
+
+(defn get [{:keys [ds path-params] :as _request}]
+  (->> path-params
+       (services/selection-schema ds)
+       (res/response)))
diff --git a/src/source/routes/selection_schemas.clj b/src/source/routes/selection_schemas.clj
new file mode 100644
index 00000000..8ed6540e
--- /dev/null
+++ b/src/source/routes/selection_schemas.clj
@@ -0,0 +1,7 @@
+(ns source.routes.selection-schemas
+  (:require [source.services.interface :as services]
+            [ring.util.response :as res]))
+
+(defn get [{:keys [ds] :as _request}]
+  (-> (services/selection-schemas ds)
+      (res/response)))
diff --git a/src/source/services/interface.clj b/src/source/services/interface.clj
index e0f33479..b27318d2 100644
--- a/src/source/services/interface.clj
+++ b/src/source/services/interface.clj
@@ -23,9 +23,12 @@
 (defn register [ds user]
   (auth/register ds user))
 
-(defn add-selection-schema! [store db {:keys [schema record] :as opts}]
+(defn add-selection-schema! [store db {:keys [_schema _record] :as opts}]
   (xml/add-selection-schema! store db opts))
 
+(defn selection-schema [ds {:keys [_id] :as opts}]
+  (xml/selection-schema ds opts))
+
 (comment
   (users (db/ds :master))
   (user (db/ds :master) {:id 2})

From a971979c1c43e67495f825f9fd6ce0a1302b55e1 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 9 Jul 2025 13:56:45 +0200
Subject: [PATCH 006/391] implemented swagger docs for routes

---
 deps.edn                     |   7 +-
 src/source/routes/reitit.clj | 218 +++++++++++++++++++++++++++++++----
 2 files changed, 201 insertions(+), 24 deletions(-)

diff --git a/deps.edn b/deps.edn
index 5cd12548..bd941b38 100644
--- a/deps.edn
+++ b/deps.edn
@@ -14,7 +14,6 @@
         ring/ring-json {:mvn/version "0.5.1"}
         ring/ring-defaults {:mvn/version "0.6.0"}
         ring-cors/ring-cors {:mvn/version "0.1.13"}
-        metosin/jsonista {:mvn/version "0.3.13"}
         com.github.seancorfield/next.jdbc {:mvn/version "1.3.994"}
         org.xerial/sqlite-jdbc {:mvn/version "3.49.1.0"}
         buddy/buddy-core {:mvn/version "1.12.0-430"}
@@ -28,4 +27,8 @@
         com.kepler16/mallard {:mvn/version "3.2.1"}
         com.kepler16/mallard-sqlite-store {:mvn/version "3.2.1"}
         com.github.seancorfield/honeysql {:mvn/version "2.7.1310"}
-        metosin/reitit {:mvn/version "0.9.1"}}}
+        metosin/jsonista {:mvn/version "0.3.13"}
+        metosin/reitit {:mvn/version "0.9.1"}
+        metosin/reitit-middleware {:mvn/version "0.9.1"}
+        metosin/reitit-swagger {:mvn/version "0.9.1"}
+        metosin/reitit-swagger-ui {:mvn/version "0.9.1"}}}
diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index 7c00aa45..5e12c7a2 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -1,5 +1,11 @@
 (ns source.routes.reitit
   (:require [reitit.ring :as ring]
+            [reitit.swagger :as swagger]
+            [reitit.swagger-ui :as swagger-ui]
+            [reitit.coercion.malli]
+            [reitit.ring.malli]
+            [malli.util :as mu]
+            [muuntaja.core :as muuntaja]
             [source.middleware.interface :as mw]
             [source.db.interface :as db]
             [clojure.data.json :as json]
@@ -19,28 +25,196 @@
   (let [ds (db/ds :master)]
     (ring/ring-handler
      (ring/router
-      [["/" {:middleware [[mw/apply-generic :ds ds]]}
-        ["" (fn [_request] {:status 200 :body {:message "success"}})]
-        ["users" {:middleware [[mw/apply-auth {:required-type :admin}]]}
-         ["" {:get users/get}]
-         ["/:id" {:get user/get
-                  :patch user/patch}]]
-        ["businesses"
-         ["" {:get businesses/get
-              :post business/post}]
-         ["/:id" {:patch business/patch}]]
-        ["sectors"
-         ["" {:get sectors/get}]]
-        ["login" {:post login/post}]
-        ["register" {:post register/post}]
-        ["oauth2"
-         ["/google"
-          ["" {:get google-launch/get}]
-          ["/callback" {:get google-redirect/get}]]]
-        ["protected" {:middleware [[mw/apply-auth]]}
-         ["/authorized" {:get authorized/get}]]
-        ["admin" {:middleware [[mw/apply-auth {:required-type :admin}]]}
-         ["/add-admin" {:post admin/post}]]]]))))
+      [["/swagger.json"
+        {:get {:no-doc true
+               :swagger {:info {:title "source-api"
+                                :description "swagger docs for source api with malli and reitit-ring"}}
+               :handler (swagger/create-swagger-handler)}}]
+       ["/users" {:middleware [[mw/apply-auth {:required-type :admin}]]}
+        ["" {:get {:summary "get all users"
+                   :responses {200 {:body [:map
+                                           [:users
+                                            [:vector
+                                             [:map
+                                              [:id :int]
+                                              [:address {:optional true} :string]
+                                              [:profile-image {:optional true} :string]
+                                              [:email :string]
+                                              [:firstname {:optional true} :string]
+                                              [:lastname {:optional true} :string]
+                                              [:type [:enum "creator" "distributor" "admin"]]
+                                              [:email-verified :int]
+                                              [:onboarded :int]
+                                              [:mobile {:optional true} :string]]]]]}
+                               401 {:body [:map [:message :string]]}
+                               403 {:body [:map [:message :string]]}}
+                   :handler users/get}}]
+        ["/:id" {:get {:summary "get user by id"
+                       :parameters {:path [:map [:id {:title "id"
+                                                      :description "user id"} :int]]}
+                       :responses {200 {:body [:map
+                                               [:user
+                                                [:map
+                                                 [:id :int]
+                                                 [:address {:optional true} :string]
+                                                 [:profile-image {:optional true} :string]
+                                                 [:email :string]
+                                                 [:firstname {:optional true} :string]
+                                                 [:lastname {:optional true} :string]
+                                                 [:type [:enum "creator" "distributor" "admin"]]
+                                                 [:email-verified :int]
+                                                 [:onboarded :int]
+                                                 [:mobile {:optional true} :string]]]]}
+                                   401 {:body [:map [:message :string]]}
+                                   403 {:body [:map [:message :string]]}}
+                       :handler user/get}
+                 :patch {:summary "update user by id"
+                         :parameters {:path [:map [:id {:title "id"
+                                                        :description "user id"} :int]]
+                                      :body [:map
+                                             [:address {:optional true} :string]
+                                             [:profile-image {:optional true} :string]
+                                             [:email :string]
+                                             [:firstname {:optional true} :string]
+                                             [:lastname {:optional true} :string]
+                                             [:type [:enum "creator" "distributor" "admin"]]
+                                             [:email-verified :int]
+                                             [:onboarded :int]
+                                             [:mobile {:optional true} :string]]}
+                         :responses {200 {:body [:map
+                                                 [:user
+                                                  [:map
+                                                   [:id :int]
+                                                   [:address {:optional true} :string]
+                                                   [:profile-image {:optional true} :string]
+                                                   [:email :string]
+                                                   [:firstname {:optional true} :string]
+                                                   [:lastname {:optional true} :string]
+                                                   [:type [:enum "creator" "distributor" "admin"]]
+                                                   [:email-verified :int]
+                                                   [:onboarded :int]
+                                                   [:mobile {:optional true} :string]]]]}
+                                     401 {:body [:map [:message :string]]}
+                                     403 {:body [:map [:message :string]]}}
+                         :handler user/patch}}]]
+       ["/businesses"
+        ["" {:get {:summary "get all businesses"
+                   :parameters {:body [:map
+                                       [:name :string]
+                                       [:url {:optional true} :string]
+                                       [:linkedin {:optional true} :string]
+                                       [:twitter {:optional true} :string]]}
+                   :responses {200 {:body [:map
+                                           [:businesses
+                                            [:map
+                                             [:id :int]
+                                             [:name :string]
+                                             [:url {:optional true} :string]
+                                             [:linkedin {:optional true} :string]
+                                             [:twitter {:optional true} :string]]]]}}
+                   :handler businesses/get}
+             :post {:summary "insert a business"
+                    :parameters {:body [:map
+                                        [:name :string]
+                                        [:url {:optional true} :string]
+                                        [:linkedin {:optional true} :string]
+                                        [:twitter {:optional true} :string]]}
+                    :responses {201 {:body [:map [:message :string]]}}
+                    :handler business/post}}]
+        ["/:id" {:patch {:summary "update business by id"
+                         :parameters {:path [:map [:id {:title "id"
+                                                        :description "business id"} :int]]
+                                      :body [:map
+                                             [:name :string]
+                                             [:url {:optional true} :string]
+                                             [:linkedin {:optional true} :string]
+                                             [:twitter {:optional true} :string]]}
+                         :responses {200 {:body [:map [:message :string]]}}
+                         :handler business/patch}}]]
+       ["/sectors"
+        ["" {:get {:summary "get all sectors"
+                   :responses {200 {:body [:map
+                                           [:sectors
+                                            [:map
+                                             [:id :int]
+                                             [:name :string]]]]}}
+                   :handler sectors/get}}]]
+       ["/login" {:post {:summary "get user data and access token provided valid login credentials"
+                         :parameters {:body [:map
+                                             [:email :string]
+                                             [:password :string]]}
+                         :responses {200 {:body [:map
+                                                 [:user
+                                                  [:map
+                                                   [:id :int]
+                                                   [:address {:optional true} :string]
+                                                   [:profile-image {:optional true} :string]
+                                                   [:email :string]
+                                                   [:firstname {:optional true} :string]
+                                                   [:lastname {:optional true} :string]
+                                                   [:type [:enum "creator" "distributor" "admin"]]
+                                                   [:email-verified :int]
+                                                   [:onboarded :int]
+                                                   [:mobile {:optional true} :string]]]
+                                                 [:access-token :string]
+                                                 [:refresh-token :string]]}
+                                     401 {:body [:map [:message :string]]}}
+                         :handler login/post}}]
+       ["/register" {:post {:summary "register a new user"
+                            :parameters {:body [:map
+                                                [:email :string]
+                                                [:password :string]
+                                                [:confirm-password :string]]}
+                            :responses {200 {:body [:map
+                                                    [:user
+                                                     [:map
+                                                      [:id :int]
+                                                      [:address {:optional true} :string]
+                                                      [:profile-image {:optional true} :string]
+                                                      [:email :string]
+                                                      [:firstname {:optional true} :string]
+                                                      [:lastname {:optional true} :string]
+                                                      [:type [:enum "creator" "distributor" "admin"]]
+                                                      [:email-verified :int]
+                                                      [:onboarded :int]
+                                                      [:mobile {:optional true} :string]]]
+                                                    [:access-token :string]
+                                                    [:refresh-token :string]]}}
+                            :handler register/post}}]
+       ["/oauth2" {:no-doc true}
+        ["/google"
+         ["" {:get google-launch/get}]
+         ["/callback" {:get google-redirect/get}]]]
+       ["/protected" {:middleware [[mw/apply-auth]]}
+        ["/authorized" {:get {:summary "checks if authenticated"
+                              :responses {200 {:body [:map
+                                                      [:user
+                                                       [:map
+                                                        [:id :int]
+                                                        [:type [:enum "creator" "distributor" "admin"]]]]]}}
+                              :handler authorized/get}}]]
+       ["/admin" {:middleware [[mw/apply-auth {:required-type :admin}]]}
+        ["/add-admin" {:post {:summary "registers an admin user [admins only]"
+                              :parameters {:body [:map
+                                                  [:email :string]
+                                                  [:password :string]
+                                                  [:confirm-password :string]]}
+                              :responses {201 {:body [:map [:message :string]]}
+                                          401 {:body [:map [:message :string]]}
+                                          403 {:body [:map [:message :string]]}}
+                              :handler admin/post}}]]]
+
+      {:data {:coercion (reitit.coercion.malli/create
+                         {:error-keys #{#_:type :coercion :in :schema :value :errors :humanized #_:transformed}
+                          :compile mu/closed-schema
+                          :strip-extra-keys true
+                          :default-values true
+                          :options nil})
+              :muuntaja muuntaja/instance
+              :middleware [[mw/apply-generic :ds ds]]}})
+     (ring/routes
+      (swagger-ui/create-swagger-ui-handler {:path "/"})
+      (ring/create-default-handler)))))
 
 (comment
   (require '[source.middleware.auth.util :as auth.util])

From f240dbe610039da455f110a27a0fd07c12c4253e Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 9 Jul 2025 14:14:36 +0200
Subject: [PATCH 007/391] added route groups for protected and admin-only
 routes

---
 src/source/routes/reitit.clj | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index 5e12c7a2..579cc265 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -28,9 +28,12 @@
       [["/swagger.json"
         {:get {:no-doc true
                :swagger {:info {:title "source-api"
-                                :description "swagger docs for source api with malli and reitit-ring"}}
+                                :description "swagger docs for source api with malli and reitit-ring"}
+                         :securityDefinitions {"auth" {:type :bearer}}}
                :handler (swagger/create-swagger-handler)}}]
-       ["/users" {:middleware [[mw/apply-auth {:required-type :admin}]]}
+       ["/users" {:middleware [[mw/apply-auth {:required-type :admin}]]
+                  :tags #{"admin-only"}
+                  :swagger {:security [{"auth" []}]}}
         ["" {:get {:summary "get all users"
                    :responses {200 {:body [:map
                                            [:users
@@ -185,7 +188,9 @@
         ["/google"
          ["" {:get google-launch/get}]
          ["/callback" {:get google-redirect/get}]]]
-       ["/protected" {:middleware [[mw/apply-auth]]}
+       ["/protected" {:middleware [[mw/apply-auth]]
+                      :tags #{"protected"}
+                      :swagger {:security [{"auth" []}]}}
         ["/authorized" {:get {:summary "checks if authenticated"
                               :responses {200 {:body [:map
                                                       [:user
@@ -193,7 +198,9 @@
                                                         [:id :int]
                                                         [:type [:enum "creator" "distributor" "admin"]]]]]}}
                               :handler authorized/get}}]]
-       ["/admin" {:middleware [[mw/apply-auth {:required-type :admin}]]}
+       ["/admin" {:middleware [[mw/apply-auth {:required-type :admin}]]
+                  :tags #{"admin-only"}
+                  :swagger {:security [{"auth" []}]}}
         ["/add-admin" {:post {:summary "registers an admin user [admins only]"
                               :parameters {:body [:map
                                                   [:email :string]

From c01d118185395bc20700b8731236a569d28910fa Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 9 Jul 2025 14:48:11 +0200
Subject: [PATCH 008/391] fixed non-required fields showing up as required

---
 src/source/routes/reitit.clj | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index 579cc265..58b6046c 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -46,8 +46,8 @@
                                               [:firstname {:optional true} :string]
                                               [:lastname {:optional true} :string]
                                               [:type [:enum "creator" "distributor" "admin"]]
-                                              [:email-verified :int]
-                                              [:onboarded :int]
+                                              [:email-verified {:optional true} :int]
+                                              [:onboarded {:optional true} :int]
                                               [:mobile {:optional true} :string]]]]]}
                                401 {:body [:map [:message :string]]}
                                403 {:body [:map [:message :string]]}}
@@ -65,8 +65,8 @@
                                                  [:firstname {:optional true} :string]
                                                  [:lastname {:optional true} :string]
                                                  [:type [:enum "creator" "distributor" "admin"]]
-                                                 [:email-verified :int]
-                                                 [:onboarded :int]
+                                                 [:email-verified {:optional true} :int]
+                                                 [:onboarded {:optional true} :int]
                                                  [:mobile {:optional true} :string]]]]}
                                    401 {:body [:map [:message :string]]}
                                    403 {:body [:map [:message :string]]}}
@@ -81,8 +81,8 @@
                                              [:firstname {:optional true} :string]
                                              [:lastname {:optional true} :string]
                                              [:type [:enum "creator" "distributor" "admin"]]
-                                             [:email-verified :int]
-                                             [:onboarded :int]
+                                             [:email-verified {:optional true} :int]
+                                             [:onboarded {:optional true} :int]
                                              [:mobile {:optional true} :string]]}
                          :responses {200 {:body [:map
                                                  [:user
@@ -94,8 +94,8 @@
                                                    [:firstname {:optional true} :string]
                                                    [:lastname {:optional true} :string]
                                                    [:type [:enum "creator" "distributor" "admin"]]
-                                                   [:email-verified :int]
-                                                   [:onboarded :int]
+                                                   [:email-verified {:optional true} :int]
+                                                   [:onboarded {:optional true} :int]
                                                    [:mobile {:optional true} :string]]]]}
                                      401 {:body [:map [:message :string]]}
                                      403 {:body [:map [:message :string]]}}
@@ -156,8 +156,8 @@
                                                    [:firstname {:optional true} :string]
                                                    [:lastname {:optional true} :string]
                                                    [:type [:enum "creator" "distributor" "admin"]]
-                                                   [:email-verified :int]
-                                                   [:onboarded :int]
+                                                   [:email-verified {:optional true} :int]
+                                                   [:onboarded {:optional true} :int]
                                                    [:mobile {:optional true} :string]]]
                                                  [:access-token :string]
                                                  [:refresh-token :string]]}
@@ -178,8 +178,8 @@
                                                       [:firstname {:optional true} :string]
                                                       [:lastname {:optional true} :string]
                                                       [:type [:enum "creator" "distributor" "admin"]]
-                                                      [:email-verified :int]
-                                                      [:onboarded :int]
+                                                      [:email-verified {:optional true} :int]
+                                                      [:onboarded {:optional true} :int]
                                                       [:mobile {:optional true} :string]]]
                                                     [:access-token :string]
                                                     [:refresh-token :string]]}}

From fd5cd452d676fbfc94f634fbaae46d5ff61ffe28 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 9 Jul 2025 14:56:08 +0200
Subject: [PATCH 009/391] fixed security definition

---
 src/source/routes/reitit.clj | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index 58b6046c..64fe34da 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -29,7 +29,9 @@
         {:get {:no-doc true
                :swagger {:info {:title "source-api"
                                 :description "swagger docs for source api with malli and reitit-ring"}
-                         :securityDefinitions {"auth" {:type :bearer}}}
+                         :securityDefinitions {"auth" {:type :apiKey
+                                                       :in :header
+                                                       :name "Authorization"}}}
                :handler (swagger/create-swagger-handler)}}]
        ["/users" {:middleware [[mw/apply-auth {:required-type :admin}]]
                   :tags #{"admin-only"}

From 7b58c1d192137816210450b472e2b34cdbc9c944 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 9 Jul 2025 15:07:26 +0200
Subject: [PATCH 010/391] made new groups

---
 src/source/routes/reitit.clj | 96 +++++++++++++++++++-----------------
 1 file changed, 50 insertions(+), 46 deletions(-)

diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index 64fe34da..9c053823 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -34,7 +34,7 @@
                                                        :name "Authorization"}}}
                :handler (swagger/create-swagger-handler)}}]
        ["/users" {:middleware [[mw/apply-auth {:required-type :admin}]]
-                  :tags #{"admin-only"}
+                  :tags #{"users"}
                   :swagger {:security [{"auth" []}]}}
         ["" {:get {:summary "get all users"
                    :responses {200 {:body [:map
@@ -102,7 +102,9 @@
                                      401 {:body [:map [:message :string]]}
                                      403 {:body [:map [:message :string]]}}
                          :handler user/patch}}]]
-       ["/businesses"
+       ["/businesses" {:middleware [[mw/apply-auth {:required-type :admin}]]
+                       :tags #{"businesses"}
+                       :swagger {:security [{"auth" []}]}}
         ["" {:get {:summary "get all businesses"
                    :parameters {:body [:map
                                        [:name :string]
@@ -136,7 +138,7 @@
                                              [:twitter {:optional true} :string]]}
                          :responses {200 {:body [:map [:message :string]]}}
                          :handler business/patch}}]]
-       ["/sectors"
+       ["/sectors" {:tags #{"sectors"}}
         ["" {:get {:summary "get all sectors"
                    :responses {200 {:body [:map
                                            [:sectors
@@ -144,48 +146,50 @@
                                              [:id :int]
                                              [:name :string]]]]}}
                    :handler sectors/get}}]]
-       ["/login" {:post {:summary "get user data and access token provided valid login credentials"
-                         :parameters {:body [:map
-                                             [:email :string]
-                                             [:password :string]]}
-                         :responses {200 {:body [:map
-                                                 [:user
-                                                  [:map
-                                                   [:id :int]
-                                                   [:address {:optional true} :string]
-                                                   [:profile-image {:optional true} :string]
-                                                   [:email :string]
-                                                   [:firstname {:optional true} :string]
-                                                   [:lastname {:optional true} :string]
-                                                   [:type [:enum "creator" "distributor" "admin"]]
-                                                   [:email-verified {:optional true} :int]
-                                                   [:onboarded {:optional true} :int]
-                                                   [:mobile {:optional true} :string]]]
-                                                 [:access-token :string]
-                                                 [:refresh-token :string]]}
-                                     401 {:body [:map [:message :string]]}}
-                         :handler login/post}}]
-       ["/register" {:post {:summary "register a new user"
-                            :parameters {:body [:map
-                                                [:email :string]
-                                                [:password :string]
-                                                [:confirm-password :string]]}
-                            :responses {200 {:body [:map
-                                                    [:user
-                                                     [:map
-                                                      [:id :int]
-                                                      [:address {:optional true} :string]
-                                                      [:profile-image {:optional true} :string]
-                                                      [:email :string]
-                                                      [:firstname {:optional true} :string]
-                                                      [:lastname {:optional true} :string]
-                                                      [:type [:enum "creator" "distributor" "admin"]]
-                                                      [:email-verified {:optional true} :int]
-                                                      [:onboarded {:optional true} :int]
-                                                      [:mobile {:optional true} :string]]]
-                                                    [:access-token :string]
-                                                    [:refresh-token :string]]}}
-                            :handler register/post}}]
+       ["/login" {:tags #{"auth"}}
+        {:post {:summary "get user data and access token provided valid login credentials"
+                :parameters {:body [:map
+                                    [:email :string]
+                                    [:password :string]]}
+                :responses {200 {:body [:map
+                                        [:user
+                                         [:map
+                                          [:id :int]
+                                          [:address {:optional true} :string]
+                                          [:profile-image {:optional true} :string]
+                                          [:email :string]
+                                          [:firstname {:optional true} :string]
+                                          [:lastname {:optional true} :string]
+                                          [:type [:enum "creator" "distributor" "admin"]]
+                                          [:email-verified {:optional true} :int]
+                                          [:onboarded {:optional true} :int]
+                                          [:mobile {:optional true} :string]]]
+                                        [:access-token :string]
+                                        [:refresh-token :string]]}
+                            401 {:body [:map [:message :string]]}}
+                :handler login/post}}]
+       ["/register" {:tags #{"auth"}}
+        {:post {:summary "register a new user"
+                :parameters {:body [:map
+                                    [:email :string]
+                                    [:password :string]
+                                    [:confirm-password :string]]}
+                :responses {200 {:body [:map
+                                        [:user
+                                         [:map
+                                          [:id :int]
+                                          [:address {:optional true} :string]
+                                          [:profile-image {:optional true} :string]
+                                          [:email :string]
+                                          [:firstname {:optional true} :string]
+                                          [:lastname {:optional true} :string]
+                                          [:type [:enum "creator" "distributor" "admin"]]
+                                          [:email-verified {:optional true} :int]
+                                          [:onboarded {:optional true} :int]
+                                          [:mobile {:optional true} :string]]]
+                                        [:access-token :string]
+                                        [:refresh-token :string]]}}
+                :handler register/post}}]
        ["/oauth2" {:no-doc true}
         ["/google"
          ["" {:get google-launch/get}]
@@ -201,7 +205,7 @@
                                                         [:type [:enum "creator" "distributor" "admin"]]]]]}}
                               :handler authorized/get}}]]
        ["/admin" {:middleware [[mw/apply-auth {:required-type :admin}]]
-                  :tags #{"admin-only"}
+                  :tags #{"admin"}
                   :swagger {:security [{"auth" []}]}}
         ["/add-admin" {:post {:summary "registers an admin user [admins only]"
                               :parameters {:body [:map

From a91c2ecb43270d8f2e56cb5f6293b3804e0c4b48 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 9 Jul 2025 15:21:54 +0200
Subject: [PATCH 011/391] removed unneeded dependencies

---
 deps.edn                     |  1 -
 src/source/routes/reitit.clj | 11 +----------
 2 files changed, 1 insertion(+), 11 deletions(-)

diff --git a/deps.edn b/deps.edn
index bd941b38..56750121 100644
--- a/deps.edn
+++ b/deps.edn
@@ -29,6 +29,5 @@
         com.github.seancorfield/honeysql {:mvn/version "2.7.1310"}
         metosin/jsonista {:mvn/version "0.3.13"}
         metosin/reitit {:mvn/version "0.9.1"}
-        metosin/reitit-middleware {:mvn/version "0.9.1"}
         metosin/reitit-swagger {:mvn/version "0.9.1"}
         metosin/reitit-swagger-ui {:mvn/version "0.9.1"}}}
diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index 9c053823..d200ddd4 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -4,8 +4,6 @@
             [reitit.swagger-ui :as swagger-ui]
             [reitit.coercion.malli]
             [reitit.ring.malli]
-            [malli.util :as mu]
-            [muuntaja.core :as muuntaja]
             [source.middleware.interface :as mw]
             [source.db.interface :as db]
             [clojure.data.json :as json]
@@ -217,14 +215,7 @@
                                           403 {:body [:map [:message :string]]}}
                               :handler admin/post}}]]]
 
-      {:data {:coercion (reitit.coercion.malli/create
-                         {:error-keys #{#_:type :coercion :in :schema :value :errors :humanized #_:transformed}
-                          :compile mu/closed-schema
-                          :strip-extra-keys true
-                          :default-values true
-                          :options nil})
-              :muuntaja muuntaja/instance
-              :middleware [[mw/apply-generic :ds ds]]}})
+      {:data {:middleware [[mw/apply-generic :ds ds]]}})
      (ring/routes
       (swagger-ui/create-swagger-ui-handler {:path "/"})
       (ring/create-default-handler)))))

From 97bcfa1143ec1dd2e62d6bdfb3bffbed609ac34e Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 9 Jul 2025 15:42:26 +0200
Subject: [PATCH 012/391] fixed location of tag in login and register

---
 src/source/routes/reitit.clj | 88 ++++++++++++++++++------------------
 1 file changed, 44 insertions(+), 44 deletions(-)

diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index d200ddd4..564abd1e 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -144,50 +144,50 @@
                                              [:id :int]
                                              [:name :string]]]]}}
                    :handler sectors/get}}]]
-       ["/login" {:tags #{"auth"}}
-        {:post {:summary "get user data and access token provided valid login credentials"
-                :parameters {:body [:map
-                                    [:email :string]
-                                    [:password :string]]}
-                :responses {200 {:body [:map
-                                        [:user
-                                         [:map
-                                          [:id :int]
-                                          [:address {:optional true} :string]
-                                          [:profile-image {:optional true} :string]
-                                          [:email :string]
-                                          [:firstname {:optional true} :string]
-                                          [:lastname {:optional true} :string]
-                                          [:type [:enum "creator" "distributor" "admin"]]
-                                          [:email-verified {:optional true} :int]
-                                          [:onboarded {:optional true} :int]
-                                          [:mobile {:optional true} :string]]]
-                                        [:access-token :string]
-                                        [:refresh-token :string]]}
-                            401 {:body [:map [:message :string]]}}
-                :handler login/post}}]
-       ["/register" {:tags #{"auth"}}
-        {:post {:summary "register a new user"
-                :parameters {:body [:map
-                                    [:email :string]
-                                    [:password :string]
-                                    [:confirm-password :string]]}
-                :responses {200 {:body [:map
-                                        [:user
-                                         [:map
-                                          [:id :int]
-                                          [:address {:optional true} :string]
-                                          [:profile-image {:optional true} :string]
-                                          [:email :string]
-                                          [:firstname {:optional true} :string]
-                                          [:lastname {:optional true} :string]
-                                          [:type [:enum "creator" "distributor" "admin"]]
-                                          [:email-verified {:optional true} :int]
-                                          [:onboarded {:optional true} :int]
-                                          [:mobile {:optional true} :string]]]
-                                        [:access-token :string]
-                                        [:refresh-token :string]]}}
-                :handler register/post}}]
+       ["/login" {:tags #{"auth"}
+                  :post {:summary "get user data and access token provided valid login credentials"
+                         :parameters {:body [:map
+                                             [:email :string]
+                                             [:password :string]]}
+                         :responses {200 {:body [:map
+                                                 [:user
+                                                  [:map
+                                                   [:id :int]
+                                                   [:address {:optional true} :string]
+                                                   [:profile-image {:optional true} :string]
+                                                   [:email :string]
+                                                   [:firstname {:optional true} :string]
+                                                   [:lastname {:optional true} :string]
+                                                   [:type [:enum "creator" "distributor" "admin"]]
+                                                   [:email-verified {:optional true} :int]
+                                                   [:onboarded {:optional true} :int]
+                                                   [:mobile {:optional true} :string]]]
+                                                 [:access-token :string]
+                                                 [:refresh-token :string]]}
+                                     401 {:body [:map [:message :string]]}}
+                         :handler login/post}}]
+       ["/register" {:tags #{"auth"}
+                     :post {:summary "register a new user"
+                            :parameters {:body [:map
+                                                [:email :string]
+                                                [:password :string]
+                                                [:confirm-password :string]]}
+                            :responses {200 {:body [:map
+                                                    [:user
+                                                     [:map
+                                                      [:id :int]
+                                                      [:address {:optional true} :string]
+                                                      [:profile-image {:optional true} :string]
+                                                      [:email :string]
+                                                      [:firstname {:optional true} :string]
+                                                      [:lastname {:optional true} :string]
+                                                      [:type [:enum "creator" "distributor" "admin"]]
+                                                      [:email-verified {:optional true} :int]
+                                                      [:onboarded {:optional true} :int]
+                                                      [:mobile {:optional true} :string]]]
+                                                    [:access-token :string]
+                                                    [:refresh-token :string]]}}
+                            :handler register/post}}]
        ["/oauth2" {:no-doc true}
         ["/google"
          ["" {:get google-launch/get}]

From 6682a2c1c06baba10bc4e7d77557277d440dc03c Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 9 Jul 2025 15:55:52 +0200
Subject: [PATCH 013/391] fixed single user get returning password in response

---
 src/source/routes/user.clj | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/src/source/routes/user.clj b/src/source/routes/user.clj
index a35761f8..45d5ab78 100644
--- a/src/source/routes/user.clj
+++ b/src/source/routes/user.clj
@@ -3,10 +3,9 @@
             [ring.util.response :as res]))
 
 (defn get [{:keys [ds path-params] :as _request}]
-  (->> path-params
-       (services/user ds)
-       (assoc {} :user)
-       (res/response)))
+  (let [user (->> path-params
+                  (services/user ds))]
+    (res/response (assoc {} :user (dissoc user :password)))))
 
 (defn patch [{:keys [ds body path-params] :as _request}]
   (services/update-user! ds

From 0f6b5ac5136c398b0010fd8d2a4d304cd7df2e8b Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Thu, 10 Jul 2025 16:08:02 +0200
Subject: [PATCH 014/391] refactored params and responses to be in route files

---
 src/source/routes/admin.clj      |  12 ++-
 src/source/routes/authorized.clj |   5 +
 src/source/routes/business.clj   |  20 +++-
 src/source/routes/businesses.clj |  15 +++
 src/source/routes/login.clj      |  21 ++++
 src/source/routes/register.clj   |  21 ++++
 src/source/routes/reitit.clj     | 163 ++++---------------------------
 src/source/routes/sectors.clj    |   6 ++
 src/source/routes/user.clj       |  48 +++++++++
 src/source/routes/users.clj      |  17 ++++
 10 files changed, 182 insertions(+), 146 deletions(-)

diff --git a/src/source/routes/admin.clj b/src/source/routes/admin.clj
index a598f8f7..ab41dad9 100644
--- a/src/source/routes/admin.clj
+++ b/src/source/routes/admin.clj
@@ -19,9 +19,17 @@
       :else
       (let [pw (pw/hash-password password)
             new-user (-> (assoc body
-                            :password pw
-                            :type "admin")
+                                :password pw
+                                :type "admin")
                          (dissoc :confirm-password))]
         (users/insert-user! ds {:data new-user})
         {:status 200 :body {:message "successfully created user"}}))))
 
+(def post-parameters {:body [:map
+                             [:email :string]
+                             [:password :string]
+                             [:confirm-password :string]]})
+
+(def post-responses {201 {:body [:map [:message :string]]}
+                     401 {:body [:map [:message :string]]}
+                     403 {:body [:map [:message :string]]}})
diff --git a/src/source/routes/authorized.clj b/src/source/routes/authorized.clj
index bb8658cd..5a51ba96 100644
--- a/src/source/routes/authorized.clj
+++ b/src/source/routes/authorized.clj
@@ -4,3 +4,8 @@
   {:status 200
    :body {:user user}})
 
+(def get-responses {200 {:body [:map
+                                [:user
+                                 [:map
+                                  [:id :int]
+                                  [:type [:enum "creator" "distributor" "admin"]]]]]}})
diff --git a/src/source/routes/business.clj b/src/source/routes/business.clj
index f63c3644..ff69ac6a 100644
--- a/src/source/routes/business.clj
+++ b/src/source/routes/business.clj
@@ -3,11 +3,29 @@
             [ring.util.response :as res]))
 
 (defn post [{:keys [ds body] :as _request}]
-  (businesses/insert-business! ds {:data body})
+  (businesses/insert-business! ds body)
   (res/response {:message "successfully added business"}))
 
+(def post-parameters {:body [:map
+                             [:name :string]
+                             [:url {:optional true} :string]
+                             [:linkedin {:optional true} :string]
+                             [:twitter {:optional true} :string]]})
+
+(def post-responses {201 {:body [:map [:message :string]]}})
+
 (defn patch [{:keys [ds body path-params] :as _request}]
   (businesses/update-business! ds {:id (:id path-params)
                                    :values body})
   (res/response {:message "successfully updated business"}))
 
+(def patch-parameters {:path [:map [:id {:title "id"
+                                         :description "business id"} :int]]
+                       :body [:map
+                              [:name :string]
+                              [:url {:optional true} :string]
+                              [:linkedin {:optional true} :string]
+                              [:twitter {:optional true} :string]]})
+
+(def patch-responses {200 {:body [:map [:message :string]]}})
+
diff --git a/src/source/routes/businesses.clj b/src/source/routes/businesses.clj
index d502ff20..0b99ccac 100644
--- a/src/source/routes/businesses.clj
+++ b/src/source/routes/businesses.clj
@@ -5,6 +5,21 @@
 (defn get [{:keys [ds] :as _request}]
   (res/response {:businesses (businesses/businesses ds)}))
 
+(def get-parameters {:body [:map
+                            [:name :string]
+                            [:url {:optional true} :string]
+                            [:linkedin {:optional true} :string]
+                            [:twitter {:optional true} :string]]})
+
+(def get-responses {200 {:body [:map
+                                [:businesses
+                                 [:map
+                                  [:id :int]
+                                  [:name :string]
+                                  [:url {:optional true} :string]
+                                  [:linkedin {:optional true} :string]
+                                  [:twitter {:optional true} :string]]]]}})
+
 (comment
   (require '[source.db.util :as db.util])
   (get {:ds (db.util/conn)})
diff --git a/src/source/routes/login.clj b/src/source/routes/login.clj
index 7bc08fee..ae7bbc92 100644
--- a/src/source/routes/login.clj
+++ b/src/source/routes/login.clj
@@ -14,6 +14,27 @@
 
       (res/response (auth/login ds {:user user})))))
 
+(def post-parameters {:body [:map
+                             [:email :string]
+                             [:password :string]]})
+
+(def post-responses {200 {:body [:map
+                                 [:user
+                                  [:map
+                                   [:id :int]
+                                   [:address {:optional true} :string]
+                                   [:profile-image {:optional true} :string]
+                                   [:email :string]
+                                   [:firstname {:optional true} :string]
+                                   [:lastname {:optional true} :string]
+                                   [:type [:enum "creator" "distributor" "admin"]]
+                                   [:email-verified {:optional true} :int]
+                                   [:onboarded {:optional true} :int]
+                                   [:mobile {:optional true} :string]]]
+                                 [:access-token :string]
+                                 [:refresh-token :string]]}
+                     401 {:body [:map [:message :string]]}})
+
 (comment
   (require '[source.db.interface :as db])
   (post {:ds (db/ds :master) :body {:email "toast@toast.com" :password "poop"}})
diff --git a/src/source/routes/register.clj b/src/source/routes/register.clj
index dacb5532..8d60404b 100644
--- a/src/source/routes/register.clj
+++ b/src/source/routes/register.clj
@@ -17,6 +17,27 @@
       (-> (services/register ds body)
           (res/response)))))
 
+(def post-parameters {:body [:map
+                             [:email :string]
+                             [:password :string]
+                             [:confirm-password :string]]})
+
+(def post-responses {200 {:body [:map
+                                 [:user
+                                  [:map
+                                   [:id :int]
+                                   [:address {:optional true} :string]
+                                   [:profile-image {:optional true} :string]
+                                   [:email :string]
+                                   [:firstname {:optional true} :string]
+                                   [:lastname {:optional true} :string]
+                                   [:type [:enum "creator" "distributor" "admin"]]
+                                   [:email-verified {:optional true} :int]
+                                   [:onboarded {:optional true} :int]
+                                   [:mobile {:optional true} :string]]]
+                                 [:access-token :string]
+                                 [:refresh-token :string]]}})
+
 (comment
   (require '[source.db.interface :as db])
   (post {:ds (db/ds :master) :body {:email "test@test.com"
diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index 564abd1e..800540b2 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -35,158 +35,44 @@
                   :tags #{"users"}
                   :swagger {:security [{"auth" []}]}}
         ["" {:get {:summary "get all users"
-                   :responses {200 {:body [:map
-                                           [:users
-                                            [:vector
-                                             [:map
-                                              [:id :int]
-                                              [:address {:optional true} :string]
-                                              [:profile-image {:optional true} :string]
-                                              [:email :string]
-                                              [:firstname {:optional true} :string]
-                                              [:lastname {:optional true} :string]
-                                              [:type [:enum "creator" "distributor" "admin"]]
-                                              [:email-verified {:optional true} :int]
-                                              [:onboarded {:optional true} :int]
-                                              [:mobile {:optional true} :string]]]]]}
-                               401 {:body [:map [:message :string]]}
-                               403 {:body [:map [:message :string]]}}
+                   :responses users/get-responses
                    :handler users/get}}]
         ["/:id" {:get {:summary "get user by id"
-                       :parameters {:path [:map [:id {:title "id"
-                                                      :description "user id"} :int]]}
-                       :responses {200 {:body [:map
-                                               [:user
-                                                [:map
-                                                 [:id :int]
-                                                 [:address {:optional true} :string]
-                                                 [:profile-image {:optional true} :string]
-                                                 [:email :string]
-                                                 [:firstname {:optional true} :string]
-                                                 [:lastname {:optional true} :string]
-                                                 [:type [:enum "creator" "distributor" "admin"]]
-                                                 [:email-verified {:optional true} :int]
-                                                 [:onboarded {:optional true} :int]
-                                                 [:mobile {:optional true} :string]]]]}
-                                   401 {:body [:map [:message :string]]}
-                                   403 {:body [:map [:message :string]]}}
+                       :parameters user/get-parameters
+                       :responses user/get-responses
                        :handler user/get}
                  :patch {:summary "update user by id"
-                         :parameters {:path [:map [:id {:title "id"
-                                                        :description "user id"} :int]]
-                                      :body [:map
-                                             [:address {:optional true} :string]
-                                             [:profile-image {:optional true} :string]
-                                             [:email :string]
-                                             [:firstname {:optional true} :string]
-                                             [:lastname {:optional true} :string]
-                                             [:type [:enum "creator" "distributor" "admin"]]
-                                             [:email-verified {:optional true} :int]
-                                             [:onboarded {:optional true} :int]
-                                             [:mobile {:optional true} :string]]}
-                         :responses {200 {:body [:map
-                                                 [:user
-                                                  [:map
-                                                   [:id :int]
-                                                   [:address {:optional true} :string]
-                                                   [:profile-image {:optional true} :string]
-                                                   [:email :string]
-                                                   [:firstname {:optional true} :string]
-                                                   [:lastname {:optional true} :string]
-                                                   [:type [:enum "creator" "distributor" "admin"]]
-                                                   [:email-verified {:optional true} :int]
-                                                   [:onboarded {:optional true} :int]
-                                                   [:mobile {:optional true} :string]]]]}
-                                     401 {:body [:map [:message :string]]}
-                                     403 {:body [:map [:message :string]]}}
+                         :parameters user/patch-parameters
+                         :responses user/patch-responses
                          :handler user/patch}}]]
        ["/businesses" {:middleware [[mw/apply-auth {:required-type :admin}]]
                        :tags #{"businesses"}
                        :swagger {:security [{"auth" []}]}}
         ["" {:get {:summary "get all businesses"
-                   :parameters {:body [:map
-                                       [:name :string]
-                                       [:url {:optional true} :string]
-                                       [:linkedin {:optional true} :string]
-                                       [:twitter {:optional true} :string]]}
-                   :responses {200 {:body [:map
-                                           [:businesses
-                                            [:map
-                                             [:id :int]
-                                             [:name :string]
-                                             [:url {:optional true} :string]
-                                             [:linkedin {:optional true} :string]
-                                             [:twitter {:optional true} :string]]]]}}
+                   :parameters businesses/get-parameters
+                   :responses businesses/get-responses
                    :handler businesses/get}
              :post {:summary "insert a business"
-                    :parameters {:body [:map
-                                        [:name :string]
-                                        [:url {:optional true} :string]
-                                        [:linkedin {:optional true} :string]
-                                        [:twitter {:optional true} :string]]}
-                    :responses {201 {:body [:map [:message :string]]}}
+                    :parameters business/post-parameters
+                    :responses business/post-responses
                     :handler business/post}}]
         ["/:id" {:patch {:summary "update business by id"
-                         :parameters {:path [:map [:id {:title "id"
-                                                        :description "business id"} :int]]
-                                      :body [:map
-                                             [:name :string]
-                                             [:url {:optional true} :string]
-                                             [:linkedin {:optional true} :string]
-                                             [:twitter {:optional true} :string]]}
-                         :responses {200 {:body [:map [:message :string]]}}
+                         :parameters business/patch-parameters
+                         :responses business/patch-responses
                          :handler business/patch}}]]
        ["/sectors" {:tags #{"sectors"}}
         ["" {:get {:summary "get all sectors"
-                   :responses {200 {:body [:map
-                                           [:sectors
-                                            [:map
-                                             [:id :int]
-                                             [:name :string]]]]}}
+                   :responses sectors/get-responses
                    :handler sectors/get}}]]
        ["/login" {:tags #{"auth"}
                   :post {:summary "get user data and access token provided valid login credentials"
-                         :parameters {:body [:map
-                                             [:email :string]
-                                             [:password :string]]}
-                         :responses {200 {:body [:map
-                                                 [:user
-                                                  [:map
-                                                   [:id :int]
-                                                   [:address {:optional true} :string]
-                                                   [:profile-image {:optional true} :string]
-                                                   [:email :string]
-                                                   [:firstname {:optional true} :string]
-                                                   [:lastname {:optional true} :string]
-                                                   [:type [:enum "creator" "distributor" "admin"]]
-                                                   [:email-verified {:optional true} :int]
-                                                   [:onboarded {:optional true} :int]
-                                                   [:mobile {:optional true} :string]]]
-                                                 [:access-token :string]
-                                                 [:refresh-token :string]]}
-                                     401 {:body [:map [:message :string]]}}
+                         :parameters login/post-parameters
+                         :responses login/post-responses
                          :handler login/post}}]
        ["/register" {:tags #{"auth"}
                      :post {:summary "register a new user"
-                            :parameters {:body [:map
-                                                [:email :string]
-                                                [:password :string]
-                                                [:confirm-password :string]]}
-                            :responses {200 {:body [:map
-                                                    [:user
-                                                     [:map
-                                                      [:id :int]
-                                                      [:address {:optional true} :string]
-                                                      [:profile-image {:optional true} :string]
-                                                      [:email :string]
-                                                      [:firstname {:optional true} :string]
-                                                      [:lastname {:optional true} :string]
-                                                      [:type [:enum "creator" "distributor" "admin"]]
-                                                      [:email-verified {:optional true} :int]
-                                                      [:onboarded {:optional true} :int]
-                                                      [:mobile {:optional true} :string]]]
-                                                    [:access-token :string]
-                                                    [:refresh-token :string]]}}
+                            :parameters register/post-parameters
+                            :responses register/post-responses
                             :handler register/post}}]
        ["/oauth2" {:no-doc true}
         ["/google"
@@ -196,23 +82,14 @@
                       :tags #{"protected"}
                       :swagger {:security [{"auth" []}]}}
         ["/authorized" {:get {:summary "checks if authenticated"
-                              :responses {200 {:body [:map
-                                                      [:user
-                                                       [:map
-                                                        [:id :int]
-                                                        [:type [:enum "creator" "distributor" "admin"]]]]]}}
+                              :responses authorized/get-responses
                               :handler authorized/get}}]]
        ["/admin" {:middleware [[mw/apply-auth {:required-type :admin}]]
                   :tags #{"admin"}
                   :swagger {:security [{"auth" []}]}}
-        ["/add-admin" {:post {:summary "registers an admin user [admins only]"
-                              :parameters {:body [:map
-                                                  [:email :string]
-                                                  [:password :string]
-                                                  [:confirm-password :string]]}
-                              :responses {201 {:body [:map [:message :string]]}
-                                          401 {:body [:map [:message :string]]}
-                                          403 {:body [:map [:message :string]]}}
+        ["/add-admin" {:post {:summary "registers an admin user"
+                              :parameters admin/post-parameters
+                              :responses admin/post-responses
                               :handler admin/post}}]]]
 
       {:data {:middleware [[mw/apply-generic :ds ds]]}})
diff --git a/src/source/routes/sectors.clj b/src/source/routes/sectors.clj
index f6aaf7ff..925c629c 100644
--- a/src/source/routes/sectors.clj
+++ b/src/source/routes/sectors.clj
@@ -5,6 +5,12 @@
 (defn get [{:keys [ds] :as _request}]
   (res/response {:sectors (sectors/sectors ds)}))
 
+(def get-responses {200 {:body [:map
+                                [:sectors
+                                 [:map
+                                  [:id :int]
+                                  [:name :string]]]]}})
+
 (comment
   (require '[source.db.util :as db.util])
   (get {:ds (db.util/conn)})
diff --git a/src/source/routes/user.clj b/src/source/routes/user.clj
index 45d5ab78..475de201 100644
--- a/src/source/routes/user.clj
+++ b/src/source/routes/user.clj
@@ -7,12 +7,60 @@
                   (services/user ds))]
     (res/response (assoc {} :user (dissoc user :password)))))
 
+(def get-parameters {:path [:map [:id {:title "id"
+                                       :description "user id"} :int]]})
+
+(def get-responses {200 {:body [:map
+                                [:user
+                                 [:map
+                                  [:id :int]
+                                  [:address {:optional true} :string]
+                                  [:profile-image {:optional true} :string]
+                                  [:email :string]
+                                  [:firstname {:optional true} :string]
+                                  [:lastname {:optional true} :string]
+                                  [:type [:enum "creator" "distributor" "admin"]]
+                                  [:email-verified {:optional true} :int]
+                                  [:onboarded {:optional true} :int]
+                                  [:mobile {:optional true} :string]]]]}
+                    401 {:body [:map [:message :string]]}
+                    403 {:body [:map [:message :string]]}})
+
 (defn patch [{:keys [ds body path-params] :as _request}]
   (services/update-user! ds
                          {:id (:id path-params)
                           :values body})
   (res/response {:message "successfully updated user"}))
 
+(def patch-parameters {:path [:map [:id {:title "id"
+                                         :description "user id"} :int]]
+                       :body [:map
+                              [:address {:optional true} :string]
+                              [:profile-image {:optional true} :string]
+                              [:email :string]
+                              [:firstname {:optional true} :string]
+                              [:lastname {:optional true} :string]
+                              [:type [:enum "creator" "distributor" "admin"]]
+                              [:email-verified {:optional true} :int]
+                              [:onboarded {:optional true} :int]
+                              [:mobile {:optional true} :string]]})
+
+(def patch-responses {200 {:body [:map
+                                  [:user
+                                   [:map
+                                    [:id :int]
+                                    [:address {:optional true} :string]
+                                    [:profile-image {:optional true} :string]
+                                    [:email :string]
+                                    [:firstname {:optional true} :string]
+                                    [:lastname {:optional true} :string]
+                                    [:type [:enum "creator" "distributor" "admin"]]
+                                    [:email-verified {:optional true} :int]
+                                    [:onboarded {:optional true} :int]
+                                    [:mobile {:optional true} :string]]]]}
+                      401 {:body [:map [:message :string]]}
+                      403 {:body [:map [:message :string]]}})
+
 (comment
   (require '[source.db.interface :as db])
   (get {:ds (db/ds :master) :path-params {:id 5}})
diff --git a/src/source/routes/users.clj b/src/source/routes/users.clj
index f883bfdb..b8433042 100644
--- a/src/source/routes/users.clj
+++ b/src/source/routes/users.clj
@@ -5,6 +5,23 @@
 (defn get [{:keys [ds] :as _request}]
   (res/response {:users (users/users ds)}))
 
+(def get-responses {200 {:body [:map
+                                    [:users
+                                     [:vector
+                                      [:map
+                                       [:id :int]
+                                       [:address {:optional true} :string]
+                                       [:profile-image {:optional true} :string]
+                                       [:email :string]
+                                       [:firstname {:optional true} :string]
+                                       [:lastname {:optional true} :string]
+                                       [:type [:enum "creator" "distributor" "admin"]]
+                                       [:email-verified {:optional true} :int]
+                                       [:onboarded {:optional true} :int]
+                                       [:mobile {:optional true} :string]]]]]}
+                        401 {:body [:map [:message :string]]}
+                        403 {:body [:map [:message :string]]}})
+
 (comment
   (require '[source.db.interface :as db])
   (get {:ds (db/ds :master)})

From 3cda4f6ec89a0a8346fb4e2137549d0b4f6ce2b8 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Fri, 11 Jul 2025 13:56:57 +0200
Subject: [PATCH 015/391] added utility to fetch metadata from a function

---
 src/source/util.clj | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/src/source/util.clj b/src/source/util.clj
index 504d046d..fa783ab9 100644
--- a/src/source/util.clj
+++ b/src/source/util.clj
@@ -1,6 +1,7 @@
 (ns source.util
   (:require [buddy.core.codecs :as codecs]
-            [buddy.core.nonce :as nonce]))
+            [buddy.core.nonce :as nonce]
+            [clojure.main :refer [demunge]]))
 
 (defn content-type [request]
   (or (get-in request [:headers "Content-Type"])
@@ -25,3 +26,12 @@
   (->
    (nonce/random-bytes 8)
    (codecs/bytes->hex)))
+
+(defn metadata [func]
+  (-> (class func)
+      (print-str)
+      (demunge)
+      (symbol)
+      (find-var)
+      (meta)))
+

From 74b00f29a1d654df8ef3e8748e84cc5c31182d11 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Fri, 11 Jul 2025 13:58:04 +0200
Subject: [PATCH 016/391] implemented refactor to fetch metadata from handlers
 as schemas

---
 src/source/routes/admin.clj      |  21 ++++---
 src/source/routes/authorized.clj |  16 ++---
 src/source/routes/business.clj   |  41 +++++++------
 src/source/routes/businesses.clj |  33 +++++-----
 src/source/routes/login.clj      |  45 +++++++-------
 src/source/routes/register.clj   |  45 +++++++-------
 src/source/routes/reitit.clj     |  81 +++++++++++-------------
 src/source/routes/sectors.clj    |  16 ++---
 src/source/routes/user.clj       | 102 ++++++++++++++++---------------
 src/source/routes/users.clj      |  38 ++++++------
 10 files changed, 222 insertions(+), 216 deletions(-)

diff --git a/src/source/routes/admin.clj b/src/source/routes/admin.clj
index ab41dad9..4215ee20 100644
--- a/src/source/routes/admin.clj
+++ b/src/source/routes/admin.clj
@@ -3,7 +3,17 @@
             [source.db.util :as db.util]
             [source.password :as pw]))
 
-(defn post [{:keys [body] :as _request}]
+(defn post
+  {:summary "registers an admin user"
+   :parameters {:body [:map
+                       [:email :string]
+                       [:password :string]
+                       [:confirm-password :string]]}
+   :responses {201 {:body [:map [:message :string]]}
+               401 {:body [:map [:message :string]]}
+               403 {:body [:map [:message :string]]}}}
+
+  [{:keys [body] :as _request}]
   (let [ds (db.util/conn :master)
         user (users/user
               ds
@@ -24,12 +34,3 @@
                          (dissoc :confirm-password))]
         (users/insert-user! ds {:data new-user})
         {:status 200 :body {:message "successfully created user"}}))))
-
-(def post-parameters {:body [:map
-                             [:email :string]
-                             [:password :string]
-                             [:confirm-password :string]]})
-
-(def post-responses {201 {:body [:map [:message :string]]}
-                     401 {:body [:map [:message :string]]}
-                     403 {:body [:map [:message :string]]}})
diff --git a/src/source/routes/authorized.clj b/src/source/routes/authorized.clj
index 5a51ba96..0356a2df 100644
--- a/src/source/routes/authorized.clj
+++ b/src/source/routes/authorized.clj
@@ -1,11 +1,13 @@
 (ns source.routes.authorized)
 
-(defn get [{:keys [user] :as _request}]
+(defn get
+  {:summary "checks if authenticated"
+   :responses {200 {:body [:map
+                           [:user
+                            [:map
+                             [:id :int]
+                             [:type [:enum "creator" "distributor" "admin"]]]]]}}}
+
+  [{:keys [user] :as _request}]
   {:status 200
    :body {:user user}})
-
-(def get-responses {200 {:body [:map
-                                [:user
-                                 [:map
-                                  [:id :int]
-                                  [:type [:enum "creator" "distributor" "admin"]]]]]}})
diff --git a/src/source/routes/business.clj b/src/source/routes/business.clj
index ff69ac6a..0ea57885 100644
--- a/src/source/routes/business.clj
+++ b/src/source/routes/business.clj
@@ -2,30 +2,31 @@
   (:require [source.services.businesses :as businesses]
             [ring.util.response :as res]))
 
-(defn post [{:keys [ds body] :as _request}]
+(defn post
+  {:summary "insert a business"
+   :parameters {:body [:map
+                       [:name :string]
+                       [:url {:optional true} :string]
+                       [:linkedin {:optional true} :string]
+                       [:twitter {:optional true} :string]]}
+   :responses {201 {:body [:map [:message :string]]}}}
+
+  [{:keys [ds body] :as _request}]
   (businesses/insert-business! ds body)
   (res/response {:message "successfully added business"}))
 
-(def post-parameters {:body [:map
-                             [:name :string]
-                             [:url {:optional true} :string]
-                             [:linkedin {:optional true} :string]
-                             [:twitter {:optional true} :string]]})
-
-(def post-responses {201 {:body [:map [:message :string]]}})
+(defn patch
+  {:summary "update a business by id"
+   :parameters {:path [:map [:id {:title "id"
+                                  :description "business id"} :int]]
+                :body [:map
+                       [:name :string]
+                       [:url {:optional true} :string]
+                       [:linkedin {:optional true} :string]
+                       [:twitter {:optional true} :string]]}
+   :responses {200 {:body [:map [:message :string]]}}}
 
-(defn patch [{:keys [ds body path-params] :as _request}]
+  [{:keys [ds body path-params] :as _request}]
   (businesses/update-business! ds {:id (:id path-params)
                                    :values body})
   (res/response {:message "successfully updated business"}))
-
-(def patch-parameters {:path [:map [:id {:title "id"
-                                         :description "business id"} :int]]
-                       :body [:map
-                              [:name :string]
-                              [:url {:optional true} :string]
-                              [:linkedin {:optional true} :string]
-                              [:twitter {:optional true} :string]]})
-
-(def patch-responses {200 {:body [:map [:message :string]]}})
-
diff --git a/src/source/routes/businesses.clj b/src/source/routes/businesses.clj
index 0b99ccac..569834cc 100644
--- a/src/source/routes/businesses.clj
+++ b/src/source/routes/businesses.clj
@@ -2,23 +2,24 @@
   (:require [source.services.businesses :as businesses]
             [ring.util.response :as res]))
 
-(defn get [{:keys [ds] :as _request}]
-  (res/response {:businesses (businesses/businesses ds)}))
-
-(def get-parameters {:body [:map
-                            [:name :string]
-                            [:url {:optional true} :string]
-                            [:linkedin {:optional true} :string]
-                            [:twitter {:optional true} :string]]})
+(defn get
+  {:summary "get all businesses"
+   :parameters {:body [:map
+                       [:name :string]
+                       [:url {:optional true} :string]
+                       [:linkedin {:optional true} :string]
+                       [:twitter {:optional true} :string]]}
+   :responses {200 {:body [:map
+                           [:businesses
+                            [:map
+                             [:id :int]
+                             [:name :string]
+                             [:url {:optional true} :string]
+                             [:linkedin {:optional true} :string]
+                             [:twitter {:optional true} :string]]]]}}}
 
-(def get-responses {200 {:body [:map
-                                [:businesses
-                                 [:map
-                                  [:id :int]
-                                  [:name :string]
-                                  [:url {:optional true} :string]
-                                  [:linkedin {:optional true} :string]
-                                  [:twitter {:optional true} :string]]]]}})
+  [{:keys [ds] :as _request}]
+  (res/response {:businesses (businesses/businesses ds)}))
 
 (comment
   (require '[source.db.util :as db.util])
diff --git a/src/source/routes/login.clj b/src/source/routes/login.clj
index ae7bbc92..85dc8dc6 100644
--- a/src/source/routes/login.clj
+++ b/src/source/routes/login.clj
@@ -4,7 +4,29 @@
             [source.services.users :as users]
             [source.password :as pw]))
 
-(defn post [{:keys [ds body] :as _request}]
+(defn post
+  {:summary "get user data and access token provided valid credentials"
+   :parameters {:body [:map
+                       [:email :string]
+                       [:password :string]]}
+   :responses {200 {:body [:map
+                           [:user
+                            [:map
+                             [:id :int]
+                             [:address {:optional true} :string]
+                             [:profile-image {:optional true} :string]
+                             [:email :string]
+                             [:firstname {:optional true} :string]
+                             [:lastname {:optional true} :string]
+                             [:type [:enum "creator" "distributor" "admin"]]
+                             [:email-verified {:optional true} :int]
+                             [:onboarded {:optional true} :int]
+                             [:mobile {:optional true} :string]]]
+                           [:access-token :string]
+                           [:refresh-token :string]]}
+               401 {:body [:map [:message :string]]}}}
+
+  [{:keys [ds body] :as _request}]
   (let [{:keys [email password]} body
         user (users/user ds {:where [:= :email email]})]
     (if
@@ -14,27 +36,6 @@
 
       (res/response (auth/login ds {:user user})))))
 
-(def post-parameters {:body [:map
-                             [:email :string]
-                             [:password :string]]})
-
-(def post-responses {200 {:body [:map
-                                 [:user
-                                  [:map
-                                   [:id :int]
-                                   [:address {:optional true} :string]
-                                   [:profile-image {:optional true} :string]
-                                   [:email :string]
-                                   [:firstname {:optional true} :string]
-                                   [:lastname {:optional true} :string]
-                                   [:type [:enum "creator" "distributor" "admin"]]
-                                   [:email-verified {:optional true} :int]
-                                   [:onboarded {:optional true} :int]
-                                   [:mobile {:optional true} :string]]]
-                                 [:access-token :string]
-                                 [:refresh-token :string]]}
-                     401 {:body [:map [:message :string]]}})
-
 (comment
   (require '[source.db.interface :as db])
   (post {:ds (db/ds :master) :body {:email "toast@toast.com" :password "poop"}})
diff --git a/src/source/routes/register.clj b/src/source/routes/register.clj
index 8d60404b..50007f1b 100644
--- a/src/source/routes/register.clj
+++ b/src/source/routes/register.clj
@@ -2,7 +2,29 @@
   (:require [source.services.interface :as services]
             [ring.util.response :as res]))
 
-(defn post [{:keys [ds body] :as _request}]
+(defn post
+  {:summary "register a new user"
+   :parameters {:body [:map
+                       [:email :string]
+                       [:password :string]
+                       [:confirm-password :string]]}
+   :responses {200 {:body [:map
+                           [:user
+                            [:map
+                             [:id :int]
+                             [:address {:optional true} :string]
+                             [:profile-image {:optional true} :string]
+                             [:email :string]
+                             [:firstname {:optional true} :string]
+                             [:lastname {:optional true} :string]
+                             [:type [:enum "creator" "distributor" "admin"]]
+                             [:email-verified {:optional true} :int]
+                             [:onboarded {:optional true} :int]
+                             [:mobile {:optional true} :string]]]
+                           [:access-token :string]
+                           [:refresh-token :string]]}}}
+
+  [{:keys [ds body] :as _request}]
   ;;TODO: needs schema validation here
   (let [{:keys [email password confirm-password]} body
         existing-user (services/user ds {:where [:= :email email]})]
@@ -17,27 +39,6 @@
       (-> (services/register ds body)
           (res/response)))))
 
-(def post-parameters {:body [:map
-                             [:email :string]
-                             [:password :string]
-                             [:confirm-password :string]]})
-
-(def post-responses {200 {:body [:map
-                                 [:user
-                                  [:map
-                                   [:id :int]
-                                   [:address {:optional true} :string]
-                                   [:profile-image {:optional true} :string]
-                                   [:email :string]
-                                   [:firstname {:optional true} :string]
-                                   [:lastname {:optional true} :string]
-                                   [:type [:enum "creator" "distributor" "admin"]]
-                                   [:email-verified {:optional true} :int]
-                                   [:onboarded {:optional true} :int]
-                                   [:mobile {:optional true} :string]]]
-                                 [:access-token :string]
-                                 [:refresh-token :string]]}})
-
 (comment
   (require '[source.db.interface :as db])
   (post {:ds (db/ds :master) :body {:email "test@test.com"
diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index 800540b2..12fc8199 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -4,6 +4,7 @@
             [reitit.swagger-ui :as swagger-ui]
             [reitit.coercion.malli]
             [reitit.ring.malli]
+            [malli.util :as mu]
             [source.middleware.interface :as mw]
             [source.db.interface :as db]
             [clojure.data.json :as json]
@@ -17,7 +18,17 @@
             [source.routes.authorized :as authorized]
             [source.routes.business :as business]
             [source.routes.businesses :as businesses]
-            [source.routes.sectors :as sectors]))
+            [source.routes.sectors :as sectors]
+            [source.util :as util]))
+
+(defn route [handlers]
+  (reduce (fn [acc [k v]]
+            (let [{:keys [summary parameters responses]} (util/metadata v)]
+              (merge acc {k {:summary summary
+                             :parameters parameters
+                             :responses responses
+                             :handler v}})))
+          {} handlers))
 
 (defn create-app []
   (let [ds (db/ds :master)]
@@ -34,46 +45,21 @@
        ["/users" {:middleware [[mw/apply-auth {:required-type :admin}]]
                   :tags #{"users"}
                   :swagger {:security [{"auth" []}]}}
-        ["" {:get {:summary "get all users"
-                   :responses users/get-responses
-                   :handler users/get}}]
-        ["/:id" {:get {:summary "get user by id"
-                       :parameters user/get-parameters
-                       :responses user/get-responses
-                       :handler user/get}
-                 :patch {:summary "update user by id"
-                         :parameters user/patch-parameters
-                         :responses user/patch-responses
-                         :handler user/patch}}]]
+        ["" (route {:get users/get})]
+        ["/:id" (route {:get user/get
+                        :patch user/patch})]]
        ["/businesses" {:middleware [[mw/apply-auth {:required-type :admin}]]
                        :tags #{"businesses"}
                        :swagger {:security [{"auth" []}]}}
-        ["" {:get {:summary "get all businesses"
-                   :parameters businesses/get-parameters
-                   :responses businesses/get-responses
-                   :handler businesses/get}
-             :post {:summary "insert a business"
-                    :parameters business/post-parameters
-                    :responses business/post-responses
-                    :handler business/post}}]
-        ["/:id" {:patch {:summary "update business by id"
-                         :parameters business/patch-parameters
-                         :responses business/patch-responses
-                         :handler business/patch}}]]
+        ["" (route {:get businesses/get
+                    :post business/post})]
+        ["/:id" (route {:patch business/patch})]]
        ["/sectors" {:tags #{"sectors"}}
-        ["" {:get {:summary "get all sectors"
-                   :responses sectors/get-responses
-                   :handler sectors/get}}]]
-       ["/login" {:tags #{"auth"}
-                  :post {:summary "get user data and access token provided valid login credentials"
-                         :parameters login/post-parameters
-                         :responses login/post-responses
-                         :handler login/post}}]
-       ["/register" {:tags #{"auth"}
-                     :post {:summary "register a new user"
-                            :parameters register/post-parameters
-                            :responses register/post-responses
-                            :handler register/post}}]
+        ["" (route {:get sectors/get})]]
+       ["/login" {:tags #{"auth"}}
+        ["" (route {:post login/post})]]
+       ["/register" {:tags #{"auth"}}
+        ["" (route {:post register/post})]]
        ["/oauth2" {:no-doc true}
         ["/google"
          ["" {:get google-launch/get}]
@@ -81,18 +67,19 @@
        ["/protected" {:middleware [[mw/apply-auth]]
                       :tags #{"protected"}
                       :swagger {:security [{"auth" []}]}}
-        ["/authorized" {:get {:summary "checks if authenticated"
-                              :responses authorized/get-responses
-                              :handler authorized/get}}]]
+        ["/authorized" (route {:get authorized/get})]]
        ["/admin" {:middleware [[mw/apply-auth {:required-type :admin}]]
                   :tags #{"admin"}
                   :swagger {:security [{"auth" []}]}}
-        ["/add-admin" {:post {:summary "registers an admin user"
-                              :parameters admin/post-parameters
-                              :responses admin/post-responses
-                              :handler admin/post}}]]]
+        ["/add-admin" (route {:post admin/post})]]]
 
-      {:data {:middleware [[mw/apply-generic :ds ds]]}})
+      {:data {:coercion (reitit.coercion.malli/create
+                         {:error-keys #{#_:type :coercion :in :schema :value :errors :humanized #_:transformed}
+                          :compile mu/closed-schema
+                          :strip-extra-keys true
+                          :default-values true
+                          :options nil})
+              :middleware [[mw/apply-generic :ds ds]]}})
      (ring/routes
       (swagger-ui/create-swagger-ui-handler {:path "/"})
       (ring/create-default-handler)))))
@@ -100,6 +87,10 @@
 (comment
   (require '[source.middleware.auth.util :as auth.util])
 
+  (route {:get #'user/get
+          :patch #'user/patch})
+  (util/metadata user/get)
+
   (let [app (create-app)
         request {:uri "/users" :request-method :get}]
     (-> request
diff --git a/src/source/routes/sectors.clj b/src/source/routes/sectors.clj
index 925c629c..1e22d4b0 100644
--- a/src/source/routes/sectors.clj
+++ b/src/source/routes/sectors.clj
@@ -2,14 +2,16 @@
   (:require [source.services.sectors :as sectors]
             [ring.util.response :as res]))
 
-(defn get [{:keys [ds] :as _request}]
-  (res/response {:sectors (sectors/sectors ds)}))
+(defn get
+  {:summary "get all sectors"
+   :responses {200 {:body [:map
+                           [:sectors
+                            [:map
+                             [:id :int]
+                             [:name :string]]]]}}}
 
-(def get-responses {200 {:body [:map
-                                [:sectors
-                                 [:map
-                                  [:id :int]
-                                  [:name :string]]]]}})
+  [{:keys [ds] :as _request}]
+  (res/response {:sectors (sectors/sectors ds)}))
 
 (comment
   (require '[source.db.util :as db.util])
diff --git a/src/source/routes/user.clj b/src/source/routes/user.clj
index 475de201..c40e4d12 100644
--- a/src/source/routes/user.clj
+++ b/src/source/routes/user.clj
@@ -2,65 +2,67 @@
   (:require [source.services.interface :as services]
             [ring.util.response :as res]))
 
-(defn get [{:keys [ds path-params] :as _request}]
+(defn get
+  {:summary "get user by id"
+   :parameters {:path [:map [:id {:title "id"
+                                  :description "user id"} :int]]}
+   :responses {200 {:body [:map
+                           [:user
+                            [:map
+                             [:id :int]
+                             [:address {:optional true} :string]
+                             [:profile-image {:optional true} :string]
+                             [:email :string]
+                             [:firstname {:optional true} :string]
+                             [:lastname {:optional true} :string]
+                             [:type [:enum "creator" "distributor" "admin"]]
+                             [:email-verified {:optional true} :int]
+                             [:onboarded {:optional true} :int]
+                             [:mobile {:optional true} :string]]]]}
+               401 {:body [:map [:message :string]]}
+               403 {:body [:map [:message :string]]}}}
+
+  [{:keys [ds path-params] :as _request}]
   (let [user (->> path-params
                   (services/user ds))]
     (res/response (assoc {} :user (dissoc user :password)))))
 
-(def get-parameters {:path [:map [:id {:title "id"
-                                       :description "user id"} :int]]})
-
-(def get-responses {200 {:body [:map
-                                [:user
-                                 [:map
-                                  [:id :int]
-                                  [:address {:optional true} :string]
-                                  [:profile-image {:optional true} :string]
-                                  [:email :string]
-                                  [:firstname {:optional true} :string]
-                                  [:lastname {:optional true} :string]
-                                  [:type [:enum "creator" "distributor" "admin"]]
-                                  [:email-verified {:optional true} :int]
-                                  [:onboarded {:optional true} :int]
-                                  [:mobile {:optional true} :string]]]]}
-                    401 {:body [:map [:message :string]]}
-                    403 {:body [:map [:message :string]]}})
+(defn patch
+  {:summary "update user by id"
+   :parameters {:path [:map [:id {:title "id"
+                                  :description "user id"} :int]]
+                :body [:map
+                       [:address {:optional true} :string]
+                       [:profile-image {:optional true} :string]
+                       [:email :string]
+                       [:firstname {:optional true} :string]
+                       [:lastname {:optional true} :string]
+                       [:type [:enum "creator" "distributor" "admin"]]
+                       [:email-verified {:optional true} :int]
+                       [:onboarded {:optional true} :int]
+                       [:mobile {:optional true} :string]]}
+   :responses {200 {:body [:map
+                           [:user
+                            [:map
+                             [:id :int]
+                             [:address {:optional true} :string]
+                             [:profile-image {:optional true} :string]
+                             [:email :string]
+                             [:firstname {:optional true} :string]
+                             [:lastname {:optional true} :string]
+                             [:type [:enum "creator" "distributor" "admin"]]
+                             [:email-verified {:optional true} :int]
+                             [:onboarded {:optional true} :int]
+                             [:mobile {:optional true} :string]]]]}
+               401 {:body [:map [:message :string]]}
+               403 {:body [:map [:message :string]]}}}
 
-(defn patch [{:keys [ds body path-params] :as _request}]
+  [{:keys [ds body path-params] :as _request}]
   (services/update-user! ds
                          {:id (:id path-params)
                           :values body})
   (res/response {:message "successfully updated user"}))
 
-(def patch-parameters {:path [:map [:id {:title "id"
-                                         :description "user id"} :int]]
-                       :body [:map
-                              [:address {:optional true} :string]
-                              [:profile-image {:optional true} :string]
-                              [:email :string]
-                              [:firstname {:optional true} :string]
-                              [:lastname {:optional true} :string]
-                              [:type [:enum "creator" "distributor" "admin"]]
-                              [:email-verified {:optional true} :int]
-                              [:onboarded {:optional true} :int]
-                              [:mobile {:optional true} :string]]})
-
-(def patch-responses {200 {:body [:map
-                                  [:user
-                                   [:map
-                                    [:id :int]
-                                    [:address {:optional true} :string]
-                                    [:profile-image {:optional true} :string]
-                                    [:email :string]
-                                    [:firstname {:optional true} :string]
-                                    [:lastname {:optional true} :string]
-                                    [:type [:enum "creator" "distributor" "admin"]]
-                                    [:email-verified {:optional true} :int]
-                                    [:onboarded {:optional true} :int]
-                                    [:mobile {:optional true} :string]]]]}
-                      401 {:body [:map [:message :string]]}
-                      403 {:body [:map [:message :string]]}})
-
 (comment
   (require '[source.db.interface :as db])
   (get {:ds (db/ds :master) :path-params {:id 5}})
@@ -68,4 +70,6 @@
           :path-params {:id 5}
           :body {:firstname "kiigan"
                  :lastname "korinzu"}})
+
+  (meta #'get)
   ())
diff --git a/src/source/routes/users.clj b/src/source/routes/users.clj
index b8433042..a641eb06 100644
--- a/src/source/routes/users.clj
+++ b/src/source/routes/users.clj
@@ -2,25 +2,27 @@
   (:require [ring.util.response :as res]
             [source.services.users :as users]))
 
-(defn get [{:keys [ds] :as _request}]
-  (res/response {:users (users/users ds)}))
+(defn get
+  {:summary "get all users"
+   :responses  {200 {:body [:map
+                            [:users
+                             [:vector
+                              [:map
+                               [:id :int]
+                               [:address {:optional true} :string]
+                               [:profile-image {:optional true} :string]
+                               [:email :string]
+                               [:firstname {:optional true} :string]
+                               [:lastname {:optional true} :string]
+                               [:type [:enum "creator" "distributor" "admin"]]
+                               [:email-verified {:optional true} :int]
+                               [:onboarded {:optional true} :int]
+                               [:mobile {:optional true} :string]]]]]}
+                401 {:body [:map [:message :string]]}
+                403 {:body [:map [:message :string]]}}}
 
-(def get-responses {200 {:body [:map
-                                    [:users
-                                     [:vector
-                                      [:map
-                                       [:id :int]
-                                       [:address {:optional true} :string]
-                                       [:profile-image {:optional true} :string]
-                                       [:email :string]
-                                       [:firstname {:optional true} :string]
-                                       [:lastname {:optional true} :string]
-                                       [:type [:enum "creator" "distributor" "admin"]]
-                                       [:email-verified {:optional true} :int]
-                                       [:onboarded {:optional true} :int]
-                                       [:mobile {:optional true} :string]]]]]}
-                        401 {:body [:map [:message :string]]}
-                        403 {:body [:map [:message :string]]}})
+  [{:keys [ds] :as _request}]
+  (res/response {:users (users/users ds)}))
 
 (comment
   (require '[source.db.interface :as db])

From dc42241cf52aaaa698a765653f40422929917b3d Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Fri, 11 Jul 2025 15:13:48 +0200
Subject: [PATCH 017/391] made get user route use thread last

---
 src/source/routes/user.clj | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/source/routes/user.clj b/src/source/routes/user.clj
index c40e4d12..5f813341 100644
--- a/src/source/routes/user.clj
+++ b/src/source/routes/user.clj
@@ -25,7 +25,9 @@
   [{:keys [ds path-params] :as _request}]
   (let [user (->> path-params
                   (services/user ds))]
-    (res/response (assoc {} :user (dissoc user :password)))))
+    (->> (dissoc user :password)
+         (assoc {} :user)
+         (res/response))))
 
 (defn patch
   {:summary "update user by id"

From 6367951cf78041bd85576654ebafb6359b0591d3 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 14 Jul 2025 13:18:25 +0200
Subject: [PATCH 018/391] updated admin to remove unnecessary db/conn call

---
 src/source/routes/admin.clj | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/src/source/routes/admin.clj b/src/source/routes/admin.clj
index 4215ee20..bcb107aa 100644
--- a/src/source/routes/admin.clj
+++ b/src/source/routes/admin.clj
@@ -1,6 +1,5 @@
 (ns source.routes.admin
   (:require [source.services.users :as users]
-            [source.db.util :as db.util]
             [source.password :as pw]))
 
 (defn post
@@ -13,9 +12,8 @@
                401 {:body [:map [:message :string]]}
                403 {:body [:map [:message :string]]}}}
 
-  [{:keys [body] :as _request}]
-  (let [ds (db.util/conn :master)
-        user (users/user
+  [{:keys [ds body] :as _request}]
+  (let [user (users/user
               ds
               {:where [:= :email (:email body)]})
         {:keys [password confirm-password]} body]

From bcb364aa36e9d9211ada88e8957bd746d946478f Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 14 Jul 2025 13:21:09 +0200
Subject: [PATCH 019/391] added intermediary step in google oauth2 flow to
 redirect to frontend

---
 src/source/routes/google_redirect.clj | 25 ++----------
 src/source/routes/google_user.clj     | 25 ++++++++++++
 src/source/routes/reitit.clj          | 58 ++++++++++++++++-----------
 3 files changed, 62 insertions(+), 46 deletions(-)
 create mode 100644 src/source/routes/google_user.clj

diff --git a/src/source/routes/google_redirect.clj b/src/source/routes/google_redirect.clj
index 873a1709..c7f4e496 100644
--- a/src/source/routes/google_redirect.clj
+++ b/src/source/routes/google_redirect.clj
@@ -1,26 +1,7 @@
 (ns source.routes.google-redirect
-  (:require [source.oauth2.google.interface :as google]
-            [source.middleware.auth.core :as auth]
-            [source.services.users :as users]
-            [source.db.util :as db.util]
-            [ring.util.response :as res]
+  (:require [ring.util.response :as res]
             [source.config :as conf]))
 
-(defn get [req]
-  (let [{:keys [uuid _uri]} (:body req)
-        email (google/google-session-user uuid (:params req))
-        ds (db.util/conn :master)
-        user (users/user ds {:where [:= :email email]})
-        user-type (get-in req [:cookies "user_type" :value])]
+(defn get [{:keys [query-string] :as req}]
+  (res/redirect (str (conf/read-value :cors-origin) "/oauth?" query-string)))
 
-    (if (some? user)
-      (let [payload (dissoc user :password)
-            {:keys [access-token]} (auth/create-session payload)]
-        (res/redirect (str (conf/read-value :cors-origin) "/api/oauth/google?token=" access-token)))
-      (do
-        (users/insert-user! ds {:data {:email email
-                                       :type user-type}})
-        (let [new-user (users/user ds {:where [:= :email email]})
-              payload (dissoc new-user :password)
-              {:keys [access-token]} (auth/create-session payload)]
-          (res/redirect (str (conf/read-value :cors-origin) "/api/oauth/google?token=" access-token)))))))
diff --git a/src/source/routes/google_user.clj b/src/source/routes/google_user.clj
new file mode 100644
index 00000000..e8342363
--- /dev/null
+++ b/src/source/routes/google_user.clj
@@ -0,0 +1,25 @@
+(ns source.routes.google-user
+  (:require [source.oauth2.google.interface :as google]
+            [source.middleware.auth.core :as auth]
+            [source.services.users :as users]
+            [source.db.util :as db.util]
+            [ring.util.response :as res]))
+
+(defn get [req]
+  (let [{:keys [uuid _uri]} (:body req)
+        email (google/google-session-user uuid (:params req))
+        ds (db.util/conn :master)
+        user (users/user ds {:where [:= :email email]})
+        user-type (get-in req [:cookies "user_type" :value])]
+
+    (if (some? user)
+      (let [payload (dissoc user :password)
+            session (auth/create-session payload)]
+        (res/response (merge {:user payload} session)))
+      (do
+        (users/insert-user! ds {:data {:email email
+                                       :type user-type}})
+        (let [new-user (users/user ds {:where [:= :email email]})
+              payload (dissoc new-user :password)
+              session (auth/create-session payload)]
+          (res/response (merge {:user payload} session)))))))
diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index 12fc8199..05cb9873 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -14,6 +14,7 @@
             [source.routes.register :as register]
             [source.routes.google-launch :as google-launch]
             [source.routes.google-redirect :as google-redirect]
+            [source.routes.google-user :as google-user]
             [source.routes.admin :as admin]
             [source.routes.authorized :as authorized]
             [source.routes.business :as business]
@@ -42,35 +43,44 @@
                                                        :in :header
                                                        :name "Authorization"}}}
                :handler (swagger/create-swagger-handler)}}]
-       ["/users" {:middleware [[mw/apply-auth {:required-type :admin}]]
-                  :tags #{"users"}
-                  :swagger {:security [{"auth" []}]}}
-        ["" (route {:get users/get})]
-        ["/:id" (route {:get user/get
-                        :patch user/patch})]]
+
+       ["/users"  {:middleware [[mw/apply-auth {:required-type :admin}]]
+                   :tags #{"users"}
+                   :swagger {:security [{"auth" []}]}}
+        [""       (route {:get users/get})]
+        ["/:id"   (route {:get user/get
+                          :patch user/patch})]]
+
        ["/businesses" {:middleware [[mw/apply-auth {:required-type :admin}]]
                        :tags #{"businesses"}
                        :swagger {:security [{"auth" []}]}}
-        ["" (route {:get businesses/get
-                    :post business/post})]
-        ["/:id" (route {:patch business/patch})]]
-       ["/sectors" {:tags #{"sectors"}}
-        ["" (route {:get sectors/get})]]
-       ["/login" {:tags #{"auth"}}
-        ["" (route {:post login/post})]]
+        [""           (route {:get businesses/get
+                              :post business/post})]
+        ["/:id"       (route {:patch business/patch})]]
+
+       ["/sectors"  {:tags #{"sectors"}}
+        [""         (route {:get sectors/get})]]
+
+       ["/login"  {:tags #{"auth"}}
+        [""       (route {:post login/post})]]
+
        ["/register" {:tags #{"auth"}}
-        ["" (route {:post register/post})]]
-       ["/oauth2" {:no-doc true}
+        [""         (route {:post register/post})]]
+
+       ["/oauth2"     {:no-doc true}
         ["/google"
-         ["" {:get google-launch/get}]
-         ["/callback" {:get google-redirect/get}]]]
-       ["/protected" {:middleware [[mw/apply-auth]]
-                      :tags #{"protected"}
-                      :swagger {:security [{"auth" []}]}}
-        ["/authorized" (route {:get authorized/get})]]
-       ["/admin" {:middleware [[mw/apply-auth {:required-type :admin}]]
-                  :tags #{"admin"}
-                  :swagger {:security [{"auth" []}]}}
+         [""          {:get google-launch/get}]
+         ["/callback" {:get google-redirect/get}]
+         ["/user"     {:get google-user/get}]]]
+
+       ["/protected"    {:middleware [[mw/apply-auth]]
+                         :tags #{"protected"}
+                         :swagger {:security [{"auth" []}]}}
+        ["/authorized"  (route {:get authorized/get})]]
+
+       ["/admin"      {:middleware [[mw/apply-auth {:required-type :admin}]]
+                       :tags #{"admin"}
+                       :swagger {:security [{"auth" []}]}}
         ["/add-admin" (route {:post admin/post})]]]
 
       {:data {:coercion (reitit.coercion.malli/create

From b9fd985c1d609cb60306503d0fc5fa274782b59b Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Tue, 15 Jul 2025 09:27:31 +0200
Subject: [PATCH 020/391] fixed table-names to return all

---
 src/source/db/tables.clj | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/source/db/tables.clj b/src/source/db/tables.clj
index 88b0f6e6..267c1741 100644
--- a/src/source/db/tables.clj
+++ b/src/source/db/tables.clj
@@ -46,7 +46,8 @@
   "returns all current tables in a sqlite datasource"
   [ds]
   (->> {:tname :sqlite-master
-        :where [:and [:= :type "table"] [:<> :name "sqlite_sequence"]]}
+        :where [:and [:= :type "table"] [:<> :name "sqlite_sequence"]]
+        :ret :*}
        (hon/find ds)))
 
 (defn table-name
@@ -110,11 +111,13 @@
    :source.db.master
    [:users
     :cadences
+    :sectors
     :businesses])
 
   (drop-tables!
    (db.util/conn :master)
    [:users
+    :sectors
     :cadences])
 
   (drop-all-tables! (db.util/conn :master))

From 56f5df020e69e859857235ce47b65ebe4ba8b8a5 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Tue, 15 Jul 2025 09:28:04 +0200
Subject: [PATCH 021/391] added case conversion middleware

---
 src/source/middleware/core.clj | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/src/source/middleware/core.clj b/src/source/middleware/core.clj
index 88c9986c..5f04387b 100644
--- a/src/source/middleware/core.clj
+++ b/src/source/middleware/core.clj
@@ -2,6 +2,8 @@
   (:require [source.middleware.auth.core :as auth]
             [source.middleware.content-type :as content-type]
             [source.config :as conf]
+            [camel-snake-kebab.core :as csk]
+            [camel-snake-kebab.extras :as cske]
             [ring.middleware.cors :refer [wrap-cors]]
             [ring.middleware.params :refer [wrap-params]]
             [ring.middleware.defaults :refer [wrap-defaults site-defaults]]
@@ -14,6 +16,21 @@
         (assoc :ds ds)
         (handler))))
 
+(defn wrap-request->kebab [handler]
+  (fn [request]
+    (-> request
+        (assoc :body (cske/transform-keys
+                      csk/->kebab-case
+                      (:body request)))
+        (handler))))
+
+(defn wrap-response->snake [handler]
+  (fn [request]
+    (let [response (handler request)]
+      (assoc response :body (cske/transform-keys
+                             csk/->snake_case
+                             (:body response))))))
+
 (defn apply-ds [app ds]
   (-> app
       (wrap-ds ds)))
@@ -26,7 +43,9 @@
                  :access-control-allow-methods [:get :put :post :delete])
       (wrap-params)
       (wrap-defaults (assoc site-defaults :session false :security {:anti-forgery false}))
+      (wrap-response->snake)
       (ring/wrap-json-response)
+      (wrap-request->kebab)
       (ring/wrap-json-body {:keywords? true})
       (cookies/wrap-cookies)))
 

From 262b886b288af0811bef20eda6810d022e8b2b5c Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Tue, 15 Jul 2025 09:55:43 +0200
Subject: [PATCH 022/391] fixed case conversion to only apply to keywords and
 strings

---
 src/source/middleware/core.clj | 29 ++++++++++++++++++++---------
 1 file changed, 20 insertions(+), 9 deletions(-)

diff --git a/src/source/middleware/core.clj b/src/source/middleware/core.clj
index 5f04387b..e1dc0202 100644
--- a/src/source/middleware/core.clj
+++ b/src/source/middleware/core.clj
@@ -18,18 +18,29 @@
 
 (defn wrap-request->kebab [handler]
   (fn [request]
-    (-> request
-        (assoc :body (cske/transform-keys
-                      csk/->kebab-case
-                      (:body request)))
-        (handler))))
+    (if (map? (:body request))
+      (-> request
+          (assoc :body (cske/transform-keys
+                        (fn [k]
+                          (if (or (keyword? k) (string? k))
+                            (csk/->kebab-case k)
+                            k))
+                        (:body request)))
+          (handler))
+      (handler request))))
 
 (defn wrap-response->snake [handler]
   (fn [request]
-    (let [response (handler request)]
-      (assoc response :body (cske/transform-keys
-                             csk/->snake_case
-                             (:body response))))))
+    (let [response (handler request)
+          body (:body response)]
+      (if (map? body)
+        (assoc response :body (cske/transform-keys
+                               (fn [k]
+                                 (if (or (keyword? k) (string? k))
+                                   (csk/->snake_case k)
+                                   k))
+                               (:body response)))
+        response))))
 
 (defn apply-ds [app ds]
   (-> app

From adbeab258ac8fe0a9d3e95304d9ee7f450145003 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Tue, 15 Jul 2025 12:42:05 +0200
Subject: [PATCH 023/391] addressed comments

---
 src/source/routes/google_redirect.clj |  8 ++-
 src/source/routes/google_user.clj     |  6 +-
 src/source/routes/reitit.clj          | 79 +++++++++++++--------------
 3 files changed, 46 insertions(+), 47 deletions(-)

diff --git a/src/source/routes/google_redirect.clj b/src/source/routes/google_redirect.clj
index c7f4e496..3bd82285 100644
--- a/src/source/routes/google_redirect.clj
+++ b/src/source/routes/google_redirect.clj
@@ -2,6 +2,8 @@
   (:require [ring.util.response :as res]
             [source.config :as conf]))
 
-(defn get [{:keys [query-string] :as req}]
-  (res/redirect (str (conf/read-value :cors-origin) "/oauth?" query-string)))
-
+(defn get [{:keys [query-string] :as _req}]
+  (-> :cors-origin
+      (conf/read-value)
+      (str "/oauth?" query-string)
+      (res/redirect)))
diff --git a/src/source/routes/google_user.clj b/src/source/routes/google_user.clj
index e8342363..70d6b808 100644
--- a/src/source/routes/google_user.clj
+++ b/src/source/routes/google_user.clj
@@ -2,13 +2,11 @@
   (:require [source.oauth2.google.interface :as google]
             [source.middleware.auth.core :as auth]
             [source.services.users :as users]
-            [source.db.util :as db.util]
             [ring.util.response :as res]))
 
-(defn get [req]
-  (let [{:keys [uuid _uri]} (:body req)
+(defn get [{:keys [ds body] :as req}]
+  (let [{:keys [uuid _uri]} body
         email (google/google-session-user uuid (:params req))
-        ds (db.util/conn :master)
         user (users/user ds {:where [:= :email email]})
         user-type (get-in req [:cookies "user_type" :value])]
 
diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index 05cb9873..33d95d34 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -35,53 +35,52 @@
   (let [ds (db/ds :master)]
     (ring/ring-handler
      (ring/router
-      [["/swagger.json"
-        {:get {:no-doc true
-               :swagger {:info {:title "source-api"
-                                :description "swagger docs for source api with malli and reitit-ring"}
-                         :securityDefinitions {"auth" {:type :apiKey
-                                                       :in :header
-                                                       :name "Authorization"}}}
-               :handler (swagger/create-swagger-handler)}}]
-
-       ["/users"  {:middleware [[mw/apply-auth {:required-type :admin}]]
-                   :tags #{"users"}
-                   :swagger {:security [{"auth" []}]}}
-        [""       (route {:get users/get})]
-        ["/:id"   (route {:get user/get
-                          :patch user/patch})]]
-
-       ["/businesses" {:middleware [[mw/apply-auth {:required-type :admin}]]
-                       :tags #{"businesses"}
-                       :swagger {:security [{"auth" []}]}}
-        [""           (route {:get businesses/get
-                              :post business/post})]
-        ["/:id"       (route {:patch business/patch})]]
-
-       ["/sectors"  {:tags #{"sectors"}}
-        [""         (route {:get sectors/get})]]
-
-       ["/login"  {:tags #{"auth"}}
-        [""       (route {:post login/post})]]
-
-       ["/register" {:tags #{"auth"}}
-        [""         (route {:post register/post})]]
-
-       ["/oauth2"     {:no-doc true}
+      [["/swagger.json" {:get {:no-doc true
+                               :swagger {:info {:title "source-api"
+                                                :description "swagger docs for source api with malli and reitit-ring"}
+                                         :securityDefinitions {"auth" {:type :apiKey
+                                                                       :in :header
+                                                                       :name "Authorization"}}}
+                               :handler (swagger/create-swagger-handler)}}]
+
+       ["/users"        {:middleware [[mw/apply-auth {:required-type :admin}]]
+                         :tags #{"users"}
+                         :swagger {:security [{"auth" []}]}}
+        [""             (route {:get users/get})]
+        ["/:id"         (route {:get user/get
+                                :patch user/patch})]]
+
+       ["/businesses"   {:middleware [[mw/apply-auth {:required-type :admin}]]
+                         :tags #{"businesses"}
+                         :swagger {:security [{"auth" []}]}}
+        [""             (route {:get businesses/get
+                                :post business/post})]
+        ["/:id"         (route {:patch business/patch})]]
+
+       ["/sectors"      {:tags #{"sectors"}}
+        [""             (route {:get sectors/get})]]
+
+       ["/login"        {:tags #{"auth"}}
+        [""             (route {:post login/post})]]
+
+       ["/register"     {:tags #{"auth"}}
+        [""             (route {:post register/post})]]
+
+       ["/oauth2"       {:no-doc true}
         ["/google"
-         [""          {:get google-launch/get}]
-         ["/callback" {:get google-redirect/get}]
-         ["/user"     {:get google-user/get}]]]
+         [""            {:get google-launch/get}]
+         ["/callback"   {:get google-redirect/get}]
+         ["/user"       {:get google-user/get}]]]
 
        ["/protected"    {:middleware [[mw/apply-auth]]
                          :tags #{"protected"}
                          :swagger {:security [{"auth" []}]}}
         ["/authorized"  (route {:get authorized/get})]]
 
-       ["/admin"      {:middleware [[mw/apply-auth {:required-type :admin}]]
-                       :tags #{"admin"}
-                       :swagger {:security [{"auth" []}]}}
-        ["/add-admin" (route {:post admin/post})]]]
+       ["/admin"        {:middleware [[mw/apply-auth {:required-type :admin}]]
+                         :tags #{"admin"}
+                         :swagger {:security [{"auth" []}]}}
+        ["/add-admin"   (route {:post admin/post})]]]
 
       {:data {:coercion (reitit.coercion.malli/create
                          {:error-keys #{#_:type :coercion :in :schema :value :errors :humanized #_:transformed}

From 81baac570aebc33da5d0991618454f1b886602d8 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Tue, 15 Jul 2025 13:33:58 +0200
Subject: [PATCH 024/391] refactored case conversion middleware

---
 src/source/middleware/core.clj | 43 ++++++++++++++--------------------
 1 file changed, 17 insertions(+), 26 deletions(-)

diff --git a/src/source/middleware/core.clj b/src/source/middleware/core.clj
index e1dc0202..033af752 100644
--- a/src/source/middleware/core.clj
+++ b/src/source/middleware/core.clj
@@ -16,31 +16,23 @@
         (assoc :ds ds)
         (handler))))
 
-(defn wrap-request->kebab [handler]
+(defn process-body [{:keys [body] :as req} t-fn]
+  (assoc req
+         :body
+         (if
+          (or (map? body) (vector? body) (seq? body))
+           (cske/transform-keys (fn [k]
+                                  (if (or (keyword? k) (string? k))
+                                    (t-fn k)
+                                    k)) body)
+           body)))
+
+(defn wrap-case-conversion [handler]
   (fn [request]
-    (if (map? (:body request))
-      (-> request
-          (assoc :body (cske/transform-keys
-                        (fn [k]
-                          (if (or (keyword? k) (string? k))
-                            (csk/->kebab-case k)
-                            k))
-                        (:body request)))
-          (handler))
-      (handler request))))
-
-(defn wrap-response->snake [handler]
-  (fn [request]
-    (let [response (handler request)
-          body (:body response)]
-      (if (map? body)
-        (assoc response :body (cske/transform-keys
-                               (fn [k]
-                                 (if (or (keyword? k) (string? k))
-                                   (csk/->snake_case k)
-                                   k))
-                               (:body response)))
-        response))))
+    (-> request
+        (process-body csk/->kebab-case-keyword)
+        (handler)
+        (process-body csk/->camelCaseKeyword))))
 
 (defn apply-ds [app ds]
   (-> app
@@ -49,14 +41,13 @@
 (defn apply-generic [app & {:keys [ds]}]
   (-> app
       (apply-ds ds)
+      (wrap-case-conversion)
       (content-type/wrap-content-type)
       (wrap-cors :access-control-allow-origin [(re-pattern (conf/read-value :cors-origin))]
                  :access-control-allow-methods [:get :put :post :delete])
       (wrap-params)
       (wrap-defaults (assoc site-defaults :session false :security {:anti-forgery false}))
-      (wrap-response->snake)
       (ring/wrap-json-response)
-      (wrap-request->kebab)
       (ring/wrap-json-body {:keywords? true})
       (cookies/wrap-cookies)))
 

From 2f897ed564a82199647e4b104c91cdaa7dc9cd4f Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Tue, 15 Jul 2025 13:59:24 +0200
Subject: [PATCH 025/391] fixed function parameters

---
 src/source/routes/google_redirect.clj | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/source/routes/google_redirect.clj b/src/source/routes/google_redirect.clj
index 3bd82285..d0da42ca 100644
--- a/src/source/routes/google_redirect.clj
+++ b/src/source/routes/google_redirect.clj
@@ -2,8 +2,7 @@
   (:require [ring.util.response :as res]
             [source.config :as conf]))
 
-(defn get [{:keys [query-string] :as _req}]
-  (-> :cors-origin
-      (conf/read-value)
+(defn get [{:keys [query-string] :as req}]
+  (-> (conf/read-value :cors-origin)
       (str "/oauth?" query-string)
       (res/redirect)))

From 5691c1c4a04785ffaf31233f3a0b20c3bef77248 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 16 Jul 2025 13:54:21 +0200
Subject: [PATCH 026/391] added malli schema validation function and
 implemented its use in endpoints

---
 src/source/routes/admin.clj    | 45 ++++++++++++++++++++--------------
 src/source/routes/business.clj | 34 ++++++++++++++++++++-----
 src/source/routes/login.clj    | 24 ++++++++++++------
 src/source/routes/register.clj | 35 +++++++++++++++-----------
 src/source/routes/user.clj     | 18 ++++++++++----
 src/source/util.clj            | 20 ++++++++++++++-
 6 files changed, 124 insertions(+), 52 deletions(-)

diff --git a/src/source/routes/admin.clj b/src/source/routes/admin.clj
index bcb107aa..2788283c 100644
--- a/src/source/routes/admin.clj
+++ b/src/source/routes/admin.clj
@@ -1,6 +1,8 @@
 (ns source.routes.admin
   (:require [source.services.users :as users]
-            [source.password :as pw]))
+            [source.password :as pw]
+            [source.util :as util]
+            [ring.util.response :as res]))
 
 (defn post
   {:summary "registers an admin user"
@@ -13,22 +15,29 @@
                403 {:body [:map [:message :string]]}}}
 
   [{:keys [ds body] :as _request}]
-  (let [user (users/user
-              ds
-              {:where [:= :email (:email body)]})
-        {:keys [password confirm-password]} body]
-    (cond
-      (not (= password confirm-password))
-      {:status 400 :body {:message "passwords do not match!"}}
 
-      (some? user)
-      {:status 400 :body {:message "an account for this email already exists!"}}
+  (let [{:keys [data error success]} (util/validate post body)]
+    (if (not success)
 
-      :else
-      (let [pw (pw/hash-password password)
-            new-user (-> (assoc body
-                                :password pw
-                                :type "admin")
-                         (dissoc :confirm-password))]
-        (users/insert-user! ds {:data new-user})
-        {:status 200 :body {:message "successfully created user"}}))))
+      (-> (res/response error)
+          (res/status 400))
+
+      (let [user (users/user
+                  ds
+                  {:where [:= :email (:email data)]})
+            {:keys [password confirm-password]} data]
+        (cond
+          (not (= password confirm-password))
+          {:status 400 :body {:message "passwords do not match!"}}
+
+          (some? user)
+          {:status 400 :body {:message "an account for this email already exists!"}}
+
+          :else
+          (let [pw (pw/hash-password password)
+                new-user (-> (assoc body
+                                    :password pw
+                                    :type "admin")
+                             (dissoc :confirm-password))]
+            (users/insert-user! ds {:data new-user})
+            {:status 200 :body {:message "successfully created user"}}))))))
diff --git a/src/source/routes/business.clj b/src/source/routes/business.clj
index 0ea57885..92396eda 100644
--- a/src/source/routes/business.clj
+++ b/src/source/routes/business.clj
@@ -1,6 +1,7 @@
 (ns source.routes.business
   (:require [source.services.businesses :as businesses]
-            [ring.util.response :as res]))
+            [ring.util.response :as res]
+            [source.util :as utils]))
 
 (defn post
   {:summary "insert a business"
@@ -12,8 +13,14 @@
    :responses {201 {:body [:map [:message :string]]}}}
 
   [{:keys [ds body] :as _request}]
-  (businesses/insert-business! ds body)
-  (res/response {:message "successfully added business"}))
+
+  (let [{:keys [data error success]} (utils/validate post body)]
+    (cond
+      (not success) (-> (res/response error)
+                        (res/status 400))
+
+      :else (do (businesses/insert-business! ds {:values data})
+                (res/response {:message "successfully added business"})))))
 
 (defn patch
   {:summary "update a business by id"
@@ -27,6 +34,21 @@
    :responses {200 {:body [:map [:message :string]]}}}
 
   [{:keys [ds body path-params] :as _request}]
-  (businesses/update-business! ds {:id (:id path-params)
-                                   :values body})
-  (res/response {:message "successfully updated business"}))
+
+  (let [{:keys [data error success]} (utils/validate patch body)]
+    (if (not success)
+
+      (-> (res/response error)
+          (res/status 400))
+
+      (do (businesses/update-business! ds {:id (:id path-params)
+                                           :values data})
+          (res/response {:message "successfully updated business"})))))
+
+(comment
+  (require '[source.db.util :as db.util])
+  (post {:ds (db.util/conn :master)
+         :body {:name "modulr"
+                :url "https://modulr.com"}})
+  ())
+
diff --git a/src/source/routes/login.clj b/src/source/routes/login.clj
index 85dc8dc6..3944ff9d 100644
--- a/src/source/routes/login.clj
+++ b/src/source/routes/login.clj
@@ -2,7 +2,8 @@
   (:require [source.services.auth :as auth]
             [ring.util.response :as res]
             [source.services.users :as users]
-            [source.password :as pw]))
+            [source.password :as pw]
+            [source.util :as util]))
 
 (defn post
   {:summary "get user data and access token provided valid credentials"
@@ -27,14 +28,21 @@
                401 {:body [:map [:message :string]]}}}
 
   [{:keys [ds body] :as _request}]
-  (let [{:keys [email password]} body
-        user (users/user ds {:where [:= :email email]})]
-    (if
-     (or (not (pw/verify-password password (:password user)))
-         (not (some? user)))
-      {:status 401 :body {:message "Invalid username or password!"}}
 
-      (res/response (auth/login ds {:user user})))))
+  (let [{:keys [data error success]} (util/validate post body)]
+    (if (not success)
+
+      (-> (res/response error)
+          (res/status 400))
+
+      (let [{:keys [email password]} data
+            user (users/user ds {:where [:= :email email]})]
+        (if
+         (or (not (pw/verify-password password (:password user)))
+             (not (some? user)))
+          {:status 401 :body {:message "Invalid username or password!"}}
+
+          (res/response (auth/login ds {:user user})))))))
 
 (comment
   (require '[source.db.interface :as db])
diff --git a/src/source/routes/register.clj b/src/source/routes/register.clj
index 50007f1b..64a53d7c 100644
--- a/src/source/routes/register.clj
+++ b/src/source/routes/register.clj
@@ -1,6 +1,7 @@
 (ns source.routes.register
   (:require [source.services.interface :as services]
-            [ring.util.response :as res]))
+            [ring.util.response :as res]
+            [source.util :as util]))
 
 (defn post
   {:summary "register a new user"
@@ -25,19 +26,25 @@
                            [:refresh-token :string]]}}}
 
   [{:keys [ds body] :as _request}]
-  ;;TODO: needs schema validation here
-  (let [{:keys [email password confirm-password]} body
-        existing-user (services/user ds {:where [:= :email email]})]
-    (cond
-      (not (= password confirm-password))
-      (-> (res/response {:error "Passwords do not match!"}))
-
-      (some? existing-user)
-      (-> (res/response {:error "An account for this email already exists!"}))
-
-      :else
-      (-> (services/register ds body)
-          (res/response)))))
+
+  (let [{:keys [data error success]} (util/validate post body)]
+    (if (not success)
+
+      (-> (res/response error)
+          (res/status 400))
+
+      (let [{:keys [email password confirm-password]} data
+            existing-user (services/user ds {:where [:= :email email]})]
+        (cond
+          (not (= password confirm-password))
+          (-> (res/response {:error "Passwords do not match!"}))
+
+          (some? existing-user)
+          (-> (res/response {:error "An account for this email already exists!"}))
+
+          :else
+          (-> (services/register ds body)
+              (res/response)))))))
 
 (comment
   (require '[source.db.interface :as db])
diff --git a/src/source/routes/user.clj b/src/source/routes/user.clj
index 5f813341..7a6255ba 100644
--- a/src/source/routes/user.clj
+++ b/src/source/routes/user.clj
@@ -1,6 +1,7 @@
 (ns source.routes.user
   (:require [source.services.interface :as services]
-            [ring.util.response :as res]))
+            [ring.util.response :as res]
+            [source.util :as util]))
 
 (defn get
   {:summary "get user by id"
@@ -60,10 +61,17 @@
                403 {:body [:map [:message :string]]}}}
 
   [{:keys [ds body path-params] :as _request}]
-  (services/update-user! ds
-                         {:id (:id path-params)
-                          :values body})
-  (res/response {:message "successfully updated user"}))
+
+  (let [{:keys [data error success]} (util/validate patch body)]
+    (if (not success)
+
+      (-> (res/response error)
+          (res/status 400))
+
+      (do (services/update-user! ds
+                                 {:id (:id path-params)
+                                  :values data})
+          (res/response {:message "successfully updated user"})))))
 
 (comment
   (require '[source.db.interface :as db])
diff --git a/src/source/util.clj b/src/source/util.clj
index fa783ab9..5bf051f8 100644
--- a/src/source/util.clj
+++ b/src/source/util.clj
@@ -1,7 +1,10 @@
 (ns source.util
   (:require [buddy.core.codecs :as codecs]
             [buddy.core.nonce :as nonce]
-            [clojure.main :refer [demunge]]))
+            [clojure.main :refer [demunge]]
+            [malli.core :as m]
+            [malli.transform :as mt]
+            [malli.error :as me]))
 
 (defn content-type [request]
   (or (get-in request [:headers "Content-Type"])
@@ -35,3 +38,18 @@
       (find-var)
       (meta)))
 
+(defn validate [handler data]
+  (let [schema (get-in (metadata handler) [:parameters :body])
+        decoded (m/decode schema data mt/string-transformer)
+        success (m/validate schema decoded)]
+    {:data decoded
+     :success success
+     :error (when-not success (->> decoded
+                                   (m/explain schema)
+                                   (me/humanize)))}))
+
+(comment 
+  (require '[source.routes.business :as business])
+  (validate business/post {:name "modulr"})
+  ())
+

From 0c5703e3b0a92306f65b08d39a090ce22e4e5f0a Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 16 Jul 2025 13:55:37 +0200
Subject: [PATCH 027/391] added reitit exception middleware

---
 deps.edn                     | 1 +
 src/source/routes/reitit.clj | 4 +++-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/deps.edn b/deps.edn
index 56750121..bd941b38 100644
--- a/deps.edn
+++ b/deps.edn
@@ -29,5 +29,6 @@
         com.github.seancorfield/honeysql {:mvn/version "2.7.1310"}
         metosin/jsonista {:mvn/version "0.3.13"}
         metosin/reitit {:mvn/version "0.9.1"}
+        metosin/reitit-middleware {:mvn/version "0.9.1"}
         metosin/reitit-swagger {:mvn/version "0.9.1"}
         metosin/reitit-swagger-ui {:mvn/version "0.9.1"}}}
diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index 33d95d34..13e02f25 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -4,6 +4,7 @@
             [reitit.swagger-ui :as swagger-ui]
             [reitit.coercion.malli]
             [reitit.ring.malli]
+            [reitit.ring.middleware.exception :as exception]
             [malli.util :as mu]
             [source.middleware.interface :as mw]
             [source.db.interface :as db]
@@ -88,7 +89,8 @@
                           :strip-extra-keys true
                           :default-values true
                           :options nil})
-              :middleware [[mw/apply-generic :ds ds]]}})
+              :middleware [[mw/apply-generic :ds ds]
+                           [exception/exception-middleware]]}})
      (ring/routes
       (swagger-ui/create-swagger-ui-handler {:path "/"})
       (ring/create-default-handler)))))

From 72f7e956f79abb2e836dc41076595a8d2a356ce2 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 16 Jul 2025 14:16:37 +0200
Subject: [PATCH 028/391] updated validation util to remove decode and return
 nil data on error

---
 src/source/util.clj | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/src/source/util.clj b/src/source/util.clj
index 5bf051f8..2f4882b5 100644
--- a/src/source/util.clj
+++ b/src/source/util.clj
@@ -40,16 +40,15 @@
 
 (defn validate [handler data]
   (let [schema (get-in (metadata handler) [:parameters :body])
-        decoded (m/decode schema data mt/string-transformer)
-        success (m/validate schema decoded)]
-    {:data decoded
+        success (m/validate schema data)]
+    {:data (when success data)
      :success success
-     :error (when-not success (->> decoded
+     :error (when-not success (->> data
                                    (m/explain schema)
                                    (me/humanize)))}))
 
 (comment 
   (require '[source.routes.business :as business])
-  (validate business/post {:name "modulr"})
+  (validate business/post {:cheese "modulr"})
   ())
 

From 15673d7854185290adf2f2a8d5ef8bb33efbb6a8 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 16 Jul 2025 14:37:23 +0200
Subject: [PATCH 029/391] stopped auth service from returning user with
 password included

---
 src/source/services/auth.clj | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/source/services/auth.clj b/src/source/services/auth.clj
index a96270a3..92d31020 100644
--- a/src/source/services/auth.clj
+++ b/src/source/services/auth.clj
@@ -14,7 +14,7 @@
                                     (assoc :password (pw/hash-password password)))})
   (let [user (users/user ds {:where [:= :email email]})]
     (merge
-     {:user user}
+     {:user (dissoc user :password)}
      (auth/create-session (select-keys user [:id :type])))))
 
 (comment

From 8eb30fbcfb068a1d2077764291f0e7907761fcb2 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 16 Jul 2025 14:38:24 +0200
Subject: [PATCH 030/391] refactored endpoints to remove redundant lets and
 conds

---
 src/source/routes/admin.clj    | 39 ++++++++++++++++------------------
 src/source/routes/business.clj |  9 ++++----
 src/source/routes/login.clj    | 24 ++++++++++-----------
 src/source/routes/register.clj | 29 ++++++++++++-------------
 4 files changed, 48 insertions(+), 53 deletions(-)

diff --git a/src/source/routes/admin.clj b/src/source/routes/admin.clj
index 2788283c..51e36ce2 100644
--- a/src/source/routes/admin.clj
+++ b/src/source/routes/admin.clj
@@ -16,28 +16,25 @@
 
   [{:keys [ds body] :as _request}]
 
-  (let [{:keys [data error success]} (util/validate post body)]
-    (if (not success)
+  (let [{:keys [data error success]} (util/validate post body)
+        user (users/user ds {:where [:= :email (:email data)]})
+        {:keys [password confirm-password]} data
+        pw (pw/hash-password password)
+        new-user (-> (assoc data
+                            :password pw
+                            :type "admin")
+                     (dissoc :confirm-password))]
+    (cond
 
-      (-> (res/response error)
-          (res/status 400))
+      (not success) (-> (res/response error)
+                        (res/status 400))
 
-      (let [user (users/user
-                  ds
-                  {:where [:= :email (:email data)]})
-            {:keys [password confirm-password]} data]
-        (cond
-          (not (= password confirm-password))
-          {:status 400 :body {:message "passwords do not match!"}}
+      (not (= password confirm-password))
+      {:status 400 :body {:message "passwords do not match!"}}
 
-          (some? user)
-          {:status 400 :body {:message "an account for this email already exists!"}}
+      (some? user)
+      {:status 400 :body {:message "an account for this email already exists!"}}
 
-          :else
-          (let [pw (pw/hash-password password)
-                new-user (-> (assoc body
-                                    :password pw
-                                    :type "admin")
-                             (dissoc :confirm-password))]
-            (users/insert-user! ds {:data new-user})
-            {:status 200 :body {:message "successfully created user"}}))))))
+      :else
+      (do (users/insert-user! ds {:data new-user})
+          (res/response {:message "successfully created user"})))))
diff --git a/src/source/routes/business.clj b/src/source/routes/business.clj
index 92396eda..16b77838 100644
--- a/src/source/routes/business.clj
+++ b/src/source/routes/business.clj
@@ -15,12 +15,11 @@
   [{:keys [ds body] :as _request}]
 
   (let [{:keys [data error success]} (utils/validate post body)]
-    (cond
-      (not success) (-> (res/response error)
-                        (res/status 400))
+    (if (not success) (-> (res/response error)
+                          (res/status 400))
 
-      :else (do (businesses/insert-business! ds {:values data})
-                (res/response {:message "successfully added business"})))))
+        (do (businesses/insert-business! ds {:values data})
+            (res/response {:message "successfully added business"})))))
 
 (defn patch
   {:summary "update a business by id"
diff --git a/src/source/routes/login.clj b/src/source/routes/login.clj
index 3944ff9d..3abbe8f0 100644
--- a/src/source/routes/login.clj
+++ b/src/source/routes/login.clj
@@ -3,7 +3,8 @@
             [ring.util.response :as res]
             [source.services.users :as users]
             [source.password :as pw]
-            [source.util :as util]))
+            [source.util :as util]
+            [source.db.util :as db.util]))
 
 (defn post
   {:summary "get user data and access token provided valid credentials"
@@ -29,20 +30,19 @@
 
   [{:keys [ds body] :as _request}]
 
-  (let [{:keys [data error success]} (util/validate post body)]
-    (if (not success)
+  (let [{:keys [data error success]} (util/validate post body)
+        {:keys [email password]} data
+        user (users/user ds {:where [:= :email email]})]
 
-      (-> (res/response error)
-          (res/status 400))
+    (cond
+      (not success) (-> (res/response error)
+                        (res/status 400))
 
-      (let [{:keys [email password]} data
-            user (users/user ds {:where [:= :email email]})]
-        (if
-         (or (not (pw/verify-password password (:password user)))
-             (not (some? user)))
-          {:status 401 :body {:message "Invalid username or password!"}}
+      (or (not (some? user))
+          (not (pw/verify-password password (:password user))))
+      {:status 401 :body {:message "Invalid username or password!"}}
 
-          (res/response (auth/login ds {:user user})))))))
+      :else (res/response (auth/login ds {:user user})))))
 
 (comment
   (require '[source.db.interface :as db])
diff --git a/src/source/routes/register.clj b/src/source/routes/register.clj
index 64a53d7c..e271175c 100644
--- a/src/source/routes/register.clj
+++ b/src/source/routes/register.clj
@@ -27,28 +27,27 @@
 
   [{:keys [ds body] :as _request}]
 
-  (let [{:keys [data error success]} (util/validate post body)]
-    (if (not success)
+  (let [{:keys [data error success]} (util/validate post body)
+        {:keys [email password confirm-password]} data
+        existing-user (services/user ds {:where [:= :email email]})]
+    (cond
 
-      (-> (res/response error)
-          (res/status 400))
+      (not success) (-> (res/response error)
+                        (res/status 400))
 
-      (let [{:keys [email password confirm-password]} data
-            existing-user (services/user ds {:where [:= :email email]})]
-        (cond
-          (not (= password confirm-password))
-          (-> (res/response {:error "Passwords do not match!"}))
+      (not (= password confirm-password))
+      (-> (res/response {:error "Passwords do not match!"}))
 
-          (some? existing-user)
-          (-> (res/response {:error "An account for this email already exists!"}))
+      (some? existing-user)
+      (-> (res/response {:error "An account for this email already exists!"}))
 
-          :else
-          (-> (services/register ds body)
-              (res/response)))))))
+      :else
+      (-> (services/register ds data)
+          (res/response)))))
 
 (comment
   (require '[source.db.interface :as db])
-  (post {:ds (db/ds :master) :body {:email "test@test.com"
+  (post {:ds (db/ds :master) :body {:email "poop@test.com"
                                     :password "test"
                                     :type "distributor"
                                     :confirm-password "test"}})

From 49f9a3fae7a9c78f29e022bc735689c34e2e45e0 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Thu, 17 Jul 2025 09:29:37 +0200
Subject: [PATCH 031/391] updated admin route to fix potential nil exception in
 password hashing

---
 src/source/routes/admin.clj | 22 ++++++++++++----------
 1 file changed, 12 insertions(+), 10 deletions(-)

diff --git a/src/source/routes/admin.clj b/src/source/routes/admin.clj
index 51e36ce2..8486907f 100644
--- a/src/source/routes/admin.clj
+++ b/src/source/routes/admin.clj
@@ -18,23 +18,25 @@
 
   (let [{:keys [data error success]} (util/validate post body)
         user (users/user ds {:where [:= :email (:email data)]})
-        {:keys [password confirm-password]} data
-        pw (pw/hash-password password)
-        new-user (-> (assoc data
-                            :password pw
-                            :type "admin")
-                     (dissoc :confirm-password))]
+        {:keys [password confirm-password]} data]
     (cond
 
       (not success) (-> (res/response error)
                         (res/status 400))
 
       (not (= password confirm-password))
-      {:status 400 :body {:message "passwords do not match!"}}
+      (-> (res/response {:message "passwords do not match!"})
+          (res/status 400))
 
       (some? user)
-      {:status 400 :body {:message "an account for this email already exists!"}}
+      (-> (res/response {:message "an account for this email already exists!"})
+          (res/status 400))
 
       :else
-      (do (users/insert-user! ds {:data new-user})
-          (res/response {:message "successfully created user"})))))
+      (let [pw (pw/hash-password password)
+            new-user (-> (assoc data
+                                :password pw
+                                :type "admin")
+                         (dissoc :confirm-password))]
+        (users/insert-user! ds {:data new-user})
+        (res/response {:message "successfully created user"})))))

From bfa1934908b91246379a47cb1e702c8ca96fe595 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Thu, 17 Jul 2025 09:31:55 +0200
Subject: [PATCH 032/391] removed unused dependency

---
 src/source/routes/login.clj | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/source/routes/login.clj b/src/source/routes/login.clj
index 3abbe8f0..530a528e 100644
--- a/src/source/routes/login.clj
+++ b/src/source/routes/login.clj
@@ -3,8 +3,7 @@
             [ring.util.response :as res]
             [source.services.users :as users]
             [source.password :as pw]
-            [source.util :as util]
-            [source.db.util :as db.util]))
+            [source.util :as util]))
 
 (defn post
   {:summary "get user data and access token provided valid credentials"

From 02bfc54a4bec1d4c747e2a53413e43acf45e1336 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Fri, 18 Jul 2025 12:54:35 +0200
Subject: [PATCH 033/391] added protected route (me) to get user data by access
 token

---
 src/source/routes/me.clj     | 25 +++++++++++++++++++++++++
 src/source/routes/reitit.clj |  6 ++++++
 2 files changed, 31 insertions(+)
 create mode 100644 src/source/routes/me.clj

diff --git a/src/source/routes/me.clj b/src/source/routes/me.clj
new file mode 100644
index 00000000..b6af7516
--- /dev/null
+++ b/src/source/routes/me.clj
@@ -0,0 +1,25 @@
+(ns source.routes.me
+  (:require [source.services.interface :as services]
+            [ring.util.response :as res]))
+
+(defn get
+  {:summary "get logged in user by access token"
+   :responses {200 {:body [:map
+                           [:id :int]
+                           [:address {:optional true} :string]
+                           [:profile-image {:optional true} :string]
+                           [:email :string]
+                           [:firstname {:optional true} :string]
+                           [:lastname {:optional true} :string]
+                           [:type [:enum "creator" "distributor" "admin"]]
+                           [:email-verified {:optional true} :int]
+                           [:onboarded {:optional true} :int]
+                           [:mobile {:optional true} :string]]}
+               401 {:body [:map [:message :string]]}
+               403 {:body [:map [:message :string]]}}}
+
+  [{:keys [ds user] :as _request}]
+  (let [user (->> user
+                  (services/user ds))]
+    (->> (dissoc user :password)
+         (res/response))))
diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index 13e02f25..a7de1dba 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -11,6 +11,7 @@
             [clojure.data.json :as json]
             [source.routes.user :as user]
             [source.routes.users :as users]
+            [source.routes.me :as me]
             [source.routes.login :as login]
             [source.routes.register :as register]
             [source.routes.google-launch :as google-launch]
@@ -51,6 +52,11 @@
         ["/:id"         (route {:get user/get
                                 :patch user/patch})]]
 
+       ["/me"           {:middleware [[mw/apply-auth]]
+                         :tags #{"me"}
+                         :swagger {:security [{"auth" []}]}}
+        [""             (route {:get me/get})]]
+
        ["/businesses"   {:middleware [[mw/apply-auth {:required-type :admin}]]
                          :tags #{"businesses"}
                          :swagger {:security [{"auth" []}]}}

From 144be7ac104d443604e52c9ba8eb3ecb8f040b86 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 21 Jul 2025 11:19:26 +0200
Subject: [PATCH 034/391] updated origin

---
 fly.dev.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fly.dev.toml b/fly.dev.toml
index 3fb8325e..63533503 100644
--- a/fly.dev.toml
+++ b/fly.dev.toml
@@ -9,7 +9,7 @@ primary_region = 'jnb'
 [build]
 
 [env]
-  CORS_ORIGIN = 'http://localhost:3001'
+  CORS_ORIGIN = 'https://source-staging.fly.dev'
   ENV = 'prod'
   GOOGLE_CLIENT_ID = '449412212863-h1rre1gdmfgq96jt160fdd30bfatjiqb.apps.googleusercontent.com'
   DATABASE_DIR = '/data'

From d3d4bfd48560eec47d735e0ef54b19aeb53ccd57 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 21 Jul 2025 11:34:43 +0200
Subject: [PATCH 035/391] updated fly toml to keep machines online

---
 fly.dev.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fly.dev.toml b/fly.dev.toml
index 63533503..964ec77c 100644
--- a/fly.dev.toml
+++ b/fly.dev.toml
@@ -17,7 +17,7 @@ primary_region = 'jnb'
 [http_service]
   internal_port = 3000
   force_https = true
-  auto_stop_machines = 'stop'
+  auto_stop_machines = 'off'
   min_machines_running = 0
   processes = ['app']
 

From 4ae1d4126112c673e07a3b4fc25d185e922a6e06 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Tue, 22 Jul 2025 09:15:37 +0200
Subject: [PATCH 036/391] implemented openapi support

---
 fly.dev.toml                 |  1 +
 fly.prod.toml                |  3 ++-
 src/source/routes/reitit.clj | 20 ++++++++++++++++++--
 3 files changed, 21 insertions(+), 3 deletions(-)

diff --git a/fly.dev.toml b/fly.dev.toml
index 964ec77c..75fee297 100644
--- a/fly.dev.toml
+++ b/fly.dev.toml
@@ -12,6 +12,7 @@ primary_region = 'jnb'
   CORS_ORIGIN = 'https://source-staging.fly.dev'
   ENV = 'prod'
   GOOGLE_CLIENT_ID = '449412212863-h1rre1gdmfgq96jt160fdd30bfatjiqb.apps.googleusercontent.com'
+  GOOGLE_REDIRECT_URI = 'https://source-be-staging.fly.dev/oauth2/google/callback'
   DATABASE_DIR = '/data'
 
 [http_service]
diff --git a/fly.prod.toml b/fly.prod.toml
index 269b438a..ac35715c 100644
--- a/fly.prod.toml
+++ b/fly.prod.toml
@@ -22,6 +22,7 @@ primary_region = 'jnb'
   cpus = 1
 
 [env]
+  GOOGLE_REDIRECT_URI = "https://source-be.fly.dev"
   GOOGLE_CLIENT_ID = "449412212863-h1rre1gdmfgq96jt160fdd30bfatjiqb.apps.googleusercontent.com"
-  CORS_ORIGIN = "http://localhost:3001"
+  CORS_ORIGIN = "https://source.fly.dev"
   ENV = "prod"
diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index a7de1dba..9e05ceaa 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -2,6 +2,7 @@
   (:require [reitit.ring :as ring]
             [reitit.swagger :as swagger]
             [reitit.swagger-ui :as swagger-ui]
+            [reitit.openapi :as openapi]
             [reitit.coercion.malli]
             [reitit.ring.malli]
             [reitit.ring.middleware.exception :as exception]
@@ -39,12 +40,22 @@
      (ring/router
       [["/swagger.json" {:get {:no-doc true
                                :swagger {:info {:title "source-api"
-                                                :description "swagger docs for source api with malli and reitit-ring"}
+                                                :description "swagger docs for source api with malli and reitit-ring"
+                                                :version "0.0.1"}
                                          :securityDefinitions {"auth" {:type :apiKey
                                                                        :in :header
                                                                        :name "Authorization"}}}
                                :handler (swagger/create-swagger-handler)}}]
 
+       ["/openapi.json" {:get {:no-doc true
+                               :openapi {:info {:title "source-api"
+                                                :description "openapi3 docs for source api with malli and reitit-ring"
+                                                :version "0.0.1"}
+                                         :components {:securitySchemas {"auth" {:type :apiKey
+                                                                                :in :header
+                                                                                :name "Authorization"}}}}
+                               :handler (openapi/create-openapi-handler)}}]
+
        ["/users"        {:middleware [[mw/apply-auth {:required-type :admin}]]
                          :tags #{"users"}
                          :swagger {:security [{"auth" []}]}}
@@ -98,7 +109,12 @@
               :middleware [[mw/apply-generic :ds ds]
                            [exception/exception-middleware]]}})
      (ring/routes
-      (swagger-ui/create-swagger-ui-handler {:path "/"})
+      (swagger-ui/create-swagger-ui-handler {:path "/"
+                                             :config {:validatorUrl nil
+                                                      :urls [{:name "swagger", :url "swagger.json"}
+                                                             {:name "openapi", :url "openapi.json"}]
+                                                      :urls.primaryName "swagger"
+                                                      :operationsSorter "alpha"}})
       (ring/create-default-handler)))))
 
 (comment

From 970650b4cac8d5e5a2b1b6dca6a54a914cd5ec86 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 23 Jul 2025 15:17:14 +0200
Subject: [PATCH 037/391] fixed malli schemas to use [:maybe] instead of
 {:optional true} to allow nulls instead of only undefined

---
 src/source/routes/businesses.clj |  6 +++---
 src/source/routes/login.clj      | 14 +++++++-------
 src/source/routes/me.clj         | 14 +++++++-------
 src/source/routes/register.clj   | 14 +++++++-------
 src/source/routes/user.clj       | 28 ++++++++++++++--------------
 src/source/routes/users.clj      | 14 +++++++-------
 6 files changed, 45 insertions(+), 45 deletions(-)

diff --git a/src/source/routes/businesses.clj b/src/source/routes/businesses.clj
index 569834cc..b7e1e056 100644
--- a/src/source/routes/businesses.clj
+++ b/src/source/routes/businesses.clj
@@ -14,9 +14,9 @@
                             [:map
                              [:id :int]
                              [:name :string]
-                             [:url {:optional true} :string]
-                             [:linkedin {:optional true} :string]
-                             [:twitter {:optional true} :string]]]]}}}
+                             [:url [:maybe :string]]
+                             [:linkedin [:maybe :string]]
+                             [:twitter [:maybe :string]]]]]}}}
 
   [{:keys [ds] :as _request}]
   (res/response {:businesses (businesses/businesses ds)}))
diff --git a/src/source/routes/login.clj b/src/source/routes/login.clj
index 530a528e..e45452f1 100644
--- a/src/source/routes/login.clj
+++ b/src/source/routes/login.clj
@@ -14,15 +14,15 @@
                            [:user
                             [:map
                              [:id :int]
-                             [:address {:optional true} :string]
-                             [:profile-image {:optional true} :string]
+                             [:address [:maybe :string]]
+                             [:profile-image [:maybe :string]]
                              [:email :string]
-                             [:firstname {:optional true} :string]
-                             [:lastname {:optional true} :string]
+                             [:firstname [:maybe :string]]
+                             [:lastname [:maybe :string]]
                              [:type [:enum "creator" "distributor" "admin"]]
-                             [:email-verified {:optional true} :int]
-                             [:onboarded {:optional true} :int]
-                             [:mobile {:optional true} :string]]]
+                             [:email-verified [:maybe :int]]
+                             [:onboarded [:maybe :int]]
+                             [:mobile [:maybe :string]]]]
                            [:access-token :string]
                            [:refresh-token :string]]}
                401 {:body [:map [:message :string]]}}}
diff --git a/src/source/routes/me.clj b/src/source/routes/me.clj
index b6af7516..6f496b4c 100644
--- a/src/source/routes/me.clj
+++ b/src/source/routes/me.clj
@@ -6,15 +6,15 @@
   {:summary "get logged in user by access token"
    :responses {200 {:body [:map
                            [:id :int]
-                           [:address {:optional true} :string]
-                           [:profile-image {:optional true} :string]
+                           [:address [:maybe :string]]
+                           [:profile-image [:maybe :string]]
                            [:email :string]
-                           [:firstname {:optional true} :string]
-                           [:lastname {:optional true} :string]
+                           [:firstname [:maybe :string]]
+                           [:lastname [:maybe :string]]
                            [:type [:enum "creator" "distributor" "admin"]]
-                           [:email-verified {:optional true} :int]
-                           [:onboarded {:optional true} :int]
-                           [:mobile {:optional true} :string]]}
+                           [:email-verified [:maybe :int]]
+                           [:onboarded [:maybe :int]]
+                           [:mobile [:maybe :string]]]}
                401 {:body [:map [:message :string]]}
                403 {:body [:map [:message :string]]}}}
 
diff --git a/src/source/routes/register.clj b/src/source/routes/register.clj
index e271175c..531605ed 100644
--- a/src/source/routes/register.clj
+++ b/src/source/routes/register.clj
@@ -13,15 +13,15 @@
                            [:user
                             [:map
                              [:id :int]
-                             [:address {:optional true} :string]
-                             [:profile-image {:optional true} :string]
+                             [:address [:maybe :string]]
+                             [:profile-image [:maybe :string]]
                              [:email :string]
-                             [:firstname {:optional true} :string]
-                             [:lastname {:optional true} :string]
+                             [:firstname [:maybe :string]]
+                             [:lastname [:maybe :string]]
                              [:type [:enum "creator" "distributor" "admin"]]
-                             [:email-verified {:optional true} :int]
-                             [:onboarded {:optional true} :int]
-                             [:mobile {:optional true} :string]]]
+                             [:email-verified [:maybe :int]]
+                             [:onboarded [:maybe :int]]
+                             [:mobile [:maybe :string]]]]
                            [:access-token :string]
                            [:refresh-token :string]]}}}
 
diff --git a/src/source/routes/user.clj b/src/source/routes/user.clj
index 7a6255ba..413ddd48 100644
--- a/src/source/routes/user.clj
+++ b/src/source/routes/user.clj
@@ -11,15 +11,15 @@
                            [:user
                             [:map
                              [:id :int]
-                             [:address {:optional true} :string]
-                             [:profile-image {:optional true} :string]
+                             [:address [:maybe :string]]
+                             [:profile-image [:maybe :string]]
                              [:email :string]
-                             [:firstname {:optional true} :string]
-                             [:lastname {:optional true} :string]
+                             [:firstname [:maybe :string]]
+                             [:lastname [:maybe :string]]
                              [:type [:enum "creator" "distributor" "admin"]]
-                             [:email-verified {:optional true} :int]
-                             [:onboarded {:optional true} :int]
-                             [:mobile {:optional true} :string]]]]}
+                             [:email-verified [:maybe :int]]
+                             [:onboarded [:maybe :int]]
+                             [:mobile [:maybe :string]]]]]}
                401 {:body [:map [:message :string]]}
                403 {:body [:map [:message :string]]}}}
 
@@ -48,15 +48,15 @@
                            [:user
                             [:map
                              [:id :int]
-                             [:address {:optional true} :string]
-                             [:profile-image {:optional true} :string]
+                             [:address [:maybe :string]]
+                             [:profile-image [:maybe :string]]
                              [:email :string]
-                             [:firstname {:optional true} :string]
-                             [:lastname {:optional true} :string]
+                             [:firstname [:maybe :string]]
+                             [:lastname [:maybe :string]]
                              [:type [:enum "creator" "distributor" "admin"]]
-                             [:email-verified {:optional true} :int]
-                             [:onboarded {:optional true} :int]
-                             [:mobile {:optional true} :string]]]]}
+                             [:email-verified [:maybe :int]]
+                             [:onboarded [:maybe :int]]
+                             [:mobile [:maybe :string]]]]]}
                401 {:body [:map [:message :string]]}
                403 {:body [:map [:message :string]]}}}
 
diff --git a/src/source/routes/users.clj b/src/source/routes/users.clj
index a641eb06..2a05f65b 100644
--- a/src/source/routes/users.clj
+++ b/src/source/routes/users.clj
@@ -9,15 +9,15 @@
                              [:vector
                               [:map
                                [:id :int]
-                               [:address {:optional true} :string]
-                               [:profile-image {:optional true} :string]
+                               [:address [:maybe :string]]
+                               [:profile-image [:maybe :string]]
                                [:email :string]
-                               [:firstname {:optional true} :string]
-                               [:lastname {:optional true} :string]
+                               [:firstname [:maybe :string]]
+                               [:lastname [:maybe :string]]
                                [:type [:enum "creator" "distributor" "admin"]]
-                               [:email-verified {:optional true} :int]
-                               [:onboarded {:optional true} :int]
-                               [:mobile {:optional true} :string]]]]]}
+                               [:email-verified [:maybe :int]]
+                               [:onboarded [:maybe :int]]
+                               [:mobile [:maybe :int]]]]]]}
                 401 {:body [:map [:message :string]]}
                 403 {:body [:map [:message :string]]}}}
 

From f2b4b3a43219f0a8f571efd62c55b34e443c5f9b Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Thu, 24 Jul 2025 13:21:25 +0200
Subject: [PATCH 038/391] updated register parameters

---
 src/source/routes/register.clj | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/source/routes/register.clj b/src/source/routes/register.clj
index 531605ed..505cc5ab 100644
--- a/src/source/routes/register.clj
+++ b/src/source/routes/register.clj
@@ -8,7 +8,8 @@
    :parameters {:body [:map
                        [:email :string]
                        [:password :string]
-                       [:confirm-password :string]]}
+                       [:confirm-password :string]
+                       [:type [:enum "creator" "distributor"]]]}
    :responses {200 {:body [:map
                            [:user
                             [:map

From 4634dea61302e72c038fc2dc36314de5fb726652 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Thu, 24 Jul 2025 13:22:59 +0200
Subject: [PATCH 039/391] added docs to google routes and updated security for
 openapi

---
 src/source/routes/google_launch.clj |  9 ++++++--
 src/source/routes/google_user.clj   | 23 +++++++++++++++++++-
 src/source/routes/reitit.clj        | 33 +++++++++++++++++------------
 3 files changed, 49 insertions(+), 16 deletions(-)

diff --git a/src/source/routes/google_launch.clj b/src/source/routes/google_launch.clj
index efccfd4b..a179ebfc 100644
--- a/src/source/routes/google_launch.clj
+++ b/src/source/routes/google_launch.clj
@@ -2,6 +2,11 @@
   (:require [source.oauth2.google.interface :as google]
             [ring.util.response :as response]))
 
-(defn get [_req]
-  (response/response (google/auth-uri)))
+(defn get
+  {:summary "begins google federated login flow"
+   :responses {200 {:body [:map
+                           [:uuid :string]
+                           [:uri :string]]}}}
 
+  [_req]
+  (response/response (google/auth-uri)))
diff --git a/src/source/routes/google_user.clj b/src/source/routes/google_user.clj
index 70d6b808..b5e19541 100644
--- a/src/source/routes/google_user.clj
+++ b/src/source/routes/google_user.clj
@@ -4,7 +4,28 @@
             [source.services.users :as users]
             [ring.util.response :as res]))
 
-(defn get [{:keys [ds body] :as req}]
+(defn get
+  {:summary "completes the google oauth2 flow and returns the authenticated user"
+   :parameters {:query [:map
+                        [:code :string]
+                        [:scope :string]]}
+   :responses {200 {:body [:map
+                           [:user
+                            [:map
+                             [:id :int]
+                             [:address [:maybe :string]]
+                             [:profile-image [:maybe :string]]
+                             [:email :string]
+                             [:firstname [:maybe :string]]
+                             [:lastname [:maybe :string]]
+                             [:type [:enum "creator" "distributor" "admin"]]
+                             [:email-verified [:maybe :int]]
+                             [:onboarded [:maybe :int]]
+                             [:mobile [:maybe :string]]]]
+                           [:access-token :string]
+                           [:refresh-token :string]]}}}
+  [{:keys [ds body] :as req}]
+
   (let [{:keys [uuid _uri]} body
         email (google/google-session-user uuid (:params req))
         user (users/user ds {:where [:= :email email]})
diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index 9e05ceaa..5d7bb086 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -51,26 +51,30 @@
                                :openapi {:info {:title "source-api"
                                                 :description "openapi3 docs for source api with malli and reitit-ring"
                                                 :version "0.0.1"}
-                                         :components {:securitySchemas {"auth" {:type :apiKey
-                                                                                :in :header
-                                                                                :name "Authorization"}}}}
+                                         :components {:securitySchemes {"bearerAuth" {:type :http
+                                                                                      :scheme :bearer
+                                                                                      :bearerFormat "JWT"
+                                                                                      :description "JWT Authorization using the Bearer scheme"}}}}
                                :handler (openapi/create-openapi-handler)}}]
 
        ["/users"        {:middleware [[mw/apply-auth {:required-type :admin}]]
                          :tags #{"users"}
-                         :swagger {:security [{"auth" []}]}}
+                         :swagger {:security [{"auth" []}]}
+                         :openapi {:security [{:bearerAuth []}]}}
         [""             (route {:get users/get})]
         ["/:id"         (route {:get user/get
                                 :patch user/patch})]]
 
        ["/me"           {:middleware [[mw/apply-auth]]
                          :tags #{"me"}
-                         :swagger {:security [{"auth" []}]}}
+                         :swagger {:security [{"auth" []}]}
+                         :openapi {:security [{:bearerAuth []}]}}
         [""             (route {:get me/get})]]
 
        ["/businesses"   {:middleware [[mw/apply-auth {:required-type :admin}]]
                          :tags #{"businesses"}
-                         :swagger {:security [{"auth" []}]}}
+                         :swagger {:security [{"auth" []}]}
+                         :openapi {:security [{:bearerAuth []}]}}
         [""             (route {:get businesses/get
                                 :post business/post})]
         ["/:id"         (route {:patch business/patch})]]
@@ -84,20 +88,23 @@
        ["/register"     {:tags #{"auth"}}
         [""             (route {:post register/post})]]
 
-       ["/oauth2"       {:no-doc true}
-        ["/google"
-         [""            {:get google-launch/get}]
-         ["/callback"   {:get google-redirect/get}]
-         ["/user"       {:get google-user/get}]]]
+       ["/oauth2"
+        ["/google"      {:tags #{"google"}}
+         [""            (route {:get google-launch/get})]
+         ["/callback"   {:no-doc true
+                         :get google-redirect/get}]
+         ["/user"       (route {:get google-user/get})]]]
 
        ["/protected"    {:middleware [[mw/apply-auth]]
                          :tags #{"protected"}
-                         :swagger {:security [{"auth" []}]}}
+                         :swagger {:security [{"auth" []}]}
+                         :openapi {:security [{:bearerAuth []}]}}
         ["/authorized"  (route {:get authorized/get})]]
 
        ["/admin"        {:middleware [[mw/apply-auth {:required-type :admin}]]
                          :tags #{"admin"}
-                         :swagger {:security [{"auth" []}]}}
+                         :swagger {:security [{"auth" []}]}
+                         :openapi {:security [{:bearerAuth []}]}}
         ["/add-admin"   (route {:post admin/post})]]]
 
       {:data {:coercion (reitit.coercion.malli/create

From 14534ff8988668181b8b94cc70f07cdbd18c3b2c Mon Sep 17 00:00:00 2001
From: Kaidan Theron <kaidan13th@gmail.com>
Date: Fri, 25 Jul 2025 10:26:06 +0200
Subject: [PATCH 040/391] resolve conflicts

---
 src/source/datastore/datalevin.clj  | 37 ++++++++++-----
 src/source/datastore/tables.clj     |  2 +
 src/source/datastore/util.clj       |  2 +-
 src/source/routes/reitit.clj        | 72 +++++++++++++++++++++++++++--
 src/source/services/interface.clj   |  9 ++++
 src/source/services/xml_schemas.clj | 20 +++++++-
 6 files changed, 125 insertions(+), 17 deletions(-)

diff --git a/src/source/datastore/datalevin.clj b/src/source/datastore/datalevin.clj
index 9ef934fc..4925f2ec 100644
--- a/src/source/datastore/datalevin.clj
+++ b/src/source/datastore/datalevin.clj
@@ -1,5 +1,6 @@
 (ns source.datastore.datalevin
   (:require [datalevin.core :as d]
+            [source.datastore.tables :as tables]
             [source.util :as util]))
 
 ;; If we want to have higher write speed in future we can use transact-kv-async
@@ -22,6 +23,8 @@
 (defn find
   "Returns the value for a key in the kv-store"
   [ds {:keys [tname key]}]
+  (tables/open-table! ds (name tname))
+  (println "finding for " key " in " (name tname))
   (d/get-value ds (name tname) key))
 
 (defn exists?
@@ -33,11 +36,13 @@
 (defn entries
   "Get the number of entries in a table"
   [ds {:keys [tname]}]
+  (tables/open-table! ds (name tname))
   (d/entries ds tname))
 
 (defn delete!
   "Removes one or multiple keys from kv store."
   [ds {:keys [tname keys]}]
+  (tables/open-table! ds (name tname))
   (->> (mapv (fn [key] [:del key]) keys)
        (d/transact-kv ds (name tname))))
 
@@ -45,6 +50,7 @@
   "Inserts kv's into store. Skips keys that already exist.
   Returns the key-value pairs that were inserted."
   [ds {:keys [tname data]}]
+  (tables/open-table! ds (name tname))
   (let [multi? (util/vectors? data)
         input-kvs (if multi? data [data])
         kvs-to-insert (->> input-kvs
@@ -58,26 +64,29 @@
 (defn update!
   "Replaces values for keys in store. Skips keys that don't exist"
   [ds {:keys [tname data]}]
-  (let [transducer (comp
-                    (filter (fn [[k _]]
-                              (exists? ds {:tname tname :key k})))
-                    (map (fn [[k v]]
-                           [:put k v])))]
-    (->> data
-         (into [] transducer)
-         (d/transact-kv ds (name tname)))))
+  (tables/open-table! ds (name tname))
+  (->> data
+       (filter (fn [[k _]]
+                 (exists? ds {:tname tname :key k})))
+       (map (fn [[k v]]
+              [:put k v]))
+       (d/transact-kv ds (name tname))))
 
 (defn get-all
   "Returns all key value pairs in table in kv-store."
   [ds {:keys [tname]}]
+  (tables/open-table! ds (name tname))
+  (println "finding all in " (name tname))
   (d/get-range ds (name tname) [:all]))
 
 (comment
   (require '[source.datastore.util :as ds.util])
-  (let [ds (ds.util/conn "datalevin")
-        key "somekey"
-        value "somestringvalue"
-        tname :some-table]
+  (let [ds (ds.util/conn :store)
+        key "test-key"
+        value {:title {:path ["tag/body" "tag/feed" "tag/title" "content/0"]}}
+        tname :test-table]
+    (println (get-all ds {:tname :selection-schemas}))
+    (println (find ds {:tname :selection-schemas :key 4}))
     (println "Putting a value")
     (d/open-dbi ds (name tname))
     (insert! ds {:tname tname
@@ -87,6 +96,10 @@
                          :key key})))
     (insert! ds {:tname tname
                  :data [key value]})
+    (println "1 " (get-all ds {:tname :selection-schemas}))
+    (println "2 " (find ds {:tname :selection-schemas :key 4}))
+    (println "3 " (get-all ds {:tname tname}))
+    (println "4 " (find ds {:tname tname :key "test-key"}))
     (println "Test passed")
     (println "Deleting a value")
     (delete! ds {:tname tname
diff --git a/src/source/datastore/tables.clj b/src/source/datastore/tables.clj
index d17c411c..1449ce2f 100644
--- a/src/source/datastore/tables.clj
+++ b/src/source/datastore/tables.clj
@@ -20,3 +20,5 @@
   "Returns a vector of all tables in store"
   [store]
   (d/list-dbis store))
+
+(open-table! (d/open-kv "store") "some-table")
diff --git a/src/source/datastore/util.clj b/src/source/datastore/util.clj
index 47efc9d3..6d0a70e2 100644
--- a/src/source/datastore/util.clj
+++ b/src/source/datastore/util.clj
@@ -15,7 +15,7 @@
 
 (defn conn
   "Open a connection to a datalevin kv store"
-  [store-name] (d/open-kv (store-path store-name)))
+  [store-name] (d/open-kv (store-path (name store-name))))
 
 (defn close
   "Close a connection to a datalevin store"
diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index de9d4eb5..c8b3b4a2 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -24,6 +24,12 @@
             [source.routes.business :as business]
             [source.routes.businesses :as businesses]
             [source.routes.sectors :as sectors]
+            [source.routes.selection-schemas :as selection-schemas]
+            [source.routes.xml :as xml]
+            [source.routes.data :as data]
+            [source.datastore.tables :as store.tables]
+            [source.datastore.interface :as store]
+            [source.datastore.util :as store.util]
             [source.util :as util]))
 
 (defn route [handlers]
@@ -109,6 +115,8 @@
                          :openapi {:security [{:bearerAuth []}]}}
         ["/add-admin"   (route {:post admin/post})]
         ["/selection-schema" {:post selection-schema/post}]]]
+        ["/ast" {:post xml/post}]
+        ["/extract-data" {:post data/post}]
 
       {:data {:coercion (reitit.coercion.malli/create
                          {:error-keys #{#_:type :coercion :in :schema :value :errors :humanized #_:transformed}
@@ -129,8 +137,9 @@
 
 (comment
   (require '[source.middleware.auth.util :as auth.util]
-           '[source.datastore.util :as store.util]
-           '[datalevin.core :as dl])
+           '[source.datastore.interface :as store]
+           '[source.datastore.tables :as store.tables]
+           '[source.rss.youtube :as yt])
 
   (route {:get #'user/get
           :patch #'user/patch})
@@ -190,6 +199,7 @@
         (json/read-json {:key-fn keyword})))
 
   (let [app (create-app)
+<<<<<<< Updated upstream
         request {:uri "/businesses"
                  :request-method :get}]
     (-> request
@@ -230,14 +240,70 @@
         ;; open table before operation
         _thing (dl/open-dbi store "selection-schemas")
         request {:uri "/admin/selection-schema"
+=======
+        store (store/ds :store)
+        request {:uri "/admin/selection-schemas"
+>>>>>>> Stashed changes
                  :request-method :post
                  :store store
                  :body {:record {:provider-id 1
                                  :output-schema-id 1}
-                        :schema {:hi "hello"}}
+                        :schema {:title {:path ["tag/body" "tag/feed" "tag/title" "content/0"]}}}
                  :headers {"authorization" (str "Bearer " (auth.util/sign-jwt {:id 1 :type "admin"}))}}]
     (-> request
         app
         :body
         (json/read-json {:key-fn keyword})))
+
+  (let [app (create-app)
+        request {:uri "/admin/selection-schema/1"
+                 :request-method :get
+                 :headers {"authorization" (str "Bearer " (auth.util/sign-jwt {:id 1 :type "admin"}))}}]
+    (-> request
+        app
+        :body
+        (json/read-json {:key-fn keyword})))
+
+  (let [app (create-app)
+        store (store/ds :store)
+        request {:uri "/admin/selection-schemas"
+                 :store store
+                 :request-method :get
+                 :headers {"authorization" (str "Bearer " (auth.util/sign-jwt {:id 1 :type "admin"}))}}]
+    (-> request
+        app
+        :body
+        (json/read-json {:key-fn keyword})))
+
+  (defn get-url []
+    (->> "https://www.youtube.com/@ThePrimeTimeagen"
+         (yt/find-channel-id)
+         (str "https://www.youtube.com/feeds/videos.xml?channel_id=")))
+
+  (let [app (create-app)
+        request {:uri "/admin/ast"
+                 :request-method :post
+                 :body {:url (get-url)}
+                 :headers {"authorization" (str "Bearer " (auth.util/sign-jwt {:id 1 :type "admin"}))}}]
+    (-> request
+        app
+        :body
+        (json/read-json {:key-fn keyword})))
+
+  (let [app (create-app)
+        store (store/ds :store)
+        request {:uri "/admin/extract-data"
+                 :store store
+                 :request-method :post
+                 :body {:schema-id 1
+                        :url (get-url)}
+                 :headers {"authorization" (str "Bearer " (auth.util/sign-jwt {:id 1 :type "admin"}))}}]
+    (println (store/get-all store {:tname :selection-schemas}))
+    (println (store/find store {:tname :selection-schemas
+                                :key 1}))
+    (-> request
+        app
+        :body
+        (json/read-json {:key-fn keyword})))
+
   ())
diff --git a/src/source/services/interface.clj b/src/source/services/interface.clj
index 82417429..449f3959 100644
--- a/src/source/services/interface.clj
+++ b/src/source/services/interface.clj
@@ -33,6 +33,15 @@
 (defn bundle [ds {:keys [_id _where] :as opts}]
   (bundles/bundle ds opts))
 
+(defn selection-schemas [ds]
+  (xml/selection-schemas ds))
+
+(defn ast [url]
+  (xml/ast url))
+
+(defn extract-data [store {:keys [schema-id url]}]
+  (xml/extract-data store schema-id url))
+
 (comment
   (users (db/ds :master))
   (user (db/ds :master) {:id 2})
diff --git a/src/source/services/xml_schemas.clj b/src/source/services/xml_schemas.clj
index fcb450b3..6b6daed8 100644
--- a/src/source/services/xml_schemas.clj
+++ b/src/source/services/xml_schemas.clj
@@ -1,6 +1,7 @@
 (ns source.services.xml-schemas
   (:require [source.db.interface :as db]
-            [source.datastore.interface :as store]))
+            [source.datastore.interface :as store]
+            [source.rss.core :as rss]))
 
 (defn get-all
   [ds {:keys [tname]}]
@@ -45,3 +46,20 @@
   (store/find store {:tname :output-schemas
                      :key id}))
 
+(defn ast
+  [url]
+  (-> url
+      slurp
+      rss/get-ast
+      rss/collect-leaf-paths))
+
+(defn extract-data
+  [store schema-id url]
+  (let [schema (store/find store {:tname :selection-schemas
+                                  :key schema-id})]
+    (println (store/get-all store {:tname :selection-schemas}))
+    (println "schema: " schema)
+    (->> url
+         slurp
+         rss/get-ast
+         (rss/extract-data schema))))

From 7f2e1ece95784dcb6fb94bbffd82b2fddcc5be38 Mon Sep 17 00:00:00 2001
From: Kaidan Theron <kaidan13th@gmail.com>
Date: Fri, 25 Jul 2025 10:27:21 +0200
Subject: [PATCH 041/391] add data and xml routes

---
 src/source/routes/data.clj | 8 ++++++++
 src/source/routes/xml.clj  | 8 ++++++++
 2 files changed, 16 insertions(+)
 create mode 100644 src/source/routes/data.clj
 create mode 100644 src/source/routes/xml.clj

diff --git a/src/source/routes/data.clj b/src/source/routes/data.clj
new file mode 100644
index 00000000..17b196ef
--- /dev/null
+++ b/src/source/routes/data.clj
@@ -0,0 +1,8 @@
+(ns source.routes.data
+  (:require [source.services.interface :as services]
+            [ring.util.response :as res]))
+
+(defn post [{:keys [store body] :as _request}]
+  (let [{:keys [_schema-id _url] :as opts} body]
+    (-> (services/extract-data store opts)
+        (res/response))))
diff --git a/src/source/routes/xml.clj b/src/source/routes/xml.clj
new file mode 100644
index 00000000..b2d944e9
--- /dev/null
+++ b/src/source/routes/xml.clj
@@ -0,0 +1,8 @@
+(ns source.routes.xml
+  (:require [source.services.interface :as services]
+            [ring.util.response :as res]))
+
+(defn post [{:keys [body] :as _request}]
+  (let [{:keys [url]} body]
+    (-> (services/ast url)
+        (res/response))))

From 4b5f38fe526c9786bb90219c6f1c5ac521696150 Mon Sep 17 00:00:00 2001
From: Kaidan Theron <kaidan13th@gmail.com>
Date: Fri, 25 Jul 2025 10:32:15 +0200
Subject: [PATCH 042/391] resolve more conflicts

---
 src/source/routes/reitit.clj | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index c8b3b4a2..476dc92e 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -115,8 +115,8 @@
                          :openapi {:security [{:bearerAuth []}]}}
         ["/add-admin"   (route {:post admin/post})]
         ["/selection-schema" {:post selection-schema/post}]]]
-        ["/ast" {:post xml/post}]
-        ["/extract-data" {:post data/post}]
+      ["/ast" {:post xml/post}]
+      ["/extract-data" {:post data/post}]
 
       {:data {:coercion (reitit.coercion.malli/create
                          {:error-keys #{#_:type :coercion :in :schema :value :errors :humanized #_:transformed}
@@ -199,7 +199,6 @@
         (json/read-json {:key-fn keyword})))
 
   (let [app (create-app)
-<<<<<<< Updated upstream
         request {:uri "/businesses"
                  :request-method :get}]
     (-> request
@@ -240,10 +239,6 @@
         ;; open table before operation
         _thing (dl/open-dbi store "selection-schemas")
         request {:uri "/admin/selection-schema"
-=======
-        store (store/ds :store)
-        request {:uri "/admin/selection-schemas"
->>>>>>> Stashed changes
                  :request-method :post
                  :store store
                  :body {:record {:provider-id 1

From 637d509331bb692b2e22af30db7b11c1bb5460cd Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Tue, 29 Jul 2025 14:59:14 +0200
Subject: [PATCH 043/391] refactored current implementation to use datahike
 instead of datalevin

---
 README.md                               |   9 +-
 deps.edn                                |   2 +-
 src/source/datastore/datahike.clj       | 143 ++++++++++++++++++++++++
 src/source/datastore/datalevin.clj      | 112 -------------------
 src/source/datastore/interface.clj      |  40 ++++---
 src/source/datastore/tables.clj         |  24 ----
 src/source/datastore/util.clj           |  29 ++++-
 src/source/db/master.clj                |   1 +
 src/source/routes/data.clj              |  13 +++
 src/source/routes/reitit.clj            |  48 ++++----
 src/source/routes/selection_schema.clj  |   6 +
 src/source/routes/selection_schemas.clj |   5 +
 src/source/services/xml_schemas.clj     |  75 +++++++++----
 13 files changed, 288 insertions(+), 219 deletions(-)
 create mode 100644 src/source/datastore/datahike.clj
 delete mode 100644 src/source/datastore/datalevin.clj
 delete mode 100644 src/source/datastore/tables.clj

diff --git a/README.md b/README.md
index 5ea62489..f0bbef6f 100644
--- a/README.md
+++ b/README.md
@@ -4,14 +4,7 @@ This is the backend for the Source platform. You can find documentation on setup
 
 ## Dependencies
 
-
-- >= openjdk version 17.
-- libc, libomp, libmvec (Refer to the Datalevin [installation docs](https://github.com/juji-io/datalevin/blob/master/doc/install.md#native-dependencies) for more info)
-
-If you are on MacOS you can run:
-```
-brew install libomp llvm openjdk@17
-```
+- >= openjdk version 11.
 
 ## Development setup
 
diff --git a/deps.edn b/deps.edn
index d193fc75..b0b9fe3f 100644
--- a/deps.edn
+++ b/deps.edn
@@ -27,7 +27,7 @@
         com.kepler16/mallard {:mvn/version "3.2.1"}
         com.kepler16/mallard-sqlite-store {:mvn/version "3.2.1"}
         com.github.seancorfield/honeysql {:mvn/version "2.7.1310"}
-        datalevin/datalevin {:mvn/version "0.9.22"}
+        io.replikativ/datahike {:mvn/version "0.6.1599"}
         hickory/hickory {:mvn/version "0.7.1"}
         metosin/jsonista {:mvn/version "0.3.13"}
         metosin/reitit {:mvn/version "0.9.1"}
diff --git a/src/source/datastore/datahike.clj b/src/source/datastore/datahike.clj
new file mode 100644
index 00000000..a13c51ae
--- /dev/null
+++ b/src/source/datastore/datahike.clj
@@ -0,0 +1,143 @@
+(ns source.datastore.datahike
+  (:require [datahike.api :as d]
+            [source.util :as util]))
+
+(defn lookup
+  "Returns one or more entities associated with the provided entity id(s)"
+  [ds ids]
+  (let [multi? (vector? ids)
+        ids-vec (if multi? ids [ids])]
+    (mapv (fn [id]
+            (->> (d/entity @ds id)
+                 (into {}))) ids-vec)))
+
+(defn find
+  "Returns one ore more entity ids where the provided key matches the provided value"
+  [ds {:keys [key value]}]
+  (->> (d/q '[:find ?e
+              :in $ ?k ?v
+              :where [?e ?k ?v]]
+            @ds key value)
+       (vec)
+       (flatten)
+       (into [])))
+
+(defn find-entities
+  "Same as find, but returns a vector of entities"
+  [ds {:keys [_key _value] :as query}]
+  (->> query
+       (find ds)
+       (lookup ds)))
+
+(defn exists?
+  "Returns true if value exists for key in kv-store"
+  [ds k]
+  (-> (d/q '[:find ?e
+             :in $ ?k
+             :where [?e ?k _]]
+           @ds k)
+      (seq)
+      (boolean)))
+
+(defn entries
+  "Get eids of all entities in which the provided attribute is present"
+  [ds key]
+  (->> (d/q '[:find ?e
+              :in $ ?k
+              :where [?e ?k _]]
+            @ds key)
+       (vec)
+       (flatten)
+       (into [])))
+
+(defn entities-with
+  "Gets all entities in which the provided attribute is present"
+  [ds key]
+  (->> key
+       (entries ds)
+       (lookup ds)))
+
+(defn delete!
+  "Accepts an entity id or a vec of entity ids and removes all the entities associated thereby"
+  [ds ids]
+  (let [multi? (vector? ids)
+        ids-vec (if multi? ids [ids])
+        query (mapv (fn [id] [:db/retractEntity id]) ids-vec)]
+    (d/transact ds query)))
+
+(defn insert!
+  "Inserts kv's into store. Returns the key-value pairs that were inserted."
+  [ds data]
+  (let [multi? (util/vectors? data)
+        input-kvs (if multi? data [data])]
+    (d/transact ds input-kvs)
+    input-kvs))
+
+(defn update!
+  "Update attributes and values for a given entity"
+  [ds id data]
+  (->> (merge {:db/add id} data)
+       (vec)
+       (flatten)
+       (conj [])
+       (d/transact ds)))
+
+(defn get-all
+  "Returns all key value pairs in table in kv-store."
+  [ds]
+  (->> (d/q '[:find ?e :where [?e _ _]] @ds)
+       (map (comp (partial into {}) (partial d/entity @ds) first))))
+
+(comment
+  (require '[source.datastore.util :as ds.util])
+
+  (def ds (ds.util/conn :datahike))
+  (get-all ds)
+  (entries ds :user/name)
+  (delete! ds (entries ds :user/name))
+  (update! ds 3 {:user/age 21})
+
+  (exists? ds :user/name)
+  (lookup ds [6 7])
+  (find ds {:key :user/name
+            :value "Keagan"})
+  (find-entities ds {:key :user/age
+                     :value 23})
+  (insert! ds {:user/name "Shani"
+               :user/age 23})
+
+  (insert! ds {:selection-schemas/id 4
+               :selection-schemas/schema {:title {:path ["tag/body" "tag/feed" "tag/title" "content/0"]}}})
+  (get-all ds)
+  (find ds {:key :selection-schemas/id
+            :value 4})
+
+  (let [ds (ds.util/conn :datahike)
+        id :selection-schemas/id
+        k :selection-schemas/schema
+        v {:title {:path ["tag/body" "tag/feed" "tag/title" "content/0"]}}]
+    (println (get-all ds))
+    (println (entities-with ds k))
+    (println "Putting a value")
+    (insert! ds {id 1
+                 k v})
+    (assert (= v
+               (->> k
+                    (entities-with k)
+                    (first)
+                    (k))))
+    (insert! ds {k v})
+    (println "1 " (get-all ds))
+    (println "2 " (find ds {:key id
+                            :value 1}))
+    (println "3 " (get-all ds))
+    (println "4 " (find ds {:key id
+                            :value 1}))
+    (println "Test passed")
+    (println "Deleting a value")
+    (delete! ds (entries ds k))
+    (assert (= []
+               (entries ds k)))
+    (println "Test passed"))
+
+  ())
diff --git a/src/source/datastore/datalevin.clj b/src/source/datastore/datalevin.clj
deleted file mode 100644
index 4925f2ec..00000000
--- a/src/source/datastore/datalevin.clj
+++ /dev/null
@@ -1,112 +0,0 @@
-(ns source.datastore.datalevin
-  (:require [datalevin.core :as d]
-            [source.datastore.tables :as tables]
-            [source.util :as util]))
-
-;; If we want to have higher write speed in future we can use transact-kv-async
-
-;; TODO:
-;; - when operations on kv store are done the relevant tables need to be opened beforehand
-;;   using (datalevin.core/open-dbi "table-name")
-;; - we need a nice way of opening and closing a connection to the datastore before and after
-;;  an operation is performed. Closing a connection to the datastore will also close the tables
-;;  in that datastore. Although it isn't documented as far as I could see in the datalevin or
-;;  LMDB docs, I think we should avoid opening the same table multiple times. So we need to figure
-;;  out a way to open a connection once (both on a datastore level and a datastore level), reuse it
-;;  where required, and then close it.
-;;
-;;  A possible solution here is to open up the kv-store and the tables we use on server start and
-;;  pass the kv-store connection around. As you might notice from the implementation, we don't need
-;;  to "def" a table connection and pass it around, only the store connection. As long as you call
-;;  (datalevin.core/open-dbi "table-name"). Refer to the comment block below.
-
-(defn find
-  "Returns the value for a key in the kv-store"
-  [ds {:keys [tname key]}]
-  (tables/open-table! ds (name tname))
-  (println "finding for " key " in " (name tname))
-  (d/get-value ds (name tname) key))
-
-(defn exists?
-  "Returns true if value exists for key in kv-store"
-  [ds opts]
-  (-> (find ds opts)
-      (some?)))
-
-(defn entries
-  "Get the number of entries in a table"
-  [ds {:keys [tname]}]
-  (tables/open-table! ds (name tname))
-  (d/entries ds tname))
-
-(defn delete!
-  "Removes one or multiple keys from kv store."
-  [ds {:keys [tname keys]}]
-  (tables/open-table! ds (name tname))
-  (->> (mapv (fn [key] [:del key]) keys)
-       (d/transact-kv ds (name tname))))
-
-(defn insert!
-  "Inserts kv's into store. Skips keys that already exist.
-  Returns the key-value pairs that were inserted."
-  [ds {:keys [tname data]}]
-  (tables/open-table! ds (name tname))
-  (let [multi? (util/vectors? data)
-        input-kvs (if multi? data [data])
-        kvs-to-insert (->> input-kvs
-                           (filterv (fn [[k _]]
-                                      (not (exists? ds {:tname tname :key k})))))]
-    (->> kvs-to-insert
-         (mapv (fn [[k v]] [:put k v]))
-         (d/transact-kv ds (name tname)))
-    kvs-to-insert))
-
-(defn update!
-  "Replaces values for keys in store. Skips keys that don't exist"
-  [ds {:keys [tname data]}]
-  (tables/open-table! ds (name tname))
-  (->> data
-       (filter (fn [[k _]]
-                 (exists? ds {:tname tname :key k})))
-       (map (fn [[k v]]
-              [:put k v]))
-       (d/transact-kv ds (name tname))))
-
-(defn get-all
-  "Returns all key value pairs in table in kv-store."
-  [ds {:keys [tname]}]
-  (tables/open-table! ds (name tname))
-  (println "finding all in " (name tname))
-  (d/get-range ds (name tname) [:all]))
-
-(comment
-  (require '[source.datastore.util :as ds.util])
-  (let [ds (ds.util/conn :store)
-        key "test-key"
-        value {:title {:path ["tag/body" "tag/feed" "tag/title" "content/0"]}}
-        tname :test-table]
-    (println (get-all ds {:tname :selection-schemas}))
-    (println (find ds {:tname :selection-schemas :key 4}))
-    (println "Putting a value")
-    (d/open-dbi ds (name tname))
-    (insert! ds {:tname tname
-                 :data [key value]})
-    (assert (= value
-               (find ds {:tname tname
-                         :key key})))
-    (insert! ds {:tname tname
-                 :data [key value]})
-    (println "1 " (get-all ds {:tname :selection-schemas}))
-    (println "2 " (find ds {:tname :selection-schemas :key 4}))
-    (println "3 " (get-all ds {:tname tname}))
-    (println "4 " (find ds {:tname tname :key "test-key"}))
-    (println "Test passed")
-    (println "Deleting a value")
-    (delete! ds {:tname tname
-                 :keys [key]})
-    (assert (= nil
-               (find ds {:tname tname
-                         :key key})))
-    (println "Test passed")
-    ()
-    (ds.util/close ds)))
diff --git a/src/source/datastore/interface.clj b/src/source/datastore/interface.clj
index a955478c..abf4c3a6 100644
--- a/src/source/datastore/interface.clj
+++ b/src/source/datastore/interface.clj
@@ -1,30 +1,36 @@
 (ns source.datastore.interface
-  (:require [source.datastore.datalevin :as dl]
+  (:require [source.datastore.datahike :as dh]
             [source.datastore.util :as store.util]))
 
 (defn ds [store-name]
   (store.util/conn store-name))
 
-(defn store-name [& args]
-  (apply store.util/store-name args))
+(defn lookup [ds eids]
+  (dh/lookup ds eids))
 
-(defn find [ds opts]
-  (dl/find ds opts))
+(defn find [ds {:keys [_key _value] :as opts}]
+  (dh/find ds opts))
 
-(defn exists? [ds opts]
-  (dl/exists? ds opts))
+(defn find-entities [ds {:keys [_key _value] :as opts}]
+  (dh/find-entities ds opts))
 
-(defn insert! [ds opts]
-  (dl/insert! ds opts))
+(defn exists? [ds k]
+  (dh/exists? ds k))
 
-(defn update! [ds opts]
-  (dl/update! ds opts))
+(defn insert! [ds data]
+  (dh/insert! ds data))
 
-(defn get-all [ds opts]
-  (dl/get-all ds opts))
+(defn update! [ds eid data]
+  (dh/update! ds eid data))
 
-(defn delete! [ds opts]
-  (dl/delete! ds opts))
+(defn get-all [ds]
+  (dh/get-all ds))
 
-(defn entries [ds opts]
-  (dl/entries ds opts))
+(defn delete! [ds eids]
+  (dh/delete! ds eids))
+
+(defn entries [ds k]
+  (dh/entries ds k))
+
+(defn entities-with [ds k]
+  (dh/entities-with ds k))
diff --git a/src/source/datastore/tables.clj b/src/source/datastore/tables.clj
deleted file mode 100644
index 1449ce2f..00000000
--- a/src/source/datastore/tables.clj
+++ /dev/null
@@ -1,24 +0,0 @@
-(ns source.datastore.tables
-  (:require [datalevin.core :as d]))
-
-(defn open-table!
-  "Opens table in the store."
-  [store tname]
-  (d/open-dbi store (name tname)))
-
-(defn drop-table!
-  "Clears data from and deleted table"
-  [store tname]
-  (d/drop-dbi store (name tname)))
-
-(defn clear-table!
-  "Clear data from table"
-  [store tname]
-  (d/clear-dbi store (name tname)))
-
-(defn tables
-  "Returns a vector of all tables in store"
-  [store]
-  (d/list-dbis store))
-
-(open-table! (d/open-kv "store") "some-table")
diff --git a/src/source/datastore/util.clj b/src/source/datastore/util.clj
index 6d0a70e2..9bb6c9a7 100644
--- a/src/source/datastore/util.clj
+++ b/src/source/datastore/util.clj
@@ -1,5 +1,5 @@
 (ns source.datastore.util
-  (:require [datalevin.core :as d]
+  (:require [datahike.api :as d]
             [source.config :as conf]
             [clojure.string :as string]))
 
@@ -13,17 +13,36 @@
   (-> (str (conf/read-value :database :dir) store-name)
       (absolute)))
 
+(defn config [store-name]
+  {:store {:backend :file
+           :path (store-path (name store-name))}
+   :schema-flexibility :read})
+
 (defn conn
-  "Open a connection to a datalevin kv store"
-  [store-name] (d/open-kv (store-path (name store-name))))
+  "Open a connection to a datahike store"
+  [store-name]
+  (d/connect (config store-name)))
 
 (defn close
-  "Close a connection to a datalevin store"
+  "Close a connection to a datahike store"
   [conn]
-  (d/close-kv conn))
+  (d/release conn))
 
 (defn store-name
   ([type]
    (name type))
   ([type id]
    (str (name type) "_" id)))
+
+(comment
+  (d/create-database (config :datahike))
+  (d/delete-database (config :datahike))
+
+  (defonce ds (conn :datahike))
+  (d/transact ds [{:user/name "Keagan"
+                   :user/age 23}])
+  (d/q '{:find [?e ?n ?a]
+         :where [[?e :user/name ?n]
+                 [?e :user/age ?a]]}
+       @ds)
+  ())
diff --git a/src/source/db/master.clj b/src/source/db/master.clj
index b2f9795f..b8b0419b 100644
--- a/src/source/db/master.clj
+++ b/src/source/db/master.clj
@@ -153,4 +153,5 @@
   (sql/format businesses)
   (sql/format user-sectors)
   (sql/format feed-sectors)
+  (sql/format selection-schemas)
   ())
diff --git a/src/source/routes/data.clj b/src/source/routes/data.clj
index 17b196ef..124a26a5 100644
--- a/src/source/routes/data.clj
+++ b/src/source/routes/data.clj
@@ -6,3 +6,16 @@
   (let [{:keys [_schema-id _url] :as opts} body]
     (-> (services/extract-data store opts)
         (res/response))))
+
+(comment
+  (require '[source.rss.youtube :as yt]
+           '[source.datastore.interface :as store])
+
+  (def url (->> "https://www.youtube.com/@ThePrimeTimeagen"
+                (yt/find-channel-id)
+                (str "https://www.youtube.com/feeds/videos.xml?channel_id=")))
+
+  (post {:store (store/ds :datahike)
+         :body {:schema-id 100
+                :url url}})
+  ())
diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index 476dc92e..38aeded4 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -27,9 +27,6 @@
             [source.routes.selection-schemas :as selection-schemas]
             [source.routes.xml :as xml]
             [source.routes.data :as data]
-            [source.datastore.tables :as store.tables]
-            [source.datastore.interface :as store]
-            [source.datastore.util :as store.util]
             [source.util :as util]))
 
 (defn route [handlers]
@@ -108,15 +105,17 @@
                          :openapi {:security [{:bearerAuth []}]}}
         ["/authorized"  (route {:get authorized/get})]]
 
-       ["/admin"        {:middleware [[mw/apply-auth {:required-type :admin}]]
-                         :no-doc true
-                         :tags #{"admin"}
-                         :swagger {:security [{"auth" []}]}
-                         :openapi {:security [{:bearerAuth []}]}}
-        ["/add-admin"   (route {:post admin/post})]
-        ["/selection-schema" {:post selection-schema/post}]]]
-      ["/ast" {:post xml/post}]
-      ["/extract-data" {:post data/post}]
+       ["/admin"                  {:middleware [[mw/apply-auth {:required-type :admin}]]
+                                   :no-doc true
+                                   :tags #{"admin"}
+                                   :swagger {:security [{"auth" []}]}
+                                   :openapi {:security [{:bearerAuth []}]}}
+        ["/add-admin"             (route {:post admin/post})]
+        ["/selection-schemas"     {:get selection-schemas/get
+                                   :post selection-schema/post}]
+        ["/selection-schemas/:id" {:get selection-schema/get}]
+        ["/ast"                   {:post xml/post}]
+        ["/extract-data"          {:post data/post}]]]
 
       {:data {:coercion (reitit.coercion.malli/create
                          {:error-keys #{#_:type :coercion :in :schema :value :errors :humanized #_:transformed}
@@ -138,13 +137,8 @@
 (comment
   (require '[source.middleware.auth.util :as auth.util]
            '[source.datastore.interface :as store]
-           '[source.datastore.tables :as store.tables]
            '[source.rss.youtube :as yt])
 
-  (route {:get #'user/get
-          :patch #'user/patch})
-  (util/metadata user/get)
-
   (let [app (create-app)
         request {:uri "/users" :request-method :get}]
     (-> request
@@ -235,10 +229,8 @@
         (json/read-json {:key-fn keyword})))
 
   (let [app (create-app)
-        store (store.util/conn "datalevin")
-        ;; open table before operation
-        _thing (dl/open-dbi store "selection-schemas")
-        request {:uri "/admin/selection-schema"
+        store (store/ds :datahike)
+        request {:uri "/admin/selection-schemas"
                  :request-method :post
                  :store store
                  :body {:record {:provider-id 1
@@ -251,7 +243,7 @@
         (json/read-json {:key-fn keyword})))
 
   (let [app (create-app)
-        request {:uri "/admin/selection-schema/1"
+        request {:uri "/admin/selection-schemas/1"
                  :request-method :get
                  :headers {"authorization" (str "Bearer " (auth.util/sign-jwt {:id 1 :type "admin"}))}}]
     (-> request
@@ -260,7 +252,7 @@
         (json/read-json {:key-fn keyword})))
 
   (let [app (create-app)
-        store (store/ds :store)
+        store (store/ds :datahike)
         request {:uri "/admin/selection-schemas"
                  :store store
                  :request-method :get
@@ -286,16 +278,16 @@
         (json/read-json {:key-fn keyword})))
 
   (let [app (create-app)
-        store (store/ds :store)
+        store (store/ds :datahike)
         request {:uri "/admin/extract-data"
                  :store store
                  :request-method :post
-                 :body {:schema-id 1
+                 :body {:schema-id 2
                         :url (get-url)}
                  :headers {"authorization" (str "Bearer " (auth.util/sign-jwt {:id 1 :type "admin"}))}}]
-    (println (store/get-all store {:tname :selection-schemas}))
-    (println (store/find store {:tname :selection-schemas
-                                :key 1}))
+    (println (store/entities-with store :selection-schemas/id))
+    (println (store/find-entities store {:key :selection-schemas/id
+                                         :value 1}))
     (-> request
         app
         :body
diff --git a/src/source/routes/selection_schema.clj b/src/source/routes/selection_schema.clj
index 10dfbc9d..178fcd04 100644
--- a/src/source/routes/selection_schema.clj
+++ b/src/source/routes/selection_schema.clj
@@ -11,3 +11,9 @@
   (->> path-params
        (services/selection-schema ds)
        (res/response)))
+
+(comment
+  (require '[source.db.util :as db.util])
+  (get {:ds (db.util/conn)
+        :path-params {:id 1}})
+  ())
diff --git a/src/source/routes/selection_schemas.clj b/src/source/routes/selection_schemas.clj
index 8ed6540e..b774ad7b 100644
--- a/src/source/routes/selection_schemas.clj
+++ b/src/source/routes/selection_schemas.clj
@@ -5,3 +5,8 @@
 (defn get [{:keys [ds] :as _request}]
   (-> (services/selection-schemas ds)
       (res/response)))
+
+(comment
+  (require '[source.db.util :as db.util])
+  (get {:ds (db.util/conn)})
+  ())
diff --git a/src/source/services/xml_schemas.clj b/src/source/services/xml_schemas.clj
index 6b6daed8..acfcbc25 100644
--- a/src/source/services/xml_schemas.clj
+++ b/src/source/services/xml_schemas.clj
@@ -3,28 +3,27 @@
             [source.datastore.interface :as store]
             [source.rss.core :as rss]))
 
-(defn get-all
-  [ds {:keys [tname]}]
-  (store/get-all ds {:tname tname}))
-
 (defn add-output-schema!
   [store {:keys [schema]}]
-  (let [id (-> (store/entries store {:tname :output-schemas})
-               inc)]
-    (store/insert! store {:tname :selection-schemas
-                          :data [id schema]})))
+  (let [id (-> (store/entries store :output-schemas/id)
+               (count)
+               (inc))]
+    (store/insert! store {:selection-schemas/id id
+                          :selection-schemas/schema schema})))
 
 (defn add-selection-schema!
   [store db {:keys [schema record]}]
   (let [db-result (db/insert! db {:tname :selection-schemas
-                                  :data record})]
-    (store/insert! store {:tname :selection-schemas
-                          :data [(:id db-result) schema]})))
+                                  :data record
+                                  :ret :1})]
+    (store/insert! store {:selection-schemas/id (:id db-result)
+                          :selection-schemas/schema schema})))
 
 (defn selection-schemas
   ([ds] (selection-schemas ds {}))
   ([ds opts]
-   (->> {:tname :selection-schemas}
+   (->> {:tname :selection-schemas
+         :ret :*}
         (merge opts)
         (db/find ds))))
 
@@ -39,27 +38,55 @@
 
 (defn output-schemas
   [store]
-  (store/get-all store {:tname :output-schemas}))
+  (store/entities-with store :output-schemas/id))
 
 (defn output-schema
-  [store id]
-  (store/find store {:tname :output-schemas
-                     :key id}))
+  [store output-schema-id]
+  (->> {:key :output-schemas/id
+        :value output-schema-id}
+       (store/find-entities store)
+       (first)))
 
 (defn ast
   [url]
   (-> url
-      slurp
-      rss/get-ast
-      rss/collect-leaf-paths))
+      (slurp)
+      (rss/get-ast)))
 
 (defn extract-data
   [store schema-id url]
-  (let [schema (store/find store {:tname :selection-schemas
-                                  :key schema-id})]
-    (println (store/get-all store {:tname :selection-schemas}))
+  (let [schema (->> {:key :selection-schemas/id
+                     :value schema-id}
+                    (store/find-entities store)
+                    (first)
+                    (:selection-schemas/schema))]
+    (println (store/entities-with store :selection-schemas/id))
     (println "schema: " schema)
     (->> url
-         slurp
-         rss/get-ast
+         (slurp)
+         (rss/get-ast)
          (rss/extract-data schema))))
+
+(comment
+  (require '[source.db.util :as db.util])
+
+  (def ds (store/ds :datahike))
+
+  (count (store/entries ds :selection-schemas/id))
+  (store/entities-with ds :selection-schemas/id)
+  (store/find-entities ds {:key :selection-schemas/id
+                           :value 100})
+
+  (extract-data
+   ds
+   1
+   "https://www.youtube.com/feeds/videos.xml?channel_id=UCUyeluBRhGPCW4rPe_UvBZQ")
+
+  (add-selection-schema!
+   ds
+   (db.util/conn)
+   {:record {:provider-id 1
+             :output-schema-id 1}
+    :schema {:title {:path ["tag/body" "tag/feed" "tag/title" "content/0"]}}})
+
+  ())

From 6eaabba4701e98dd73b1553d23c2cb3dc69600a7 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Tue, 29 Jul 2025 19:26:03 +0200
Subject: [PATCH 044/391] added selection-schema, output-schema and provider
 routes

---
 src/source/routes/output_schema.clj           |  8 +++
 src/source/routes/output_schemas.clj          | 12 +++++
 src/source/routes/provider.clj                |  8 +++
 .../routes/provider_selection_schemas.clj     |  8 +++
 src/source/routes/providers.clj               | 12 +++++
 src/source/routes/reitit.clj                  | 34 +++++++++----
 src/source/routes/selection_schema.clj        |  5 --
 src/source/routes/selection_schemas.clj       |  5 ++
 src/source/services/interface.clj             | 21 ++++++++
 src/source/services/xml_schemas.clj           | 49 ++++++++++++++-----
 10 files changed, 137 insertions(+), 25 deletions(-)
 create mode 100644 src/source/routes/output_schema.clj
 create mode 100644 src/source/routes/output_schemas.clj
 create mode 100644 src/source/routes/provider.clj
 create mode 100644 src/source/routes/provider_selection_schemas.clj
 create mode 100644 src/source/routes/providers.clj

diff --git a/src/source/routes/output_schema.clj b/src/source/routes/output_schema.clj
new file mode 100644
index 00000000..1e2894e0
--- /dev/null
+++ b/src/source/routes/output_schema.clj
@@ -0,0 +1,8 @@
+(ns source.routes.output-schema
+  (:require [source.services.interface :as services]
+            [ring.util.response :as res]))
+
+(defn get [{:keys [store path-params] :as _request}]
+  (->> (:id path-params)
+       (services/output-schema store)
+       (res/response)))
diff --git a/src/source/routes/output_schemas.clj b/src/source/routes/output_schemas.clj
new file mode 100644
index 00000000..bd7444bf
--- /dev/null
+++ b/src/source/routes/output_schemas.clj
@@ -0,0 +1,12 @@
+(ns source.routes.output-schemas
+  (:require [source.services.interface :as services]
+            [ring.util.response :as res]))
+
+(defn get [{:keys [store] :as _request}]
+  (-> (services/output-schemas store)
+      (res/response)))
+
+(defn post [{:keys [store body] :as _request}]
+  (let [{:keys [schema]} body]
+    (services/add-output-schema! store schema)
+    (res/response {:message "successfully added output schema"})))
diff --git a/src/source/routes/provider.clj b/src/source/routes/provider.clj
new file mode 100644
index 00000000..966910e6
--- /dev/null
+++ b/src/source/routes/provider.clj
@@ -0,0 +1,8 @@
+(ns source.routes.provider
+  (:require [source.services.interface :as services]
+            [ring.util.response :as res]))
+
+(defn get [{:keys [store path-params] :as _request}]
+  (->> (:id path-params)
+       (services/provider store)
+       (res/response)))
diff --git a/src/source/routes/provider_selection_schemas.clj b/src/source/routes/provider_selection_schemas.clj
new file mode 100644
index 00000000..a4411245
--- /dev/null
+++ b/src/source/routes/provider_selection_schemas.clj
@@ -0,0 +1,8 @@
+(ns source.routes.provider-selection-schemas
+  (:require [source.services.interface :as services]
+            [ring.util.response :as res]))
+
+(defn get [{:keys [ds path-params] :as _request}]
+  (->> {:provider-id (:id path-params)}
+       (services/selection-schemas-by-provider ds)
+       (res/response)))
diff --git a/src/source/routes/providers.clj b/src/source/routes/providers.clj
new file mode 100644
index 00000000..fdd4d21e
--- /dev/null
+++ b/src/source/routes/providers.clj
@@ -0,0 +1,12 @@
+(ns source.routes.providers
+  (:require [source.services.interface :as services]
+            [ring.util.response :as res]))
+
+(defn get [{:keys [store] :as _request}]
+  (-> (services/providers store)
+      (res/response)))
+
+(defn post [{:keys [store body] :as _request}]
+  (let [{:keys [_name]} body]
+    (services/add-provider! store name)
+    (res/response {:message "successfully added provider"})))
diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index 38aeded4..07ce3c70 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -20,14 +20,20 @@
             [source.routes.google-user :as google-user]
             [source.routes.admin :as admin]
             [source.routes.authorized :as authorized]
-            [source.routes.selection-schema :as selection-schema]
             [source.routes.business :as business]
             [source.routes.businesses :as businesses]
             [source.routes.sectors :as sectors]
             [source.routes.selection-schemas :as selection-schemas]
+            [source.routes.selection-schema :as selection-schema]
+            [source.routes.provider-selection-schemas :as provider-selection-schemas]
+            [source.routes.output-schemas :as output-schemas]
+            [source.routes.output-schema :as output-schema]
+            [source.routes.providers :as providers]
+            [source.routes.provider :as provider]
             [source.routes.xml :as xml]
             [source.routes.data :as data]
-            [source.util :as util]))
+            [source.util :as util]
+            [source.datastore.interface :as store]))
 
 (defn route [handlers]
   (reduce (fn [acc [k v]]
@@ -39,7 +45,8 @@
           {} handlers))
 
 (defn create-app []
-  (let [ds (db/ds :master)]
+  (let [ds (db/ds :master)
+        store (store/ds :datahike)]
     (ring/ring-handler
      (ring/router
       [["/swagger.json" {:get {:no-doc true
@@ -111,9 +118,19 @@
                                    :swagger {:security [{"auth" []}]}
                                    :openapi {:security [{:bearerAuth []}]}}
         ["/add-admin"             (route {:post admin/post})]
-        ["/selection-schemas"     {:get selection-schemas/get
-                                   :post selection-schema/post}]
-        ["/selection-schemas/:id" {:get selection-schema/get}]
+        ["/selection-schemas"
+         [""                      {:get selection-schemas/get
+                                   :post selection-schemas/post}]
+         ["/:id"                  {:get selection-schema/get}]
+         ["/providers/:id"        {:get provider-selection-schemas/get}]]
+        ["/output-schemas"
+         [""                      {:get output-schemas/get
+                                   :post output-schemas/post}]
+         ["/:id"                  {:get output-schema/get}]]
+        ["/providers"
+         [""                      {:get providers/get
+                                   :post providers/post}]
+         ["/:id"                  {:get provider/get}]]
         ["/ast"                   {:post xml/post}]
         ["/extract-data"          {:post data/post}]]]
 
@@ -123,7 +140,7 @@
                           :strip-extra-keys true
                           :default-values true
                           :options nil})
-              :middleware [[mw/apply-generic :ds ds]
+              :middleware [[mw/apply-generic :ds ds :store store]
                            [exception/exception-middleware]]}})
      (ring/routes
       (swagger-ui/create-swagger-ui-handler {:path "/"
@@ -136,7 +153,6 @@
 
 (comment
   (require '[source.middleware.auth.util :as auth.util]
-           '[source.datastore.interface :as store]
            '[source.rss.youtube :as yt])
 
   (let [app (create-app)
@@ -282,7 +298,7 @@
         request {:uri "/admin/extract-data"
                  :store store
                  :request-method :post
-                 :body {:schema-id 2
+                 :body {:schema-id 1
                         :url (get-url)}
                  :headers {"authorization" (str "Bearer " (auth.util/sign-jwt {:id 1 :type "admin"}))}}]
     (println (store/entities-with store :selection-schemas/id))
diff --git a/src/source/routes/selection_schema.clj b/src/source/routes/selection_schema.clj
index 178fcd04..d75925f1 100644
--- a/src/source/routes/selection_schema.clj
+++ b/src/source/routes/selection_schema.clj
@@ -2,11 +2,6 @@
   (:require [source.services.interface :as services]
             [ring.util.response :as res]))
 
-(defn post [{:keys [store ds body] :as _request}]
-  (let [{:keys [_schema _record] :as opts} body]
-    (-> (services/add-selection-schema! store ds opts)
-        (res/response))))
-
 (defn get [{:keys [ds path-params] :as _request}]
   (->> path-params
        (services/selection-schema ds)
diff --git a/src/source/routes/selection_schemas.clj b/src/source/routes/selection_schemas.clj
index b774ad7b..9d4531f2 100644
--- a/src/source/routes/selection_schemas.clj
+++ b/src/source/routes/selection_schemas.clj
@@ -6,6 +6,11 @@
   (-> (services/selection-schemas ds)
       (res/response)))
 
+(defn post [{:keys [store ds body] :as _request}]
+  (let [{:keys [_schema _record] :as opts} body]
+    (-> (services/add-selection-schema! store ds opts)
+        (res/response))))
+
 (comment
   (require '[source.db.util :as db.util])
   (get {:ds (db.util/conn)})
diff --git a/src/source/services/interface.clj b/src/source/services/interface.clj
index 449f3959..d875ce09 100644
--- a/src/source/services/interface.clj
+++ b/src/source/services/interface.clj
@@ -36,12 +36,33 @@
 (defn selection-schemas [ds]
   (xml/selection-schemas ds))
 
+(defn selection-schemas-by-provider [ds {:keys [_provider-id] :as opts}]
+  (xml/selection-schemas-by-provider ds opts))
+
 (defn ast [url]
   (xml/ast url))
 
 (defn extract-data [store {:keys [schema-id url]}]
   (xml/extract-data store schema-id url))
 
+(defn output-schemas [store]
+  (xml/output-schemas store))
+
+(defn output-schema [store output-schema-id]
+  (xml/output-schema store output-schema-id))
+
+(defn add-output-schema! [store schema]
+  (xml/add-output-schema! store schema))
+
+(defn providers [store]
+  (xml/providers store))
+
+(defn provider [store provider-id]
+  (xml/provider store provider-id))
+
+(defn add-provider! [store name]
+  (xml/add-provider! store name))
+
 (comment
   (users (db/ds :master))
   (user (db/ds :master) {:id 2})
diff --git a/src/source/services/xml_schemas.clj b/src/source/services/xml_schemas.clj
index acfcbc25..217debaa 100644
--- a/src/source/services/xml_schemas.clj
+++ b/src/source/services/xml_schemas.clj
@@ -4,12 +4,23 @@
             [source.rss.core :as rss]))
 
 (defn add-output-schema!
-  [store {:keys [schema]}]
+  [store schema]
   (let [id (-> (store/entries store :output-schemas/id)
                (count)
                (inc))]
-    (store/insert! store {:selection-schemas/id id
-                          :selection-schemas/schema schema})))
+    (store/insert! store {:output-schemas/id id
+                          :output-schemas/schema schema})))
+
+(defn output-schemas
+  [store]
+  (store/entities-with store :output-schemas/id))
+
+(defn output-schema
+  [store output-schema-id]
+  (->> {:key :output-schemas/id
+        :value output-schema-id}
+       (store/find-entities store)
+       (first)))
 
 (defn add-selection-schema!
   [store db {:keys [schema record]}]
@@ -27,6 +38,14 @@
         (merge opts)
         (db/find ds))))
 
+(defn selection-schemas-by-provider
+  [ds {:keys [provider-id] :as opts}]
+  (->> {:tname :selection-schemas
+        :where [:= :provider-id provider-id]
+        :ret :*}
+       (merge opts)
+       (db/find ds)))
+
 (defn selection-schema [ds {:keys [id where] :as opts}]
   (->> {:tname :selection-schemas
         :where (if (some? id)
@@ -36,14 +55,22 @@
        (merge opts)
        (db/find ds)))
 
-(defn output-schemas
+(defn add-provider!
+  [store name]
+  (let [id (-> (store/entries store :providers/id)
+               (count)
+               (inc))]
+    (store/insert! store {:providers/id id
+                          :providers/name name})))
+
+(defn providers
   [store]
-  (store/entities-with store :output-schemas/id))
+  (store/entities-with store :providers/id))
 
-(defn output-schema
-  [store output-schema-id]
-  (->> {:key :output-schemas/id
-        :value output-schema-id}
+(defn provider
+  [store provider-id]
+  (->> {:key :providers/id
+        :value provider-id}
        (store/find-entities store)
        (first)))
 
@@ -60,8 +87,6 @@
                     (store/find-entities store)
                     (first)
                     (:selection-schemas/schema))]
-    (println (store/entities-with store :selection-schemas/id))
-    (println "schema: " schema)
     (->> url
          (slurp)
          (rss/get-ast)
@@ -72,6 +97,8 @@
 
   (def ds (store/ds :datahike))
 
+  (selection-schemas-by-provider (db.util/conn) {:provider-id 1})
+
   (count (store/entries ds :selection-schemas/id))
   (store/entities-with ds :selection-schemas/id)
   (store/find-entities ds {:key :selection-schemas/id

From c0169240207bf44fff3291ee6bf0c181d40e4edc Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Tue, 29 Jul 2025 19:27:07 +0200
Subject: [PATCH 045/391] added middleware to attach datastore connection to
 request

---
 src/source/middleware/core.clj      | 13 ++++++++++++-
 src/source/middleware/interface.clj |  4 ++--
 2 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/src/source/middleware/core.clj b/src/source/middleware/core.clj
index 033af752..344e6c15 100644
--- a/src/source/middleware/core.clj
+++ b/src/source/middleware/core.clj
@@ -16,6 +16,16 @@
         (assoc :ds ds)
         (handler))))
 
+(defn wrap-store [handler store]
+  (fn [request]
+    (-> request
+        (assoc :store store)
+        (handler))))
+
+(defn apply-store [app store]
+  (-> app
+      (wrap-store store)))
+
 (defn process-body [{:keys [body] :as req} t-fn]
   (assoc req
          :body
@@ -38,9 +48,10 @@
   (-> app
       (wrap-ds ds)))
 
-(defn apply-generic [app & {:keys [ds]}]
+(defn apply-generic [app & {:keys [ds store]}]
   (-> app
       (apply-ds ds)
+      (apply-store store)
       (wrap-case-conversion)
       (content-type/wrap-content-type)
       (wrap-cors :access-control-allow-origin [(re-pattern (conf/read-value :cors-origin))]
diff --git a/src/source/middleware/interface.clj b/src/source/middleware/interface.clj
index ffb97174..fb2c9096 100644
--- a/src/source/middleware/interface.clj
+++ b/src/source/middleware/interface.clj
@@ -1,8 +1,8 @@
 (ns source.middleware.interface
   (:require [source.middleware.core :as mw]))
 
-(defn apply-generic [app & {:keys [ds]}]
-  (mw/apply-generic app :ds ds))
+(defn apply-generic [app & {:keys [ds store]}]
+  (mw/apply-generic app :ds ds :store store))
 
 (defn apply-auth
   "accepts required-type as an optional parameter to authorize the route only for the specified user type"

From 44e706ee18b70c9c0b910f8675f7d2829c2cd327 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Tue, 29 Jul 2025 19:27:48 +0200
Subject: [PATCH 046/391] allow for datastore creation and deletion in
 migration

---
 src/source/datastore/util.clj                | 42 ++++++++++++++++++--
 src/source/migrations/001_init_master_db.clj |  5 +++
 2 files changed, 43 insertions(+), 4 deletions(-)

diff --git a/src/source/datastore/util.clj b/src/source/datastore/util.clj
index 9bb6c9a7..29dcaaf6 100644
--- a/src/source/datastore/util.clj
+++ b/src/source/datastore/util.clj
@@ -9,14 +9,48 @@
       (clojure.string/replace "/." "/")
       (str path)))
 
-(defn store-path [store-name]
-  (-> (str (conf/read-value :database :dir) store-name)
-      (absolute)))
+(defn store-path
+  ([store-name]
+   (-> (str (conf/read-value :database :dir) store-name)
+       (absolute))))
 
 (defn config [store-name]
   {:store {:backend :file
            :path (store-path (name store-name))}
-   :schema-flexibility :read})
+   :schema-flexibility :read
+   :initial-tx [{:db/ident        :selection-schemas/id
+                 :db/valueType    :db.type/long
+                 :db/cardinality  :db.cardinality/one
+                 :db/unique       :db.unique/identity}
+                {:db/ident        :selection-schemas/schema
+                 :db/valueType    :db.type/any
+                 :db/cardinality  :db.cardinality/one}
+
+                {:db/ident        :output-schemas/id
+                 :db/valueType    :db.type/long
+                 :db/cardinality  :db.cardinality/one
+                 :db/unique       :db.unique/identity}
+                {:db/ident        :output-schemas/schema
+                 :db/valueType    :db.type/any
+                 :db/cardinality  :db.cardinality/one}
+
+                {:db/ident        :providers/id
+                 :db/valueType    :db.type/long
+                 :db/cardinality  :db.cardinality/one
+                 :db/unique       :db.unique/identity}
+                {:db/ident        :providers/name
+                 :db/valueType    :db.type/string
+                 :db/cardinality  :db.cardinality/one}]})
+
+(defn create-datastore [store-name]
+  (when-not (d/database-exists? (config store-name))
+    (println "Creating datastore...")
+    (d/create-database (config store-name))))
+
+(defn delete-datastore [store-name]
+  (when (d/database-exists? (config store-name))
+    (println "Deleting datastore...")
+    (d/delete-database (config store-name))))
 
 (defn conn
   "Open a connection to a datahike store"
diff --git a/src/source/migrations/001_init_master_db.clj b/src/source/migrations/001_init_master_db.clj
index 01cb8a27..6d4f7395 100644
--- a/src/source/migrations/001_init_master_db.clj
+++ b/src/source/migrations/001_init_master_db.clj
@@ -3,6 +3,7 @@
             [source.db.master]
             [source.db.honey :as db]
             [source.db.tables :as tables]
+            [source.datastore.util :as ds.util]
             [source.config :as conf]))
 
 (def baselines-seed
@@ -58,6 +59,8 @@
 (defn run-up! [context]
   (let [ds-master (:db-master context)]
 
+    (ds.util/create-datastore :datahike)
+
     (tables/create-tables!
      ds-master
      :source.db.master
@@ -96,6 +99,8 @@
 
 (defn run-down! [context]
   (let [ds-master (:db-master context)]
+    (ds.util/delete-datastore :datahike)
+
     (tables/drop-all-tables! ds-master)))
 
 (comment

From 7cf61480ae6a71a6b9a919172b3678d8bbc5f184 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Tue, 29 Jul 2025 20:40:00 +0200
Subject: [PATCH 047/391] fixed ast route

---
 src/source/services/xml_schemas.clj | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/source/services/xml_schemas.clj b/src/source/services/xml_schemas.clj
index 217debaa..2cd7c92e 100644
--- a/src/source/services/xml_schemas.clj
+++ b/src/source/services/xml_schemas.clj
@@ -78,7 +78,8 @@
   [url]
   (-> url
       (slurp)
-      (rss/get-ast)))
+      (rss/get-ast)
+      (rss/collect-leaf-paths)))
 
 (defn extract-data
   [store schema-id url]

From c4f76a954028068137d1be28d411ce854ae73207 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 30 Jul 2025 16:39:39 +0200
Subject: [PATCH 048/391] fixed ids being passed as strings in :id datahike
 routes

---
 src/source/routes/output_schema.clj              | 1 +
 src/source/routes/provider.clj                   | 1 +
 src/source/routes/provider_selection_schemas.clj | 3 ++-
 3 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/source/routes/output_schema.clj b/src/source/routes/output_schema.clj
index 1e2894e0..95fe3943 100644
--- a/src/source/routes/output_schema.clj
+++ b/src/source/routes/output_schema.clj
@@ -4,5 +4,6 @@
 
 (defn get [{:keys [store path-params] :as _request}]
   (->> (:id path-params)
+       (Integer/parseInt)
        (services/output-schema store)
        (res/response)))
diff --git a/src/source/routes/provider.clj b/src/source/routes/provider.clj
index 966910e6..9c67895a 100644
--- a/src/source/routes/provider.clj
+++ b/src/source/routes/provider.clj
@@ -4,5 +4,6 @@
 
 (defn get [{:keys [store path-params] :as _request}]
   (->> (:id path-params)
+       (Integer/parseInt)
        (services/provider store)
        (res/response)))
diff --git a/src/source/routes/provider_selection_schemas.clj b/src/source/routes/provider_selection_schemas.clj
index a4411245..ca0094ed 100644
--- a/src/source/routes/provider_selection_schemas.clj
+++ b/src/source/routes/provider_selection_schemas.clj
@@ -3,6 +3,7 @@
             [ring.util.response :as res]))
 
 (defn get [{:keys [ds path-params] :as _request}]
-  (->> {:provider-id (:id path-params)}
+  (->> (:id path-params)
+       (assoc {} :provider-id)
        (services/selection-schemas-by-provider ds)
        (res/response)))

From da4a021303a757d063394d42fa18802e8ab33a81 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Thu, 31 Jul 2025 16:25:22 +0200
Subject: [PATCH 049/391] fixed bugs in providers

---
 src/source/datastore/datahike.clj   | 2 ++
 src/source/datastore/util.clj       | 3 +++
 src/source/routes/providers.clj     | 8 +++++++-
 src/source/services/xml_schemas.clj | 3 +++
 4 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/src/source/datastore/datahike.clj b/src/source/datastore/datahike.clj
index a13c51ae..065b4b87 100644
--- a/src/source/datastore/datahike.clj
+++ b/src/source/datastore/datahike.clj
@@ -105,6 +105,8 @@
                      :value 23})
   (insert! ds {:user/name "Shani"
                :user/age 23})
+  (insert! ds {:providers/name "YouTube"})
+  (delete! ds (entries ds :providers/id))
 
   (insert! ds {:selection-schemas/id 4
                :selection-schemas/schema {:title {:path ["tag/body" "tag/feed" "tag/title" "content/0"]}}})
diff --git a/src/source/datastore/util.clj b/src/source/datastore/util.clj
index 29dcaaf6..efaaa512 100644
--- a/src/source/datastore/util.clj
+++ b/src/source/datastore/util.clj
@@ -33,6 +33,9 @@
                 {:db/ident        :output-schemas/schema
                  :db/valueType    :db.type/any
                  :db/cardinality  :db.cardinality/one}
+                {:db/ident        :output-schemas/version
+                 :db/valueType    :db.type/long
+                 :db/cardinality  :db.cardinality/one}
 
                 {:db/ident        :providers/id
                  :db/valueType    :db.type/long
diff --git a/src/source/routes/providers.clj b/src/source/routes/providers.clj
index fdd4d21e..2cdd478a 100644
--- a/src/source/routes/providers.clj
+++ b/src/source/routes/providers.clj
@@ -7,6 +7,12 @@
       (res/response)))
 
 (defn post [{:keys [store body] :as _request}]
-  (let [{:keys [_name]} body]
+  (let [{:keys [name]} body]
     (services/add-provider! store name)
     (res/response {:message "successfully added provider"})))
+
+(comment 
+  (require '[source.datastore.interface :as store])
+
+  (services/add-provider! (store/ds :datahike) "YouTube")
+  )
diff --git a/src/source/services/xml_schemas.clj b/src/source/services/xml_schemas.clj
index 2cd7c92e..bfca6ffb 100644
--- a/src/source/services/xml_schemas.clj
+++ b/src/source/services/xml_schemas.clj
@@ -99,6 +99,9 @@
   (def ds (store/ds :datahike))
 
   (selection-schemas-by-provider (db.util/conn) {:provider-id 1})
+  (add-provider! ds "poop")
+  (add-provider! ds "YouTube")
+  (providers ds)
 
   (count (store/entries ds :selection-schemas/id))
   (store/entities-with ds :selection-schemas/id)

From 57127123017c943d225121c66a1b158d4c25aa33 Mon Sep 17 00:00:00 2001
From: Merveille van Eck <merveillevaneck@gmail.com>
Date: Fri, 1 Aug 2025 09:53:02 +0200
Subject: [PATCH 050/391] added some shitness

---
 resources/admins_encrypted.json               |  2 +-
 src/source/db/honey.clj                       | 11 +++++-----
 .../routes/provider_selection_schemas.clj     | 21 ++++++++++++-------
 src/source/routes/reitit.clj                  |  3 ++-
 src/source/routes/selection_schema.clj        | 14 +++++++------
 src/source/routes/selection_schemas.clj       | 10 ++++-----
 src/source/services/xml_schemas.clj           |  6 ++++--
 7 files changed, 40 insertions(+), 27 deletions(-)

diff --git a/resources/admins_encrypted.json b/resources/admins_encrypted.json
index 50d35965..03531385 100644
--- a/resources/admins_encrypted.json
+++ b/resources/admins_encrypted.json
@@ -1 +1 @@
-205f937d56f65fb4f74c29508e2dca11bd4747257dceb2840a4a9610fa086aa55996313547f4fafa6c319b463049fc6d11d781a8f1d97ecc03353b6c3c3c80e2ab99e78cab2728cc9a208bf7d922a6d9a2fd9cc0d0d19d1dbe43f1e0e7de06ed8a7f3132e53d72939b99c7e524ed791872fcf019c200eb5cf81d963462235c6970dcf425351e4d491f997885302513066ad671891a4ea95e4436d5f3f10fdbb861cb26c107d6ad103ec774f86b332b56d135a8b53fb4b4dc95599638ab07753cf4b4675e689609edfcbbc2c6694d1945d539c9e00b4e21667d48586c3c08277dab000afae8c76352164b4771bf169166bf71338e1cb38dc6af02a01d2a715cecec93f7b66f8f4e0387947918e2c3f56f2406443936bf5147491b77de91d2267de864a7316e8ef59971cfde209d1a86b1a022a8af0f653d1d0f380dfb6ca85f610aa6ed74a4d59041cd481d498d0e8bf6bf2d674f07b750a798ee8c4c7720b165eb2710170aab171c93a73967ac037ce00442793e4b11c74d32415b895ab101126af9db912befc603bf2bbe2d7aa651cbb9adb044b54737d6163986d460156840ead2d5e2b9ab80eddd047da9f319865a
\ No newline at end of file
+1f849fdbcaaa4d5cf82b56107541fa49174399ba9cc32f67fd7eceb7f0847b8050f85973837676f7d7f8d92fa4d6cacf9e175a8fd6fff154212d1187c2888ec3bd091e838d7719d1d920ea70d9c230806642de10bfedcc2b4512c127f839eae51cf766a96a7a9fbc9e3f17680e077c5eb408459ad866d7f6ed37f7ec6a5fd87757b9265552d481b087becef00af7cdadf8d4da15f0e4365b4b6ce3e7e6ea7717b0fe10240e36bafb1ec29fc45495076b502573651b1a025b718a09121c009d4c7dcc53386ba6a133d736dc4fc35ed01e681cfce6c1fab6212e5442b3c4430cae496390e7fdbc67e5a7d0154f81d805bdd1396c217c5a4f2d4d75ca920dd5ed0022d3776a4df151e2af3f22183fe815df3e429fce7bfb4f8885656863e841b4ba239d35682dd4e0e5d4e1f9ecf0a975c7833f60fc6511021b4c571f42907e559553093ec9f7cfe8ba88af64eaba1a408e7ae83cc2569cfb2ccfab05fbcf3350a929ec4dd85bb95d7f66ab5149d4d589f4d7cd7f4962ab0e046a88eceb0e3f3fb583b606c2bdde0b90539e41fde19aa7bd93a995ca6a6ed2dbd448e72e880332f10c4179863749308e6468a252c08206683af33ee5b7bc1f306af72c1a31495347478957a8925ff07d1a175d9ef83fd1c98a369f0ab8a06997b27527f645484a113eaf23f709a8594514ea72d04fe4cc307787030e14f9aad2df63bc8cd36917a3
\ No newline at end of file
diff --git a/src/source/db/honey.clj b/src/source/db/honey.clj
index cb626077..ee5458eb 100644
--- a/src/source/db/honey.clj
+++ b/src/source/db/honey.clj
@@ -1,11 +1,11 @@
 (ns source.db.honey
-  (:require [honey.sql :as sql]
-            [honey.sql.helpers :as hsql]
-            [source.db.util :as db.util]
-            [camel-snake-kebab.core :as csk]
+  (:require [camel-snake-kebab.core :as csk]
             [camel-snake-kebab.extras :as cske]
+            [honey.sql :as sql]
+            [honey.sql.helpers :as hsql]
             [next.jdbc :as jdbc]
-            [next.jdbc.result-set :as rs]))
+            [next.jdbc.result-set :as rs]
+            [source.db.util :as db.util]))
 
 (defn execute!
   "computes a prepared statement for an sql map and executes select one
@@ -52,6 +52,7 @@
   (let [values' (or data values)
         multi? (vector? values')
         vals (if multi? values' [values'])]
+    (prn "vals" vals)
     (execute! ds
               (-> (hsql/insert-into (csk/->snake_case_keyword tname))
                   (hsql/values vals)
diff --git a/src/source/routes/provider_selection_schemas.clj b/src/source/routes/provider_selection_schemas.clj
index ca0094ed..59b9c837 100644
--- a/src/source/routes/provider_selection_schemas.clj
+++ b/src/source/routes/provider_selection_schemas.clj
@@ -1,9 +1,16 @@
 (ns source.routes.provider-selection-schemas
-  (:require [source.services.interface :as services]
-            [ring.util.response :as res]))
+  (:require [ring.util.response :as res]
+            [source.services.interface :as services]))
 
-(defn get [{:keys [ds path-params] :as _request}]
-  (->> (:id path-params)
-       (assoc {} :provider-id)
-       (services/selection-schemas-by-provider ds)
-       (res/response)))
+(defn get [{:keys [ds path-params store] :as _request}]
+  (let [selection-schemas (->>
+                           (:id path-params)
+                           (assoc {} :provider-id)
+                           (services/selection-schemas-by-provider ds))
+        results (mapv (fn [{:keys [output-schema-id] :as ss}]
+                        (merge
+                         ss
+                         {:output-schema
+                          (services/output-schema store output-schema-id)}))
+                      selection-schemas)]
+    (res/response results)))
diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index 07ce3c70..4550ddd8 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -141,7 +141,8 @@
                           :default-values true
                           :options nil})
               :middleware [[mw/apply-generic :ds ds :store store]
-                           [exception/exception-middleware]]}})
+                           ;;[exception/exception-middleware]
+                           ]}})
      (ring/routes
       (swagger-ui/create-swagger-ui-handler {:path "/"
                                              :config {:validatorUrl nil
diff --git a/src/source/routes/selection_schema.clj b/src/source/routes/selection_schema.clj
index d75925f1..32354936 100644
--- a/src/source/routes/selection_schema.clj
+++ b/src/source/routes/selection_schema.clj
@@ -1,11 +1,13 @@
 (ns source.routes.selection-schema
-  (:require [source.services.interface :as services]
-            [ring.util.response :as res]))
+  (:require [ring.util.response :as res]
+            [source.services.interface :as services]))
 
-(defn get [{:keys [ds path-params] :as _request}]
-  (->> path-params
-       (services/selection-schema ds)
-       (res/response)))
+(defn get [{:keys [ds path-params store] :as _request}]
+  (let [{:keys [output-schema-id] :as selection-schema}
+        (services/selection-schema ds (:id path-params))
+        output-schema (services/output-schema store output-schema-id)]
+    (res/response (merge {:output-schema output-schema}
+                         selection-schema))))
 
 (comment
   (require '[source.db.util :as db.util])
diff --git a/src/source/routes/selection_schemas.clj b/src/source/routes/selection_schemas.clj
index 9d4531f2..f795446f 100644
--- a/src/source/routes/selection_schemas.clj
+++ b/src/source/routes/selection_schemas.clj
@@ -1,15 +1,15 @@
 (ns source.routes.selection-schemas
-  (:require [source.services.interface :as services]
-            [ring.util.response :as res]))
+  (:require [ring.util.response :as res]
+            [source.services.interface :as services]))
 
 (defn get [{:keys [ds] :as _request}]
   (-> (services/selection-schemas ds)
       (res/response)))
 
 (defn post [{:keys [store ds body] :as _request}]
-  (let [{:keys [_schema _record] :as opts} body]
-    (-> (services/add-selection-schema! store ds opts)
-        (res/response))))
+  (-> (services/add-selection-schema! store ds body)
+      (res/response)))
+
 
 (comment
   (require '[source.db.util :as db.util])
diff --git a/src/source/services/xml_schemas.clj b/src/source/services/xml_schemas.clj
index bfca6ffb..b0a615b7 100644
--- a/src/source/services/xml_schemas.clj
+++ b/src/source/services/xml_schemas.clj
@@ -1,6 +1,6 @@
 (ns source.services.xml-schemas
-  (:require [source.db.interface :as db]
-            [source.datastore.interface :as store]
+  (:require [source.datastore.interface :as store]
+            [source.db.interface :as db]
             [source.rss.core :as rss]))
 
 (defn add-output-schema!
@@ -24,6 +24,8 @@
 
 (defn add-selection-schema!
   [store db {:keys [schema record]}]
+  (prn "schema" schema)
+  (prn "record" record)
   (let [db-result (db/insert! db {:tname :selection-schemas
                                   :data record
                                   :ret :1})]

From 5d79934d7eccd4345367cde1b7d335e9965b4e64 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 4 Aug 2025 09:14:10 +0200
Subject: [PATCH 051/391] added partial delete route for providers and a
 versioning system for schemas

---
 src/source/datastore/datahike.clj       |  2 +-
 src/source/db/honey.clj                 |  1 -
 src/source/db/master.clj                |  1 +
 src/source/routes/provider.clj          |  6 ++++
 src/source/routes/reitit.clj            |  3 +-
 src/source/routes/selection_schemas.clj |  1 +
 src/source/services/interface.clj       |  3 ++
 src/source/services/xml_schemas.clj     | 37 ++++++++++++++++++++++---
 8 files changed, 47 insertions(+), 7 deletions(-)

diff --git a/src/source/datastore/datahike.clj b/src/source/datastore/datahike.clj
index 065b4b87..61f02467 100644
--- a/src/source/datastore/datahike.clj
+++ b/src/source/datastore/datahike.clj
@@ -94,7 +94,7 @@
   (def ds (ds.util/conn :datahike))
   (get-all ds)
   (entries ds :user/name)
-  (delete! ds (entries ds :user/name))
+  (delete! ds (entries ds :output-schemas/id))
   (update! ds 3 {:user/age 21})
 
   (exists? ds :user/name)
diff --git a/src/source/db/honey.clj b/src/source/db/honey.clj
index ee5458eb..f8dec2e1 100644
--- a/src/source/db/honey.clj
+++ b/src/source/db/honey.clj
@@ -52,7 +52,6 @@
   (let [values' (or data values)
         multi? (vector? values')
         vals (if multi? values' [values'])]
-    (prn "vals" vals)
     (execute! ds
               (-> (hsql/insert-into (csk/->snake_case_keyword tname))
                   (hsql/values vals)
diff --git a/src/source/db/master.clj b/src/source/db/master.clj
index b8b0419b..65b3a9e6 100644
--- a/src/source/db/master.clj
+++ b/src/source/db/master.clj
@@ -133,6 +133,7 @@
   (tables/create-table-sql
    :selection-schemas
    (tables/table-id)
+   [:version :integer :not nil]
    [:output-schema-id :integer :not nil]
    [:provider-id :integer :not nil]
    (tables/foreign-key :provider-id :providers :id)))
diff --git a/src/source/routes/provider.clj b/src/source/routes/provider.clj
index 9c67895a..d126afaa 100644
--- a/src/source/routes/provider.clj
+++ b/src/source/routes/provider.clj
@@ -7,3 +7,9 @@
        (Integer/parseInt)
        (services/provider store)
        (res/response)))
+
+(defn delete [{:keys [store path-params] :as _request}]
+  (->> (:id path-params)
+       (Integer/parseInt)
+       (services/delete-provider! store))
+  (res/response {:message "successfully deleted provider"}))
diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index 4550ddd8..f713ca67 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -130,7 +130,8 @@
         ["/providers"
          [""                      {:get providers/get
                                    :post providers/post}]
-         ["/:id"                  {:get provider/get}]]
+         ["/:id"                  {:get provider/get
+                                   :delete provider/delete}]]
         ["/ast"                   {:post xml/post}]
         ["/extract-data"          {:post data/post}]]]
 
diff --git a/src/source/routes/selection_schemas.clj b/src/source/routes/selection_schemas.clj
index f795446f..34aaa5a3 100644
--- a/src/source/routes/selection_schemas.clj
+++ b/src/source/routes/selection_schemas.clj
@@ -8,6 +8,7 @@
 
 (defn post [{:keys [store ds body] :as _request}]
   (-> (services/add-selection-schema! store ds body)
+      (first)
       (res/response)))
 
 
diff --git a/src/source/services/interface.clj b/src/source/services/interface.clj
index d875ce09..d658031f 100644
--- a/src/source/services/interface.clj
+++ b/src/source/services/interface.clj
@@ -60,6 +60,9 @@
 (defn provider [store provider-id]
   (xml/provider store provider-id))
 
+(defn delete-provider! [store provider-id]
+  (xml/delete-provider! store provider-id))
+
 (defn add-provider! [store name]
   (xml/add-provider! store name))
 
diff --git a/src/source/services/xml_schemas.clj b/src/source/services/xml_schemas.clj
index b0a615b7..0b7b928f 100644
--- a/src/source/services/xml_schemas.clj
+++ b/src/source/services/xml_schemas.clj
@@ -22,12 +22,24 @@
        (store/find-entities store)
        (first)))
 
+(defn highest-version
+  [previous-records]
+  (->> (reduce (fn [acc {:keys [version]}]
+                 (conj acc version)) [] previous-records)
+       (apply max 0)))
+
 (defn add-selection-schema!
   [store db {:keys [schema record]}]
-  (prn "schema" schema)
-  (prn "record" record)
-  (let [db-result (db/insert! db {:tname :selection-schemas
-                                  :data record
+  (let [{:keys [output-schema-id provider-id]} record
+        previous-versions (db/find db {:tname :selection-schemas
+                                       :where [:= :provider-id provider-id]
+                                       :ret :*})
+        next-version (-> (highest-version previous-versions)
+                         (inc))
+        db-result (db/insert! db {:tname :selection-schemas
+                                  :data {:output-schema-id output-schema-id
+                                         :provider-id provider-id
+                                         :version next-version}
                                   :ret :1})]
     (store/insert! store {:selection-schemas/id (:id db-result)
                           :selection-schemas/schema schema})))
@@ -76,6 +88,13 @@
        (store/find-entities store)
        (first)))
 
+(defn delete-provider!
+  [store provider-id]
+  (->> {:key :providers/id
+        :value provider-id}
+       (store/find store)
+       (store/delete! store)))
+
 (defn ast
   [url]
   (-> url
@@ -97,6 +116,16 @@
 
 (comment
   (require '[source.db.util :as db.util])
+  (ast "https://www.youtube.com/feeds/videos.xml?channel_id=UCUyeluBRhGPCW4rPe_UvBZQ")
+
+  (reduce (fn [acc {:keys [version]}]
+            (conj acc version)) [] [{:version 2} {:version 3}])
+
+  (db/find (db.util/conn) {:tname :selection-schemas
+                           :where [:and
+                                   [:= :output-schema-id 1]
+                                   [:= :provider-id 1]]
+                           :ret :*})
 
   (def ds (store/ds :datahike))
 

From 6bc275780cd3ab0a7bbb4e1ecb43bbaac54fd9c7 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Fri, 8 Aug 2025 13:33:51 +0200
Subject: [PATCH 052/391] updated providers routes to use sqlite db and added
 content-types routes

---
 src/source/routes/content_type.clj    |  8 +++++
 src/source/routes/content_types.clj   |  7 ++++
 src/source/routes/provider.clj        | 14 ++++----
 src/source/routes/providers.clj       | 15 +++++----
 src/source/routes/reitit.clj          | 48 ++++++++++++++++++++++++---
 src/source/services/content_types.clj | 26 +++++++++++++++
 src/source/services/interface.clj     | 29 +++++++++++-----
 src/source/services/providers.clj     | 20 ++++++++---
 src/source/services/xml_schemas.clj   | 29 ----------------
 9 files changed, 134 insertions(+), 62 deletions(-)
 create mode 100644 src/source/routes/content_type.clj
 create mode 100644 src/source/routes/content_types.clj
 create mode 100644 src/source/services/content_types.clj

diff --git a/src/source/routes/content_type.clj b/src/source/routes/content_type.clj
new file mode 100644
index 00000000..0eb0b039
--- /dev/null
+++ b/src/source/routes/content_type.clj
@@ -0,0 +1,8 @@
+(ns source.routes.content-type
+  (:require [source.services.interface :as services]
+            [ring.util.response :as res]))
+
+(defn get [{:keys [ds path-params] :as _request}]
+  (->> path-params
+       (services/content-type ds)
+       (res/response)))
diff --git a/src/source/routes/content_types.clj b/src/source/routes/content_types.clj
new file mode 100644
index 00000000..9b226e1c
--- /dev/null
+++ b/src/source/routes/content_types.clj
@@ -0,0 +1,7 @@
+(ns source.routes.content-types
+  (:require [source.services.interface :as services]
+            [ring.util.response :as res]))
+
+(defn get [{:keys [ds] :as _request}]
+  (-> (services/content-types ds)
+      (res/response)))
diff --git a/src/source/routes/provider.clj b/src/source/routes/provider.clj
index d126afaa..d02480a6 100644
--- a/src/source/routes/provider.clj
+++ b/src/source/routes/provider.clj
@@ -2,14 +2,12 @@
   (:require [source.services.interface :as services]
             [ring.util.response :as res]))
 
-(defn get [{:keys [store path-params] :as _request}]
-  (->> (:id path-params)
-       (Integer/parseInt)
-       (services/provider store)
+(defn get [{:keys [ds path-params] :as _request}]
+  (->> path-params
+       (services/provider ds)
        (res/response)))
 
-(defn delete [{:keys [store path-params] :as _request}]
-  (->> (:id path-params)
-       (Integer/parseInt)
-       (services/delete-provider! store))
+(defn delete [{:keys [ds path-params] :as _request}]
+  (->> path-params
+       (services/delete-provider! ds))
   (res/response {:message "successfully deleted provider"}))
diff --git a/src/source/routes/providers.clj b/src/source/routes/providers.clj
index 2cdd478a..2ba7de8c 100644
--- a/src/source/routes/providers.clj
+++ b/src/source/routes/providers.clj
@@ -2,17 +2,18 @@
   (:require [source.services.interface :as services]
             [ring.util.response :as res]))
 
-(defn get [{:keys [store] :as _request}]
-  (-> (services/providers store)
+(defn get [{:keys [ds] :as _request}]
+  (-> (services/providers ds)
       (res/response)))
 
-(defn post [{:keys [store body] :as _request}]
-  (let [{:keys [name]} body]
-    (services/add-provider! store name)
+(defn post [{:keys [ds body] :as _request}]
+  (let [{:keys [provider]} body]
+    (services/insert-provider! ds {:data provider
+                                   :ret :1})
     (res/response {:message "successfully added provider"})))
 
-(comment 
+(comment
   (require '[source.datastore.interface :as store])
 
   (services/add-provider! (store/ds :datahike) "YouTube")
-  )
+  ())
diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index f713ca67..8b68f1c0 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -30,6 +30,8 @@
             [source.routes.output-schema :as output-schema]
             [source.routes.providers :as providers]
             [source.routes.provider :as provider]
+            [source.routes.content-types :as content-types]
+            [source.routes.content-type :as content-type]
             [source.routes.xml :as xml]
             [source.routes.data :as data]
             [source.util :as util]
@@ -112,6 +114,14 @@
                          :openapi {:security [{:bearerAuth []}]}}
         ["/authorized"  (route {:get authorized/get})]]
 
+       ["/providers"
+        [""             {:get providers/get}]
+        ["/:id"         {:get provider/get}]]
+
+       ["/content-types"
+        [""             {:get content-types/get}]
+        ["/:id"         {:get content-type/get}]]
+
        ["/admin"                  {:middleware [[mw/apply-auth {:required-type :admin}]]
                                    :no-doc true
                                    :tags #{"admin"}
@@ -128,10 +138,8 @@
                                    :post output-schemas/post}]
          ["/:id"                  {:get output-schema/get}]]
         ["/providers"
-         [""                      {:get providers/get
-                                   :post providers/post}]
-         ["/:id"                  {:get provider/get
-                                   :delete provider/delete}]]
+         [""                      {:post providers/post}]
+         ["/:id"                  {:delete provider/delete}]]
         ["/ast"                   {:post xml/post}]
         ["/extract-data"          {:post data/post}]]]
 
@@ -246,6 +254,38 @@
         :body
         (json/read-json {:key-fn keyword})))
 
+  (let [app (create-app)
+        request {:uri "/providers"
+                 :request-method :get}]
+    (-> request
+        app
+        :body
+        (json/read-json {:key-fn keyword})))
+
+  (let [app (create-app)
+        request {:uri "/providers/1"
+                 :request-method :get}]
+    (-> request
+        app
+        :body
+        (json/read-json {:key-fn keyword})))
+
+  (let [app (create-app)
+        request {:uri "/content-types"
+                 :request-method :get}]
+    (-> request
+        app
+        :body
+        (json/read-json {:key-fn keyword})))
+
+  (let [app (create-app)
+        request {:uri "/content-types/1"
+                 :request-method :get}]
+    (-> request
+        app
+        :body
+        (json/read-json {:key-fn keyword})))
+
   (let [app (create-app)
         store (store/ds :datahike)
         request {:uri "/admin/selection-schemas"
diff --git a/src/source/services/content_types.clj b/src/source/services/content_types.clj
new file mode 100644
index 00000000..de8a0520
--- /dev/null
+++ b/src/source/services/content_types.clj
@@ -0,0 +1,26 @@
+(ns source.services.content-types
+  (:require [source.db.interface :as db]))
+
+(defn add-content-type! [ds {:keys [data ret] :as opts}]
+  (->> {:tname :providers
+        :data data
+        :ret ret}
+       (merge opts)
+       (db/insert! ds)))
+
+(defn content-types
+  ([ds] (content-types ds {}))
+  ([ds opts]
+   (->> {:tname :content-types
+         :ret :*}
+        (merge opts)
+        (db/find ds))))
+
+(defn content-type [ds {:keys [id where] :as opts}]
+  (->> {:tname :providers
+        :where (if (some? id)
+                 [:= :id id]
+                 where)
+        :ret :1}
+       (merge opts)
+       (db/find ds)))
diff --git a/src/source/services/interface.clj b/src/source/services/interface.clj
index d658031f..0f8740a3 100644
--- a/src/source/services/interface.clj
+++ b/src/source/services/interface.clj
@@ -3,7 +3,9 @@
              [source.db.interface :as db]
              [source.services.auth :as auth]
              [source.services.xml-schemas :as xml]
-             [source.services.bundles :as bundles]))
+             [source.services.bundles :as bundles]
+             [source.services.providers :as providers]
+             [source.services.content-types :as content-types]))
 
 (defn users
   [& args]
@@ -54,17 +56,26 @@
 (defn add-output-schema! [store schema]
   (xml/add-output-schema! store schema))
 
-(defn providers [store]
-  (xml/providers store))
+(defn providers [ds]
+  (providers/providers ds))
 
-(defn provider [store provider-id]
-  (xml/provider store provider-id))
+(defn provider [ds provider-id]
+  (providers/provider ds provider-id))
 
-(defn delete-provider! [store provider-id]
-  (xml/delete-provider! store provider-id))
+(defn delete-provider! [ds provider-id]
+  (providers/delete-provider! ds provider-id))
 
-(defn add-provider! [store name]
-  (xml/add-provider! store name))
+(defn insert-provider! [ds {:keys [_values _ret] :as opts}]
+  (providers/insert-provider! ds opts))
+
+(defn content-types [ds]
+  (content-types/content-types ds))
+
+(defn content-type [ds id]
+  (content-types/content-type ds id))
+
+(defn add-content-type! [ds {:keys [_values _ret] :as opts}]
+  (content-types/add-content-type! ds opts))
 
 (comment
   (users (db/ds :master))
diff --git a/src/source/services/providers.clj b/src/source/services/providers.clj
index 7bafe60c..61fe4af4 100644
--- a/src/source/services/providers.clj
+++ b/src/source/services/providers.clj
@@ -1,16 +1,18 @@
 (ns source.services.providers
-  (:require [source.db.interface :as db]
-            [source.datastore.interface :as store]))
+  (:require [source.db.interface :as db]))
 
-(defn insert-provider! [ds provider]
+(defn insert-provider! [ds {:keys [data ret] :as opts}]
   (->> {:tname :providers
-        :data provider}
+        :data data
+        :ret ret}
+       (merge opts)
        (db/insert! ds)))
 
 (defn providers
   ([ds] (providers ds {}))
   ([ds opts]
-   (->> {:tname :providers}
+   (->> {:tname :providers
+         :ret :*}
         (merge opts)
         (db/find ds))))
 
@@ -23,3 +25,11 @@
        (merge opts)
        (db/find ds)))
 
+(defn delete-provider! [ds {:keys [id where] :as opts}]
+  (->> {:tname :providers
+        :where (if (some? id)
+                 [:= :id id]
+                 where)
+        :ret :1}
+       (merge opts)
+       (db/delete! ds)))
diff --git a/src/source/services/xml_schemas.clj b/src/source/services/xml_schemas.clj
index 0b7b928f..f33cfe51 100644
--- a/src/source/services/xml_schemas.clj
+++ b/src/source/services/xml_schemas.clj
@@ -69,32 +69,6 @@
        (merge opts)
        (db/find ds)))
 
-(defn add-provider!
-  [store name]
-  (let [id (-> (store/entries store :providers/id)
-               (count)
-               (inc))]
-    (store/insert! store {:providers/id id
-                          :providers/name name})))
-
-(defn providers
-  [store]
-  (store/entities-with store :providers/id))
-
-(defn provider
-  [store provider-id]
-  (->> {:key :providers/id
-        :value provider-id}
-       (store/find-entities store)
-       (first)))
-
-(defn delete-provider!
-  [store provider-id]
-  (->> {:key :providers/id
-        :value provider-id}
-       (store/find store)
-       (store/delete! store)))
-
 (defn ast
   [url]
   (-> url
@@ -130,9 +104,6 @@
   (def ds (store/ds :datahike))
 
   (selection-schemas-by-provider (db.util/conn) {:provider-id 1})
-  (add-provider! ds "poop")
-  (add-provider! ds "YouTube")
-  (providers ds)
 
   (count (store/entries ds :selection-schemas/id))
   (store/entities-with ds :selection-schemas/id)

From 1c77a2cc0b39aea3bd53b9d7d7954164b05287ab Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 11 Aug 2025 13:20:22 +0200
Subject: [PATCH 053/391] removed explicit selection schema versioning

---
 src/source/services/xml_schemas.clj | 14 +-------------
 1 file changed, 1 insertion(+), 13 deletions(-)

diff --git a/src/source/services/xml_schemas.clj b/src/source/services/xml_schemas.clj
index f33cfe51..9b9f3c76 100644
--- a/src/source/services/xml_schemas.clj
+++ b/src/source/services/xml_schemas.clj
@@ -22,24 +22,12 @@
        (store/find-entities store)
        (first)))
 
-(defn highest-version
-  [previous-records]
-  (->> (reduce (fn [acc {:keys [version]}]
-                 (conj acc version)) [] previous-records)
-       (apply max 0)))
-
 (defn add-selection-schema!
   [store db {:keys [schema record]}]
   (let [{:keys [output-schema-id provider-id]} record
-        previous-versions (db/find db {:tname :selection-schemas
-                                       :where [:= :provider-id provider-id]
-                                       :ret :*})
-        next-version (-> (highest-version previous-versions)
-                         (inc))
         db-result (db/insert! db {:tname :selection-schemas
                                   :data {:output-schema-id output-schema-id
-                                         :provider-id provider-id
-                                         :version next-version}
+                                         :provider-id provider-id}
                                   :ret :1})]
     (store/insert! store {:selection-schemas/id (:id db-result)
                           :selection-schemas/schema schema})))

From 471b0b2bef3895f1f2b7b535abc80fec611ea47e Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Thu, 21 Aug 2025 11:53:48 +0200
Subject: [PATCH 054/391] addressed validation and removed redundant service

---
 src/source/routes/output_schema.clj              | 13 +++++++++----
 src/source/routes/provider_selection_schemas.clj |  7 +++----
 src/source/services/interface.clj                | 10 +++++-----
 src/source/services/xml_schemas.clj              | 10 ----------
 4 files changed, 17 insertions(+), 23 deletions(-)

diff --git a/src/source/routes/output_schema.clj b/src/source/routes/output_schema.clj
index 95fe3943..0ff89175 100644
--- a/src/source/routes/output_schema.clj
+++ b/src/source/routes/output_schema.clj
@@ -3,7 +3,12 @@
             [ring.util.response :as res]))
 
 (defn get [{:keys [store path-params] :as _request}]
-  (->> (:id path-params)
-       (Integer/parseInt)
-       (services/output-schema store)
-       (res/response)))
+  (let [id (try
+             (Integer/parseInt (:id path-params))
+             (catch Exception _ nil))]
+    (if (some? id)
+      (->> id
+           (services/output-schema store)
+           (res/response))
+      (-> (res/response {:message "invalid id"})
+          (res/status 400)))))
diff --git a/src/source/routes/provider_selection_schemas.clj b/src/source/routes/provider_selection_schemas.clj
index 59b9c837..bc146c1d 100644
--- a/src/source/routes/provider_selection_schemas.clj
+++ b/src/source/routes/provider_selection_schemas.clj
@@ -3,10 +3,9 @@
             [source.services.interface :as services]))
 
 (defn get [{:keys [ds path-params store] :as _request}]
-  (let [selection-schemas (->>
-                           (:id path-params)
-                           (assoc {} :provider-id)
-                           (services/selection-schemas-by-provider ds))
+  (let [selection-schemas (services/selection-schemas 
+                            ds 
+                            {:where [:= :provider-id (:id path-params)]})
         results (mapv (fn [{:keys [output-schema-id] :as ss}]
                         (merge
                          ss
diff --git a/src/source/services/interface.clj b/src/source/services/interface.clj
index 0f8740a3..e2f3e0f8 100644
--- a/src/source/services/interface.clj
+++ b/src/source/services/interface.clj
@@ -35,11 +35,11 @@
 (defn bundle [ds {:keys [_id _where] :as opts}]
   (bundles/bundle ds opts))
 
-(defn selection-schemas [ds]
-  (xml/selection-schemas ds))
-
-(defn selection-schemas-by-provider [ds {:keys [_provider-id] :as opts}]
-  (xml/selection-schemas-by-provider ds opts))
+(defn selection-schemas
+  ([ds]
+   (selection-schemas ds {}))
+  ([ds opts]
+   (xml/selection-schemas ds opts)))
 
 (defn ast [url]
   (xml/ast url))
diff --git a/src/source/services/xml_schemas.clj b/src/source/services/xml_schemas.clj
index 9b9f3c76..63ceac24 100644
--- a/src/source/services/xml_schemas.clj
+++ b/src/source/services/xml_schemas.clj
@@ -40,14 +40,6 @@
         (merge opts)
         (db/find ds))))
 
-(defn selection-schemas-by-provider
-  [ds {:keys [provider-id] :as opts}]
-  (->> {:tname :selection-schemas
-        :where [:= :provider-id provider-id]
-        :ret :*}
-       (merge opts)
-       (db/find ds)))
-
 (defn selection-schema [ds {:keys [id where] :as opts}]
   (->> {:tname :selection-schemas
         :where (if (some? id)
@@ -91,8 +83,6 @@
 
   (def ds (store/ds :datahike))
 
-  (selection-schemas-by-provider (db.util/conn) {:provider-id 1})
-
   (count (store/entries ds :selection-schemas/id))
   (store/entities-with ds :selection-schemas/id)
   (store/find-entities ds {:key :selection-schemas/id

From c95fd410acdd4f3f19aa613a8aeee9b3a3384fef Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Thu, 21 Aug 2025 13:05:16 +0200
Subject: [PATCH 055/391] addressed datastore changes

---
 src/source/datastore/config.clj              | 56 ++++++++++++++++++
 src/source/datastore/datahike.clj            | 31 +++++-----
 src/source/datastore/interface.clj           |  7 ++-
 src/source/datastore/util.clj                | 61 ++------------------
 src/source/migrations/001_init_master_db.clj |  6 +-
 src/source/services/xml_schemas.clj          |  6 +-
 6 files changed, 83 insertions(+), 84 deletions(-)
 create mode 100644 src/source/datastore/config.clj

diff --git a/src/source/datastore/config.clj b/src/source/datastore/config.clj
new file mode 100644
index 00000000..41dacc5f
--- /dev/null
+++ b/src/source/datastore/config.clj
@@ -0,0 +1,56 @@
+(ns source.datastore.config
+  (:require [datahike.api :as d]
+            [source.config :as conf]
+            [clojure.string :as string]))
+
+(defn absolute [path]
+  (-> (java.io.File. ".")
+      .getAbsolutePath
+      (clojure.string/replace "/." "/")
+      (str path)))
+
+(defn store-path
+  ([store-name]
+   (-> (str (conf/read-value :database :dir) store-name)
+       (absolute))))
+
+(defn config [store-name]
+  {:store {:backend :file
+           :path (store-path (name store-name))}
+   :schema-flexibility :read
+   :initial-tx [{:db/ident        :selection-schemas/id
+                 :db/valueType    :db.type/long
+                 :db/cardinality  :db.cardinality/one
+                 :db/unique       :db.unique/identity}
+                {:db/ident        :selection-schemas/schema
+                 :db/valueType    :db.type/any
+                 :db/cardinality  :db.cardinality/one}
+
+                {:db/ident        :output-schemas/id
+                 :db/valueType    :db.type/long
+                 :db/cardinality  :db.cardinality/one
+                 :db/unique       :db.unique/identity}
+                {:db/ident        :output-schemas/schema
+                 :db/valueType    :db.type/any
+                 :db/cardinality  :db.cardinality/one}
+                {:db/ident        :output-schemas/version
+                 :db/valueType    :db.type/long
+                 :db/cardinality  :db.cardinality/one}
+
+                {:db/ident        :providers/id
+                 :db/valueType    :db.type/long
+                 :db/cardinality  :db.cardinality/one
+                 :db/unique       :db.unique/identity}
+                {:db/ident        :providers/name
+                 :db/valueType    :db.type/string
+                 :db/cardinality  :db.cardinality/one}]})
+
+(defn create-datastore [store-name]
+  (when-not (d/database-exists? (config store-name))
+    (println "Creating datastore...")
+    (d/create-database (config store-name))))
+
+(defn delete-datastore [store-name]
+  (when (d/database-exists? (config store-name))
+    (println "Deleting datastore...")
+    (d/delete-database (config store-name))))
diff --git a/src/source/datastore/datahike.clj b/src/source/datastore/datahike.clj
index 61f02467..e590811d 100644
--- a/src/source/datastore/datahike.clj
+++ b/src/source/datastore/datahike.clj
@@ -2,7 +2,7 @@
   (:require [datahike.api :as d]
             [source.util :as util]))
 
-(defn lookup
+(defn entities
   "Returns one or more entities associated with the provided entity id(s)"
   [ds ids]
   (let [multi? (vector? ids)
@@ -12,7 +12,7 @@
                  (into {}))) ids-vec)))
 
 (defn find
-  "Returns one ore more entity ids where the provided key matches the provided value"
+  "Returns entity ids where the provided key matches the provided value"
   [ds {:keys [key value]}]
   (->> (d/q '[:find ?e
               :in $ ?k ?v
@@ -27,17 +27,14 @@
   [ds {:keys [_key _value] :as query}]
   (->> query
        (find ds)
-       (lookup ds)))
-
-(defn exists?
-  "Returns true if value exists for key in kv-store"
-  [ds k]
-  (-> (d/q '[:find ?e
-             :in $ ?k
-             :where [?e ?k _]]
-           @ds k)
-      (seq)
-      (boolean)))
+       (entities ds)))
+
+(defn lookup 
+  "Returns the first entity where the provided key matches the provided value"
+  [ds {:keys [_key _value] :as query}]
+  (->> query
+       (find-entities ds)
+       (first)))
 
 (defn entries
   "Get eids of all entities in which the provided attribute is present"
@@ -55,7 +52,7 @@
   [ds key]
   (->> key
        (entries ds)
-       (lookup ds)))
+       (entities ds)))
 
 (defn delete!
   "Accepts an entity id or a vec of entity ids and removes all the entities associated thereby"
@@ -97,8 +94,7 @@
   (delete! ds (entries ds :output-schemas/id))
   (update! ds 3 {:user/age 21})
 
-  (exists? ds :user/name)
-  (lookup ds [6 7])
+  (entities ds [6 7])
   (find ds {:key :user/name
             :value "Keagan"})
   (find-entities ds {:key :user/age
@@ -125,8 +121,7 @@
                  k v})
     (assert (= v
                (->> k
-                    (entities-with k)
-                    (first)
+                    (lookup ds)
                     (k))))
     (insert! ds {k v})
     (println "1 " (get-all ds))
diff --git a/src/source/datastore/interface.clj b/src/source/datastore/interface.clj
index abf4c3a6..686c46b9 100644
--- a/src/source/datastore/interface.clj
+++ b/src/source/datastore/interface.clj
@@ -5,8 +5,8 @@
 (defn ds [store-name]
   (store.util/conn store-name))
 
-(defn lookup [ds eids]
-  (dh/lookup ds eids))
+(defn entities [ds eids]
+  (dh/entities ds eids))
 
 (defn find [ds {:keys [_key _value] :as opts}]
   (dh/find ds opts))
@@ -14,6 +14,9 @@
 (defn find-entities [ds {:keys [_key _value] :as opts}]
   (dh/find-entities ds opts))
 
+(defn lookup [ds {:keys [_key _value] :as opts}]
+  (dh/lookup ds opts))
+
 (defn exists? [ds k]
   (dh/exists? ds k))
 
diff --git a/src/source/datastore/util.clj b/src/source/datastore/util.clj
index efaaa512..c4151c2a 100644
--- a/src/source/datastore/util.clj
+++ b/src/source/datastore/util.clj
@@ -1,64 +1,11 @@
 (ns source.datastore.util
   (:require [datahike.api :as d]
-            [source.config :as conf]
-            [clojure.string :as string]))
-
-(defn absolute [path]
-  (-> (java.io.File. ".")
-      .getAbsolutePath
-      (clojure.string/replace "/." "/")
-      (str path)))
-
-(defn store-path
-  ([store-name]
-   (-> (str (conf/read-value :database :dir) store-name)
-       (absolute))))
-
-(defn config [store-name]
-  {:store {:backend :file
-           :path (store-path (name store-name))}
-   :schema-flexibility :read
-   :initial-tx [{:db/ident        :selection-schemas/id
-                 :db/valueType    :db.type/long
-                 :db/cardinality  :db.cardinality/one
-                 :db/unique       :db.unique/identity}
-                {:db/ident        :selection-schemas/schema
-                 :db/valueType    :db.type/any
-                 :db/cardinality  :db.cardinality/one}
-
-                {:db/ident        :output-schemas/id
-                 :db/valueType    :db.type/long
-                 :db/cardinality  :db.cardinality/one
-                 :db/unique       :db.unique/identity}
-                {:db/ident        :output-schemas/schema
-                 :db/valueType    :db.type/any
-                 :db/cardinality  :db.cardinality/one}
-                {:db/ident        :output-schemas/version
-                 :db/valueType    :db.type/long
-                 :db/cardinality  :db.cardinality/one}
-
-                {:db/ident        :providers/id
-                 :db/valueType    :db.type/long
-                 :db/cardinality  :db.cardinality/one
-                 :db/unique       :db.unique/identity}
-                {:db/ident        :providers/name
-                 :db/valueType    :db.type/string
-                 :db/cardinality  :db.cardinality/one}]})
-
-(defn create-datastore [store-name]
-  (when-not (d/database-exists? (config store-name))
-    (println "Creating datastore...")
-    (d/create-database (config store-name))))
-
-(defn delete-datastore [store-name]
-  (when (d/database-exists? (config store-name))
-    (println "Deleting datastore...")
-    (d/delete-database (config store-name))))
+            [source.datastore.config :as c]))
 
 (defn conn
   "Open a connection to a datahike store"
   [store-name]
-  (d/connect (config store-name)))
+  (d/connect (c/config store-name)))
 
 (defn close
   "Close a connection to a datahike store"
@@ -72,8 +19,8 @@
    (str (name type) "_" id)))
 
 (comment
-  (d/create-database (config :datahike))
-  (d/delete-database (config :datahike))
+  (d/create-database (c/config :datahike))
+  (d/delete-database (c/config :datahike))
 
   (defonce ds (conn :datahike))
   (d/transact ds [{:user/name "Keagan"
diff --git a/src/source/migrations/001_init_master_db.clj b/src/source/migrations/001_init_master_db.clj
index 6d4f7395..42dc4573 100644
--- a/src/source/migrations/001_init_master_db.clj
+++ b/src/source/migrations/001_init_master_db.clj
@@ -3,7 +3,7 @@
             [source.db.master]
             [source.db.honey :as db]
             [source.db.tables :as tables]
-            [source.datastore.util :as ds.util]
+            [source.datastore.config :as ds]
             [source.config :as conf]))
 
 (def baselines-seed
@@ -59,7 +59,7 @@
 (defn run-up! [context]
   (let [ds-master (:db-master context)]
 
-    (ds.util/create-datastore :datahike)
+    (ds/create-datastore :datahike)
 
     (tables/create-tables!
      ds-master
@@ -99,7 +99,7 @@
 
 (defn run-down! [context]
   (let [ds-master (:db-master context)]
-    (ds.util/delete-datastore :datahike)
+    (ds/delete-datastore :datahike)
 
     (tables/drop-all-tables! ds-master)))
 
diff --git a/src/source/services/xml_schemas.clj b/src/source/services/xml_schemas.clj
index 63ceac24..40236898 100644
--- a/src/source/services/xml_schemas.clj
+++ b/src/source/services/xml_schemas.clj
@@ -19,8 +19,7 @@
   [store output-schema-id]
   (->> {:key :output-schemas/id
         :value output-schema-id}
-       (store/find-entities store)
-       (first)))
+       (store/lookup store)))
 
 (defn add-selection-schema!
   [store db {:keys [schema record]}]
@@ -60,8 +59,7 @@
   [store schema-id url]
   (let [schema (->> {:key :selection-schemas/id
                      :value schema-id}
-                    (store/find-entities store)
-                    (first)
+                    (store/lookup store)
                     (:selection-schemas/schema))]
     (->> url
          (slurp)

From fa42b46164297417a2601f469a9f0ad616d30f45 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Thu, 21 Aug 2025 13:48:29 +0200
Subject: [PATCH 056/391] integrated congest added jobs and job-metadata tables
 added services for sql tables added handlers for congest

---
 deps.edn                             |  3 +-
 src/source/db/master.clj             | 46 ++++++++++++-
 src/source/jobs/core.clj             | 98 ++++++++++++++++++++++++++++
 src/source/jobs/handlers.clj         | 28 ++++++++
 src/source/jobs/oplog.clj            | 65 ++++++++++++++++++
 src/source/routes/job.clj            | 17 +++++
 src/source/routes/job_deregister.clj |  7 ++
 src/source/routes/job_start.clj      |  7 ++
 src/source/routes/job_stop.clj       |  7 ++
 src/source/routes/jobs.clj           | 16 +++++
 src/source/services/jobs.clj         | 72 ++++++++++++++++++++
 11 files changed, 364 insertions(+), 2 deletions(-)
 create mode 100644 src/source/jobs/core.clj
 create mode 100644 src/source/jobs/handlers.clj
 create mode 100644 src/source/jobs/oplog.clj
 create mode 100644 src/source/routes/job.clj
 create mode 100644 src/source/routes/job_deregister.clj
 create mode 100644 src/source/routes/job_start.clj
 create mode 100644 src/source/routes/job_stop.clj
 create mode 100644 src/source/routes/jobs.clj
 create mode 100644 src/source/services/jobs.clj

diff --git a/deps.edn b/deps.edn
index b0b9fe3f..0245ba4e 100644
--- a/deps.edn
+++ b/deps.edn
@@ -14,6 +14,7 @@
         ring/ring-json {:mvn/version "0.5.1"}
         ring/ring-defaults {:mvn/version "0.6.0"}
         ring-cors/ring-cors {:mvn/version "0.1.13"}
+        io.github.modulr-software/congest {:mvn/version "0.1.5"}
         com.github.seancorfield/next.jdbc {:mvn/version "1.3.994"}
         org.xerial/sqlite-jdbc {:mvn/version "3.49.1.0"}
         buddy/buddy-core {:mvn/version "1.12.0-430"}
@@ -30,7 +31,7 @@
         io.replikativ/datahike {:mvn/version "0.6.1599"}
         hickory/hickory {:mvn/version "0.7.1"}
         metosin/jsonista {:mvn/version "0.3.13"}
-        metosin/reitit {:mvn/version "0.9.1"}
+        metosin/reitit {:mvn/version "0.9.1"} 
         metosin/reitit-middleware {:mvn/version "0.9.1"}
         metosin/reitit-swagger {:mvn/version "0.9.1"}
         metosin/reitit-swagger-ui {:mvn/version "0.9.1"}}}
diff --git a/src/source/db/master.clj b/src/source/db/master.clj
index 65b3a9e6..a2ca5641 100644
--- a/src/source/db/master.clj
+++ b/src/source/db/master.clj
@@ -133,11 +133,52 @@
   (tables/create-table-sql
    :selection-schemas
    (tables/table-id)
-   [:version :integer :not nil]
    [:output-schema-id :integer :not nil]
    [:provider-id :integer :not nil]
    (tables/foreign-key :provider-id :providers :id)))
 
+(def incoming-posts
+  (tables/create-table-sql
+   :incoming-posts
+   (tables/table-id)
+   [:feed-id :integer :not nil]
+   [:title :text :not nil]
+   [:info :text]
+   [:url :text]
+   [:stream-url :text]
+   [:creator-id :integer :not nil]
+   [:season :integer]
+   [:episode :integer]
+   [:content-type-id :integer :not nil]
+   [:redacted :integer]
+   [:posted-at :datetime]))
+
+(def jobs
+  (tables/create-table-sql
+   :jobs
+   (tables/table-id)
+   [:status :text [:check [:in :status ["running" "stopped"]]]]
+   [:args :text]
+   [:handler :text :not nil]
+   [:last-heartbeat :datetime]
+   [:job-metadata-id :integer :not nil]
+   (tables/foreign-key :job-metadata-id :job-metadata :id)
+   [[:foreign-key :job-metadata-id] [:references :job-metadata :id] :on-delete :cascade]))
+
+(def job-metadata
+  (tables/create-table-sql
+   :job-metadata
+   (tables/table-id)
+   [:initial-delay :integer]
+   [:auto-start :integer]
+   [:stop-after-fail :integer]
+   [:kill-after :integer]
+   [:num-calls :integer]
+   [:interval :integer]
+   [:recurring :integer]
+   [:created-at :datetime]
+   [:sleep :integer]))
+
 (comment
   (require '[honey.sql :as sql])
 
@@ -155,4 +196,7 @@
   (sql/format user-sectors)
   (sql/format feed-sectors)
   (sql/format selection-schemas)
+  (sql/format incoming-posts)
+  (sql/format jobs)
+  (sql/format job-metadata)
   ())
diff --git a/src/source/jobs/core.clj b/src/source/jobs/core.clj
new file mode 100644
index 00000000..4e0f95aa
--- /dev/null
+++ b/src/source/jobs/core.clj
@@ -0,0 +1,98 @@
+(ns source.jobs.core
+  (:require [congest.jobs :as congest]
+            [clojure.data.json :as json]
+            [source.services.interface :as services]
+            [source.jobs.oplog :as oplog]
+            [source.jobs.handlers :as handlers]))
+
+(defn start! [js ds store job-id]
+  (let [i->b (fn [i] (if (= i 1) true false))
+        job (services/job ds {:id job-id})
+        job-metadata (services/job-metadata ds {:id (:job-metadata-id job)})
+        metadata (-> (assoc job-metadata
+                            :id (str (:id job))
+                            :auto-start (i->b (:auto-start job-metadata))
+                            :stop-after-fail (i->b (:stop-after-fail job-metadata))
+                            :recurring? (i->b (:recurring job-metadata))
+                            :sleep (i->b (:sleep job-metadata))
+                            :args (json/read-str (:args job) {:key-fn keyword})
+                            :handler-name (:handler job)
+                            :handler (handlers/handler job)
+                            :logger oplog/operation-logger
+                            :ds ds
+                            :store store)
+                     (dissoc :recurring))]
+    (congest/deregister! js (str (:id job)))
+    (congest/register! js metadata)))
+
+(defn register!
+  "Registers a new job with the given job metadata, handler and arguments"
+  [js ds store job-metadata args]
+  (let [handler-fn (handlers/handler job-metadata)
+        job-id (-> (services/jobs ds)
+                   (count)
+                   (inc))
+        job-metadata (assoc job-metadata
+                            :id (str job-id)
+                            :args args
+                            :ds ds
+                            :store store
+                            :handler-name (:handler job-metadata)
+                            :handler handler-fn
+                            :logger oplog/operation-logger)]
+    (congest/register! js job-metadata)))
+
+(defn deregister! [js job-id]
+  (congest/deregister! js (str job-id)))
+
+(defn stop!
+  ([js job-id]
+   (stop! js job-id false))
+  ([js job-id kill?]
+   (congest/stop! js (str job-id) kill?)))
+
+(defn kill! [js job-id]
+  (stop! js job-id true))
+
+(defn kill-all! [js]
+  (congest/kill! js))
+
+(defn start-interrupted-jobs!
+  "Starts all jobs with a status of 'running'. Intended for server startup to restart interrupted jobs."
+  [js ds store]
+  (let [jobs (services/jobs ds)]
+    (run! (fn [job]
+            (when (= (:status job) "running")
+              (start! js ds store (:id job)))) jobs)))
+
+(comment
+  (require '[source.db.util :as db.util]
+           '[source.datastore.util :as store.util])
+  (def ds (db.util/conn))
+  (def store (store.util/conn :datahike))
+
+  (def testjob {:initial-delay 10
+                :auto-start true
+                :stop-after-fail false,
+                ;:kill-after 5
+                ;:num-calls nil
+                :interval 3000
+                :recurring? true
+                :handler :test
+                :created-at nil
+                :sleep false})
+
+  (services/jobs ds)
+  (services/job-metadata ds {:id 3})
+
+  (services/delete-job! ds {})
+  (services/delete-job-metadata! ds {})
+
+  (def js (congest/create-job-service []))
+  (register! js ds store testjob {:name "congest"})
+  (deregister! js 1)
+  (stop! js 1)
+  (start! js ds store 1)
+  (start-interrupted-jobs! js ds store)
+
+  ())
diff --git a/src/source/jobs/handlers.clj b/src/source/jobs/handlers.clj
new file mode 100644
index 00000000..1deb0e09
--- /dev/null
+++ b/src/source/jobs/handlers.clj
@@ -0,0 +1,28 @@
+(ns source.jobs.handlers
+  (:require [source.services.interface :as services]
+            [source.util :as util]))
+
+(defmulti handler
+  (fn [opts]
+    (keyword (:handler opts))))
+
+(defmethod handler :default [opts]
+  (throw (IllegalArgumentException.
+          (str "Handler with name " (:handler opts) " does not exist."))))
+
+(defmethod handler :test [_]
+  (fn [{:keys [args]}]
+    (println "hello" (get args :name) args)))
+
+(defmethod handler :update-feed-post [_]
+  (fn [{:keys [args ds store]}]
+    (try
+      (let [{:keys [feed-id post-id schema-id url]} args
+            extracted (services/extract-data store {:schema-id schema-id
+                                                    :url url})]
+        (services/update-feed! ds {:id feed-id
+                                   :data {:updated-at (util/get-utc-timestamp-string)}})
+        (services/update-incoming-post! ds {:id post-id
+                                            :data extracted}))
+      (catch Exception _ :fail))))
+
diff --git a/src/source/jobs/oplog.clj b/src/source/jobs/oplog.clj
new file mode 100644
index 00000000..32428bd1
--- /dev/null
+++ b/src/source/jobs/oplog.clj
@@ -0,0 +1,65 @@
+(ns source.jobs.oplog
+  (:require [source.services.interface :as services]
+            [source.util :as util]
+            [clojure.data.json :as json]))
+
+(defn delete-job! [{:keys [ds id]}]
+  (let [metadata-id (:job-metadata-id (services/job ds {:id id}))]
+    (services/delete-job-metadata! ds {:id metadata-id})
+    (services/delete-job! ds {:id id})))
+
+(defn update-job! [{:keys [ds id]} status]
+  (services/update-job! ds {:id id
+                            :data {:status status
+                                   :last-heartbeat (util/get-utc-timestamp-string)}}))
+
+(defn update-job-metadata! [{:keys [ds id num-calls]}]
+  (let [metadata-id (:job-metadata-id (services/job ds {:id id}))]
+    (services/update-job-metadata! ds {:id metadata-id
+                                       :data {:num-calls num-calls}})))
+
+(defn create-job! [{:keys [ds id args handler-name] :as job-metadata}]
+  (let [b->i (fn [b] (if b 1 0))
+        metadata {:initial-delay (:initial-delay job-metadata)
+                  :auto-start (b->i (:auto-start job-metadata))
+                  :stop-after-fail (b->i (:stop-after-fail job-metadata))
+                  :kill-after (:kill-after job-metadata)
+                  :num-calls (:num-calls job-metadata)
+                  :interval (:interval job-metadata)
+                  :recurring (b->i (:recurring? job-metadata))
+                  :created-at (:created-at job-metadata)
+                  :sleep (b->i (:sleep job-metadata))}
+        job (services/job ds {:id id})
+        metadata-id (when-not (some? job)
+                      (:id (services/insert-job-metadata! ds {:data metadata
+                                                              :ret :1})))]
+    (if (some? job)
+      (services/update-job! ds {:id id
+                                :data {:status "running"
+                                       :last-heartbeat (util/get-utc-timestamp-string)}})
+      (services/insert-job! ds {:data {:id id
+                                       :status "running"
+                                       :args (json/json-str args)
+                                       :handler (name handler-name)
+                                       :last-heartbeat (util/get-utc-timestamp-string)
+                                       :job-metadata-id metadata-id}
+                                :ret :1}))))
+
+(defn operation-logger [{:keys [action] :as metadata}]
+  (cond
+    (= action "register")
+    (create-job! metadata)
+
+    (= action "deregister")
+    (delete-job! metadata)
+
+    (= action "start")
+    (update-job! metadata "running")
+
+    (or (= action "stop") (= action "kill"))
+    (update-job! metadata "stopped")
+
+    (= action "run")
+    (do
+      (update-job! metadata "running")
+      (update-job-metadata! metadata))))
diff --git a/src/source/routes/job.clj b/src/source/routes/job.clj
new file mode 100644
index 00000000..1ccf2ddf
--- /dev/null
+++ b/src/source/routes/job.clj
@@ -0,0 +1,17 @@
+(ns source.routes.job
+  (:require [source.services.interface :as services]
+            [ring.util.response :as res]))
+
+(defn get [{:keys [ds path-params] :as _req}]
+  (let [job (services/job ds path-params)
+        metadata (services/job-metadata ds {:id (:job-metadata-id job)})]
+    (res/response (merge (dissoc job :job-metadata-id)
+                         {:metadata metadata}))))
+
+(comment
+  (require '[source.db.util :as db.util])
+  (def ds (db.util/conn))
+
+  (get {:ds ds :path-params {:id 1}})
+
+  ())
diff --git a/src/source/routes/job_deregister.clj b/src/source/routes/job_deregister.clj
new file mode 100644
index 00000000..4fe88f05
--- /dev/null
+++ b/src/source/routes/job_deregister.clj
@@ -0,0 +1,7 @@
+(ns source.routes.job-deregister
+  (:require [source.jobs.core :as jobs]
+            [ring.util.response :as res]))
+
+(defn get [{:keys [js path-params] :as _req}]
+  (jobs/deregister! js (:id path-params))
+  (res/response {:message "successfully deregistered job"}))
diff --git a/src/source/routes/job_start.clj b/src/source/routes/job_start.clj
new file mode 100644
index 00000000..f3852028
--- /dev/null
+++ b/src/source/routes/job_start.clj
@@ -0,0 +1,7 @@
+(ns source.routes.job-start
+  (:require [source.jobs.core :as jobs]
+            [ring.util.response :as res]))
+
+(defn get [{:keys [js ds store path-params]}]
+  (jobs/start! js ds store (:id path-params))
+  (res/response {:message "successfully started job"}))
diff --git a/src/source/routes/job_stop.clj b/src/source/routes/job_stop.clj
new file mode 100644
index 00000000..cf0c3c4e
--- /dev/null
+++ b/src/source/routes/job_stop.clj
@@ -0,0 +1,7 @@
+(ns source.routes.job-stop
+  (:require [source.jobs.core :as jobs]
+            [ring.util.response :as res]))
+
+(defn get [{:keys [js path-params]}]
+  (jobs/stop! js (:id path-params))
+  (res/response {:message "successfully stopped job"}))
diff --git a/src/source/routes/jobs.clj b/src/source/routes/jobs.clj
new file mode 100644
index 00000000..b5e64528
--- /dev/null
+++ b/src/source/routes/jobs.clj
@@ -0,0 +1,16 @@
+(ns source.routes.jobs
+  (:require [source.services.interface :as services]
+            [source.jobs.core :as jobs]
+            [ring.util.response :as res]))
+
+(defn get [{:keys [ds] :as _req}]
+  (res/response
+   (mapv (fn [job]
+           (let [metadata (services/job-metadata ds {:id (:job-metadata-id job)})]
+             (merge (dissoc job :job-metadata-id)
+                    {:metadata metadata}))) (services/jobs ds))))
+
+(defn post [{:keys [js ds store body] :as _req}]
+  (let [{:keys [metadata args]} body]
+    (jobs/register! js ds store metadata args)
+    (res/response {:message "successfully registered job"})))
diff --git a/src/source/services/jobs.clj b/src/source/services/jobs.clj
new file mode 100644
index 00000000..7a7b1574
--- /dev/null
+++ b/src/source/services/jobs.clj
@@ -0,0 +1,72 @@
+(ns source.services.jobs
+  (:require [source.db.interface :as db]))
+
+(defn insert-job! [ds {:keys [data ret] :as opts}]
+  (->> {:tname :jobs
+        :data data
+        :ret ret}
+       (merge opts)
+       (db/insert! ds)))
+
+(defn update-job! [ds {:keys [id data where] :as opts}]
+  (->> {:tname :jobs
+        :values data
+        :where (if (some? id) [:= :id id] where)}
+       (merge opts)
+       (db/update! ds)))
+
+(defn delete-job! [ds {:keys [id where] :as opts}]
+  (->> {:tname :jobs
+        :where (if (some? id)
+                 [:= :id id]
+                 where)
+        :ret :1}
+       (merge opts)
+       (db/delete! ds)))
+
+(defn jobs
+  ([ds] (jobs ds {}))
+  ([ds opts]
+   (->> {:tname :jobs
+         :ret :*}
+        (merge opts)
+        (db/find ds))))
+
+(defn job [ds {:keys [id where] :as opts}]
+  (->> {:tname :jobs
+        :where (if (some? id) [:= :id id] where)
+        :ret :1}
+       (merge opts)
+       (db/find ds)))
+
+(defn insert-job-metadata! [ds {:keys [data ret] :as opts}]
+  (->> {:tname :job-metadata
+        :data data
+        :ret ret}
+       (merge opts)
+       (db/insert! ds)))
+
+(defn update-job-metadata! [ds {:keys [id data where] :as opts}]
+  (->> {:tname :job-metadata
+        :values data
+        :where (if (some? id) [:= :id id] where)}
+       (merge opts)
+       (db/update! ds)))
+
+(defn delete-job-metadata! [ds {:keys [id where] :as opts}]
+  (->> {:tname :job-metadata
+        :where (if (some? id)
+                 [:= :id id]
+                 where)
+        :ret :1}
+       (merge opts)
+       (db/delete! ds)))
+
+(defn job-metadata [ds {:keys [id where] :as opts}]
+  (->> {:tname :job-metadata
+        :where (if (some? id)
+                 [:= :id id]
+                 where)
+        :ret :1}
+       (merge opts)
+       (db/find ds)))

From bf0388cd638025e521ee976f7b2552d6d9c64eea Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Thu, 21 Aug 2025 13:52:18 +0200
Subject: [PATCH 057/391] added middleware to include congest job service

---
 src/source/middleware/core.clj      | 33 +++++++++++++++++++++++------
 src/source/middleware/interface.clj |  5 ++---
 2 files changed, 29 insertions(+), 9 deletions(-)

diff --git a/src/source/middleware/core.clj b/src/source/middleware/core.clj
index 344e6c15..31f27831 100644
--- a/src/source/middleware/core.clj
+++ b/src/source/middleware/core.clj
@@ -4,6 +4,7 @@
             [source.config :as conf]
             [camel-snake-kebab.core :as csk]
             [camel-snake-kebab.extras :as cske]
+            [ring.util.response :as res]
             [ring.middleware.cors :refer [wrap-cors]]
             [ring.middleware.params :refer [wrap-params]]
             [ring.middleware.defaults :refer [wrap-defaults site-defaults]]
@@ -22,10 +23,33 @@
         (assoc :store store)
         (handler))))
 
+(defn wrap-js [handler js]
+  (fn [request]
+    (-> request
+        (assoc :js js)
+        (handler))))
+
+(defn apply-ds [app ds]
+  (-> app
+      (wrap-ds ds)))
+
 (defn apply-store [app store]
   (-> app
       (wrap-store store)))
 
+(defn apply-js [app js]
+  (-> app
+      (wrap-js js)))
+
+(defn wrap-exception-logger [handler]
+  (fn [req]
+    (try
+      (handler req)
+      (catch Exception e
+        (println "Unhandled Exception:\n" e)
+        (-> (res/response {:message "Internal Server Error"})
+            (res/status 500))))))
+
 (defn process-body [{:keys [body] :as req} t-fn]
   (assoc req
          :body
@@ -44,14 +68,12 @@
         (handler)
         (process-body csk/->camelCaseKeyword))))
 
-(defn apply-ds [app ds]
-  (-> app
-      (wrap-ds ds)))
-
-(defn apply-generic [app & {:keys [ds store]}]
+(defn apply-generic [app & {:keys [ds store js]}]
   (-> app
+      (wrap-exception-logger)
       (apply-ds ds)
       (apply-store store)
+      (apply-js js)
       (wrap-case-conversion)
       (content-type/wrap-content-type)
       (wrap-cors :access-control-allow-origin [(re-pattern (conf/read-value :cors-origin))]
@@ -70,4 +92,3 @@
 (defn apply-bundle [app]
   (-> app
       (auth/wrap-bundle-id)))
-
diff --git a/src/source/middleware/interface.clj b/src/source/middleware/interface.clj
index fb2c9096..27b1db52 100644
--- a/src/source/middleware/interface.clj
+++ b/src/source/middleware/interface.clj
@@ -1,8 +1,8 @@
 (ns source.middleware.interface
   (:require [source.middleware.core :as mw]))
 
-(defn apply-generic [app & {:keys [ds store]}]
-  (mw/apply-generic app :ds ds :store store))
+(defn apply-generic [app & {:keys [ds store js]}]
+  (mw/apply-generic app :ds ds :store store :js js))
 
 (defn apply-auth
   "accepts required-type as an optional parameter to authorize the route only for the specified user type"
@@ -11,4 +11,3 @@
 
 (defn apply-bundle [app]
   (mw/apply-bundle app))
-

From e89ffd15a872e32ac4a5087f8614e8cd7818821c Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Thu, 21 Aug 2025 13:53:54 +0200
Subject: [PATCH 058/391] updated migrations to include new tables

---
 src/source/migrations/001_init_master_db.clj | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/source/migrations/001_init_master_db.clj b/src/source/migrations/001_init_master_db.clj
index 6d4f7395..6fb7e9f8 100644
--- a/src/source/migrations/001_init_master_db.clj
+++ b/src/source/migrations/001_init_master_db.clj
@@ -77,7 +77,10 @@
       :businesses
       :user-sectors
       :feed-sectors
-      :selection-schemas])
+      :selection-schemas
+      :incoming-posts
+      :jobs
+      :job-metadata])
 
     (db/insert! ds-master baselines-seed)
     (db/insert! ds-master cadences-seed)

From d627e17ed777829ea6bf138fe52981adf5237e40 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Thu, 21 Aug 2025 13:57:53 +0200
Subject: [PATCH 059/391] added and updated db services

---
 src/source/services/baselines.clj      | 26 +++++++
 src/source/services/cadences.clj       | 26 +++++++
 src/source/services/content_types.clj  |  6 +-
 src/source/services/feeds.clj          | 35 +++++++++
 src/source/services/incoming_posts.clj | 35 +++++++++
 src/source/services/interface.clj      | 99 ++++++++++++++++++++++++--
 src/source/services/xml_schemas.clj    | 22 ++++--
 7 files changed, 234 insertions(+), 15 deletions(-)
 create mode 100644 src/source/services/baselines.clj
 create mode 100644 src/source/services/cadences.clj
 create mode 100644 src/source/services/feeds.clj
 create mode 100644 src/source/services/incoming_posts.clj

diff --git a/src/source/services/baselines.clj b/src/source/services/baselines.clj
new file mode 100644
index 00000000..4bb168dd
--- /dev/null
+++ b/src/source/services/baselines.clj
@@ -0,0 +1,26 @@
+(ns source.services.baselines
+  (:require [source.db.interface :as db]))
+
+(defn insert-baseline! [ds {:keys [data ret] :as opts}]
+  (->> {:tname :baselines
+        :data data
+        :ret ret}
+       (merge opts)
+       (db/insert! ds)))
+
+(defn baselines
+  ([ds] (baselines ds {}))
+  ([ds opts]
+   (->> {:tname :baselines
+         :ret :*}
+        (merge opts)
+        (db/find ds))))
+
+(defn baseline [ds {:keys [id where] :as opts}]
+  (->> {:tname :baselines
+        :where (if (some? id)
+                 [:= :id id]
+                 where)
+        :ret :1}
+       (merge opts)
+       (db/find ds)))
diff --git a/src/source/services/cadences.clj b/src/source/services/cadences.clj
new file mode 100644
index 00000000..d806938b
--- /dev/null
+++ b/src/source/services/cadences.clj
@@ -0,0 +1,26 @@
+(ns source.services.cadences
+  (:require [source.db.interface :as db]))
+
+(defn insert-cadence! [ds {:keys [data ret] :as opts}]
+  (->> {:tname :cadences
+        :data data
+        :ret ret}
+       (merge opts)
+       (db/insert! ds)))
+
+(defn cadences
+  ([ds] (cadences ds {}))
+  ([ds opts]
+   (->> {:tname :cadences
+         :ret :*}
+        (merge opts)
+        (db/find ds))))
+
+(defn cadence [ds {:keys [id where] :as opts}]
+  (->> {:tname :cadences
+        :where (if (some? id)
+                 [:= :id id]
+                 where)
+        :ret :1}
+       (merge opts)
+       (db/find ds)))
diff --git a/src/source/services/content_types.clj b/src/source/services/content_types.clj
index de8a0520..f124dec4 100644
--- a/src/source/services/content_types.clj
+++ b/src/source/services/content_types.clj
@@ -1,8 +1,8 @@
 (ns source.services.content-types
   (:require [source.db.interface :as db]))
 
-(defn add-content-type! [ds {:keys [data ret] :as opts}]
-  (->> {:tname :providers
+(defn insert-content-type! [ds {:keys [data ret] :as opts}]
+  (->> {:tname :content-types
         :data data
         :ret ret}
        (merge opts)
@@ -17,7 +17,7 @@
         (db/find ds))))
 
 (defn content-type [ds {:keys [id where] :as opts}]
-  (->> {:tname :providers
+  (->> {:tname :content-types
         :where (if (some? id)
                  [:= :id id]
                  where)
diff --git a/src/source/services/feeds.clj b/src/source/services/feeds.clj
new file mode 100644
index 00000000..91251770
--- /dev/null
+++ b/src/source/services/feeds.clj
@@ -0,0 +1,35 @@
+(ns source.services.feeds
+  (:require [source.db.interface :as db]))
+
+(defn insert-feed! [ds {:keys [data] :as opts}]
+  (->> {:tname :feeds
+        :data data
+        :ret :1}
+       (merge opts)
+       (db/insert! ds)))
+
+(defn update-feed! [ds {:keys [id data where] :as opts}]
+  (->> {:tname :feeds
+        :values data
+        :where (if (some? id) [:= :id id] where)
+        :ret :1}
+       (merge opts)
+       (db/update! ds)))
+
+(defn feeds
+  ([ds] (feeds ds {}))
+  ([ds {:keys [where] :as opts}]
+   (->> {:tname :feeds
+         :where where
+         :ret :*}
+        (merge opts)
+        (db/find ds))))
+
+(defn feed [ds {:keys [id where] :as opts}]
+  (->> {:tname :feeds
+        :where (if (some? id)
+                 [:= :id id]
+                 where)
+        :ret :1}
+       (merge opts)
+       (db/find ds)))
diff --git a/src/source/services/incoming_posts.clj b/src/source/services/incoming_posts.clj
new file mode 100644
index 00000000..99f79134
--- /dev/null
+++ b/src/source/services/incoming_posts.clj
@@ -0,0 +1,35 @@
+(ns source.services.incoming-posts
+  (:require [source.db.interface :as db]))
+
+(defn insert-incoming-post! [ds {:keys [data] :as opts}]
+  (->> {:tname :incoming-posts
+        :data data
+        :ret :1}
+       (merge opts)
+       (db/insert! ds)))
+
+(defn update-incoming-post! [ds {:keys [id data where] :as opts}]
+  (->> {:tname :incoming-posts
+        :data data
+        :where (if (some? id) [:= :id id] where)
+        :ret :1}
+       (merge opts)
+       (db/update! ds)))
+
+(defn incoming-posts
+  ([ds] (incoming-posts ds {}))
+  ([ds {:keys [where] :as opts}]
+   (->> {:tname :incoming-posts
+         :where where
+         :ret :*}
+        (merge opts)
+        (db/find ds))))
+
+(defn incoming-post [ds {:keys [id where] :as opts}]
+  (->> {:tname :incoming-posts
+        :where (if (some? id)
+                 [:= :id id]
+                 where)
+        :ret :1}
+       (merge opts)
+       (db/find ds)))
diff --git a/src/source/services/interface.clj b/src/source/services/interface.clj
index 0f8740a3..6e902453 100644
--- a/src/source/services/interface.clj
+++ b/src/source/services/interface.clj
@@ -5,7 +5,12 @@
              [source.services.xml-schemas :as xml]
              [source.services.bundles :as bundles]
              [source.services.providers :as providers]
-             [source.services.content-types :as content-types]))
+             [source.services.feeds :as feeds]
+             [source.services.incoming-posts :as incoming-posts]
+             [source.services.cadences :as cadences]
+             [source.services.baselines :as baselines]
+             [source.services.content-types :as content-types]
+             [source.services.jobs :as jobs]))
 
 (defn users
   [& args]
@@ -26,8 +31,8 @@
 (defn register [ds user]
   (auth/register ds user))
 
-(defn add-selection-schema! [store db {:keys [_schema _record] :as opts}]
-  (xml/add-selection-schema! store db opts))
+(defn insert-selection-schema! [store db {:keys [_schema _record] :as opts}]
+  (xml/insert-selection-schema! store db opts))
 
 (defn selection-schema [ds {:keys [_id] :as opts}]
   (xml/selection-schema ds opts))
@@ -41,6 +46,9 @@
 (defn selection-schemas-by-provider [ds {:keys [_provider-id] :as opts}]
   (xml/selection-schemas-by-provider ds opts))
 
+(defn delete-selection-schemas-by-provider! [store db provider-id]
+  (xml/delete-selection-schemas-by-provider! store db provider-id))
+
 (defn ast [url]
   (xml/ast url))
 
@@ -53,8 +61,8 @@
 (defn output-schema [store output-schema-id]
   (xml/output-schema store output-schema-id))
 
-(defn add-output-schema! [store schema]
-  (xml/add-output-schema! store schema))
+(defn insert-output-schema! [store schema]
+  (xml/insert-output-schema! store schema))
 
 (defn providers [ds]
   (providers/providers ds))
@@ -74,8 +82,85 @@
 (defn content-type [ds id]
   (content-types/content-type ds id))
 
-(defn add-content-type! [ds {:keys [_values _ret] :as opts}]
-  (content-types/add-content-type! ds opts))
+(defn insert-content-type! [ds {:keys [_values _ret] :as opts}]
+  (content-types/insert-content-type! ds opts))
+
+(defn insert-feed! [ds {:keys [_data] :as opts}]
+  (feeds/insert-feed! ds opts))
+
+(defn update-feed! [ds {:keys [_id _data _where] :as opts}]
+  (feeds/update-feed! ds opts))
+
+(defn feeds
+  ([ds]
+   (feeds/feeds ds))
+  ([ds {:keys [_where] :as opts}]
+   (feeds/feeds ds opts)))
+
+(defn feed [ds id]
+  (feeds/feed ds id))
+
+(defn insert-incoming-post! [ds {:keys [_data] :as opts}]
+  (incoming-posts/insert-incoming-post! ds opts))
+
+(defn update-incoming-post! [ds {:keys [_id _data _where] :as opts}]
+  (incoming-posts/update-incoming-post! ds opts))
+
+(defn incoming-posts
+  ([ds]
+   (incoming-posts/incoming-posts ds))
+  ([ds {:keys [_where] :as opts}]
+   (incoming-posts/incoming-posts ds opts)))
+
+(defn incoming-post [ds id]
+  (incoming-posts/incoming-post ds id))
+
+(defn insert-cadence! [ds {:keys [_values _ret] :as opts}]
+  (cadences/insert-cadence! ds opts))
+
+(defn cadences [ds]
+  (cadences/cadences ds))
+
+(defn cadence [ds id]
+  (cadences/cadence ds id))
+
+(defn insert-baseline! [ds {:keys [_values _ret] :as opts}]
+  (baselines/insert-baseline! ds opts))
+
+(defn baselines [ds]
+  (baselines/baselines ds))
+
+(defn baseline [ds id]
+  (baselines/baseline ds id))
+
+(defn insert-job! [ds {:keys [_data _ret] :as opts}]
+  (jobs/insert-job! ds opts))
+
+(defn update-job! [ds {:keys [_id _data _where] :as opts}]
+  (jobs/update-job! ds opts))
+
+(defn delete-job! [ds {:keys [_id _where] :as opts}]
+  (jobs/delete-job! ds opts))
+
+(defn jobs
+  ([ds] (jobs ds {}))
+  ([ds opts]
+   (jobs/jobs ds opts)))
+
+(defn job [ds {:keys [_id _where] :as opts}]
+  (jobs/job ds opts))
+
+(defn insert-job-metadata! [ds {:keys [_data _ret] :as opts}]
+  (jobs/insert-job-metadata! ds opts))
+
+(defn update-job-metadata! [ds {:keys [_id _data _where] :as opts}]
+  (jobs/update-job-metadata! ds opts))
+
+(defn delete-job-metadata! [ds {:keys [_id _where] :as opts}]
+  (jobs/delete-job-metadata! ds opts))
+
+(defn job-metadata [ds {:keys [_id _where] :as opts}]
+  (jobs/job-metadata ds opts))
 
 (comment
   (users (db/ds :master))
diff --git a/src/source/services/xml_schemas.clj b/src/source/services/xml_schemas.clj
index 9b9f3c76..b34ffab6 100644
--- a/src/source/services/xml_schemas.clj
+++ b/src/source/services/xml_schemas.clj
@@ -3,7 +3,7 @@
             [source.db.interface :as db]
             [source.rss.core :as rss]))
 
-(defn add-output-schema!
+(defn insert-output-schema!
   [store schema]
   (let [id (-> (store/entries store :output-schemas/id)
                (count)
@@ -22,7 +22,7 @@
        (store/find-entities store)
        (first)))
 
-(defn add-selection-schema!
+(defn insert-selection-schema!
   [store db {:keys [schema record]}]
   (let [{:keys [output-schema-id provider-id]} record
         db-result (db/insert! db {:tname :selection-schemas
@@ -32,6 +32,18 @@
     (store/insert! store {:selection-schemas/id (:id db-result)
                           :selection-schemas/schema schema})))
 
+(defn delete-selection-schemas-by-provider!
+  [store db provider-id]
+  (let [selection-schemas (db/find db {:tname :selection-schemas
+                                       :where [:= :provider-id provider-id]
+                                       :ret :*})]
+    (run! (fn [{:keys [id]}]
+            (->> (store/find store {:key :selection-schemas/id
+                                    :value id})
+                 (store/delete! store))) selection-schemas)
+    (db/delete! db {:tname :selection-schemas
+                    :where [:= :provider-id provider-id]})))
+
 (defn selection-schemas
   ([ds] (selection-schemas ds {}))
   ([ds opts]
@@ -80,8 +92,8 @@
   (require '[source.db.util :as db.util])
   (ast "https://www.youtube.com/feeds/videos.xml?channel_id=UCUyeluBRhGPCW4rPe_UvBZQ")
 
-  (reduce (fn [acc {:keys [version]}]
-            (conj acc version)) [] [{:version 2} {:version 3}])
+  (reduce (fn [acc {:keys [id]}]
+            (conj acc id)) [] [{:id 2} {:id 3}])
 
   (db/find (db.util/conn) {:tname :selection-schemas
                            :where [:and
@@ -103,7 +115,7 @@
    1
    "https://www.youtube.com/feeds/videos.xml?channel_id=UCUyeluBRhGPCW4rPe_UvBZQ")
 
-  (add-selection-schema!
+  (insert-selection-schema!
    ds
    (db.util/conn)
    {:record {:provider-id 1

From cc38fb3f0f6aa22ae5fa443914834ad22e9fb92b Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Thu, 21 Aug 2025 14:01:56 +0200
Subject: [PATCH 060/391] added schemas to provider routes

---
 src/source/routes/provider.clj  | 21 +++++++++++++++++++--
 src/source/routes/providers.clj | 30 ++++++++++++++++++++++--------
 2 files changed, 41 insertions(+), 10 deletions(-)

diff --git a/src/source/routes/provider.clj b/src/source/routes/provider.clj
index d02480a6..a6f4f4c0 100644
--- a/src/source/routes/provider.clj
+++ b/src/source/routes/provider.clj
@@ -2,12 +2,29 @@
   (:require [source.services.interface :as services]
             [ring.util.response :as res]))
 
-(defn get [{:keys [ds path-params] :as _request}]
+(defn get
+  {:summary "get provider by id"
+   :parameters {:path [:map [:id {:title "id"
+                                  :description "provider id"} :int]]}
+   :responses {200 {:body [:map
+                           [:id :int]
+                           [:name :string]
+                           [:domain :string]
+                           [:content-type-id :int]]}}}
+
+  [{:keys [ds path-params] :as _request}]
   (->> path-params
        (services/provider ds)
        (res/response)))
 
-(defn delete [{:keys [ds path-params] :as _request}]
+(defn delete 
+  {:summary "given a provider id, delete provider and associated selection schemas"
+   :parameters {:path [:map [:id {:title "id"
+                                  :description "provider id"} :int]]}
+   :responses {200 {:body [:map [:message :string]]}}}
+  
+  [{:keys [store ds path-params] :as _request}]
   (->> path-params
        (services/delete-provider! ds))
+  (services/delete-selection-schemas-by-provider! store ds (:id path-params))
   (res/response {:message "successfully deleted provider"}))
diff --git a/src/source/routes/providers.clj b/src/source/routes/providers.clj
index 2ba7de8c..4ce4548e 100644
--- a/src/source/routes/providers.clj
+++ b/src/source/routes/providers.clj
@@ -2,18 +2,32 @@
   (:require [source.services.interface :as services]
             [ring.util.response :as res]))
 
-(defn get [{:keys [ds] :as _request}]
+(defn get
+  {:summary "get all providers"
+   :responses {200 {:body [:vector
+                           [:map
+                            [:id :int]
+                            [:name :string]
+                            [:domain :string]
+                            [:content-type-id :int]]]}}}
+
+  [{:keys [ds] :as _request}]
   (-> (services/providers ds)
       (res/response)))
 
-(defn post [{:keys [ds body] :as _request}]
+(defn post
+  {:summary "add a provider"
+   :parameters {:body [:map
+                       [:provider
+                        [:map
+                         [:id :int]
+                         [:name :string]
+                         [:domain :string]
+                         [:content-type-id :int]]]]}
+   :responses {200 {:body [:map [:message :string]]}}}
+
+  [{:keys [ds body] :as _request}]
   (let [{:keys [provider]} body]
     (services/insert-provider! ds {:data provider
                                    :ret :1})
     (res/response {:message "successfully added provider"})))
-
-(comment
-  (require '[source.datastore.interface :as store])
-
-  (services/add-provider! (store/ds :datahike) "YouTube")
-  ())

From 156e3b3d859657e05ca872a43e0db502321eace3 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Thu, 21 Aug 2025 14:03:06 +0200
Subject: [PATCH 061/391] updated reitit with new routes updated routes to fit
 naming conventions added feeds routes

---
 src/source/routes/feeds.clj             | 120 +++++++++++
 src/source/routes/output_schemas.clj    |   2 +-
 src/source/routes/reitit.clj            | 256 ++++++++++++++++--------
 src/source/routes/selection_schema.clj  |   6 +-
 src/source/routes/selection_schemas.clj |   2 +-
 5 files changed, 304 insertions(+), 82 deletions(-)
 create mode 100644 src/source/routes/feeds.clj

diff --git a/src/source/routes/feeds.clj b/src/source/routes/feeds.clj
new file mode 100644
index 00000000..1d17ead9
--- /dev/null
+++ b/src/source/routes/feeds.clj
@@ -0,0 +1,120 @@
+(ns source.routes.feeds
+  (:require [source.services.interface :as services]
+            [source.util :as utils]
+            [source.jobs.core :as jobs]
+            [ring.util.response :as res]))
+
+(defn get
+  {:summary "get all feeds"
+   :responses  {200 {:body [:map
+                            [:users
+                             [:vector
+                              [:map
+                               [:id :int]
+                               [:title :string]
+                               [:display-picture [:maybe :string]]
+                               [:url [:maybe :string]]
+                               [:rss-url :string]
+                               [:user-id [:maybe :int]]
+                               [:provider-id [:maybe :int]]
+                               [:created-at :string]
+                               [:updated-at [:maybe :string]]
+                               [:content-type-id :int]
+                               [:cadence-id :int]
+                               [:baseline-id :int]
+                               [:ts-and-cs [:maybe :string]]
+                               [:state [:maybe :string]]]]]]}}}
+
+  [{:keys [ds user] :as _request}]
+  (-> (services/feeds ds {:where [:= :user-id (:id user)]})
+      (res/response)))
+
+(defn post
+  {:summary "adds a feed and extracts data from RSS feed URL into a post and schedules a job to keep them updated"
+   :parameters {:body [:map
+                       [:title :string]
+                       [:display-picture {:optional true} :string]
+                       [:url {:optional true} :string]
+                       [:rss-url :string]
+                       [:provider-id :int]
+                       [:content-type-id :int]
+                       [:cadence-id :int]
+                       [:baseline-id :int]
+                       [:ts-and-cs {:optional true} :string]
+                       [:state {:optional true} :string]]}
+   :responses {200 {:body [:map
+                           [:id :int]
+                           [:title :string]
+                           [:display-picture [:maybe :string]]
+                           [:url [:maybe :string]]
+                           [:rss-url :string]
+                           [:user-id [:maybe :int]]
+                           [:provider-id [:maybe :int]]
+                           [:created-at :string]
+                           [:updated-at [:maybe :string]]
+                           [:content-type-id :int]
+                           [:cadence-id :int]
+                           [:baseline-id :int]
+                           [:ts-and-cs [:maybe :string]]
+                           [:state [:maybe :string]]]}}}
+
+  [{:keys [js ds store user body] :as _request}]
+  (let [{:keys [provider-id rss-url content-type-id]} body
+        datetime (utils/get-utc-timestamp-string)
+        selection-schemas (->> [:= :provider-id provider-id]
+                               (assoc {} :where)
+                               (services/selection-schemas ds))
+        latest-ss (->> selection-schemas
+                       (reduce (fn [acc {:keys [id]}]
+                                 (conj acc id)) [])
+                       (apply max -1))
+        extracted (when-not (= latest-ss -1)
+                    (services/extract-data store {:schema-id latest-ss
+                                                  :url rss-url}))
+        new-feed (services/insert-feed!
+                  ds
+                  {:data (merge body {:user-id (:id user)
+                                      :created-at datetime})})
+        new-post (when (some? extracted)
+                   (services/insert-incoming-post! ds {:data (merge extracted
+                                                                    {:feed-id (:id new-feed)
+                                                                     :creator-id (:id user)
+                                                                     :content-type-id content-type-id})}))]
+
+    (if (some? extracted)
+      (do (jobs/register! js ds store
+                          {:initial-delay 10;(* 1000 60 60 24)
+                           :auto-start true
+                           :stop-after-fail false,
+                           :interval (* 1000 60) ;(* 1000 60 60 24)
+                           :recurring? true
+                           :handler :update-feed-post
+                           :created-at (utils/get-utc-timestamp-string)
+                           :sleep false}
+                          {:feed-id (:id new-feed)
+                           :post-id (:id new-post)
+                           :schema-id latest-ss
+                           :url rss-url})
+          (res/response new-feed))
+
+      (-> (res/response {:message "Failed to extract data, did you provide a selection schema?"})
+          (res/status 500)))))
+
+(comment
+  (require '[source.db.util :as db.util]
+           '[source.datastore.util :as store.util]
+           '[congest.jobs :as js])
+
+  (get {:ds (db.util/conn) :user {:id 5}})
+  (post {:ds (db.util/conn)
+         :js (js/create-job-service [])
+         :store (store.util/conn :datahike)
+         :user {:id 5}
+         :body {:title "primeagen test"
+                :rss-url "https://www.youtube.com/feeds/videos.xml?channel_id=UCUyeluBRhGPCW4rPe_UvBZQ"
+                :provider-id 1
+                :content-type-id 1
+                :cadence-id 1
+                :baseline-id 1}})
+
+  ())
diff --git a/src/source/routes/output_schemas.clj b/src/source/routes/output_schemas.clj
index bd7444bf..ac6d7a0f 100644
--- a/src/source/routes/output_schemas.clj
+++ b/src/source/routes/output_schemas.clj
@@ -8,5 +8,5 @@
 
 (defn post [{:keys [store body] :as _request}]
   (let [{:keys [schema]} body]
-    (services/add-output-schema! store schema)
+    (services/insert-output-schema! store schema)
     (res/response {:message "successfully added output schema"})))
diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index 8b68f1c0..3506e09b 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -5,10 +5,10 @@
             [reitit.openapi :as openapi]
             [reitit.coercion.malli]
             [reitit.ring.malli]
-            [reitit.ring.middleware.exception :as exception]
             [malli.util :as mu]
             [source.middleware.interface :as mw]
             [source.db.interface :as db]
+            [congest.jobs :as congest]
             [clojure.data.json :as json]
             [source.routes.user :as user]
             [source.routes.users :as users]
@@ -32,10 +32,17 @@
             [source.routes.provider :as provider]
             [source.routes.content-types :as content-types]
             [source.routes.content-type :as content-type]
+            [source.routes.feeds :as feeds]
             [source.routes.xml :as xml]
             [source.routes.data :as data]
+            [source.routes.jobs :as jobs]
+            [source.routes.job :as job]
+            [source.routes.job-deregister :as job-deregister]
+            [source.routes.job-start :as job-start]
+            [source.routes.job-stop :as job-stop]
             [source.util :as util]
-            [source.datastore.interface :as store]))
+            [source.datastore.interface :as store]
+            [source.jobs.core :as job-service]))
 
 (defn route [handlers]
   (reduce (fn [acc [k v]]
@@ -48,85 +55,102 @@
 
 (defn create-app []
   (let [ds (db/ds :master)
-        store (store/ds :datahike)]
+        store (store/ds :datahike)
+        js (congest/create-job-service [])]
+    (job-service/kill-all! js)
+    (job-service/start-interrupted-jobs! js ds)
     (ring/ring-handler
      (ring/router
-      [["/swagger.json" {:get {:no-doc true
-                               :swagger {:info {:title "source-api"
-                                                :description "swagger docs for source api with malli and reitit-ring"
-                                                :version "0.0.1"}
-                                         :securityDefinitions {"auth" {:type :apiKey
-                                                                       :in :header
-                                                                       :name "Authorization"}}}
-                               :handler (swagger/create-swagger-handler)}}]
-
-       ["/openapi.json" {:get {:no-doc true
-                               :openapi {:info {:title "source-api"
-                                                :description "openapi3 docs for source api with malli and reitit-ring"
-                                                :version "0.0.1"}
-                                         :components {:securitySchemes {"bearerAuth" {:type :http
-                                                                                      :scheme :bearer
-                                                                                      :bearerFormat "JWT"
-                                                                                      :description "JWT Authorization using the Bearer scheme"}}}}
-                               :handler (openapi/create-openapi-handler)}}]
-
-       ["/users"        {:middleware [[mw/apply-auth {:required-type :admin}]]
-                         :tags #{"users"}
-                         :swagger {:security [{"auth" []}]}
-                         :openapi {:security [{:bearerAuth []}]}}
-        [""             (route {:get users/get})]
-        ["/:id"         (route {:get user/get
-                                :patch user/patch})]]
-
-       ["/me"           {:middleware [[mw/apply-auth]]
-                         :tags #{"me"}
-                         :swagger {:security [{"auth" []}]}
-                         :openapi {:security [{:bearerAuth []}]}}
-        [""             (route {:get me/get})]]
-
-       ["/businesses"   {:middleware [[mw/apply-auth {:required-type :admin}]]
-                         :tags #{"businesses"}
-                         :swagger {:security [{"auth" []}]}
-                         :openapi {:security [{:bearerAuth []}]}}
-        [""             (route {:get businesses/get
-                                :post business/post})]
-        ["/:id"         (route {:patch business/patch})]]
-
-       ["/sectors"      {:tags #{"sectors"}}
-        [""             (route {:get sectors/get})]]
-
-       ["/login"        {:tags #{"auth"}}
-        [""             (route {:post login/post})]]
-
-       ["/register"     {:tags #{"auth"}}
-        [""             (route {:post register/post})]]
+      [["/swagger.json"   {:get {:no-doc true
+                                 :swagger {:info {:title "source-api"
+                                                  :description "swagger docs for source api with malli and reitit-ring"
+                                                  :version "0.0.1"}
+                                           :securityDefinitions {"auth" {:type :apiKey
+                                                                         :in :header
+                                                                         :name "Authorization"}}}
+                                 :handler (swagger/create-swagger-handler)}}]
+
+       ["/openapi.json"   {:get {:no-doc true
+                                 :openapi {:info {:title "source-api"
+                                                  :description "openapi3 docs for source api with malli and reitit-ring"
+                                                  :version "0.0.1"}
+                                           :components {:securitySchemes {"bearerAuth" {:type :http
+                                                                                        :scheme :bearer
+                                                                                        :bearerFormat "JWT"
+                                                                                        :description "JWT Authorization using the Bearer scheme"}}}}
+                                 :handler (openapi/create-openapi-handler)}}]
+
+       ["/users"          {:middleware [[mw/apply-auth {:required-type :admin}]]
+                           :tags #{"users"}
+                           :swagger {:security [{"auth" []}]}
+                           :openapi {:security [{:bearerAuth []}]}}
+        [""               (route {:get users/get})]
+        ["/:id"           (route {:get user/get
+                                  :patch user/patch})]]
+
+       ["/me"             {:middleware [[mw/apply-auth]]
+                           :tags #{"me"}
+                           :swagger {:security [{"auth" []}]}
+                           :openapi {:security [{:bearerAuth []}]}}
+        [""               (route {:get me/get})]]
+
+       ["/businesses"     {:middleware [[mw/apply-auth {:required-type :admin}]]
+                           :tags #{"businesses"}
+                           :swagger {:security [{"auth" []}]}
+                           :openapi {:security [{:bearerAuth []}]}}
+        [""               (route {:get businesses/get
+                                  :post business/post})]
+        ["/:id"           (route {:patch business/patch})]]
+
+       ["/sectors"        {:tags #{"sectors"}}
+        [""               (route {:get sectors/get})]]
+
+       ["/login"          {:tags #{"auth"}}
+        [""               (route {:post login/post})]]
+
+       ["/register"       {:tags #{"auth"}}
+        [""               (route {:post register/post})]]
 
        ["/oauth2"
-        ["/google"      {:tags #{"google"}}
-         [""            (route {:get google-launch/get})]
-         ["/callback"   {:no-doc true
-                         :get google-redirect/get}]
-         ["/user"       (route {:get google-user/get})]]]
-
-       ["/protected"    {:middleware [[mw/apply-auth]]
-                         :tags #{"protected"}
-                         :swagger {:security [{"auth" []}]}
-                         :openapi {:security [{:bearerAuth []}]}}
-        ["/authorized"  (route {:get authorized/get})]]
-
-       ["/providers"
-        [""             {:get providers/get}]
-        ["/:id"         {:get provider/get}]]
-
-       ["/content-types"
-        [""             {:get content-types/get}]
-        ["/:id"         {:get content-type/get}]]
+        ["/google"        {:tags #{"google"}}
+         [""              (route {:get google-launch/get})]
+         ["/callback"     {:no-doc true
+                           :get google-redirect/get}]
+         ["/user"         (route {:get google-user/get})]]]
+
+       ["/protected"      {:middleware [[mw/apply-auth]]
+                           :tags #{"protected"}
+                           :swagger {:security [{"auth" []}]}
+                           :openapi {:security [{:bearerAuth []}]}}
+        ["/authorized"    (route {:get authorized/get})]]
+
+       ["/providers"      {:tags #{"providers"}}
+        [""               (route {:get providers/get})]
+        ["/:id"           (route {:get provider/get})]]
+
+       ["/content-types"  {:tags #{"content types"}}
+        [""               {:get content-types/get}]
+        ["/:id"           {:get content-type/get}]]
+
+       ["/feeds"          {:middleware [[mw/apply-auth]]
+                           :tags #{"feeds"}}
+        [""               (route {:get feeds/get
+                                  :post feeds/post})]]
 
        ["/admin"                  {:middleware [[mw/apply-auth {:required-type :admin}]]
                                    :no-doc true
                                    :tags #{"admin"}
                                    :swagger {:security [{"auth" []}]}
                                    :openapi {:security [{:bearerAuth []}]}}
+        ["/jobs"
+         [""                      {:get jobs/get}]
+         ["/manage"
+          ["/register"            {:post jobs/post}]]
+         ["/:id"                  {:get job/get}
+          ["/manage"
+           ["/deregister"         {:get job-deregister/get}]
+           ["/start"              {:get job-start/get}]
+           ["/stop"               {:get job-stop/get}]]]]
         ["/add-admin"             (route {:post admin/post})]
         ["/selection-schemas"
          [""                      {:get selection-schemas/get
@@ -138,8 +162,8 @@
                                    :post output-schemas/post}]
          ["/:id"                  {:get output-schema/get}]]
         ["/providers"
-         [""                      {:post providers/post}]
-         ["/:id"                  {:delete provider/delete}]]
+         [""                      (route {:post providers/post})]
+         ["/:id"                  (route {:delete provider/delete})]]
         ["/ast"                   {:post xml/post}]
         ["/extract-data"          {:post data/post}]]]
 
@@ -149,9 +173,7 @@
                           :strip-extra-keys true
                           :default-values true
                           :options nil})
-              :middleware [[mw/apply-generic :ds ds :store store]
-                           ;;[exception/exception-middleware]
-                           ]}})
+              :middleware [[mw/apply-generic :ds ds :store store :js js]]}})
      (ring/routes
       (swagger-ui/create-swagger-ui-handler {:path "/"
                                              :config {:validatorUrl nil
@@ -166,14 +188,18 @@
            '[source.rss.youtube :as yt])
 
   (let [app (create-app)
-        request {:uri "/users" :request-method :get}]
+        request {:uri "/users"
+                 :request-method :get
+                 :headers {"authorization" (str "Bearer " (auth.util/sign-jwt {:id 1 :type "admin"}))}}]
     (-> request
         app
         :body
         (json/read-json {:key-fn keyword})))
 
   (let [app (create-app)
-        request {:uri "/users/3" :request-method :get}]
+        request {:uri "/users/3"
+                 :request-method :get
+                 :headers {"authorization" (str "Bearer " (auth.util/sign-jwt {:id 1 :type "admin"}))}}]
     (-> request
         app
         :body
@@ -286,6 +312,35 @@
         :body
         (json/read-json {:key-fn keyword})))
 
+  (defn get-url []
+    (->> "https://www.youtube.com/@ThePrimeTimeagen"
+         (yt/find-channel-id)
+         (str "https://www.youtube.com/feeds/videos.xml?channel_id=")))
+
+  (let [app (create-app)
+        request {:uri "/admin/feeds"
+                 :request-method :get
+                 :headers {"authorization" (str "Bearer " (auth.util/sign-jwt {:id 5 :type "creator"}))}}]
+    (-> request
+        app
+        :body
+        (json/read-json {:key-fn keyword})))
+
+  (let [app (create-app)
+        request {:uri "/admin/feeds"
+                 :request-method :post
+                 :body {:title "primeagen test"
+                        :rss-url (get-url)
+                        :provider-id 1
+                        :content-type-id 1
+                        :cadence-id 1
+                        :baseline-id 1}
+                 :headers {"authorization" (str "Bearer " (auth.util/sign-jwt {:id 5 :type "creator"}))}}]
+    (-> request
+        app
+        :body
+        (json/read-json {:key-fn keyword})))
+
   (let [app (create-app)
         store (store/ds :datahike)
         request {:uri "/admin/selection-schemas"
@@ -351,4 +406,49 @@
         :body
         (json/read-json {:key-fn keyword})))
 
+  (let [app (create-app)
+        request {:uri "/admin/jobs"
+                 :request-method :get
+                 :headers {"authorization" (str "Bearer " (auth.util/sign-jwt {:id 1 :type "admin"}))}}]
+    (-> request
+        app
+        :body
+        (json/read-json {:key-fn keyword})))
+
+  (let [app (create-app)
+        request {:uri "/admin/jobs/1"
+                 :request-method :get
+                 :headers {"authorization" (str "Bearer " (auth.util/sign-jwt {:id 1 :type "admin"}))}}]
+    (-> request
+        app
+        :body
+        (json/read-json {:key-fn keyword})))
+
+  (let [app (create-app)
+        request {:uri "/admin/jobs/manage/register"
+                 :request-method :post
+                 :body {:metadata {:initial-delay 10
+                                   :auto-start true
+                                   :stop-after-fail false,
+                                   :interval 1000
+                                   :recurring? true
+                                   :handler "test"
+                                   :created-at nil
+                                   :sleep false}
+                        :args {:name "congest"}}
+                 :headers {"authorization" (str "Bearer " (auth.util/sign-jwt {:id 1 :type "admin"}))}}]
+    (-> request
+        app
+        :body
+        (json/read-json {:key-fn keyword})))
+
+  (let [app (create-app)
+        request {:uri "/admin/jobs/1/manage/deregister"
+                 :request-method :get
+                 :headers {"authorization" (str "Bearer " (auth.util/sign-jwt {:id 1 :type "admin"}))}}]
+    (-> request
+        app
+        :body
+        (json/read-json {:key-fn keyword})))
+
   ())
diff --git a/src/source/routes/selection_schema.clj b/src/source/routes/selection_schema.clj
index 32354936..77d25d67 100644
--- a/src/source/routes/selection_schema.clj
+++ b/src/source/routes/selection_schema.clj
@@ -4,13 +4,15 @@
 
 (defn get [{:keys [ds path-params store] :as _request}]
   (let [{:keys [output-schema-id] :as selection-schema}
-        (services/selection-schema ds (:id path-params))
+        (services/selection-schema ds path-params)
         output-schema (services/output-schema store output-schema-id)]
     (res/response (merge {:output-schema output-schema}
                          selection-schema))))
 
 (comment
-  (require '[source.db.util :as db.util])
+  (require '[source.db.util :as db.util]
+           '[source.datastore.util :as store.util])
   (get {:ds (db.util/conn)
+        :store (store.util/conn :datahike)
         :path-params {:id 1}})
   ())
diff --git a/src/source/routes/selection_schemas.clj b/src/source/routes/selection_schemas.clj
index 34aaa5a3..f0a67534 100644
--- a/src/source/routes/selection_schemas.clj
+++ b/src/source/routes/selection_schemas.clj
@@ -7,7 +7,7 @@
       (res/response)))
 
 (defn post [{:keys [store ds body] :as _request}]
-  (-> (services/add-selection-schema! store ds body)
+  (-> (services/insert-selection-schema! store ds body)
       (first)
       (res/response)))
 

From f67b39d73865c6fc01d3ab8362a583c9cbb37f57 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Thu, 21 Aug 2025 14:06:00 +0200
Subject: [PATCH 062/391] updated parameters for clearer usage

---
 src/source/db/interface.clj | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/source/db/interface.clj b/src/source/db/interface.clj
index a1622be2..a36c93f7 100644
--- a/src/source/db/interface.clj
+++ b/src/source/db/interface.clj
@@ -11,7 +11,7 @@
 (defn execute! [ds opts]
   (hon/execute! ds opts))
 
-(defn find [ds opts]
+(defn find [ds {:keys [_tname _where _ret] :as opts}]
   (hon/find ds opts))
 
 (defn find-one [ds opts]

From fadcff1221a7379df1ddc409de713fee1185e1f5 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Thu, 21 Aug 2025 14:27:05 +0200
Subject: [PATCH 063/391] hotfixes

---
 src/source/datastore/interface.clj | 3 ---
 src/source/routes/feeds.clj        | 4 ++--
 src/source/routes/reitit.clj       | 2 +-
 3 files changed, 3 insertions(+), 6 deletions(-)

diff --git a/src/source/datastore/interface.clj b/src/source/datastore/interface.clj
index 686c46b9..3c065890 100644
--- a/src/source/datastore/interface.clj
+++ b/src/source/datastore/interface.clj
@@ -17,9 +17,6 @@
 (defn lookup [ds {:keys [_key _value] :as opts}]
   (dh/lookup ds opts))
 
-(defn exists? [ds k]
-  (dh/exists? ds k))
-
 (defn insert! [ds data]
   (dh/insert! ds data))
 
diff --git a/src/source/routes/feeds.clj b/src/source/routes/feeds.clj
index 1d17ead9..c1e8a2a3 100644
--- a/src/source/routes/feeds.clj
+++ b/src/source/routes/feeds.clj
@@ -83,10 +83,10 @@
 
     (if (some? extracted)
       (do (jobs/register! js ds store
-                          {:initial-delay 10;(* 1000 60 60 24)
+                          {:initial-delay (* 1000 60 60 24)
                            :auto-start true
                            :stop-after-fail false,
-                           :interval (* 1000 60) ;(* 1000 60 60 24)
+                           :interval (* 1000 60 60 24)
                            :recurring? true
                            :handler :update-feed-post
                            :created-at (utils/get-utc-timestamp-string)
diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index 3506e09b..c5c09c16 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -58,7 +58,7 @@
         store (store/ds :datahike)
         js (congest/create-job-service [])]
     (job-service/kill-all! js)
-    (job-service/start-interrupted-jobs! js ds)
+    (job-service/start-interrupted-jobs! js ds store)
     (ring/ring-handler
      (ring/router
       [["/swagger.json"   {:get {:no-doc true

From e07ff7248ea6eacebd7a463dd6ff1f2766b3e2bb Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 25 Aug 2025 11:32:25 +0200
Subject: [PATCH 064/391] refactored congest integration removed unnecessary
 usage of persistance layer services outside oplogger relying on congest
 functions instead of wrappers

---
 deps.edn                             |   2 +-
 src/source/db/master.clj             |   1 +
 src/source/jobs/core.clj             | 100 +++++++++++----------------
 src/source/jobs/oplog.clj            |  34 +++++----
 src/source/middleware/core.clj       |   8 ++-
 src/source/routes/feeds.clj          |  41 ++++++-----
 src/source/routes/job_deregister.clj |   2 +-
 src/source/routes/job_stop.clj       |   4 +-
 src/source/routes/jobs.clj           |   5 +-
 src/source/routes/reitit.clj         |   5 +-
 10 files changed, 100 insertions(+), 102 deletions(-)

diff --git a/deps.edn b/deps.edn
index 0245ba4e..3efb2cfd 100644
--- a/deps.edn
+++ b/deps.edn
@@ -14,7 +14,7 @@
         ring/ring-json {:mvn/version "0.5.1"}
         ring/ring-defaults {:mvn/version "0.6.0"}
         ring-cors/ring-cors {:mvn/version "0.1.13"}
-        io.github.modulr-software/congest {:mvn/version "0.1.5"}
+        io.github.modulr-software/congest {:mvn/version "0.1.6"}
         com.github.seancorfield/next.jdbc {:mvn/version "1.3.994"}
         org.xerial/sqlite-jdbc {:mvn/version "3.49.1.0"}
         buddy/buddy-core {:mvn/version "1.12.0-430"}
diff --git a/src/source/db/master.clj b/src/source/db/master.clj
index a2ca5641..bec159b3 100644
--- a/src/source/db/master.clj
+++ b/src/source/db/master.clj
@@ -157,6 +157,7 @@
   (tables/create-table-sql
    :jobs
    (tables/table-id)
+   [:job-id :text :not nil]
    [:status :text [:check [:in :status ["running" "stopped"]]]]
    [:args :text]
    [:handler :text :not nil]
diff --git a/src/source/jobs/core.clj b/src/source/jobs/core.clj
index 4e0f95aa..c20fcfc2 100644
--- a/src/source/jobs/core.clj
+++ b/src/source/jobs/core.clj
@@ -5,94 +5,74 @@
             [source.jobs.oplog :as oplog]
             [source.jobs.handlers :as handlers]))
 
-(defn start! [js ds store job-id]
-  (let [i->b (fn [i] (if (= i 1) true false))
-        job (services/job ds {:id job-id})
-        job-metadata (services/job-metadata ds {:id (:job-metadata-id job)})
-        metadata (-> (assoc job-metadata
-                            :id (str (:id job))
-                            :auto-start (i->b (:auto-start job-metadata))
-                            :stop-after-fail (i->b (:stop-after-fail job-metadata))
-                            :recurring? (i->b (:recurring job-metadata))
-                            :sleep (i->b (:sleep job-metadata))
-                            :args (json/read-str (:args job) {:key-fn keyword})
-                            :handler-name (:handler job)
-                            :handler (handlers/handler job)
-                            :logger oplog/operation-logger
-                            :ds ds
-                            :store store)
-                     (dissoc :recurring))]
-    (congest/deregister! js (str (:id job)))
-    (congest/register! js metadata)))
+(defn prepare-congest-metadata
+  "given raw job metadata, returns extended metadata necessary for use with congest"
+  [ds store metadata]
+  (let [i->b (fn [i] (if (integer? i)
+                       (if (= i 1) true false)
+                       i))
+        args (:args metadata)]
+    (-> metadata
+        (assoc :auto-start (i->b (:auto-start metadata))
+               :stop-after-fail (i->b (:stop-after-fail metadata))
+               :recurring? (i->b (or (:recurring metadata) (:recurring? metadata)))
+               :sleep (i->b (:sleep metadata))
+               :args (if (string? args) (json/read-str (:args metadata) {:key-fn keyword}) args)
+               :handler-name (:handler metadata)
+               :handler (handlers/handler metadata)
+               :logger oplog/operation-logger
+               :ds ds
+               :store store)
+        (dissoc :recurring))))
 
-(defn register!
-  "Registers a new job with the given job metadata, handler and arguments"
-  [js ds store job-metadata args]
-  (let [handler-fn (handlers/handler job-metadata)
-        job-id (-> (services/jobs ds)
-                   (count)
-                   (inc))
-        job-metadata (assoc job-metadata
-                            :id (str job-id)
+(defn start!
+  "given a job-id, re-registers an existing job from the database"
+  [js ds store job-id]
+  (let [{:keys [job-metadata-id args handler]} (services/job ds {:where [:= :job-id job-id]})
+        metadata (-> (services/job-metadata ds {:id job-metadata-id})
+                     (assoc :id job-id
                             :args args
-                            :ds ds
-                            :store store
-                            :handler-name (:handler job-metadata)
-                            :handler handler-fn
-                            :logger oplog/operation-logger)]
-    (congest/register! js job-metadata)))
-
-(defn deregister! [js job-id]
-  (congest/deregister! js (str job-id)))
-
-(defn stop!
-  ([js job-id]
-   (stop! js job-id false))
-  ([js job-id kill?]
-   (congest/stop! js (str job-id) kill?)))
-
-(defn kill! [js job-id]
-  (stop! js job-id true))
-
-(defn kill-all! [js]
-  (congest/kill! js))
+                            :handler handler))]
+    (congest/deregister! js job-id)
+    (congest/register! js (prepare-congest-metadata ds store metadata))))
 
 (defn start-interrupted-jobs!
   "Starts all jobs with a status of 'running'. Intended for server startup to restart interrupted jobs."
   [js ds store]
   (let [jobs (services/jobs ds)]
-    (run! (fn [job]
-            (when (= (:status job) "running")
-              (start! js ds store (:id job)))) jobs)))
+    (run! (fn [{:keys [job-id status]}]
+            (when (= status "running")
+              (start! js ds store job-id))) jobs)))
 
 (comment
   (require '[source.db.util :as db.util]
            '[source.datastore.util :as store.util])
+
   (def ds (db.util/conn))
   (def store (store.util/conn :datahike))
 
-  (def testjob {:initial-delay 10
+  (def testjob {:id "test"
+                :initial-delay 10
                 :auto-start true
                 :stop-after-fail false,
-                ;:kill-after 5
-                ;:num-calls nil
                 :interval 3000
                 :recurring? true
+                :args {:name "congest"}
                 :handler :test
                 :created-at nil
                 :sleep false})
 
   (services/jobs ds)
-  (services/job-metadata ds {:id 3})
+  (services/job-metadata ds {:id 4})
 
   (services/delete-job! ds {})
   (services/delete-job-metadata! ds {})
 
   (def js (congest/create-job-service []))
-  (register! js ds store testjob {:name "congest"})
-  (deregister! js 1)
-  (stop! js 1)
-  (start! js ds store 1)
+  (congest/register! js (prepare-congest-metadata ds store testjob))
+  (congest/deregister! js "test")
+  (congest/stop! js "test" false)
+  (start! js ds store "test")
   (start-interrupted-jobs! js ds store)
 
   ())
diff --git a/src/source/jobs/oplog.clj b/src/source/jobs/oplog.clj
index 32428bd1..60c0a92b 100644
--- a/src/source/jobs/oplog.clj
+++ b/src/source/jobs/oplog.clj
@@ -3,22 +3,30 @@
             [source.util :as util]
             [clojure.data.json :as json]))
 
-(defn delete-job! [{:keys [ds id]}]
-  (let [metadata-id (:job-metadata-id (services/job ds {:id id}))]
-    (services/delete-job-metadata! ds {:id metadata-id})
+(defn delete-job!
+  "Delete job record and metadata associated therewith"
+  [{:keys [ds id]}]
+  (let [{:keys [job-metadata-id id]} (services/job ds {:where [:= :job-id id]})]
+    (services/delete-job-metadata! ds {:id job-metadata-id})
     (services/delete-job! ds {:id id})))
 
-(defn update-job! [{:keys [ds id]} status]
-  (services/update-job! ds {:id id
+(defn update-job!
+  "Update job status and last heartbeat"
+  [{:keys [ds id]} status]
+  (services/update-job! ds {:where [:= :job-id id]
                             :data {:status status
                                    :last-heartbeat (util/get-utc-timestamp-string)}}))
 
-(defn update-job-metadata! [{:keys [ds id num-calls]}]
-  (let [metadata-id (:job-metadata-id (services/job ds {:id id}))]
+(defn update-job-metadata!
+  "Update the number of calls in job metadata"
+  [{:keys [ds id num-calls]}]
+  (let [metadata-id (:job-metadata-id (services/job ds {:where [:= :job-id id]}))]
     (services/update-job-metadata! ds {:id metadata-id
                                        :data {:num-calls num-calls}})))
 
-(defn create-job! [{:keys [ds id args handler-name] :as job-metadata}]
+(defn create-job!
+  "Inserts a new job record and its associated metadata if the job doesn't already exist in the database."
+  [{:keys [ds id args handler-name] :as job-metadata}]
   (let [b->i (fn [b] (if b 1 0))
         metadata {:initial-delay (:initial-delay job-metadata)
                   :auto-start (b->i (:auto-start job-metadata))
@@ -29,15 +37,15 @@
                   :recurring (b->i (:recurring? job-metadata))
                   :created-at (:created-at job-metadata)
                   :sleep (b->i (:sleep job-metadata))}
-        job (services/job ds {:id id})
+        job (services/job ds {:where [:= :job-id id]})
         metadata-id (when-not (some? job)
                       (:id (services/insert-job-metadata! ds {:data metadata
                                                               :ret :1})))]
     (if (some? job)
-      (services/update-job! ds {:id id
+      (services/update-job! ds {:where [:= :job-id id]
                                 :data {:status "running"
                                        :last-heartbeat (util/get-utc-timestamp-string)}})
-      (services/insert-job! ds {:data {:id id
+      (services/insert-job! ds {:data {:job-id id
                                        :status "running"
                                        :args (json/json-str args)
                                        :handler (name handler-name)
@@ -45,7 +53,9 @@
                                        :job-metadata-id metadata-id}
                                 :ret :1}))))
 
-(defn operation-logger [{:keys [action] :as metadata}]
+(defn operation-logger
+  "This function serves as the operation logger called from within congest to handle the persistence layer"
+  [{:keys [action] :as metadata}]
   (cond
     (= action "register")
     (create-job! metadata)
diff --git a/src/source/middleware/core.clj b/src/source/middleware/core.clj
index 31f27831..df88af5f 100644
--- a/src/source/middleware/core.clj
+++ b/src/source/middleware/core.clj
@@ -23,7 +23,9 @@
         (assoc :store store)
         (handler))))
 
-(defn wrap-js [handler js]
+(defn wrap-js 
+  "attaches the provided job service to the handler's request"
+  [handler js]
   (fn [request]
     (-> request
         (assoc :js js)
@@ -37,7 +39,9 @@
   (-> app
       (wrap-store store)))
 
-(defn apply-js [app js]
+(defn apply-js 
+  "middleware for attaching the job service to the request"
+  [app js]
   (-> app
       (wrap-js js)))
 
diff --git a/src/source/routes/feeds.clj b/src/source/routes/feeds.clj
index c1e8a2a3..4878bf34 100644
--- a/src/source/routes/feeds.clj
+++ b/src/source/routes/feeds.clj
@@ -1,6 +1,7 @@
 (ns source.routes.feeds
   (:require [source.services.interface :as services]
             [source.util :as utils]
+            [congest.jobs :as congest]
             [source.jobs.core :as jobs]
             [ring.util.response :as res]))
 
@@ -82,32 +83,35 @@
                                                                      :content-type-id content-type-id})}))]
 
     (if (some? extracted)
-      (do (jobs/register! js ds store
-                          {:initial-delay (* 1000 60 60 24)
-                           :auto-start true
-                           :stop-after-fail false,
-                           :interval (* 1000 60 60 24)
-                           :recurring? true
-                           :handler :update-feed-post
-                           :created-at (utils/get-utc-timestamp-string)
-                           :sleep false}
-                          {:feed-id (:id new-feed)
-                           :post-id (:id new-post)
-                           :schema-id latest-ss
-                           :url rss-url})
-          (res/response new-feed))
+      (do
+        (->> (jobs/prepare-congest-metadata
+              ds
+              store
+              {:initial-delay (* 1000 60 60 24)
+               :auto-start true
+               :stop-after-fail false,
+               :interval (* 1000 60 60 24)
+               :recurring? true
+               :args {:feed-id (:id new-feed)
+                      :post-id (:id new-post)
+                      :schema-id latest-ss
+                      :url rss-url}
+               :handler :update-feed-post
+               :created-at (utils/get-utc-timestamp-string)
+               :sleep false})
+             (congest/register! js))
+        (res/response new-feed))
 
-      (-> (res/response {:message "Failed to extract data, did you provide a selection schema?"})
+      (-> (res/response {:message "failed to extract data"})
           (res/status 500)))))
 
 (comment
   (require '[source.db.util :as db.util]
-           '[source.datastore.util :as store.util]
-           '[congest.jobs :as js])
+           '[source.datastore.util :as store.util])
 
   (get {:ds (db.util/conn) :user {:id 5}})
   (post {:ds (db.util/conn)
-         :js (js/create-job-service [])
+         :js (congest/create-job-service [])
          :store (store.util/conn :datahike)
          :user {:id 5}
          :body {:title "primeagen test"
@@ -116,5 +120,4 @@
                 :content-type-id 1
                 :cadence-id 1
                 :baseline-id 1}})
-
   ())
diff --git a/src/source/routes/job_deregister.clj b/src/source/routes/job_deregister.clj
index 4fe88f05..ce3f08cb 100644
--- a/src/source/routes/job_deregister.clj
+++ b/src/source/routes/job_deregister.clj
@@ -1,5 +1,5 @@
 (ns source.routes.job-deregister
-  (:require [source.jobs.core :as jobs]
+  (:require [congest.jobs :as jobs]
             [ring.util.response :as res]))
 
 (defn get [{:keys [js path-params] :as _req}]
diff --git a/src/source/routes/job_stop.clj b/src/source/routes/job_stop.clj
index cf0c3c4e..1805f6f0 100644
--- a/src/source/routes/job_stop.clj
+++ b/src/source/routes/job_stop.clj
@@ -1,7 +1,7 @@
 (ns source.routes.job-stop
-  (:require [source.jobs.core :as jobs]
+  (:require [congest.jobs :as jobs]
             [ring.util.response :as res]))
 
 (defn get [{:keys [js path-params]}]
-  (jobs/stop! js (:id path-params))
+  (jobs/stop! js (:id path-params) false)
   (res/response {:message "successfully stopped job"}))
diff --git a/src/source/routes/jobs.clj b/src/source/routes/jobs.clj
index b5e64528..b3943b1f 100644
--- a/src/source/routes/jobs.clj
+++ b/src/source/routes/jobs.clj
@@ -1,5 +1,6 @@
 (ns source.routes.jobs
   (:require [source.services.interface :as services]
+            [congest.jobs :as congest]
             [source.jobs.core :as jobs]
             [ring.util.response :as res]))
 
@@ -11,6 +12,6 @@
                     {:metadata metadata}))) (services/jobs ds))))
 
 (defn post [{:keys [js ds store body] :as _req}]
-  (let [{:keys [metadata args]} body]
-    (jobs/register! js ds store metadata args)
+  (let [{:keys [metadata]} body]
+    (congest/register! js (jobs/prepare-congest-metadata ds store metadata))
     (res/response {:message "successfully registered job"})))
diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index c5c09c16..c274940a 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -57,7 +57,6 @@
   (let [ds (db/ds :master)
         store (store/ds :datahike)
         js (congest/create-job-service [])]
-    (job-service/kill-all! js)
     (job-service/start-interrupted-jobs! js ds store)
     (ring/ring-handler
      (ring/router
@@ -432,10 +431,10 @@
                                    :stop-after-fail false,
                                    :interval 1000
                                    :recurring? true
+                                   :args {:name "congest"}
                                    :handler "test"
                                    :created-at nil
-                                   :sleep false}
-                        :args {:name "congest"}}
+                                   :sleep false}}
                  :headers {"authorization" (str "Bearer " (auth.util/sign-jwt {:id 1 :type "admin"}))}}]
     (-> request
         app

From aca000a570c1d5e5b9cb541c36ab3d4e44f2072b Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 25 Aug 2025 11:56:38 +0200
Subject: [PATCH 065/391] updated feeds job id

---
 src/source/routes/feeds.clj | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/source/routes/feeds.clj b/src/source/routes/feeds.clj
index 4878bf34..33c61516 100644
--- a/src/source/routes/feeds.clj
+++ b/src/source/routes/feeds.clj
@@ -80,14 +80,16 @@
                    (services/insert-incoming-post! ds {:data (merge extracted
                                                                     {:feed-id (:id new-feed)
                                                                      :creator-id (:id user)
-                                                                     :content-type-id content-type-id})}))]
+                                                                     :content-type-id content-type-id})}))
+        {:keys [email]} (services/user ds {:id (:id user)})]
 
     (if (some? extracted)
       (do
         (->> (jobs/prepare-congest-metadata
               ds
               store
-              {:initial-delay (* 1000 60 60 24)
+              {:id (str email "-" (:id new-feed) "-" (:id new-post))
+               :initial-delay (* 1000 60 60 24)
                :auto-start true
                :stop-after-fail false,
                :interval (* 1000 60 60 24)

From a355fd28ca9f94e04ec902773b6330d638e25070 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 25 Aug 2025 12:33:15 +0200
Subject: [PATCH 066/391] fixed get job route

---
 src/source/jobs/core.clj     | 2 +-
 src/source/routes/job.clj    | 8 +++++---
 src/source/routes/reitit.clj | 3 ++-
 3 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/src/source/jobs/core.clj b/src/source/jobs/core.clj
index c20fcfc2..7d0a6edb 100644
--- a/src/source/jobs/core.clj
+++ b/src/source/jobs/core.clj
@@ -63,7 +63,7 @@
                 :sleep false})
 
   (services/jobs ds)
-  (services/job-metadata ds {:id 4})
+  (services/job-metadata ds {:id 5})
 
   (services/delete-job! ds {})
   (services/delete-job-metadata! ds {})
diff --git a/src/source/routes/job.clj b/src/source/routes/job.clj
index 1ccf2ddf..ae8bec01 100644
--- a/src/source/routes/job.clj
+++ b/src/source/routes/job.clj
@@ -5,13 +5,15 @@
 (defn get [{:keys [ds path-params] :as _req}]
   (let [job (services/job ds path-params)
         metadata (services/job-metadata ds {:id (:job-metadata-id job)})]
-    (res/response (merge (dissoc job :job-metadata-id)
-                         {:metadata metadata}))))
+    (if (some? job)
+      (res/response (merge (dissoc job :job-metadata-id)
+                           {:metadata metadata}))
+      (res/response {}))))
 
 (comment
   (require '[source.db.util :as db.util])
   (def ds (db.util/conn))
 
-  (get {:ds ds :path-params {:id 1}})
+  (get {:ds ds :path-params {:id 5}})
 
   ())
diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index c274940a..0baf1d4e 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -145,7 +145,8 @@
          [""                      {:get jobs/get}]
          ["/manage"
           ["/register"            {:post jobs/post}]]
-         ["/:id"                  {:get job/get}
+         ["/:id"
+          [""                     {:get job/get}]
           ["/manage"
            ["/deregister"         {:get job-deregister/get}]
            ["/start"              {:get job-start/get}]

From bba0898b46e87ae8dd52fd152fcd54e159b141ee Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 25 Aug 2025 15:33:44 +0200
Subject: [PATCH 067/391] fixed job routes

---
 src/source/routes/job_deregister.clj |  10 +-
 src/source/routes/job_start.clj      |   8 +-
 src/source/routes/job_stop.clj       |  10 +-
 src/source/routes/jobs.clj           |   6 +-
 src/source/routes/reitit.clj         | 317 +++++++++++++--------------
 5 files changed, 179 insertions(+), 172 deletions(-)

diff --git a/src/source/routes/job_deregister.clj b/src/source/routes/job_deregister.clj
index ce3f08cb..8d39ce28 100644
--- a/src/source/routes/job_deregister.clj
+++ b/src/source/routes/job_deregister.clj
@@ -1,7 +1,9 @@
 (ns source.routes.job-deregister
   (:require [congest.jobs :as jobs]
-            [ring.util.response :as res]))
+            [ring.util.response :as res]
+            [source.services.interface :as services]))
 
-(defn get [{:keys [js path-params] :as _req}]
-  (jobs/deregister! js (:id path-params))
-  (res/response {:message "successfully deregistered job"}))
+(defn get [{:keys [js ds path-params] :as _req}]
+  (let [job (services/job ds path-params)]
+    (jobs/deregister! js (:job-id job))
+    (res/response {:message "successfully deregistered job"})))
diff --git a/src/source/routes/job_start.clj b/src/source/routes/job_start.clj
index f3852028..7a3873fe 100644
--- a/src/source/routes/job_start.clj
+++ b/src/source/routes/job_start.clj
@@ -1,7 +1,9 @@
 (ns source.routes.job-start
   (:require [source.jobs.core :as jobs]
-            [ring.util.response :as res]))
+            [ring.util.response :as res]
+            [source.services.interface :as services]))
 
 (defn get [{:keys [js ds store path-params]}]
-  (jobs/start! js ds store (:id path-params))
-  (res/response {:message "successfully started job"}))
+  (let [job (services/job ds path-params)]
+    (jobs/start! js ds store (:job-id job))
+    (res/response {:message "successfully started job"})))
diff --git a/src/source/routes/job_stop.clj b/src/source/routes/job_stop.clj
index 1805f6f0..2a3624b1 100644
--- a/src/source/routes/job_stop.clj
+++ b/src/source/routes/job_stop.clj
@@ -1,7 +1,9 @@
 (ns source.routes.job-stop
   (:require [congest.jobs :as jobs]
-            [ring.util.response :as res]))
+            [ring.util.response :as res]
+            [source.services.interface :as services]))
 
-(defn get [{:keys [js path-params]}]
-  (jobs/stop! js (:id path-params) false)
-  (res/response {:message "successfully stopped job"}))
+(defn get [{:keys [js ds path-params]}]
+  (let [job (services/job ds path-params)]
+    (jobs/stop! js (:job-id job) false)
+    (res/response {:message "successfully stopped job"})))
diff --git a/src/source/routes/jobs.clj b/src/source/routes/jobs.clj
index b3943b1f..14d19f81 100644
--- a/src/source/routes/jobs.clj
+++ b/src/source/routes/jobs.clj
@@ -5,11 +5,13 @@
             [ring.util.response :as res]))
 
 (defn get [{:keys [ds] :as _req}]
-  (res/response
+  (->>
+   (services/jobs ds)
    (mapv (fn [job]
            (let [metadata (services/job-metadata ds {:id (:job-metadata-id job)})]
              (merge (dissoc job :job-metadata-id)
-                    {:metadata metadata}))) (services/jobs ds))))
+                    {:metadata metadata}))))
+   (res/response)))
 
 (defn post [{:keys [js ds store body] :as _req}]
   (let [{:keys [metadata]} body]
diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index 0baf1d4e..67d2cac4 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -7,8 +7,6 @@
             [reitit.ring.malli]
             [malli.util :as mu]
             [source.middleware.interface :as mw]
-            [source.db.interface :as db]
-            [congest.jobs :as congest]
             [clojure.data.json :as json]
             [source.routes.user :as user]
             [source.routes.users :as users]
@@ -40,9 +38,7 @@
             [source.routes.job-deregister :as job-deregister]
             [source.routes.job-start :as job-start]
             [source.routes.job-stop :as job-stop]
-            [source.util :as util]
-            [source.datastore.interface :as store]
-            [source.jobs.core :as job-service]))
+            [source.util :as util]))
 
 (defn route [handlers]
   (reduce (fn [acc [k v]]
@@ -53,141 +49,144 @@
                              :handler v}})))
           {} handlers))
 
-(defn create-app []
-  (let [ds (db/ds :master)
-        store (store/ds :datahike)
-        js (congest/create-job-service [])]
-    (job-service/start-interrupted-jobs! js ds store)
-    (ring/ring-handler
-     (ring/router
-      [["/swagger.json"   {:get {:no-doc true
-                                 :swagger {:info {:title "source-api"
-                                                  :description "swagger docs for source api with malli and reitit-ring"
-                                                  :version "0.0.1"}
-                                           :securityDefinitions {"auth" {:type :apiKey
-                                                                         :in :header
-                                                                         :name "Authorization"}}}
-                                 :handler (swagger/create-swagger-handler)}}]
-
-       ["/openapi.json"   {:get {:no-doc true
-                                 :openapi {:info {:title "source-api"
-                                                  :description "openapi3 docs for source api with malli and reitit-ring"
-                                                  :version "0.0.1"}
-                                           :components {:securitySchemes {"bearerAuth" {:type :http
-                                                                                        :scheme :bearer
-                                                                                        :bearerFormat "JWT"
-                                                                                        :description "JWT Authorization using the Bearer scheme"}}}}
-                                 :handler (openapi/create-openapi-handler)}}]
-
-       ["/users"          {:middleware [[mw/apply-auth {:required-type :admin}]]
-                           :tags #{"users"}
-                           :swagger {:security [{"auth" []}]}
-                           :openapi {:security [{:bearerAuth []}]}}
-        [""               (route {:get users/get})]
-        ["/:id"           (route {:get user/get
-                                  :patch user/patch})]]
-
-       ["/me"             {:middleware [[mw/apply-auth]]
-                           :tags #{"me"}
-                           :swagger {:security [{"auth" []}]}
-                           :openapi {:security [{:bearerAuth []}]}}
-        [""               (route {:get me/get})]]
-
-       ["/businesses"     {:middleware [[mw/apply-auth {:required-type :admin}]]
-                           :tags #{"businesses"}
-                           :swagger {:security [{"auth" []}]}
-                           :openapi {:security [{:bearerAuth []}]}}
-        [""               (route {:get businesses/get
-                                  :post business/post})]
-        ["/:id"           (route {:patch business/patch})]]
-
-       ["/sectors"        {:tags #{"sectors"}}
-        [""               (route {:get sectors/get})]]
-
-       ["/login"          {:tags #{"auth"}}
-        [""               (route {:post login/post})]]
-
-       ["/register"       {:tags #{"auth"}}
-        [""               (route {:post register/post})]]
-
-       ["/oauth2"
-        ["/google"        {:tags #{"google"}}
-         [""              (route {:get google-launch/get})]
-         ["/callback"     {:no-doc true
-                           :get google-redirect/get}]
-         ["/user"         (route {:get google-user/get})]]]
-
-       ["/protected"      {:middleware [[mw/apply-auth]]
-                           :tags #{"protected"}
-                           :swagger {:security [{"auth" []}]}
-                           :openapi {:security [{:bearerAuth []}]}}
-        ["/authorized"    (route {:get authorized/get})]]
-
-       ["/providers"      {:tags #{"providers"}}
-        [""               (route {:get providers/get})]
-        ["/:id"           (route {:get provider/get})]]
-
-       ["/content-types"  {:tags #{"content types"}}
-        [""               {:get content-types/get}]
-        ["/:id"           {:get content-type/get}]]
-
-       ["/feeds"          {:middleware [[mw/apply-auth]]
-                           :tags #{"feeds"}}
-        [""               (route {:get feeds/get
-                                  :post feeds/post})]]
-
-       ["/admin"                  {:middleware [[mw/apply-auth {:required-type :admin}]]
-                                   :no-doc true
-                                   :tags #{"admin"}
-                                   :swagger {:security [{"auth" []}]}
-                                   :openapi {:security [{:bearerAuth []}]}}
-        ["/jobs"
-         [""                      {:get jobs/get}]
-         ["/manage"
-          ["/register"            {:post jobs/post}]]
-         ["/:id"
-          [""                     {:get job/get}]
-          ["/manage"
-           ["/deregister"         {:get job-deregister/get}]
-           ["/start"              {:get job-start/get}]
-           ["/stop"               {:get job-stop/get}]]]]
-        ["/add-admin"             (route {:post admin/post})]
-        ["/selection-schemas"
-         [""                      {:get selection-schemas/get
-                                   :post selection-schemas/post}]
-         ["/:id"                  {:get selection-schema/get}]
-         ["/providers/:id"        {:get provider-selection-schemas/get}]]
-        ["/output-schemas"
-         [""                      {:get output-schemas/get
-                                   :post output-schemas/post}]
-         ["/:id"                  {:get output-schema/get}]]
-        ["/providers"
-         [""                      (route {:post providers/post})]
-         ["/:id"                  (route {:delete provider/delete})]]
-        ["/ast"                   {:post xml/post}]
-        ["/extract-data"          {:post data/post}]]]
-
-      {:data {:coercion (reitit.coercion.malli/create
-                         {:error-keys #{#_:type :coercion :in :schema :value :errors :humanized #_:transformed}
-                          :compile mu/closed-schema
-                          :strip-extra-keys true
-                          :default-values true
-                          :options nil})
-              :middleware [[mw/apply-generic :ds ds :store store :js js]]}})
-     (ring/routes
-      (swagger-ui/create-swagger-ui-handler {:path "/"
-                                             :config {:validatorUrl nil
-                                                      :urls [{:name "swagger", :url "swagger.json"}
-                                                             {:name "openapi", :url "openapi.json"}]
-                                                      :urls.primaryName "swagger"
-                                                      :operationsSorter "alpha"}})
-      (ring/create-default-handler)))))
+(defn create-app [{:keys [ds store js]}]
+  (ring/ring-handler
+   (ring/router
+    [["/swagger.json"   {:get {:no-doc true
+                               :swagger {:info {:title "source-api"
+                                                :description "swagger docs for source api with malli and reitit-ring"
+                                                :version "0.0.1"}
+                                         :securityDefinitions {"auth" {:type :apiKey
+                                                                       :in :header
+                                                                       :name "Authorization"}}}
+                               :handler (swagger/create-swagger-handler)}}]
+
+     ["/openapi.json"   {:get {:no-doc true
+                               :openapi {:info {:title "source-api"
+                                                :description "openapi3 docs for source api with malli and reitit-ring"
+                                                :version "0.0.1"}
+                                         :components {:securitySchemes {"bearerAuth" {:type :http
+                                                                                      :scheme :bearer
+                                                                                      :bearerFormat "JWT"
+                                                                                      :description "JWT Authorization using the Bearer scheme"}}}}
+                               :handler (openapi/create-openapi-handler)}}]
+
+     ["/users"          {:middleware [[mw/apply-auth {:required-type :admin}]]
+                         :tags #{"users"}
+                         :swagger {:security [{"auth" []}]}
+                         :openapi {:security [{:bearerAuth []}]}}
+      [""               (route {:get users/get})]
+      ["/:id"           (route {:get user/get
+                                :patch user/patch})]]
+
+     ["/me"             {:middleware [[mw/apply-auth]]
+                         :tags #{"me"}
+                         :swagger {:security [{"auth" []}]}
+                         :openapi {:security [{:bearerAuth []}]}}
+      [""               (route {:get me/get})]]
+
+     ["/businesses"     {:middleware [[mw/apply-auth {:required-type :admin}]]
+                         :tags #{"businesses"}
+                         :swagger {:security [{"auth" []}]}
+                         :openapi {:security [{:bearerAuth []}]}}
+      [""               (route {:get businesses/get
+                                :post business/post})]
+      ["/:id"           (route {:patch business/patch})]]
+
+     ["/sectors"        {:tags #{"sectors"}}
+      [""               (route {:get sectors/get})]]
+
+     ["/login"          {:tags #{"auth"}}
+      [""               (route {:post login/post})]]
+
+     ["/register"       {:tags #{"auth"}}
+      [""               (route {:post register/post})]]
+
+     ["/oauth2"
+      ["/google"        {:tags #{"google"}}
+       [""              (route {:get google-launch/get})]
+       ["/callback"     {:no-doc true
+                         :get google-redirect/get}]
+       ["/user"         (route {:get google-user/get})]]]
+
+     ["/protected"      {:middleware [[mw/apply-auth]]
+                         :tags #{"protected"}
+                         :swagger {:security [{"auth" []}]}
+                         :openapi {:security [{:bearerAuth []}]}}
+      ["/authorized"    (route {:get authorized/get})]]
+
+     ["/providers"      {:tags #{"providers"}}
+      [""               (route {:get providers/get})]
+      ["/:id"           (route {:get provider/get})]]
+
+     ["/content-types"  {:tags #{"content types"}}
+      [""               {:get content-types/get}]
+      ["/:id"           {:get content-type/get}]]
+
+     ["/feeds"          {:middleware [[mw/apply-auth]]
+                         :tags #{"feeds"}}
+      [""               (route {:get feeds/get
+                                :post feeds/post})]]
+
+     ["/admin"                  {:middleware [[mw/apply-auth {:required-type :admin}]]
+                                 :no-doc true
+                                 :tags #{"admin"}
+                                 :swagger {:security [{"auth" []}]}
+                                 :openapi {:security [{:bearerAuth []}]}}
+      ["/jobs"
+       [""                      {:get jobs/get}]
+       ["/manage"
+        ["/register"            {:post jobs/post}]]
+       ["/:id"
+        [""                     {:get job/get}]
+        ["/manage"
+         ["/deregister"         {:get job-deregister/get}]
+         ["/start"              {:get job-start/get}]
+         ["/stop"               {:get job-stop/get}]]]]
+      ["/add-admin"             (route {:post admin/post})]
+      ["/selection-schemas"
+       [""                      {:get selection-schemas/get
+                                 :post selection-schemas/post}]
+       ["/:id"                  {:get selection-schema/get}]
+       ["/providers/:id"        {:get provider-selection-schemas/get}]]
+      ["/output-schemas"
+       [""                      {:get output-schemas/get
+                                 :post output-schemas/post}]
+       ["/:id"                  {:get output-schema/get}]]
+      ["/providers"
+       [""                      (route {:post providers/post})]
+       ["/:id"                  (route {:delete provider/delete})]]
+      ["/ast"                   {:post xml/post}]
+      ["/extract-data"          {:post data/post}]]]
+
+    {:data {:coercion (reitit.coercion.malli/create
+                       {:error-keys #{#_:type :coercion :in :schema :value :errors :humanized #_:transformed}
+                        :compile mu/closed-schema
+                        :strip-extra-keys true
+                        :default-values true
+                        :options nil})
+            :middleware [[mw/apply-generic :ds ds :store store :js js]]}})
+   (ring/routes
+    (swagger-ui/create-swagger-ui-handler {:path "/"
+                                           :config {:validatorUrl nil
+                                                    :urls [{:name "swagger", :url "swagger.json"}
+                                                           {:name "openapi", :url "openapi.json"}]
+                                                    :urls.primaryName "swagger"
+                                                    :operationsSorter "alpha"}})
+    (ring/create-default-handler))))
 
 (comment
   (require '[source.middleware.auth.util :as auth.util]
+           '[source.db.util :as db.util]
+           '[congest.jobs :as js]
+           '[source.datastore.interface :as store]
            '[source.rss.youtube :as yt])
 
-  (let [app (create-app)
+  (def components {:ds (db.util/conn)
+                   :store (store/ds :datahike)
+                   :js (js/create-job-service [])})
+
+  (let [app (create-app components)
         request {:uri "/users"
                  :request-method :get
                  :headers {"authorization" (str "Bearer " (auth.util/sign-jwt {:id 1 :type "admin"}))}}]
@@ -196,7 +195,7 @@
         :body
         (json/read-json {:key-fn keyword})))
 
-  (let [app (create-app)
+  (let [app (create-app components)
         request {:uri "/users/3"
                  :request-method :get
                  :headers {"authorization" (str "Bearer " (auth.util/sign-jwt {:id 1 :type "admin"}))}}]
@@ -205,7 +204,7 @@
         :body
         (json/read-json {:key-fn keyword})))
 
-  (let [app (create-app)
+  (let [app (create-app components)
         request {:uri "/users/3"
                  :request-method :patch
                  :body {:firstname "Keagan"
@@ -215,7 +214,7 @@
         :body
         (json/read-json {:key-fn keyword})))
 
-  (let [app (create-app)
+  (let [app (create-app components)
         request {:uri "/protected/authorized"
                  :request-method :get
                  :headers {"authorization" (str "Bearer " (auth.util/sign-jwt {:id 5 :type "distributor"}))}}]
@@ -224,7 +223,7 @@
         :body
         (json/read-json {:key-fn keyword})))
 
-  (let [app (create-app)
+  (let [app (create-app components)
         request {:uri "/admin/add-admin"
                  :request-method :post
                  :body {:email "test@test.com"
@@ -236,7 +235,7 @@
         :body
         (json/read-json {:key-fn keyword})))
 
-  (let [app (create-app)
+  (let [app (create-app components)
         request {:uri "/oauth2/google"
                  :request-method :get}]
     (-> request
@@ -244,7 +243,7 @@
         :body
         (json/read-json {:key-fn keyword})))
 
-  (let [app (create-app)
+  (let [app (create-app components)
         request {:uri "/businesses"
                  :request-method :get}]
     (-> request
@@ -252,7 +251,7 @@
         :body
         (json/read-json {:key-fn keyword})))
 
-  (let [app (create-app)
+  (let [app (create-app components)
         request {:uri "/businesses"
                  :request-method :post
                  :body {:name "beep"
@@ -262,7 +261,7 @@
         :body
         (json/read-json {:key-fn keyword})))
 
-  (let [app (create-app)
+  (let [app (create-app components)
         request {:uri "/businesses/1"
                  :request-method :patch
                  :body {:name "thebest"
@@ -272,7 +271,7 @@
         :body
         (json/read-json {:key-fn keyword})))
 
-  (let [app (create-app)
+  (let [app (create-app components)
         request {:uri "/sectors"
                  :request-method :get}]
     (-> request
@@ -280,7 +279,7 @@
         :body
         (json/read-json {:key-fn keyword})))
 
-  (let [app (create-app)
+  (let [app (create-app components)
         request {:uri "/providers"
                  :request-method :get}]
     (-> request
@@ -288,7 +287,7 @@
         :body
         (json/read-json {:key-fn keyword})))
 
-  (let [app (create-app)
+  (let [app (create-app components)
         request {:uri "/providers/1"
                  :request-method :get}]
     (-> request
@@ -296,7 +295,7 @@
         :body
         (json/read-json {:key-fn keyword})))
 
-  (let [app (create-app)
+  (let [app (create-app components)
         request {:uri "/content-types"
                  :request-method :get}]
     (-> request
@@ -304,7 +303,7 @@
         :body
         (json/read-json {:key-fn keyword})))
 
-  (let [app (create-app)
+  (let [app (create-app components)
         request {:uri "/content-types/1"
                  :request-method :get}]
     (-> request
@@ -317,7 +316,7 @@
          (yt/find-channel-id)
          (str "https://www.youtube.com/feeds/videos.xml?channel_id=")))
 
-  (let [app (create-app)
+  (let [app (create-app components)
         request {:uri "/admin/feeds"
                  :request-method :get
                  :headers {"authorization" (str "Bearer " (auth.util/sign-jwt {:id 5 :type "creator"}))}}]
@@ -326,7 +325,7 @@
         :body
         (json/read-json {:key-fn keyword})))
 
-  (let [app (create-app)
+  (let [app (create-app components)
         request {:uri "/admin/feeds"
                  :request-method :post
                  :body {:title "primeagen test"
@@ -341,7 +340,7 @@
         :body
         (json/read-json {:key-fn keyword})))
 
-  (let [app (create-app)
+  (let [app (create-app components)
         store (store/ds :datahike)
         request {:uri "/admin/selection-schemas"
                  :request-method :post
@@ -355,7 +354,7 @@
         :body
         (json/read-json {:key-fn keyword})))
 
-  (let [app (create-app)
+  (let [app (create-app components)
         request {:uri "/admin/selection-schemas/1"
                  :request-method :get
                  :headers {"authorization" (str "Bearer " (auth.util/sign-jwt {:id 1 :type "admin"}))}}]
@@ -364,7 +363,7 @@
         :body
         (json/read-json {:key-fn keyword})))
 
-  (let [app (create-app)
+  (let [app (create-app components)
         store (store/ds :datahike)
         request {:uri "/admin/selection-schemas"
                  :store store
@@ -380,7 +379,7 @@
          (yt/find-channel-id)
          (str "https://www.youtube.com/feeds/videos.xml?channel_id=")))
 
-  (let [app (create-app)
+  (let [app (create-app components)
         request {:uri "/admin/ast"
                  :request-method :post
                  :body {:url (get-url)}
@@ -390,7 +389,7 @@
         :body
         (json/read-json {:key-fn keyword})))
 
-  (let [app (create-app)
+  (let [app (create-app components)
         store (store/ds :datahike)
         request {:uri "/admin/extract-data"
                  :store store
@@ -406,7 +405,7 @@
         :body
         (json/read-json {:key-fn keyword})))
 
-  (let [app (create-app)
+  (let [app (create-app components)
         request {:uri "/admin/jobs"
                  :request-method :get
                  :headers {"authorization" (str "Bearer " (auth.util/sign-jwt {:id 1 :type "admin"}))}}]
@@ -415,7 +414,7 @@
         :body
         (json/read-json {:key-fn keyword})))
 
-  (let [app (create-app)
+  (let [app (create-app components)
         request {:uri "/admin/jobs/1"
                  :request-method :get
                  :headers {"authorization" (str "Bearer " (auth.util/sign-jwt {:id 1 :type "admin"}))}}]
@@ -424,7 +423,7 @@
         :body
         (json/read-json {:key-fn keyword})))
 
-  (let [app (create-app)
+  (let [app (create-app components)
         request {:uri "/admin/jobs/manage/register"
                  :request-method :post
                  :body {:metadata {:initial-delay 10
@@ -442,7 +441,7 @@
         :body
         (json/read-json {:key-fn keyword})))
 
-  (let [app (create-app)
+  (let [app (create-app components)
         request {:uri "/admin/jobs/1/manage/deregister"
                  :request-method :get
                  :headers {"authorization" (str "Bearer " (auth.util/sign-jwt {:id 1 :type "admin"}))}}]

From a97aadda5372101375502b0f64ad8434ed346ca4 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 25 Aug 2025 15:39:28 +0200
Subject: [PATCH 068/391] updated components to be initialised on server
 startup and refactored job recovery

---
 src/source/jobs/core.clj        | 18 +++++++++-------
 src/source/routes/interface.clj |  4 ++--
 src/source/server.clj           | 37 +++++++++++++++++++++++++++------
 3 files changed, 44 insertions(+), 15 deletions(-)

diff --git a/src/source/jobs/core.clj b/src/source/jobs/core.clj
index 7d0a6edb..0b25c1d4 100644
--- a/src/source/jobs/core.clj
+++ b/src/source/jobs/core.clj
@@ -36,13 +36,17 @@
     (congest/deregister! js job-id)
     (congest/register! js (prepare-congest-metadata ds store metadata))))
 
-(defn start-interrupted-jobs!
-  "Starts all jobs with a status of 'running'. Intended for server startup to restart interrupted jobs."
-  [js ds store]
+(defn interrupted-jobs
+  "Get congest-ready metadata of all jobs marked as running"
+  [ds store]
   (let [jobs (services/jobs ds)]
-    (run! (fn [{:keys [job-id status]}]
-            (when (= status "running")
-              (start! js ds store job-id))) jobs)))
+    (reduce (fn [acc {:keys [job-id job-metadata-id args handler status]}]
+              (when (= status "running")
+                (let [metadata (-> (services/job-metadata ds {:id job-metadata-id})
+                                   (assoc :id job-id
+                                          :args args
+                                          :handler handler))]
+                  (conj acc (prepare-congest-metadata ds store metadata))))) [] jobs)))
 
 (comment
   (require '[source.db.util :as db.util]
@@ -73,6 +77,6 @@
   (congest/deregister! js "test")
   (congest/stop! js "test" false)
   (start! js ds store "test")
-  (start-interrupted-jobs! js ds store)
+  (interrupted-jobs ds store)
 
   ())
diff --git a/src/source/routes/interface.clj b/src/source/routes/interface.clj
index 605252ee..baec7cf9 100644
--- a/src/source/routes/interface.clj
+++ b/src/source/routes/interface.clj
@@ -1,6 +1,6 @@
 (ns source.routes.interface
   (:require [source.routes.reitit :as reitit]))
 
-(defn create-app []
-  (reitit/create-app))
+(defn create-app [{:keys [ds store js] :as opts}]
+  (reitit/create-app opts))
 
diff --git a/src/source/server.clj b/src/source/server.clj
index 2eef3cb9..5161a0e2 100644
--- a/src/source/server.clj
+++ b/src/source/server.clj
@@ -1,11 +1,34 @@
 (ns source.server
-  (:require
-   [org.httpkit.server :as http]
-   [source.routes.interface :as routes]
-   [source.util :as util]))
+  (:require [org.httpkit.server :as http]
+            [source.db.interface :as db]
+            [source.datastore.interface :as store]
+            [congest.jobs :as congest]
+            [source.jobs.core :as jobs]
+            [source.routes.interface :as routes]
+            [source.util :as util]))
 
 (defonce ^:private *server (atom nil))
 
+(defonce ^:private *components (atom nil))
+
+(defn initialise-components! []
+  (if (some? @*components)
+    (do
+      (println "Components already initialised.")
+      (let [{:keys [ds store]} @*components]
+        (->> (jobs/interrupted-jobs ds store)
+             (congest/create-job-service)
+             (swap! *components assoc :js))))
+
+    (try
+      (println "Initialising components...")
+      (let [{:keys [ds store]} (reset! *components {:ds (db/ds :master)
+                                                    :store (store/ds :datahike)})]
+        (->> (jobs/interrupted-jobs ds store)
+             (congest/create-job-service)
+             (swap! *components assoc :js)))
+      (catch Exception e (println "Failed to initialise components: " (.getMessage e))))))
+
 (defn running? []
   (some? @*server))
 
@@ -13,14 +36,17 @@
   (cond (not (some? @*server))
         (do
           (println "Starting server on port 3000...")
+          (initialise-components!)
           (reset! *server (http/run-server
-                           (routes/create-app)
+                           (routes/create-app @*components)
                            {:port 3000})))
         :else
         (println "Server already running!")))
 
 (defn stop-server []
   (println "Stopping server...")
+  (when (some? @*components)
+    (congest/kill! (:js @*components)))
   (when (some? @*server)
     (@*server))
   (reset! *server nil))
@@ -35,4 +61,3 @@
   (test-wrapper {:status 200
                  :body "{\"value\":\"Hello, Source!\"}"
                  :headers {"Content-Type" "application/json"}}))
-

From 85ab28b4c4141cd64b0b73f755bc87f0d1fea088 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Tue, 26 Aug 2025 14:50:09 +0200
Subject: [PATCH 069/391] updated server to use a component system

---
 src/source/server.clj | 106 +++++++++++++++++++++++++++++-------------
 1 file changed, 73 insertions(+), 33 deletions(-)

diff --git a/src/source/server.clj b/src/source/server.clj
index 5161a0e2..8a874138 100644
--- a/src/source/server.clj
+++ b/src/source/server.clj
@@ -7,57 +7,97 @@
             [source.routes.interface :as routes]
             [source.util :as util]))
 
-(defonce ^:private *server (atom nil))
-
 (defonce ^:private *components (atom nil))
 
-(defn initialise-components! []
-  (if (some? @*components)
-    (do
-      (println "Components already initialised.")
-      (let [{:keys [ds store]} @*components]
-        (->> (jobs/interrupted-jobs ds store)
-             (congest/create-job-service)
-             (swap! *components assoc :js))))
-
-    (try
-      (println "Initialising components...")
-      (let [{:keys [ds store]} (reset! *components {:ds (db/ds :master)
-                                                    :store (store/ds :datahike)})]
-        (->> (jobs/interrupted-jobs ds store)
-             (congest/create-job-service)
-             (swap! *components assoc :js)))
-      (catch Exception e (println "Failed to initialise components: " (.getMessage e))))))
+(defn initialise-server! [{:keys [ds store js]}]
+  (http/run-server
+   (routes/create-app {:ds ds
+                       :store store
+                       :js js})
+   {:port 3000}))
+
+(defn initialise-job-service! [{:keys [ds store] :as _deps}]
+  (->> (jobs/interrupted-jobs ds store)
+       (congest/create-job-service)))
+
+(defn component-on? [component]
+  (if (some? (get @*components component))
+    true
+    false))
+
+(defn deps-on? [deps]
+  (every? component-on? deps))
+
+(defn initialise!
+  "executes the init-fn on the provided component and, if successful, updates the components atom with the new component"
+  [{:keys [name init-fn deps] :as _component}]
+  (try
+    (when (deps-on? deps)
+      (swap! *components assoc name (init-fn @*components)))
+    (catch Exception e (println (str "Failed to initialise " name ":") e))))
+
+(defn initialise-components! [components]
+  (run! initialise! components))
 
 (defn running? []
-  (some? @*server))
+  (some? (:server @*components)))
 
 (defn start-server []
-  (cond (not (some? @*server))
+  (cond (not (some? (:server @*components)))
         (do
           (println "Starting server on port 3000...")
-          (initialise-components!)
-          (reset! *server (http/run-server
-                           (routes/create-app @*components)
-                           {:port 3000})))
+          (initialise-components! [{:name :ds
+                                    :init-fn (fn [_deps] (db/ds :master))
+                                    :deps []}
+                                   {:name :store
+                                    :init-fn (fn [_deps] (store/ds :datahike))
+                                    :deps []}
+                                   {:name :js
+                                    :deps [:ds :store]
+                                    :init-fn initialise-job-service!}
+                                   {:name :server
+                                    :deps [:ds :store :js]
+                                    :init-fn initialise-server!}]))
         :else
         (println "Server already running!")))
 
 (defn stop-server []
   (println "Stopping server...")
-  (when (some? @*components)
+  (when (some? (:js @*components))
     (congest/kill! (:js @*components)))
-  (when (some? @*server)
-    (@*server))
-  (reset! *server nil))
+  (when (some? (:server @*components))
+    (let [server (:server @*components)]
+      (server)))
+  (reset! *components nil))
 
-(defn restart-server []
-  (stop-server)
-  (start-server))
+(defn restart-server [& {:keys [keep-js]}]
+  (if keep-js
+    (do
+      (when (some? (:server @*components))
+        (let [server (:server @*components)]
+          (server)))
+      (swap! *components select-keys [:js])
+      (initialise-components! [{:name :ds
+                                :init-fn (fn [_deps] (db/ds :master))
+                                :deps []}
+                               {:name :store
+                                :init-fn (fn [_deps] (store/ds :datahike))
+                                :deps []}
+                               {:name :server
+                                :deps [:ds :store :js]
+                                :init-fn initialise-server!}]))
+    (do
+      (stop-server)
+      (start-server))))
 
 (comment
+  (start-server)
+  (stop-server)
+  (restart-server)
+  (restart-server :keep-js true)
   (def test-wrapper
     (util/wrap-json (fn [request] request)))
   (test-wrapper {:status 200
                  :body "{\"value\":\"Hello, Source!\"}"
-                 :headers {"Content-Type" "application/json"}}))
+                 :headers {"Content-Type" "application/json"}})
+  ())

From 1ac2143d2711cbc5238950048625b47f04468006 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Tue, 26 Aug 2025 15:53:46 +0200
Subject: [PATCH 070/391] addressed comment

---
 src/source/server.clj | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/source/server.clj b/src/source/server.clj
index 8a874138..0a84b7dd 100644
--- a/src/source/server.clj
+++ b/src/source/server.clj
@@ -66,16 +66,16 @@
   (when (some? (:js @*components))
     (congest/kill! (:js @*components)))
   (when (some? (:server @*components))
-    (let [server (:server @*components)]
-      (server)))
+    (let [server-stop (:server @*components)]
+      (server-stop)))
   (reset! *components nil))
 
 (defn restart-server [& {:keys [keep-js]}]
   (if keep-js
     (do
       (when (some? (:server @*components))
-        (let [server (:server @*components)]
-          (server)))
+        (let [server-stop (:server @*components)]
+          (server-stop)))
       (swap! *components select-keys [:js])
       (initialise-components! [{:name :ds
                                 :init-fn (fn [_deps] (db/ds :master))

From db8af5b4b311bd37f1c0cdc76d712f52654dcdb9 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Thu, 28 Aug 2025 13:59:39 +0200
Subject: [PATCH 071/391] updated add feed route to pull all posts and
 update/insert posts every 24 hours

---
 src/source/db/master.clj               |  1 +
 src/source/jobs/handlers.clj           | 34 +++++++++---
 src/source/routes/feeds.clj            | 76 +++++++++++++-------------
 src/source/services/incoming_posts.clj |  4 +-
 src/source/services/interface.clj      |  2 +-
 5 files changed, 70 insertions(+), 47 deletions(-)

diff --git a/src/source/db/master.clj b/src/source/db/master.clj
index bec159b3..39cb0ba7 100644
--- a/src/source/db/master.clj
+++ b/src/source/db/master.clj
@@ -141,6 +141,7 @@
   (tables/create-table-sql
    :incoming-posts
    (tables/table-id)
+   [:post-id :text :not nil]
    [:feed-id :integer :not nil]
    [:title :text :not nil]
    [:info :text]
diff --git a/src/source/jobs/handlers.clj b/src/source/jobs/handlers.clj
index 1deb0e09..a56bedcb 100644
--- a/src/source/jobs/handlers.clj
+++ b/src/source/jobs/handlers.clj
@@ -14,15 +14,35 @@
   (fn [{:keys [args]}]
     (println "hello" (get args :name) args)))
 
-(defmethod handler :update-feed-post [_]
+(defmethod handler :update-feed-posts [_]
   (fn [{:keys [args ds store]}]
     (try
-      (let [{:keys [feed-id post-id schema-id url]} args
-            extracted (services/extract-data store {:schema-id schema-id
-                                                    :url url})]
+      (let [{:keys [feed-id creator-id content-type-id provider-id url]} args
+            selection-schemas (->> [:= :provider-id provider-id]
+                                   (assoc {} :where)
+                                   (services/selection-schemas ds))
+            latest-ss (->> selection-schemas
+                           (reduce (fn [acc {:keys [id]}]
+                                     (conj acc id)) [])
+                           (apply max -1))
+            extracted (services/extract-data store {:schema-id latest-ss
+                                                    :url url})
+            extracted-posts (get-in extracted [:feed :posts])
+            extended-posts (mapv (fn [post]
+                                   (merge post
+                                          {:feed-id feed-id
+                                           :creator-id creator-id
+                                           :content-type-id content-type-id})) extracted-posts)
+            existing-posts (services/incoming-posts ds {:where [:= :creator-id creator-id]})]
         (services/update-feed! ds {:id feed-id
-                                   :data {:updated-at (util/get-utc-timestamp-string)}})
-        (services/update-incoming-post! ds {:id post-id
-                                            :data extracted}))
+                                   :data {:title (get-in extracted [:feed :title])
+                                          :updated-at (util/get-utc-timestamp-string)}})
+        (run!
+         (fn [post]
+           (if (some #(= (:post-id post) (:post-id %)) existing-posts)
+             (services/update-incoming-post! ds {:where [:= :post-id (:post-id post)]
+                                                 :data post})
+             (services/insert-incoming-post! ds {:data post})))
+         extended-posts))
       (catch Exception _ :fail))))
 
diff --git a/src/source/routes/feeds.clj b/src/source/routes/feeds.clj
index 33c61516..c929c61f 100644
--- a/src/source/routes/feeds.clj
+++ b/src/source/routes/feeds.clj
@@ -7,33 +7,30 @@
 
 (defn get
   {:summary "get all feeds"
-   :responses  {200 {:body [:map
-                            [:users
-                             [:vector
-                              [:map
-                               [:id :int]
-                               [:title :string]
-                               [:display-picture [:maybe :string]]
-                               [:url [:maybe :string]]
-                               [:rss-url :string]
-                               [:user-id [:maybe :int]]
-                               [:provider-id [:maybe :int]]
-                               [:created-at :string]
-                               [:updated-at [:maybe :string]]
-                               [:content-type-id :int]
-                               [:cadence-id :int]
-                               [:baseline-id :int]
-                               [:ts-and-cs [:maybe :string]]
-                               [:state [:maybe :string]]]]]]}}}
+   :responses  {200 {:body [:vector
+                            [:map
+                             [:id :int]
+                             [:title :string]
+                             [:display-picture [:maybe :string]]
+                             [:url [:maybe :string]]
+                             [:rss-url :string]
+                             [:user-id [:maybe :int]]
+                             [:provider-id [:maybe :int]]
+                             [:created-at :string]
+                             [:updated-at [:maybe :string]]
+                             [:content-type-id :int]
+                             [:cadence-id :int]
+                             [:baseline-id :int]
+                             [:ts-and-cs [:maybe :string]]
+                             [:state [:maybe :string]]]]}}}
 
   [{:keys [ds user] :as _request}]
   (-> (services/feeds ds {:where [:= :user-id (:id user)]})
       (res/response)))
 
 (defn post
-  {:summary "adds a feed and extracts data from RSS feed URL into a post and schedules a job to keep them updated"
+  {:summary "adds a feed and extracts data from RSS feed URL to create incoming posts and schedules a job to keep them updated"
    :parameters {:body [:map
-                       [:title :string]
                        [:display-picture {:optional true} :string]
                        [:url {:optional true} :string]
                        [:rss-url :string]
@@ -72,33 +69,38 @@
         extracted (when-not (= latest-ss -1)
                     (services/extract-data store {:schema-id latest-ss
                                                   :url rss-url}))
+        extracted-posts (get-in extracted [:feed :posts])
         new-feed (services/insert-feed!
                   ds
-                  {:data (merge body {:user-id (:id user)
+                  {:data (merge body {:title (get-in extracted [:feed :title])
+                                      :user-id (:id user)
                                       :created-at datetime})})
-        new-post (when (some? extracted)
-                   (services/insert-incoming-post! ds {:data (merge extracted
-                                                                    {:feed-id (:id new-feed)
-                                                                     :creator-id (:id user)
-                                                                     :content-type-id content-type-id})}))
+        extended-posts (mapv (fn [post]
+                               (merge post
+                                      {:feed-id (:id new-feed)
+                                       :creator-id (:id user)
+                                       :content-type-id content-type-id})) extracted-posts)
         {:keys [email]} (services/user ds {:id (:id user)})]
 
-    (if (some? extracted)
+    (if (some? extracted-posts)
       (do
+        (services/insert-incoming-post! ds {:data extended-posts})
+
         (->> (jobs/prepare-congest-metadata
               ds
               store
-              {:id (str email "-" (:id new-feed) "-" (:id new-post))
-               :initial-delay (* 1000 60 60 24)
+              {:id (str email "-" (:id new-feed))
+               :initial-delay #_(* 1000 60 60 24) (* 1000 60)
                :auto-start true
                :stop-after-fail false,
-               :interval (* 1000 60 60 24)
+               :interval #_(* 1000 60 60 24) (* 1000 60)
                :recurring? true
                :args {:feed-id (:id new-feed)
-                      :post-id (:id new-post)
-                      :schema-id latest-ss
+                      :creator-id (:id user)
+                      :content-type-id content-type-id
+                      :provider-id provider-id
                       :url rss-url}
-               :handler :update-feed-post
+               :handler :update-feed-posts
                :created-at (utils/get-utc-timestamp-string)
                :sleep false})
              (congest/register! js))
@@ -111,15 +113,15 @@
   (require '[source.db.util :as db.util]
            '[source.datastore.util :as store.util])
 
-  (get {:ds (db.util/conn) :user {:id 5}})
+  (get {:ds (db.util/conn) :user {:id 3}})
   (post {:ds (db.util/conn)
          :js (congest/create-job-service [])
          :store (store.util/conn :datahike)
-         :user {:id 5}
-         :body {:title "primeagen test"
-                :rss-url "https://www.youtube.com/feeds/videos.xml?channel_id=UCUyeluBRhGPCW4rPe_UvBZQ"
+         :user {:id 3}
+         :body {:rss-url "https://www.youtube.com/feeds/videos.xml?channel_id=UCUyeluBRhGPCW4rPe_UvBZQ"
                 :provider-id 1
                 :content-type-id 1
                 :cadence-id 1
                 :baseline-id 1}})
+
   ())
diff --git a/src/source/services/incoming_posts.clj b/src/source/services/incoming_posts.clj
index 99f79134..cfa76bc5 100644
--- a/src/source/services/incoming_posts.clj
+++ b/src/source/services/incoming_posts.clj
@@ -1,10 +1,10 @@
 (ns source.services.incoming-posts
   (:require [source.db.interface :as db]))
 
-(defn insert-incoming-post! [ds {:keys [data] :as opts}]
+(defn insert-incoming-post! [ds {:keys [data ret] :as opts}]
   (->> {:tname :incoming-posts
         :data data
-        :ret :1}
+        :ret ret}
        (merge opts)
        (db/insert! ds)))
 
diff --git a/src/source/services/interface.clj b/src/source/services/interface.clj
index b687c7ec..a61b4248 100644
--- a/src/source/services/interface.clj
+++ b/src/source/services/interface.clj
@@ -100,7 +100,7 @@
 (defn feed [ds id]
   (feeds/feed ds id))
 
-(defn insert-incoming-post! [ds {:keys [_data] :as opts}]
+(defn insert-incoming-post! [ds {:keys [_data _ret] :as opts}]
   (incoming-posts/insert-incoming-post! ds opts))
 
 (defn update-incoming-post! [ds {:keys [_id _data _where] :as opts}]

From d7fe77c4cb54c58ecb59c73f9b6065dd7d968974 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Thu, 28 Aug 2025 14:01:44 +0200
Subject: [PATCH 072/391] added routes for getting one or many feeds and posts
 from the creators point of view

---
 src/source/routes/feed.clj   | 29 +++++++++++++++++++++++++++++
 src/source/routes/posts.clj  | 31 +++++++++++++++++++++++++++++++
 src/source/routes/reitit.clj |  9 +++++++--
 3 files changed, 67 insertions(+), 2 deletions(-)
 create mode 100644 src/source/routes/feed.clj
 create mode 100644 src/source/routes/posts.clj

diff --git a/src/source/routes/feed.clj b/src/source/routes/feed.clj
new file mode 100644
index 00000000..87a20824
--- /dev/null
+++ b/src/source/routes/feed.clj
@@ -0,0 +1,29 @@
+(ns source.routes.feed
+  (:require [source.services.interface :as services]
+            [ring.util.response :as res]))
+
+(defn get
+  {:summary "get feed by id"
+   :parameters {:path [:map [:id {:title "id"
+                                  :description "feed id"} :int]]}
+   :responses  {200 {:body [:map
+                            [:id :int]
+                            [:title :string]
+                            [:display-picture [:maybe :string]]
+                            [:url [:maybe :string]]
+                            [:rss-url :string]
+                            [:user-id [:maybe :int]]
+                            [:provider-id [:maybe :int]]
+                            [:created-at :string]
+                            [:updated-at [:maybe :string]]
+                            [:content-type-id :int]
+                            [:cadence-id :int]
+                            [:baseline-id :int]
+                            [:ts-and-cs [:maybe :string]]
+                            [:state [:maybe :string]]]}}}
+
+  [{:keys [ds user path-params] :as _request}]
+  (-> (services/feed ds {:where [:and
+                                 [:= :id (:id path-params)]
+                                 [:= :user-id (:id user)]]})
+      (res/response)))
diff --git a/src/source/routes/posts.clj b/src/source/routes/posts.clj
new file mode 100644
index 00000000..73713f1f
--- /dev/null
+++ b/src/source/routes/posts.clj
@@ -0,0 +1,31 @@
+(ns source.routes.posts
+  (:require [source.services.interface :as services]
+            [ring.util.response :as res]))
+
+(defn get
+  {:summary "get all posts by feed id"
+   :parameters {:path [:map [:id {:title "id"
+                                  :description "feed id"} :int]]}
+   :responses {200 {:body [:vector
+                           [:map
+                            [:id :int]
+                            [:post-id :string]
+                            [:feed-id :int]
+                            [:creator-id :int]
+                            [:content-type-id :int]
+                            [:title :string]
+                            [:info [:maybe :string]]
+                            [:url [:maybe :string]]
+                            [:stream-url [:maybe :string]]
+                            [:season [:maybe :int]]
+                            [:episode [:maybe :int]]
+                            [:redacted [:maybe :int]]
+                            [:posted-at [:maybe :string]]]]}
+               401 {:body [:map [:message :string]]}
+               403 {:body [:map [:message :string]]}}}
+
+  [{:keys [ds user path-params] :as _request}]
+  (-> (services/incoming-posts ds {:where [:and
+                                           [:= :creator-id (:id user)]
+                                           [:= :feed-id (:id path-params)]]})
+      (res/response)))
diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index 67d2cac4..ebebe586 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -31,6 +31,8 @@
             [source.routes.content-types :as content-types]
             [source.routes.content-type :as content-type]
             [source.routes.feeds :as feeds]
+            [source.routes.feed :as feed]
+            [source.routes.posts :as posts]
             [source.routes.xml :as xml]
             [source.routes.data :as data]
             [source.routes.jobs :as jobs]
@@ -126,7 +128,10 @@
      ["/feeds"          {:middleware [[mw/apply-auth]]
                          :tags #{"feeds"}}
       [""               (route {:get feeds/get
-                                :post feeds/post})]]
+                                :post feeds/post})]
+      ["/:id"
+       [""              (route {:get feed/get})]
+       ["/posts"        (route {:get posts/get})]]]
 
      ["/admin"                  {:middleware [[mw/apply-auth {:required-type :admin}]]
                                  :no-doc true
@@ -442,7 +447,7 @@
         (json/read-json {:key-fn keyword})))
 
   (let [app (create-app components)
-        request {:uri "/admin/jobs/1/manage/deregister"
+        request {:uri "/admin/jobs/8/manage/deregister"
                  :request-method :get
                  :headers {"authorization" (str "Bearer " (auth.util/sign-jwt {:id 1 :type "admin"}))}}]
     (-> request

From fb509832f83ff8ee3d1dc16f90c86db4cc4fb524 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Thu, 28 Aug 2025 17:26:39 +0200
Subject: [PATCH 073/391] fixed datastore path function to include forward
 slash

---
 src/source/datastore/config.clj | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/src/source/datastore/config.clj b/src/source/datastore/config.clj
index 41dacc5f..85fff782 100644
--- a/src/source/datastore/config.clj
+++ b/src/source/datastore/config.clj
@@ -10,9 +10,13 @@
       (str path)))
 
 (defn store-path
-  ([store-name]
-   (-> (str (conf/read-value :database :dir) store-name)
-       (absolute))))
+  [store-name]
+  (let [db-dir (conf/read-value :database :dir)]
+    (str
+     db-dir
+     (when (not (= (last db-dir) \/))
+       "/")
+     store-name)))
 
 (defn config [store-name]
   {:store {:backend :file

From 64349e349047ab5b352cf214c215312099339821 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 1 Sep 2025 11:54:48 +0200
Subject: [PATCH 074/391] fixed job interval and initial delay

---
 src/source/routes/feeds.clj | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/source/routes/feeds.clj b/src/source/routes/feeds.clj
index c929c61f..6a49c227 100644
--- a/src/source/routes/feeds.clj
+++ b/src/source/routes/feeds.clj
@@ -90,10 +90,10 @@
               ds
               store
               {:id (str email "-" (:id new-feed))
-               :initial-delay #_(* 1000 60 60 24) (* 1000 60)
+               :initial-delay (* 1000 60 60 24)
                :auto-start true
                :stop-after-fail false,
-               :interval #_(* 1000 60 60 24) (* 1000 60)
+               :interval (* 1000 60 60 24)
                :recurring? true
                :args {:feed-id (:id new-feed)
                       :creator-id (:id user)

From edba174e3e2a61d09c1a9ce550bf416fb56d01ff Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 1 Sep 2025 11:56:48 +0200
Subject: [PATCH 075/391] removed redundant user id checks

---
 src/source/routes/feed.clj  | 6 ++----
 src/source/routes/posts.clj | 6 ++----
 2 files changed, 4 insertions(+), 8 deletions(-)

diff --git a/src/source/routes/feed.clj b/src/source/routes/feed.clj
index 87a20824..3d112e01 100644
--- a/src/source/routes/feed.clj
+++ b/src/source/routes/feed.clj
@@ -22,8 +22,6 @@
                             [:ts-and-cs [:maybe :string]]
                             [:state [:maybe :string]]]}}}
 
-  [{:keys [ds user path-params] :as _request}]
-  (-> (services/feed ds {:where [:and
-                                 [:= :id (:id path-params)]
-                                 [:= :user-id (:id user)]]})
+  [{:keys [ds path-params] :as _request}]
+  (-> (services/feed ds path-params)
       (res/response)))
diff --git a/src/source/routes/posts.clj b/src/source/routes/posts.clj
index 73713f1f..327662e1 100644
--- a/src/source/routes/posts.clj
+++ b/src/source/routes/posts.clj
@@ -24,8 +24,6 @@
                401 {:body [:map [:message :string]]}
                403 {:body [:map [:message :string]]}}}
 
-  [{:keys [ds user path-params] :as _request}]
-  (-> (services/incoming-posts ds {:where [:and
-                                           [:= :creator-id (:id user)]
-                                           [:= :feed-id (:id path-params)]]})
+  [{:keys [ds path-params] :as _request}]
+  (-> (services/incoming-posts ds {:where [:= :feed-id (:id path-params)]})
       (res/response)))

From ec715befe6291548b684e3d793aa08d3c4f66cf8 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 1 Sep 2025 11:58:05 +0200
Subject: [PATCH 076/391] added admin endpoint to view all feeds

---
 src/source/routes/admin_feeds.clj | 26 ++++++++++++++++++++++++++
 src/source/routes/reitit.clj      |  4 +++-
 2 files changed, 29 insertions(+), 1 deletion(-)
 create mode 100644 src/source/routes/admin_feeds.clj

diff --git a/src/source/routes/admin_feeds.clj b/src/source/routes/admin_feeds.clj
new file mode 100644
index 00000000..689e1121
--- /dev/null
+++ b/src/source/routes/admin_feeds.clj
@@ -0,0 +1,26 @@
+(ns source.routes.admin-feeds
+  (:require [source.services.interface :as services]
+            [ring.util.response :as res]))
+
+(defn get
+  {:summary "get all feeds"
+   :responses  {200 {:body [:vector
+                            [:map
+                             [:id :int]
+                             [:title :string]
+                             [:display-picture [:maybe :string]]
+                             [:url [:maybe :string]]
+                             [:rss-url :string]
+                             [:user-id [:maybe :int]]
+                             [:provider-id [:maybe :int]]
+                             [:created-at :string]
+                             [:updated-at [:maybe :string]]
+                             [:content-type-id :int]
+                             [:cadence-id :int]
+                             [:baseline-id :int]
+                             [:ts-and-cs [:maybe :string]]
+                             [:state [:maybe :string]]]]}}}
+
+  [{:keys [ds] :as _request}]
+    (-> (services/feeds ds)
+        (res/response)))
diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index ebebe586..7165fcfd 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -33,6 +33,7 @@
             [source.routes.feeds :as feeds]
             [source.routes.feed :as feed]
             [source.routes.posts :as posts]
+            [source.routes.admin-feeds :as admin-feeds]
             [source.routes.xml :as xml]
             [source.routes.data :as data]
             [source.routes.jobs :as jobs]
@@ -134,10 +135,11 @@
        ["/posts"        (route {:get posts/get})]]]
 
      ["/admin"                  {:middleware [[mw/apply-auth {:required-type :admin}]]
-                                 :no-doc true
                                  :tags #{"admin"}
                                  :swagger {:security [{"auth" []}]}
                                  :openapi {:security [{:bearerAuth []}]}}
+      ["/feeds"
+       [""                      (route {:get admin-feeds/get})]]
       ["/jobs"
        [""                      {:get jobs/get}]
        ["/manage"

From 0606eeb094b3e04aff92dc4bcd43f676457b0f04 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Tue, 2 Sep 2025 15:26:34 +0200
Subject: [PATCH 077/391] added services for sending emails and templates for
 emails

---
 deps.edn                       |   2 +
 resources/config.edn           |   3 +-
 src/source/email/gmail.clj     |  60 ++++++++++++++
 src/source/email/templates.clj | 146 +++++++++++++++++++++++++++++++++
 4 files changed, 210 insertions(+), 1 deletion(-)
 create mode 100644 src/source/email/gmail.clj
 create mode 100644 src/source/email/templates.clj

diff --git a/deps.edn b/deps.edn
index 3efb2cfd..2a5d95f2 100644
--- a/deps.edn
+++ b/deps.edn
@@ -30,6 +30,8 @@
         com.github.seancorfield/honeysql {:mvn/version "2.7.1310"}
         io.replikativ/datahike {:mvn/version "0.6.1599"}
         hickory/hickory {:mvn/version "0.7.1"}
+        com.draines/postal {:mvn/version "2.0.5"}
+        hiccup/hiccup {:mvn/version "2.0.0"}
         metosin/jsonista {:mvn/version "0.3.13"}
         metosin/reitit {:mvn/version "0.9.1"} 
         metosin/reitit-middleware {:mvn/version "0.9.1"}
diff --git a/resources/config.edn b/resources/config.edn
index 993553d1..6d76608c 100644
--- a/resources/config.edn
+++ b/resources/config.edn
@@ -5,6 +5,8 @@
  :env #or [#env ENV "dev"]
  :database {:dir #or [#env DATABASE_DIR ".db/"]
             :type "sqlite"}
+ :email {:username #env EMAIL_USERNAME
+         :password #env EMAIL_PASSWORD}
  :oauth2 {:google {:authorization-uri   "https://accounts.google.com/o/oauth2/auth"
                    :access-token-uri    "https://oauth2.googleapis.com/token"
                    :redirect-uri        #or [#env GOOGLE_REDIRECT_URI "http://localhost:3000/oauth2/google/callback"]
@@ -13,4 +15,3 @@
                    :access-query-param  :access_token
                    :scope               ["https://www.googleapis.com/auth/userinfo.email"]
                    :grant-type          "authorization_code"}}}
-
diff --git a/src/source/email/gmail.clj b/src/source/email/gmail.clj
new file mode 100644
index 00000000..37a7d63e
--- /dev/null
+++ b/src/source/email/gmail.clj
@@ -0,0 +1,60 @@
+(ns source.email.gmail
+  (:require [postal.core :as postal]
+            [source.config :as conf]))
+
+(defn postal-config []
+  (let [email-username (conf/read-value :email :username)
+        gmail-password (conf/read-value :email :password)]
+    {:host "smtp.gmail.com"
+     :user email-username
+     :pass gmail-password
+     :port 587
+     :tls true}))
+
+(defn send-plaintext-email [{:keys [to subject body] :as _opts}]
+  (let [email-username (conf/read-value :email :username)]
+    (postal/send-message
+     (postal-config)
+     {:from email-username
+      :to to
+      :subject subject
+      :body [{:type "text/plain"
+              :content body}]})))
+
+(defn send-html-email [{:keys [to subject body] :as _opts}]
+  (let [email-username (conf/read-value :email :username)]
+    (postal/send-message
+     (postal-config)
+     {:from email-username
+      :to to
+      :subject subject
+      :body [{:type "text/html"
+              :content body}]})))
+
+(comment
+  (require '[source.email.templates :as templates])
+
+  (send-plaintext-email {:to "keaganncollins@gmail.com"
+                         :subject "test email"
+                         :body "hi this is a test coming from source-be"})
+
+  (send-html-email {:to "keaganncollins@gmail.com"
+                    :subject "feed rejection template"
+                    :body (templates/feed-rejection {:creator-name "Keagan"
+                                                     :feed-title "Keagan's Mukbang Channel"
+                                                     :reason "too cringe frfr"})})
+
+  (send-html-email {:to "keaganncollins@gmail.com"
+                    :subject "feed approval template"
+                    :body (templates/feed-approval {:creator-name "Keagan"
+                                                    :feed-title "Keagan's Mukbang Channel"
+                                                    :feed-id 2})})
+
+  (send-html-email {:to "keaganncollins@gmail.com"
+                    :subject "admin reported problem"
+                    :body (templates/admin-reported-problem {:user-id 3
+                                                             :user-email "keaganncollins@gmail.com"
+                                                             :user-type "creator"
+                                                             :message "no burger king foot lettuce :("})})
+
+  ())
diff --git a/src/source/email/templates.clj b/src/source/email/templates.clj
new file mode 100644
index 00000000..75f3ceb3
--- /dev/null
+++ b/src/source/email/templates.clj
@@ -0,0 +1,146 @@
+(ns source.email.templates
+  (:require [hiccup.page :as h]
+            [source.config :as conf]))
+
+(defn head-metadata []
+  [:head
+   [:meta {:charset "UTF-8"}]
+   [:meta {:name "viewport" :content "width=device-width, initial-scale=1.0"}]])
+
+(defn header []
+  [:tr
+   [:td {:class "header"
+         :style "background-color: #0F172A; padding: 40px; text-align: center; color: white; font-size: 36px;"}
+    "Source"]])
+
+(defn button [{:keys [text redirect]}]
+  [:tr
+   [:td {:style "padding: 0px 40px 0px 40px; text-align: center;"}
+    [:table {:cellspacing "0" :cellpadding "0" :style "margin: auto;"}
+     [:tr
+      [:td {:align "center"
+            :style "background-color: #0F172A; padding: 10px 20px; border-radius: 5px;"}
+       [:a {:href redirect
+            :target "_blank"
+            :style "color: #ffffff; text-decoration: none; font-weight: bold;"}
+        text]]]]]])
+
+(defn footer []
+  [:tr
+   [:td {:class "footer"
+         :style "background-color: #0F172A; padding: 20px; text-align: center; color: white; font-size: 14px;"}
+    "Copyright © 2025 | Wearesource"]])
+
+(defn feed-rejection
+  "Returns the completed HTML for a feed rejection email"
+  [{:keys [creator-name feed-title reason]}]
+  (str (h/html5
+        {:lang "en"}
+        (head-metadata)
+        [:body {:style "font-family: 'Switzer', sans-serif"}
+         [:table {:width "100%" :border "0" :cellspacing "0" :cellpadding "0"}
+          [:tr
+           [:td {:align "center" :style "padding: 20px;"}
+            [:table {:class "content"
+                     :width "600"
+                     :border "0"
+                     :cellspacing "0"
+                     :cellpadding "0"
+                     :style "border-collapse: collapse; border: 1px solid #cccccc;"}
+             (header)
+             [:tr
+              [:td {:class "body"
+                    :style "padding: 40px; text-align: left; font-size: 16px; line-height: 1.6;"}
+               (str "Hi " creator-name)
+               [:br]
+               (str "Unfortunately, the feed \"" feed-title "\" that you recently added was rejected.")
+               [:br] [:br]
+               (str reason)
+               [:br] [:br]
+               "If you believe this was in error, you can reply to this email or click on the link below to leave us a message."]]
+             (button {:text "Leave us a message"
+                               :redirect (str (conf/read-value :cors-origin) "/report-a-problem")})
+             [:tr
+              [:td {:class "body"
+                    :style "padding: 40px; text-align: left; font-size: 16px; line-height: 1.6;"}
+               "Regards,"
+               [:br]
+               "The Source Team"]]
+             (footer)]]]]])))
+
+(defn feed-approval
+  "Returns the completed HTML for a feed approval email"
+  [{:keys [creator-name feed-title feed-id]}]
+  (str (h/html5
+        {:lang "en"}
+        (head-metadata)
+        [:body {:style "font-family: 'Switzer', sans-serif"}
+         [:table {:width "100%" :border "0" :cellspacing "0" :cellpadding "0"}
+          [:tr
+           [:td {:align "center" :style "padding: 20px;"}
+            [:table {:class "content"
+                     :width "600"
+                     :border "0"
+                     :cellspacing "0"
+                     :cellpadding "0"
+                     :style "border-collapse: collapse; border: 1px solid #cccccc;"}
+             (header)
+             [:tr
+              [:td {:class "body"
+                    :style "padding: 40px; text-align: left; font-size: 16px; line-height: 1.6;"}
+               (str "Hi " creator-name)
+               [:br]
+               (str "Good news! The feed \"" feed-title "\" that you recently added was approved and is now live on the platform.")
+               [:br] [:br]
+               "Click on the link below to go to your dashboard and view your feed."]]
+             (button (str (conf/read-value :cors-origin) "/dashboard/feeds/" feed-id))
+             [:tr
+              [:td {:class "body"
+                    :style "padding: 40px; text-align: left; font-size: 16px; line-height: 1.6;"}
+               "Regards,"
+               [:br]
+               "The Source Team"]]
+             (footer)]]]]])))
+
+(defn admin-reported-problem
+  "Returns the completed HTML for an admin problem report email"
+  [{:keys [user-id user-type user-email message]}]
+  (let [shortened-message (if (> (count message) 15)
+                            (str (subs message 0 15) "...")
+                            message)]
+    (str (h/html5
+          {:lang "en"}
+          (head-metadata)
+          [:body {:style "font-family: 'Switzer', sans-serif"}
+           [:table {:width "100%" :border "0" :cellspacing "0" :cellpadding "0"}
+            [:tr
+             [:td {:align "center" :style "padding: 20px;"}
+              [:table {:class "content"
+                       :width "600"
+                       :border "0"
+                       :cellspacing "0"
+                       :cellpadding "0"
+                       :style "border-collapse: collapse; border: 1px solid #cccccc;"}
+               (header)
+               [:tr
+                [:td {:class "body"
+                      :style "padding: 40px; text-align: left; font-size: 16px; line-height: 1.6;"}
+                 "A user has reported a problem:"
+                 [:br] [:br]
+                 message
+                 [:br] [:br]
+                 (str "User ID: " user-id)
+                 [:br]
+                 (str "User email address: " user-email)
+                 [:br]
+                 (str "User type: " user-type)
+                 [:br]
+                 [:br]
+                 "Click on the link below to respond"]]
+               (button {:text "Respond"
+                                 :redirect (str "mailto:" user-email "?subject=Source Team Re:" shortened-message)})
+               [:tr
+                [:td {:class "body"
+                      :style "padding: 40px; text-align: left; font-size: 11px; line-height: 1.6;"}
+                 "This is an automated message. Please do not reply directly to this email."]]
+               (footer)]]]]]))))

From 84bdc9773cfa9a9dd6da520ab29d547320d08078 Mon Sep 17 00:00:00 2001
From: merveillevaneck <merveillevaneck@gmail.com>
Date: Tue, 2 Sep 2025 15:39:10 +0200
Subject: [PATCH 078/391] added generic function for sending email

---
 src/source/email/gmail.clj | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/src/source/email/gmail.clj b/src/source/email/gmail.clj
index 37a7d63e..1125f804 100644
--- a/src/source/email/gmail.clj
+++ b/src/source/email/gmail.clj
@@ -21,6 +21,16 @@
       :body [{:type "text/plain"
               :content body}]})))
 
+(defn send-email [{:keys [to subject body type] :as _opts}]
+  (let [email-username (conf/read-value :email :username)]
+    (-> (postal-config)
+        (postal/send-message
+         {:from email-username
+          :to to
+          :subject subject
+          :body [{:type type
+                  :content body}]}))))
+
 (defn send-html-email [{:keys [to subject body] :as _opts}]
   (let [email-username (conf/read-value :email :username)]
     (postal/send-message

From 1333943fc6299599c80d6d6be1bae8d20b82c311 Mon Sep 17 00:00:00 2001
From: merveillevaneck <merveillevaneck@gmail.com>
Date: Tue, 2 Sep 2025 15:43:06 +0200
Subject: [PATCH 079/391] updated gmail example snippets to use new function
 with keyword type

---
 src/source/email/gmail.clj | 44 +++++++++++++++++++++-----------------
 1 file changed, 24 insertions(+), 20 deletions(-)

diff --git a/src/source/email/gmail.clj b/src/source/email/gmail.clj
index 1125f804..29772d1c 100644
--- a/src/source/email/gmail.clj
+++ b/src/source/email/gmail.clj
@@ -28,7 +28,7 @@
          {:from email-username
           :to to
           :subject subject
-          :body [{:type type
+          :body [{:type (str (namespace type) "/" (name type))
                   :content body}]}))))
 
 (defn send-html-email [{:keys [to subject body] :as _opts}]
@@ -44,27 +44,31 @@
 (comment
   (require '[source.email.templates :as templates])
 
-  (send-plaintext-email {:to "keaganncollins@gmail.com"
-                         :subject "test email"
-                         :body "hi this is a test coming from source-be"})
+  (send-email {:to "keaganncollins@gmail.com"
+               :subject "test email"
+               :body "hi this is a test coming from source-be"
+               :type :text/plain})
 
-  (send-html-email {:to "keaganncollins@gmail.com"
-                    :subject "feed rejection template"
-                    :body (templates/feed-rejection {:creator-name "Keagan"
-                                                     :feed-title "Keagan's Mukbang Channel"
-                                                     :reason "too cringe frfr"})})
+  (send-email {:to "keaganncollins@gmail.com"
+               :subject "feed rejection template"
+               :body (templates/feed-rejection {:creator-name "Keagan"
+                                                :feed-title "Keagan's Mukbang Channel"
+                                                :reason "too cringe frfr"})
+               :type :text/html})
 
-  (send-html-email {:to "keaganncollins@gmail.com"
-                    :subject "feed approval template"
-                    :body (templates/feed-approval {:creator-name "Keagan"
-                                                    :feed-title "Keagan's Mukbang Channel"
-                                                    :feed-id 2})})
+  (send-email {:to "keaganncollins@gmail.com"
+              :subject "feed approval template"
+              :body (templates/feed-approval {:creator-name "Keagan"
+                                              :feed-title "Keagan's Mukbang Channel"
+                                              :feed-id 2})
+               :type :text/html})
 
-  (send-html-email {:to "keaganncollins@gmail.com"
-                    :subject "admin reported problem"
-                    :body (templates/admin-reported-problem {:user-id 3
-                                                             :user-email "keaganncollins@gmail.com"
-                                                             :user-type "creator"
-                                                             :message "no burger king foot lettuce :("})})
+  (send-email {:to "keaganncollins@gmail.com"
+               :subject "admin reported problem"
+               :type :text/html
+               :body (templates/admin-reported-problem {:user-id 3
+                                                        :user-email "keaganncollins@gmail.com"
+                                                        :user-type "creator"
+                                                        :message "no burger king foot lettuce :("})})
 
   ())

From 114b441e876e3d37abb3eb790a69e495849ee138 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Tue, 2 Sep 2025 15:48:39 +0200
Subject: [PATCH 080/391] fixed button and removed unused functions

---
 src/source/email/gmail.clj     | 20 --------------------
 src/source/email/templates.clj |  7 ++++---
 2 files changed, 4 insertions(+), 23 deletions(-)

diff --git a/src/source/email/gmail.clj b/src/source/email/gmail.clj
index 29772d1c..770caea2 100644
--- a/src/source/email/gmail.clj
+++ b/src/source/email/gmail.clj
@@ -11,16 +11,6 @@
      :port 587
      :tls true}))
 
-(defn send-plaintext-email [{:keys [to subject body] :as _opts}]
-  (let [email-username (conf/read-value :email :username)]
-    (postal/send-message
-     (postal-config)
-     {:from email-username
-      :to to
-      :subject subject
-      :body [{:type "text/plain"
-              :content body}]})))
-
 (defn send-email [{:keys [to subject body type] :as _opts}]
   (let [email-username (conf/read-value :email :username)]
     (-> (postal-config)
@@ -31,16 +21,6 @@
           :body [{:type (str (namespace type) "/" (name type))
                   :content body}]}))))
 
-(defn send-html-email [{:keys [to subject body] :as _opts}]
-  (let [email-username (conf/read-value :email :username)]
-    (postal/send-message
-     (postal-config)
-     {:from email-username
-      :to to
-      :subject subject
-      :body [{:type "text/html"
-              :content body}]})))
-
 (comment
   (require '[source.email.templates :as templates])
 
diff --git a/src/source/email/templates.clj b/src/source/email/templates.clj
index 75f3ceb3..d71901b7 100644
--- a/src/source/email/templates.clj
+++ b/src/source/email/templates.clj
@@ -59,7 +59,7 @@
                [:br] [:br]
                "If you believe this was in error, you can reply to this email or click on the link below to leave us a message."]]
              (button {:text "Leave us a message"
-                               :redirect (str (conf/read-value :cors-origin) "/report-a-problem")})
+                      :redirect (str (conf/read-value :cors-origin) "/report-a-problem")})
              [:tr
               [:td {:class "body"
                     :style "padding: 40px; text-align: left; font-size: 16px; line-height: 1.6;"}
@@ -93,7 +93,8 @@
                (str "Good news! The feed \"" feed-title "\" that you recently added was approved and is now live on the platform.")
                [:br] [:br]
                "Click on the link below to go to your dashboard and view your feed."]]
-             (button (str (conf/read-value :cors-origin) "/dashboard/feeds/" feed-id))
+             (button {:text "View your feed"
+                      :redirect (str (conf/read-value :cors-origin) "/dashboard/feeds/" feed-id)})
              [:tr
               [:td {:class "body"
                     :style "padding: 40px; text-align: left; font-size: 16px; line-height: 1.6;"}
@@ -138,7 +139,7 @@
                  [:br]
                  "Click on the link below to respond"]]
                (button {:text "Respond"
-                                 :redirect (str "mailto:" user-email "?subject=Source Team Re:" shortened-message)})
+                        :redirect (str "mailto:" user-email "?subject=Source Team Re:" shortened-message)})
                [:tr
                 [:td {:class "body"
                       :style "padding: 40px; text-align: left; font-size: 11px; line-height: 1.6;"}

From d4fcf57f33f5793d1ba737dab0e29e2fb5e0450f Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Tue, 2 Sep 2025 16:18:53 +0200
Subject: [PATCH 081/391] removed unnecessary str calls

---
 src/source/email/templates.clj | 132 ++++++++++++++++-----------------
 1 file changed, 66 insertions(+), 66 deletions(-)

diff --git a/src/source/email/templates.clj b/src/source/email/templates.clj
index d71901b7..093ce228 100644
--- a/src/source/email/templates.clj
+++ b/src/source/email/templates.clj
@@ -34,74 +34,74 @@
 (defn feed-rejection
   "Returns the completed HTML for a feed rejection email"
   [{:keys [creator-name feed-title reason]}]
-  (str (h/html5
-        {:lang "en"}
-        (head-metadata)
-        [:body {:style "font-family: 'Switzer', sans-serif"}
-         [:table {:width "100%" :border "0" :cellspacing "0" :cellpadding "0"}
-          [:tr
-           [:td {:align "center" :style "padding: 20px;"}
-            [:table {:class "content"
-                     :width "600"
-                     :border "0"
-                     :cellspacing "0"
-                     :cellpadding "0"
-                     :style "border-collapse: collapse; border: 1px solid #cccccc;"}
-             (header)
-             [:tr
-              [:td {:class "body"
-                    :style "padding: 40px; text-align: left; font-size: 16px; line-height: 1.6;"}
-               (str "Hi " creator-name)
-               [:br]
-               (str "Unfortunately, the feed \"" feed-title "\" that you recently added was rejected.")
-               [:br] [:br]
-               (str reason)
-               [:br] [:br]
-               "If you believe this was in error, you can reply to this email or click on the link below to leave us a message."]]
-             (button {:text "Leave us a message"
-                      :redirect (str (conf/read-value :cors-origin) "/report-a-problem")})
-             [:tr
-              [:td {:class "body"
-                    :style "padding: 40px; text-align: left; font-size: 16px; line-height: 1.6;"}
-               "Regards,"
-               [:br]
-               "The Source Team"]]
-             (footer)]]]]])))
+  (h/html5
+   {:lang "en"}
+   (head-metadata)
+   [:body {:style "font-family: 'Switzer', sans-serif"}
+    [:table {:width "100%" :border "0" :cellspacing "0" :cellpadding "0"}
+     [:tr
+      [:td {:align "center" :style "padding: 20px;"}
+       [:table {:class "content"
+                :width "600"
+                :border "0"
+                :cellspacing "0"
+                :cellpadding "0"
+                :style "border-collapse: collapse; border: 1px solid #cccccc;"}
+        (header)
+        [:tr
+         [:td {:class "body"
+               :style "padding: 40px; text-align: left; font-size: 16px; line-height: 1.6;"}
+          (str "Hi " creator-name)
+          [:br]
+          (str "Unfortunately, the feed \"" feed-title "\" that you recently added was rejected.")
+          [:br] [:br]
+          (str reason)
+          [:br] [:br]
+          "If you believe this was in error, you can reply to this email or click on the link below to leave us a message."]]
+        (button {:text "Leave us a message"
+                 :redirect (str (conf/read-value :cors-origin) "/report-a-problem")})
+        [:tr
+         [:td {:class "body"
+               :style "padding: 40px; text-align: left; font-size: 16px; line-height: 1.6;"}
+          "Regards,"
+          [:br]
+          "The Source Team"]]
+        (footer)]]]]]))
 
 (defn feed-approval
   "Returns the completed HTML for a feed approval email"
   [{:keys [creator-name feed-title feed-id]}]
-  (str (h/html5
-        {:lang "en"}
-        (head-metadata)
-        [:body {:style "font-family: 'Switzer', sans-serif"}
-         [:table {:width "100%" :border "0" :cellspacing "0" :cellpadding "0"}
-          [:tr
-           [:td {:align "center" :style "padding: 20px;"}
-            [:table {:class "content"
-                     :width "600"
-                     :border "0"
-                     :cellspacing "0"
-                     :cellpadding "0"
-                     :style "border-collapse: collapse; border: 1px solid #cccccc;"}
-             (header)
-             [:tr
-              [:td {:class "body"
-                    :style "padding: 40px; text-align: left; font-size: 16px; line-height: 1.6;"}
-               (str "Hi " creator-name)
-               [:br]
-               (str "Good news! The feed \"" feed-title "\" that you recently added was approved and is now live on the platform.")
-               [:br] [:br]
-               "Click on the link below to go to your dashboard and view your feed."]]
-             (button {:text "View your feed"
-                      :redirect (str (conf/read-value :cors-origin) "/dashboard/feeds/" feed-id)})
-             [:tr
-              [:td {:class "body"
-                    :style "padding: 40px; text-align: left; font-size: 16px; line-height: 1.6;"}
-               "Regards,"
-               [:br]
-               "The Source Team"]]
-             (footer)]]]]])))
+  (h/html5
+   {:lang "en"}
+   (head-metadata)
+   [:body {:style "font-family: 'Switzer', sans-serif"}
+    [:table {:width "100%" :border "0" :cellspacing "0" :cellpadding "0"}
+     [:tr
+      [:td {:align "center" :style "padding: 20px;"}
+       [:table {:class "content"
+                :width "600"
+                :border "0"
+                :cellspacing "0"
+                :cellpadding "0"
+                :style "border-collapse: collapse; border: 1px solid #cccccc;"}
+        (header)
+        [:tr
+         [:td {:class "body"
+               :style "padding: 40px; text-align: left; font-size: 16px; line-height: 1.6;"}
+          (str "Hi " creator-name)
+          [:br]
+          (str "Good news! The feed \"" feed-title "\" that you recently added was approved and is now live on the platform.")
+          [:br] [:br]
+          "Click on the link below to go to your dashboard and view your feed."]]
+        (button {:text "View your feed"
+                 :redirect (str (conf/read-value :cors-origin) "/dashboard/feeds/" feed-id)})
+        [:tr
+         [:td {:class "body"
+               :style "padding: 40px; text-align: left; font-size: 16px; line-height: 1.6;"}
+          "Regards,"
+          [:br]
+          "The Source Team"]]
+        (footer)]]]]]))
 
 (defn admin-reported-problem
   "Returns the completed HTML for an admin problem report email"
@@ -109,7 +109,7 @@
   (let [shortened-message (if (> (count message) 15)
                             (str (subs message 0 15) "...")
                             message)]
-    (str (h/html5
+    (h/html5
           {:lang "en"}
           (head-metadata)
           [:body {:style "font-family: 'Switzer', sans-serif"}
@@ -144,4 +144,4 @@
                 [:td {:class "body"
                       :style "padding: 40px; text-align: left; font-size: 11px; line-height: 1.6;"}
                  "This is an automated message. Please do not reply directly to this email."]]
-               (footer)]]]]]))))
+               (footer)]]]]])))

From fd6f7aa3fbad0da3917412009476e1f6721d3c30 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 3 Sep 2025 11:19:52 +0200
Subject: [PATCH 082/391] added endpoint to allow a user to send a report to
 admins

---
 resources/config.edn         |  3 ++-
 src/source/routes/reitit.clj | 17 +++++++++++++++++
 src/source/routes/report.clj | 34 ++++++++++++++++++++++++++++++++++
 3 files changed, 53 insertions(+), 1 deletion(-)
 create mode 100644 src/source/routes/report.clj

diff --git a/resources/config.edn b/resources/config.edn
index 6d76608c..d6d97e5e 100644
--- a/resources/config.edn
+++ b/resources/config.edn
@@ -5,7 +5,8 @@
  :env #or [#env ENV "dev"]
  :database {:dir #or [#env DATABASE_DIR ".db/"]
             :type "sqlite"}
- :email {:username #env EMAIL_USERNAME
+ :email {:address #or [#env SUPPORT_ADDRESS "lee@wearesource.earth"]
+         :username #env EMAIL_USERNAME
          :password #env EMAIL_PASSWORD}
  :oauth2 {:google {:authorization-uri   "https://accounts.google.com/o/oauth2/auth"
                    :access-token-uri    "https://oauth2.googleapis.com/token"
diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index 7165fcfd..37c224e5 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -41,6 +41,7 @@
             [source.routes.job-deregister :as job-deregister]
             [source.routes.job-start :as job-start]
             [source.routes.job-stop :as job-stop]
+            [source.routes.report :as report]
             [source.util :as util]))
 
 (defn route [handlers]
@@ -88,6 +89,12 @@
                          :openapi {:security [{:bearerAuth []}]}}
       [""               (route {:get me/get})]]
 
+     ["/mail"             {:middleware [[mw/apply-auth]]
+                           :tags #{"mail"}
+                           :swagger {:security [{"auth" []}]}
+                           :openapi {:security [{:bearerAuth []}]}}
+      ["/report"          (route {:post report/post})]]
+
      ["/businesses"     {:middleware [[mw/apply-auth {:required-type :admin}]]
                          :tags #{"businesses"}
                          :swagger {:security [{"auth" []}]}
@@ -457,4 +464,14 @@
         :body
         (json/read-json {:key-fn keyword})))
 
+  (let [app (create-app components)
+        request {:uri "/mail/report"
+                 :request-method :post
+                 :body {:message "I didn't get my cheesy fries how dare you"}
+                 :headers {"authorization" (str "Bearer " (auth.util/sign-jwt {:id 1 :type "admin"}))}}]
+    (-> request
+        app
+        :body
+        (json/read-json {:key-fn keyword})))
+
   ())
diff --git a/src/source/routes/report.clj b/src/source/routes/report.clj
new file mode 100644
index 00000000..afc222f1
--- /dev/null
+++ b/src/source/routes/report.clj
@@ -0,0 +1,34 @@
+(ns source.routes.report
+  (:require [source.config :as conf]
+            [source.email.gmail :as gmail]
+            [source.email.templates :as templates]
+            [source.services.interface :as services]
+            [ring.util.response :as res]))
+
+(defn post
+  {:summary "sends us a message to let us know of a problem"
+   :parameters {:body [:map [:message :string]]}
+   :responses {200 {:body [:map [:message :string]]}
+               401 {:body [:map [:message :string]]}
+               403 {:body [:map [:message :string]]}}}
+  [{:keys [ds user body] :as _req}]
+  (let [{:keys [email]} (services/user ds {:id (:id user)})
+        email-body (templates/admin-reported-problem {:user-id (:id user)
+                                                      :user-type (:type user)
+                                                      :user-email email
+                                                      :message (:message body)})]
+    (gmail/send-email {:to (conf/read-value :email :address)
+                       :subject (str "Report from " email)
+                       :body email-body
+                       :type :text/html})
+    (res/response {:message "successfully sent report"})))
+
+(comment
+  (require '[source.db.util :as db.util])
+
+  (post {:ds (db.util/conn)
+         :user {:id 3
+                :type "admin"}
+         :body {:message "my happy meal didn't come with a toy :("}})
+
+  ())

From 55bab7deb7dda16d372dcd5e7b091d1b14172e01 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 3 Sep 2025 11:56:45 +0200
Subject: [PATCH 083/391] updated fly dev toml to include new env variables

---
 fly.dev.toml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/fly.dev.toml b/fly.dev.toml
index 75fee297..84d2f69f 100644
--- a/fly.dev.toml
+++ b/fly.dev.toml
@@ -14,6 +14,8 @@ primary_region = 'jnb'
   GOOGLE_CLIENT_ID = '449412212863-h1rre1gdmfgq96jt160fdd30bfatjiqb.apps.googleusercontent.com'
   GOOGLE_REDIRECT_URI = 'https://source-be-staging.fly.dev/oauth2/google/callback'
   DATABASE_DIR = '/data'
+  SUPPORT_ADDRESS="keaganncollins@gmail.com"
+  EMAIL_USERNAME="merveillevaneck@gmail.com"
 
 [http_service]
   internal_port = 3000

From c8c9f254aef1fc47f3830438039a5a6d3ce04c92 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Thu, 4 Sep 2025 10:11:12 +0200
Subject: [PATCH 084/391] updated feeds table to use an enum for state
 including live, not live and pending

---
 src/source/db/master.clj          |  4 ++--
 src/source/routes/admin_feeds.clj |  8 ++++----
 src/source/routes/feed.clj        |  4 ++--
 src/source/routes/feeds.clj       | 14 +++++++-------
 4 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/src/source/db/master.clj b/src/source/db/master.clj
index 39cb0ba7..26542098 100644
--- a/src/source/db/master.clj
+++ b/src/source/db/master.clj
@@ -69,7 +69,7 @@
    [:display-picture :text]
    [:url :text]
    [:rss-url :text :not nil]
-   [:user-id :integer]
+   [:user-id :integer :not nil]
    [:provider-id :integer]
    [:created-at :datetime :not nil]
    [:updated-at :datetime]
@@ -77,7 +77,7 @@
    [:cadence-id :integer :not nil]
    [:baseline-id :integer :not nil]
    [:ts-and-cs :text]
-   [:state :text]
+   [:state :text [:check [:in :state ["live" "not live" "pending"]]]]
    (tables/foreign-key :user-id :users :id)
    (tables/foreign-key :provider-id :providers :id)
    (tables/foreign-key :cadence-id :cadences :id)
diff --git a/src/source/routes/admin_feeds.clj b/src/source/routes/admin_feeds.clj
index 689e1121..f6729f58 100644
--- a/src/source/routes/admin_feeds.clj
+++ b/src/source/routes/admin_feeds.clj
@@ -11,7 +11,7 @@
                              [:display-picture [:maybe :string]]
                              [:url [:maybe :string]]
                              [:rss-url :string]
-                             [:user-id [:maybe :int]]
+                             [:user-id :int]
                              [:provider-id [:maybe :int]]
                              [:created-at :string]
                              [:updated-at [:maybe :string]]
@@ -19,8 +19,8 @@
                              [:cadence-id :int]
                              [:baseline-id :int]
                              [:ts-and-cs [:maybe :string]]
-                             [:state [:maybe :string]]]]}}}
+                             [:state [:enum "live" "not live" "pending"]]]]}}}
 
   [{:keys [ds] :as _request}]
-    (-> (services/feeds ds)
-        (res/response)))
+  (-> (services/feeds ds)
+      (res/response)))
diff --git a/src/source/routes/feed.clj b/src/source/routes/feed.clj
index 3d112e01..56523ed3 100644
--- a/src/source/routes/feed.clj
+++ b/src/source/routes/feed.clj
@@ -12,7 +12,7 @@
                             [:display-picture [:maybe :string]]
                             [:url [:maybe :string]]
                             [:rss-url :string]
-                            [:user-id [:maybe :int]]
+                            [:user-id :int]
                             [:provider-id [:maybe :int]]
                             [:created-at :string]
                             [:updated-at [:maybe :string]]
@@ -20,7 +20,7 @@
                             [:cadence-id :int]
                             [:baseline-id :int]
                             [:ts-and-cs [:maybe :string]]
-                            [:state [:maybe :string]]]}}}
+                            [:state [:enum "live" "not live" "pending"]]]}}}
 
   [{:keys [ds path-params] :as _request}]
   (-> (services/feed ds path-params)
diff --git a/src/source/routes/feeds.clj b/src/source/routes/feeds.clj
index 6a49c227..aaea0e8a 100644
--- a/src/source/routes/feeds.clj
+++ b/src/source/routes/feeds.clj
@@ -14,7 +14,7 @@
                              [:display-picture [:maybe :string]]
                              [:url [:maybe :string]]
                              [:rss-url :string]
-                             [:user-id [:maybe :int]]
+                             [:user-id :int]
                              [:provider-id [:maybe :int]]
                              [:created-at :string]
                              [:updated-at [:maybe :string]]
@@ -22,7 +22,7 @@
                              [:cadence-id :int]
                              [:baseline-id :int]
                              [:ts-and-cs [:maybe :string]]
-                             [:state [:maybe :string]]]]}}}
+                             [:state [:enum "live" "not live" "pending"]]]]}}}
 
   [{:keys [ds user] :as _request}]
   (-> (services/feeds ds {:where [:= :user-id (:id user)]})
@@ -38,15 +38,14 @@
                        [:content-type-id :int]
                        [:cadence-id :int]
                        [:baseline-id :int]
-                       [:ts-and-cs {:optional true} :string]
-                       [:state {:optional true} :string]]}
+                       [:ts-and-cs {:optional true} :string]]}
    :responses {200 {:body [:map
                            [:id :int]
                            [:title :string]
                            [:display-picture [:maybe :string]]
                            [:url [:maybe :string]]
                            [:rss-url :string]
-                           [:user-id [:maybe :int]]
+                           [:user-id :int]
                            [:provider-id [:maybe :int]]
                            [:created-at :string]
                            [:updated-at [:maybe :string]]
@@ -54,7 +53,7 @@
                            [:cadence-id :int]
                            [:baseline-id :int]
                            [:ts-and-cs [:maybe :string]]
-                           [:state [:maybe :string]]]}}}
+                           [:state [:enum "live" "not live" "pending"]]]}}}
 
   [{:keys [js ds store user body] :as _request}]
   (let [{:keys [provider-id rss-url content-type-id]} body
@@ -74,7 +73,8 @@
                   ds
                   {:data (merge body {:title (get-in extracted [:feed :title])
                                       :user-id (:id user)
-                                      :created-at datetime})})
+                                      :created-at datetime
+                                      :state "pending"})})
         extended-posts (mapv (fn [post]
                                (merge post
                                       {:feed-id (:id new-feed)

From d0bdef3321d76770ee858be7d93fb18821122881 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Thu, 4 Sep 2025 10:11:42 +0200
Subject: [PATCH 085/391] added endpoints to approve and reject feeds

---
 src/source/routes/approve_feed.clj | 26 ++++++++++++++++++++++++++
 src/source/routes/reitit.clj       |  7 ++++++-
 src/source/routes/reject_feed.clj  | 27 +++++++++++++++++++++++++++
 src/source/services/interface.clj  |  4 ++--
 4 files changed, 61 insertions(+), 3 deletions(-)
 create mode 100644 src/source/routes/approve_feed.clj
 create mode 100644 src/source/routes/reject_feed.clj

diff --git a/src/source/routes/approve_feed.clj b/src/source/routes/approve_feed.clj
new file mode 100644
index 00000000..2c8f039c
--- /dev/null
+++ b/src/source/routes/approve_feed.clj
@@ -0,0 +1,26 @@
+(ns source.routes.approve-feed
+  (:require [source.services.interface :as services]
+            [source.email.gmail :as gmail]
+            [source.email.templates :as templates]
+            [ring.util.response :as res]))
+
+(defn patch
+  {:summary "approve the feed with the given feed-id and allow it to go live"
+   :parameters {:path [:map [:id {:title "id"
+                                  :description "feed id"} :int]]}
+   :responses {200 {:body [:map [:message :string]]}
+               401 {:body [:map [:message :string]]}
+               403 {:body [:map [:message :string]]}}}
+
+  [{:keys [ds path-params] :as _request}]
+  (let [{:keys [id user-id title]} (services/feed ds path-params)
+        {:keys [email firstname]} (services/user ds {:id user-id})]
+    (services/update-feed! ds {:id (:id path-params)
+                               :data {:state "live"}})
+    (gmail/send-email {:to email
+                       :subject "Feed Approval"
+                       :body (templates/feed-approval {:creator-name firstname
+                                                       :feed-title title
+                                                       :feed-id id})
+                       :type :text/html})
+    (res/response {:message "successfully approved feed"})))
diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index 37c224e5..dba8c919 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -42,6 +42,8 @@
             [source.routes.job-start :as job-start]
             [source.routes.job-stop :as job-stop]
             [source.routes.report :as report]
+            [source.routes.approve-feed :as approve-feed]
+            [source.routes.reject-feed :as reject-feed]
             [source.util :as util]))
 
 (defn route [handlers]
@@ -146,7 +148,10 @@
                                  :swagger {:security [{"auth" []}]}
                                  :openapi {:security [{:bearerAuth []}]}}
       ["/feeds"
-       [""                      (route {:get admin-feeds/get})]]
+       [""                      (route {:get admin-feeds/get})]
+       ["/:id"
+        ["/approve"             (route {:patch approve-feed/patch})]
+        ["/reject"              (route {:patch reject-feed/patch})]]]
       ["/jobs"
        [""                      {:get jobs/get}]
        ["/manage"
diff --git a/src/source/routes/reject_feed.clj b/src/source/routes/reject_feed.clj
new file mode 100644
index 00000000..0fa909dc
--- /dev/null
+++ b/src/source/routes/reject_feed.clj
@@ -0,0 +1,27 @@
+(ns source.routes.reject-feed
+  (:require [source.services.interface :as services]
+            [source.email.gmail :as gmail]
+            [source.email.templates :as templates]
+            [ring.util.response :as res]))
+
+(defn patch
+  {:summary "reject the feed with the given feed-id and prevent it from going live"
+   :parameters {:path [:map [:id {:title "id"
+                                  :description "feed id"} :int]]
+                :body [:map [:message :string]]}
+   :responses {200 {:body [:map [:message :string]]}
+               401 {:body [:map [:message :string]]}
+               403 {:body [:map [:message :string]]}}}
+
+  [{:keys [ds path-params body] :as _request}]
+  (let [{:keys [user-id title]} (services/feed ds path-params)
+        {:keys [email firstname]} (services/user ds {:id user-id})]
+    (services/update-feed! ds {:id (:id path-params)
+                               :data {:state "not live"}})
+    (gmail/send-email {:to email
+                       :subject "Feed Rejection"
+                       :body (templates/feed-rejection {:creator-name firstname
+                                                        :feed-title title
+                                                        :reason (:message body)})
+                       :type :text/html})
+    (res/response {:message "successfully rejected feed"})))
diff --git a/src/source/services/interface.clj b/src/source/services/interface.clj
index a61b4248..1faea62f 100644
--- a/src/source/services/interface.clj
+++ b/src/source/services/interface.clj
@@ -97,8 +97,8 @@
   ([ds {:keys [_where] :as opts}]
    (feeds/feeds ds opts)))
 
-(defn feed [ds id]
-  (feeds/feed ds id))
+(defn feed [ds {:keys [_id] :as opts}]
+  (feeds/feed ds opts))
 
 (defn insert-incoming-post! [ds {:keys [_data _ret] :as opts}]
   (incoming-posts/insert-incoming-post! ds opts))

From 4e20e5757a31a490d3a36fdb50d15e9863d23de3 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Thu, 4 Sep 2025 11:18:25 +0200
Subject: [PATCH 086/391] updated endpoints to use post instead of patch

---
 src/source/routes/approve_feed.clj | 2 +-
 src/source/routes/reitit.clj       | 4 ++--
 src/source/routes/reject_feed.clj  | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/source/routes/approve_feed.clj b/src/source/routes/approve_feed.clj
index 2c8f039c..bcad0eb5 100644
--- a/src/source/routes/approve_feed.clj
+++ b/src/source/routes/approve_feed.clj
@@ -4,7 +4,7 @@
             [source.email.templates :as templates]
             [ring.util.response :as res]))
 
-(defn patch
+(defn post
   {:summary "approve the feed with the given feed-id and allow it to go live"
    :parameters {:path [:map [:id {:title "id"
                                   :description "feed id"} :int]]}
diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index dba8c919..92cedaff 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -150,8 +150,8 @@
       ["/feeds"
        [""                      (route {:get admin-feeds/get})]
        ["/:id"
-        ["/approve"             (route {:patch approve-feed/patch})]
-        ["/reject"              (route {:patch reject-feed/patch})]]]
+        ["/approve"             (route {:post approve-feed/post})]
+        ["/reject"              (route {:post reject-feed/post})]]]
       ["/jobs"
        [""                      {:get jobs/get}]
        ["/manage"
diff --git a/src/source/routes/reject_feed.clj b/src/source/routes/reject_feed.clj
index 0fa909dc..d483a5bb 100644
--- a/src/source/routes/reject_feed.clj
+++ b/src/source/routes/reject_feed.clj
@@ -4,7 +4,7 @@
             [source.email.templates :as templates]
             [ring.util.response :as res]))
 
-(defn patch
+(defn post
   {:summary "reject the feed with the given feed-id and prevent it from going live"
    :parameters {:path [:map [:id {:title "id"
                                   :description "feed id"} :int]]

From a421bb483df3b15a15b8b0c50ebb92dd57c0f880 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Fri, 5 Sep 2025 14:01:33 +0200
Subject: [PATCH 087/391] implemented services for working with categories and
 feed categories

---
 src/source/db/master.clj                |  3 +-
 src/source/services/categories.clj      | 43 +++++++++++++++++
 src/source/services/feed_categories.clj | 61 ++++++++++++++++++++++++-
 src/source/services/interface.clj       | 36 +++++++++++++++
 4 files changed, 141 insertions(+), 2 deletions(-)
 create mode 100644 src/source/services/categories.clj

diff --git a/src/source/db/master.clj b/src/source/db/master.clj
index 26542098..b7ad184f 100644
--- a/src/source/db/master.clj
+++ b/src/source/db/master.clj
@@ -91,7 +91,8 @@
    [:feed-id :integer :not nil]
    [:category-id :integer :not nil]
    (tables/foreign-key :feed-id :feeds :id)
-   (tables/foreign-key :category-id :categories :id)))
+   (tables/foreign-key :category-id :categories :id)
+   [[:unique [:composite :feed-id :category-id]]]))
 
 (def providers
   (tables/create-table-sql
diff --git a/src/source/services/categories.clj b/src/source/services/categories.clj
new file mode 100644
index 00000000..b8f6e53b
--- /dev/null
+++ b/src/source/services/categories.clj
@@ -0,0 +1,43 @@
+(ns source.services.categories
+  (:require [source.db.interface :as db]))
+
+(defn insert-category! [ds {:keys [data ret] :as opts}]
+  (->> {:tname :categories
+        :data data
+        :ret ret}
+       (merge opts)
+       (db/insert! ds)))
+
+(defn update-category! [ds {:keys [id data where] :as opts}]
+  (->> {:tname :categories
+        :values data
+        :where (if (some? id) [:= :id id] where)
+        :ret :1}
+       (merge opts)
+       (db/update! ds)))
+
+(defn categories
+  ([ds] (categories ds {}))
+  ([ds {:keys [where] :as opts}]
+   (->> {:tname :categories
+         :where where
+         :ret :*}
+        (merge opts)
+        (db/find ds))))
+
+(defn category [ds {:keys [id where] :as opts}]
+  (->> {:tname :categories
+        :where (if (some? id)
+                 [:= :id id]
+                 where)
+        :ret :1}
+       (merge opts)
+       (db/find ds)))
+
+(comment 
+  (require '[source.db.util :as db.util])
+  (def ds (db.util/conn))
+
+  (categories ds)
+  (insert-category! ds {:data {:name "programming"}})
+  ())
diff --git a/src/source/services/feed_categories.clj b/src/source/services/feed_categories.clj
index d58b13de..8780904b 100644
--- a/src/source/services/feed_categories.clj
+++ b/src/source/services/feed_categories.clj
@@ -1,5 +1,48 @@
 (ns source.services.feed-categories
-  (:require [source.db.interface :as db]))
+  (:require [source.db.interface :as db]
+            [source.db.honey :as hon]
+            [honey.sql.helpers :as hsql]))
+
+(defn feed-categories
+  ([ds] (feed-categories ds {}))
+  ([ds {:keys [where] :as opts}]
+   (->> {:tname :feed-categories
+         :where where
+         :ret :*}
+        (merge opts)
+        (db/find ds))))
+
+(defn insert-feed-category! [ds {:keys [_data _ret] :as opts}]
+  (->> {:tname :feed-categories}
+       (merge opts)
+       (db/insert! ds)))
+
+(defn upsert-feed-categories! [ds {:keys [data]}]
+  (hon/execute!
+   ds
+   (-> (hsql/insert-into :feed-categories)
+       (hsql/values data)
+       (assoc :on-conflict [:feed-id :category-id])
+       (assoc :do-update-set {:category-id :excluded.category-id}))))
+
+(defn delete-feed-category! [ds {:keys [id where] :as opts}]
+  (->> {:tname :feed-categories
+        :where (if (some? id)
+                 [:= :id id]
+                 where)
+        :ret :1}
+       (merge opts)
+       (db/delete! ds)))
+
+(defn categories-by-feed [ds {:keys [feed-id where] :as _opts}]
+  (hon/execute! ds
+                {:select :*
+                 :from :categories
+                 :join [:feed-categories [:= :feed-categories.category-id :categories.id]]
+                 :where (if (some? feed-id)
+                          [:= :feed-id feed-id]
+                          where)}
+                {:ret :*}))
 
 (defn category-id [ds {:keys [feed-id where] :as opts}]
   (->> {:tname :feed-categories
@@ -10,3 +53,19 @@
        (merge opts)
        (db/find ds)))
 
+(comment
+  (require '[source.db.util :as db.util])
+  (def ds (db.util/conn))
+
+  (db/delete! ds {:tname :feed-categories})
+  (feed-categories ds)
+  (insert-feed-category! ds {:data {:feed-id 1
+                                    :category-id 2}})
+  (categories-by-feed ds {:feed-id 1})
+  (upsert-feed-categories! ds {:data [{:feed-id 2
+                                       :category-id 2}
+                                      {:feed-id 2
+                                       :category-id 3}
+                                      {:feed-id 2
+                                       :category-id 1}]})
+  ())
diff --git a/src/source/services/interface.clj b/src/source/services/interface.clj
index 1faea62f..e3af69ca 100644
--- a/src/source/services/interface.clj
+++ b/src/source/services/interface.clj
@@ -10,6 +10,8 @@
              [source.services.cadences :as cadences]
              [source.services.baselines :as baselines]
              [source.services.content-types :as content-types]
+             [source.services.categories :as categories]
+             [source.services.feed-categories :as feed-categories]
              [source.services.jobs :as jobs]))
 
 (defn users
@@ -133,6 +135,40 @@
 (defn baseline [ds id]
   (baselines/baseline ds id))
 
+(defn insert-category! [ds {:keys [_data _ret] :as opts}]
+  (categories/insert-category! ds opts))
+
+(defn update-category! [ds {:keys [_id _data _where] :as opts}]
+  (categories/update-category! ds opts))
+
+(defn categories
+  ([ds] (categories ds {}))
+  ([ds {:keys [_where] :as opts}]
+   (categories/categories ds opts)))
+
+(defn category [ds {:keys [_id _where] :as opts}]
+  (categories/category ds opts))
+
+(defn feed-categories
+  ([ds] (feed-categories ds {}))
+  ([ds {:keys [_where] :as opts}]
+   (feed-categories/feed-categories ds opts)))
+
+(defn insert-feed-category! [ds {:keys [_data _ret] :as opts}]
+  (feed-categories/insert-feed-category! ds opts))
+
+(defn upsert-feed-categories! [ds {:keys [_data] :as opts}]
+  (feed-categories/upsert-feed-categories! ds opts))
+
+(defn delete-feed-category! [ds {:keys [_id _where] :as opts}]
+  (feed-categories/delete-feed-category! ds opts))
+
+(defn categories-by-feed [ds {:keys [_feed-id _where] :as opts}]
+  (feed-categories/categories-by-feed ds opts))
+
+(defn category-id [ds {:keys [_feed-id _where] :as opts}]
+  (feed-categories/category-id ds opts))
+
 (defn insert-job! [ds {:keys [_data _ret] :as opts}]
   (jobs/insert-job! ds opts))
 

From 3c77dac7d13ccaa8c6c78e31b004622c7f0362bb Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Fri, 5 Sep 2025 14:03:07 +0200
Subject: [PATCH 088/391] implemented endpoints for getting baselines,
 cadences, categories and updating feeds and categories

---
 src/source/routes/baselines.clj       | 15 ++++++++++++
 src/source/routes/cadences.clj        | 14 +++++++++++
 src/source/routes/categories.clj      | 13 ++++++++++
 src/source/routes/feed.clj            | 19 +++++++++++++++
 src/source/routes/feed_categories.clj | 35 +++++++++++++++++++++++++++
 src/source/routes/reitit.clj          | 20 +++++++++++++--
 6 files changed, 114 insertions(+), 2 deletions(-)
 create mode 100644 src/source/routes/baselines.clj
 create mode 100644 src/source/routes/cadences.clj
 create mode 100644 src/source/routes/categories.clj
 create mode 100644 src/source/routes/feed_categories.clj

diff --git a/src/source/routes/baselines.clj b/src/source/routes/baselines.clj
new file mode 100644
index 00000000..b1e8e4d5
--- /dev/null
+++ b/src/source/routes/baselines.clj
@@ -0,0 +1,15 @@
+(ns source.routes.baselines
+  (:require [source.services.interface :as services]
+            [ring.util.response :as res]))
+
+(defn get
+  {:summary "get all baselines"
+   :responses {200 {:body [:vector
+                           [:map
+                            [:id :int]
+                            [:label :string]
+                            [:min :int]
+                            [:max :int]]]}}}
+  [{:keys [ds] :as _request}]
+  (->> (services/baselines ds)
+       (res/response)))
diff --git a/src/source/routes/cadences.clj b/src/source/routes/cadences.clj
new file mode 100644
index 00000000..c8809adf
--- /dev/null
+++ b/src/source/routes/cadences.clj
@@ -0,0 +1,14 @@
+(ns source.routes.cadences
+  (:require [source.services.interface :as services]
+            [ring.util.response :as res]))
+
+(defn get
+  {:summary "get all cadences"
+   :responses {200 {:body [:vector
+                           [:map
+                            [:id :int]
+                            [:label :string]
+                            [:days :int]]]}}}
+  [{:keys [ds] :as _request}]
+  (->> (services/cadences ds)
+       (res/response)))
diff --git a/src/source/routes/categories.clj b/src/source/routes/categories.clj
new file mode 100644
index 00000000..cddbb67b
--- /dev/null
+++ b/src/source/routes/categories.clj
@@ -0,0 +1,13 @@
+(ns source.routes.categories
+  (:require [source.services.interface :as services]
+            [ring.util.response :as res]))
+
+(defn get
+  {:summary "get all categories"
+   :responses {200 {:body [:vector
+                           [:map
+                            [:id :int]
+                            [:name :string]]]}}}
+  [{:keys [ds] :as _request}]
+  (->> (services/categories ds)
+       (res/response)))
diff --git a/src/source/routes/feed.clj b/src/source/routes/feed.clj
index 56523ed3..341b28cb 100644
--- a/src/source/routes/feed.clj
+++ b/src/source/routes/feed.clj
@@ -25,3 +25,22 @@
   [{:keys [ds path-params] :as _request}]
   (-> (services/feed ds path-params)
       (res/response)))
+
+(defn patch
+  {:summary "update feed by id"
+   :parameters {:path [:map [:id {:title "id"
+                                  :description "feed id"} :int]]
+                :body [:map
+                       [:title :string]
+                       [:display-picture {:optional true} :string]
+                       [:url {:optional true} :string]
+                       [:cadence-id :int]
+                       [:baseline-id :int]]}
+   :responses  {200 {:body [:map [:message :string]]}
+                401 {:body [:map [:message :string]]}
+                403 {:body [:map [:message :string]]}}}
+
+  [{:keys [ds path-params body] :as _request}]
+  (services/update-feed! ds {:id (:id path-params)
+                             :data body})
+  (res/response {:message "successfully updated feed"}))
diff --git a/src/source/routes/feed_categories.clj b/src/source/routes/feed_categories.clj
new file mode 100644
index 00000000..49466729
--- /dev/null
+++ b/src/source/routes/feed_categories.clj
@@ -0,0 +1,35 @@
+(ns source.routes.feed-categories
+  (:require [source.services.interface :as services]
+            [ring.util.response :as res]))
+
+(defn get
+  {:summary "get all categories belonging to the feed with the given id"
+   :parameters {:path [:map [:id {:title "id"
+                                  :description "feed id"} :int]]}
+   :responses {200 {:body [:vector
+                           [:map
+                            [:id :int]
+                            [:name :string]
+                            [:feed-id :int]
+                            [:category-id :int]]]}}}
+  [{:keys [ds path-params] :as _request}]
+  (->> (services/categories-by-feed ds {:feed-id (:id path-params)})
+       (res/response)))
+
+(defn patch
+  {:summary "update categories belonging to the feed with the given id"
+   :parameters {:path [:map [:id {:title "id"
+                                  :description "feed id"} :int]]
+                :body [:vector
+                       [:map
+                        [:id :int]
+                        [:name :string]]]}
+   :responses {200 {:body [:map [:message :string]]}}}
+  [{:keys [ds path-params body] :as _request}]
+  (let [update-data (reduce (fn [acc {:keys [id]}]
+                              (conj acc {:feed-id (:id path-params)
+                                         :category-id id})) [] body)]
+    (if (seq update-data)
+      (services/upsert-feed-categories! ds {:data update-data})
+      (services/delete-feed-category! ds {:where [:= :feed-id (:id path-params)]}))
+    (res/response {:message "successfully updated feed categories"})))
diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index 92cedaff..c5e1e38e 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -28,10 +28,14 @@
             [source.routes.output-schema :as output-schema]
             [source.routes.providers :as providers]
             [source.routes.provider :as provider]
+            [source.routes.cadences :as cadences]
+            [source.routes.categories :as categories]
+            [source.routes.baselines :as baselines]
             [source.routes.content-types :as content-types]
             [source.routes.content-type :as content-type]
             [source.routes.feeds :as feeds]
             [source.routes.feed :as feed]
+            [source.routes.feed-categories :as feed-categories]
             [source.routes.posts :as posts]
             [source.routes.admin-feeds :as admin-feeds]
             [source.routes.xml :as xml]
@@ -131,6 +135,15 @@
       [""               (route {:get providers/get})]
       ["/:id"           (route {:get provider/get})]]
 
+     ["/cadences"      {:tags #{"cadences"}}
+      [""               (route {:get cadences/get})]]
+
+     ["/categories"      {:tags #{"categories"}}
+      [""               (route {:get categories/get})]]
+
+     ["/baselines"      {:tags #{"baselines"}}
+      [""               (route {:get baselines/get})]]
+
      ["/content-types"  {:tags #{"content types"}}
       [""               {:get content-types/get}]
       ["/:id"           {:get content-type/get}]]
@@ -140,8 +153,11 @@
       [""               (route {:get feeds/get
                                 :post feeds/post})]
       ["/:id"
-       [""              (route {:get feed/get})]
-       ["/posts"        (route {:get posts/get})]]]
+       [""              (route {:get feed/get
+                                :patch feed/patch})]
+       ["/posts"        (route {:get posts/get})]
+       ["/categories"   (route {:get feed-categories/get
+                                :patch feed-categories/patch})]]]
 
      ["/admin"                  {:middleware [[mw/apply-auth {:required-type :admin}]]
                                  :tags #{"admin"}

From 9d72046ed56bc3788e96b4cf03bd1978d94750be Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 10 Sep 2025 15:11:39 +0200
Subject: [PATCH 089/391] changed patch to post and fixed bug in update feed
 categories route

---
 src/source/routes/businesses.clj        |  5 -----
 src/source/routes/feed.clj              |  2 +-
 src/source/routes/feed_categories.clj   |  7 +++----
 src/source/routes/reitit.clj            |  4 ++--
 src/source/services/feed_categories.clj | 10 +++-------
 5 files changed, 9 insertions(+), 19 deletions(-)

diff --git a/src/source/routes/businesses.clj b/src/source/routes/businesses.clj
index b7e1e056..4d06fc14 100644
--- a/src/source/routes/businesses.clj
+++ b/src/source/routes/businesses.clj
@@ -4,11 +4,6 @@
 
 (defn get
   {:summary "get all businesses"
-   :parameters {:body [:map
-                       [:name :string]
-                       [:url {:optional true} :string]
-                       [:linkedin {:optional true} :string]
-                       [:twitter {:optional true} :string]]}
    :responses {200 {:body [:map
                            [:businesses
                             [:map
diff --git a/src/source/routes/feed.clj b/src/source/routes/feed.clj
index 341b28cb..272d4b4c 100644
--- a/src/source/routes/feed.clj
+++ b/src/source/routes/feed.clj
@@ -26,7 +26,7 @@
   (-> (services/feed ds path-params)
       (res/response)))
 
-(defn patch
+(defn post
   {:summary "update feed by id"
    :parameters {:path [:map [:id {:title "id"
                                   :description "feed id"} :int]]
diff --git a/src/source/routes/feed_categories.clj b/src/source/routes/feed_categories.clj
index 49466729..6be698b9 100644
--- a/src/source/routes/feed_categories.clj
+++ b/src/source/routes/feed_categories.clj
@@ -16,7 +16,7 @@
   (->> (services/categories-by-feed ds {:feed-id (:id path-params)})
        (res/response)))
 
-(defn patch
+(defn post
   {:summary "update categories belonging to the feed with the given id"
    :parameters {:path [:map [:id {:title "id"
                                   :description "feed id"} :int]]
@@ -29,7 +29,6 @@
   (let [update-data (reduce (fn [acc {:keys [id]}]
                               (conj acc {:feed-id (:id path-params)
                                          :category-id id})) [] body)]
-    (if (seq update-data)
-      (services/upsert-feed-categories! ds {:data update-data})
-      (services/delete-feed-category! ds {:where [:= :feed-id (:id path-params)]}))
+    (services/delete-feed-category! ds {:where [:= :feed-id (:id path-params)]})
+    (services/insert-feed-category! ds {:data update-data})
     (res/response {:message "successfully updated feed categories"})))
diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index c5e1e38e..8e7f3ff3 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -154,10 +154,10 @@
                                 :post feeds/post})]
       ["/:id"
        [""              (route {:get feed/get
-                                :patch feed/patch})]
+                                :post feed/post})]
        ["/posts"        (route {:get posts/get})]
        ["/categories"   (route {:get feed-categories/get
-                                :patch feed-categories/patch})]]]
+                                :post feed-categories/post})]]]
 
      ["/admin"                  {:middleware [[mw/apply-auth {:required-type :admin}]]
                                  :tags #{"admin"}
diff --git a/src/source/services/feed_categories.clj b/src/source/services/feed_categories.clj
index 8780904b..b64391d1 100644
--- a/src/source/services/feed_categories.clj
+++ b/src/source/services/feed_categories.clj
@@ -36,7 +36,7 @@
 
 (defn categories-by-feed [ds {:keys [feed-id where] :as _opts}]
   (hon/execute! ds
-                {:select :*
+                {:select [[:feed-categories.category-id :id] :name]
                  :from :categories
                  :join [:feed-categories [:= :feed-categories.category-id :categories.id]]
                  :where (if (some? feed-id)
@@ -62,10 +62,6 @@
   (insert-feed-category! ds {:data {:feed-id 1
                                     :category-id 2}})
   (categories-by-feed ds {:feed-id 1})
-  (upsert-feed-categories! ds {:data [{:feed-id 2
-                                       :category-id 2}
-                                      {:feed-id 2
-                                       :category-id 3}
-                                      {:feed-id 2
-                                       :category-id 1}]})
+  (upsert-feed-categories! ds {:data [{:feed-id 1
+                                       :category-id 6}]})
   ())

From c1eb5e5fe3f9d73ddac8532b0bb74ffab7eedc40 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Thu, 11 Sep 2025 11:05:59 +0200
Subject: [PATCH 090/391] added categories to seed data and updated tsandcs
 from string to bool

---
 src/source/db/master.clj                     | 2 +-
 src/source/migrations/001_init_master_db.clj | 8 ++++++++
 src/source/routes/admin_feeds.clj            | 2 +-
 src/source/routes/feed.clj                   | 3 ++-
 src/source/routes/feeds.clj                  | 6 +++---
 5 files changed, 15 insertions(+), 6 deletions(-)

diff --git a/src/source/db/master.clj b/src/source/db/master.clj
index b7ad184f..124cd5ac 100644
--- a/src/source/db/master.clj
+++ b/src/source/db/master.clj
@@ -76,7 +76,7 @@
    [:content-type-id :integer :not nil]
    [:cadence-id :integer :not nil]
    [:baseline-id :integer :not nil]
-   [:ts-and-cs :text]
+   [:ts-and-cs :integer]
    [:state :text [:check [:in :state ["live" "not live" "pending"]]]]
    (tables/foreign-key :user-id :users :id)
    (tables/foreign-key :provider-id :providers :id)
diff --git a/src/source/migrations/001_init_master_db.clj b/src/source/migrations/001_init_master_db.clj
index c1d965d4..68ea8a2e 100644
--- a/src/source/migrations/001_init_master_db.clj
+++ b/src/source/migrations/001_init_master_db.clj
@@ -50,6 +50,13 @@
            :domain "www.medium.com"
            :content-type-id 3}]})
 
+(def categories-seed
+  {:tname :categories
+   :data [{:name "programming"}
+          {:name "game development"}
+          {:name "languages"}
+          {:name "technology"}]})
+
 (def sectors-seed
   {:tname :sectors
    :data [{:name "renewable energy"}
@@ -86,6 +93,7 @@
     (db/insert! ds-master cadences-seed)
     (db/insert! ds-master content-types-seed)
     (db/insert! ds-master providers-seed)
+    (db/insert! ds-master categories-seed)
     (db/insert! ds-master sectors-seed)
 
     (when (= (conf/read-value :env) "dev")
diff --git a/src/source/routes/admin_feeds.clj b/src/source/routes/admin_feeds.clj
index f6729f58..c67ac05c 100644
--- a/src/source/routes/admin_feeds.clj
+++ b/src/source/routes/admin_feeds.clj
@@ -18,7 +18,7 @@
                              [:content-type-id :int]
                              [:cadence-id :int]
                              [:baseline-id :int]
-                             [:ts-and-cs [:maybe :string]]
+                             [:ts-and-cs [:maybe :int]]
                              [:state [:enum "live" "not live" "pending"]]]]}}}
 
   [{:keys [ds] :as _request}]
diff --git a/src/source/routes/feed.clj b/src/source/routes/feed.clj
index 272d4b4c..077e28b4 100644
--- a/src/source/routes/feed.clj
+++ b/src/source/routes/feed.clj
@@ -19,7 +19,7 @@
                             [:content-type-id :int]
                             [:cadence-id :int]
                             [:baseline-id :int]
-                            [:ts-and-cs [:maybe :string]]
+                            [:ts-and-cs [:maybe :int]]
                             [:state [:enum "live" "not live" "pending"]]]}}}
 
   [{:keys [ds path-params] :as _request}]
@@ -34,6 +34,7 @@
                        [:title :string]
                        [:display-picture {:optional true} :string]
                        [:url {:optional true} :string]
+                       [:ts-and-cs {:optional true} :int]
                        [:cadence-id :int]
                        [:baseline-id :int]]}
    :responses  {200 {:body [:map [:message :string]]}
diff --git a/src/source/routes/feeds.clj b/src/source/routes/feeds.clj
index aaea0e8a..1394db06 100644
--- a/src/source/routes/feeds.clj
+++ b/src/source/routes/feeds.clj
@@ -21,7 +21,7 @@
                              [:content-type-id :int]
                              [:cadence-id :int]
                              [:baseline-id :int]
-                             [:ts-and-cs [:maybe :string]]
+                             [:ts-and-cs [:maybe :int]]
                              [:state [:enum "live" "not live" "pending"]]]]}}}
 
   [{:keys [ds user] :as _request}]
@@ -38,7 +38,7 @@
                        [:content-type-id :int]
                        [:cadence-id :int]
                        [:baseline-id :int]
-                       [:ts-and-cs {:optional true} :string]]}
+                       [:ts-and-cs {:optional true} :int]]}
    :responses {200 {:body [:map
                            [:id :int]
                            [:title :string]
@@ -52,7 +52,7 @@
                            [:content-type-id :int]
                            [:cadence-id :int]
                            [:baseline-id :int]
-                           [:ts-and-cs [:maybe :string]]
+                           [:ts-and-cs [:maybe :int]]
                            [:state [:enum "live" "not live" "pending"]]]}}}
 
   [{:keys [js ds store user body] :as _request}]

From 9ecd4c1db9d785284383350d0e55de54f09c9516 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 15 Sep 2025 10:01:43 +0200
Subject: [PATCH 091/391] added routes for logged in user to update their
 profile and add a business

---
 src/source/db/master.clj          |  4 +++-
 src/source/routes/me.clj          | 24 +++++++++++++++++++++++-
 src/source/routes/me_business.clj | 25 +++++++++++++++++++++++++
 src/source/routes/reitit.clj      |  5 ++++-
 src/source/services/interface.clj | 14 +++++++++++++-
 5 files changed, 68 insertions(+), 4 deletions(-)
 create mode 100644 src/source/routes/me_business.clj

diff --git a/src/source/db/master.clj b/src/source/db/master.clj
index 124cd5ac..5f62be74 100644
--- a/src/source/db/master.clj
+++ b/src/source/db/master.clj
@@ -14,7 +14,9 @@
    [:onboarded :integer [:default 0]]
    [:address :text]
    [:mobile :text]
-   [:profile-image :text]))
+   [:profile-image :text]
+   [:business-id :int]
+   (tables/foreign-key :business-id :businesses :id)))
 
 (def sectors
   (tables/create-table-sql
diff --git a/src/source/routes/me.clj b/src/source/routes/me.clj
index 6f496b4c..a1ad22e1 100644
--- a/src/source/routes/me.clj
+++ b/src/source/routes/me.clj
@@ -1,6 +1,7 @@
 (ns source.routes.me
   (:require [source.services.interface :as services]
-            [ring.util.response :as res]))
+            [ring.util.response :as res]
+            [source.util :as util]))
 
 (defn get
   {:summary "get logged in user by access token"
@@ -23,3 +24,24 @@
                   (services/user ds))]
     (->> (dissoc user :password)
          (res/response))))
+
+(defn post
+  {:summary "update logged-in user by access token"
+   :parameters {:body [:map
+                       [:address [:maybe :string]]
+                       [:profile-image [:maybe :string]]
+                       [:firstname [:maybe :string]]
+                       [:lastname [:maybe :string]]
+                       [:email-verified [:maybe :int]]
+                       [:onboarded [:maybe :int]]
+                       [:mobile [:maybe :string]]]}
+   :responses {200 {:body [:map [:message :string]]}
+               400 {:body [:map [:message :string]]}}}
+
+  [{:keys [ds user body] :as _request}]
+  (let [{:keys [data error success]} (util/validate post body)]
+    (if (not success)
+      (-> (res/response {:message error})
+          (res/status 400))
+      (services/update-user! ds {:id (:id user)
+                                 :data data}))))
diff --git a/src/source/routes/me_business.clj b/src/source/routes/me_business.clj
new file mode 100644
index 00000000..5ebba417
--- /dev/null
+++ b/src/source/routes/me_business.clj
@@ -0,0 +1,25 @@
+(ns source.routes.me-business
+  (:require [source.util :as util]
+            [ring.util.response :as res]
+            [source.services.interface :as services]))
+
+(defn post
+  {:summary "add business for logged-in user"
+   :parameters {:body [:map
+                       [:name {:optional true} :string]
+                       [:url {:optional true} :string]
+                       [:linkedin {:optional true} :string]
+                       [:twitter {:optional true} :string]]}
+   :responses {200 {:body [:map [:message :string]]}
+               400 {:body [:map [:message :string]]}}}
+
+  [{:keys [ds user body] :as _request}]
+  (let [{:keys [data error success]} (util/validate post body)]
+    (if (not success)
+      (-> (res/response {:message error})
+          (res/status 400))
+
+      (let [business (services/insert-business! ds data)]
+        (services/update-user! ds {:id (:id user)
+                                   :data {:business-id (:id business)}})
+        (res/response {:message "successfully added business"})))))
diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index 8e7f3ff3..d8dbd257 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -11,6 +11,7 @@
             [source.routes.user :as user]
             [source.routes.users :as users]
             [source.routes.me :as me]
+            [source.routes.me-business :as me-business]
             [source.routes.login :as login]
             [source.routes.register :as register]
             [source.routes.google-launch :as google-launch]
@@ -93,7 +94,9 @@
                          :tags #{"me"}
                          :swagger {:security [{"auth" []}]}
                          :openapi {:security [{:bearerAuth []}]}}
-      [""               (route {:get me/get})]]
+      [""               (route {:get me/get
+                                :post me/post})]
+      ["/business"      (route {:post me-business/post})]]
 
      ["/mail"             {:middleware [[mw/apply-auth]]
                            :tags #{"mail"}
diff --git a/src/source/services/interface.clj b/src/source/services/interface.clj
index e3af69ca..24a20bb7 100644
--- a/src/source/services/interface.clj
+++ b/src/source/services/interface.clj
@@ -12,7 +12,8 @@
              [source.services.content-types :as content-types]
              [source.services.categories :as categories]
              [source.services.feed-categories :as feed-categories]
-             [source.services.jobs :as jobs]))
+             [source.services.jobs :as jobs]
+             [source.services.businesses :as businesses]))
 
 (defn users
   [& args]
@@ -27,6 +28,17 @@
 (defn update-user! [ds {:keys [_id _values _where] :as opts}]
   (users/update-user! ds opts))
 
+(defn businesses
+  ([ds] (businesses ds {}))
+  ([ds opts]
+   (businesses/businesses ds opts)))
+
+(defn insert-business! [ds {:keys [_values _ret] :as opts}]
+  (businesses/insert-business! ds opts))
+
+(defn update-business! [ds {:keys [_id _values _where] :as opts}]
+  (businesses/update-business! ds opts))
+
 (defn login [ds {:keys [_email] :as opts}]
   (auth/login ds opts))
 

From 9bfa6483c1e1eb07c17a3580aeee72cf5c994301 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 15 Sep 2025 15:11:31 +0200
Subject: [PATCH 092/391] updated business table to include address

---
 src/source/db/master.clj          | 1 +
 src/source/routes/business.clj    | 2 ++
 src/source/routes/businesses.clj  | 1 +
 src/source/routes/me_business.clj | 4 +++-
 4 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/source/db/master.clj b/src/source/db/master.clj
index 5f62be74..850b5eee 100644
--- a/src/source/db/master.clj
+++ b/src/source/db/master.clj
@@ -110,6 +110,7 @@
    :businesses
    (tables/table-id)
    [:name :text]
+   [:address :text [:default nil]]
    [:url :text [:default nil]]
    [:linkedin :text [:default nil]]
    [:twitter :text [:default nil]]))
diff --git a/src/source/routes/business.clj b/src/source/routes/business.clj
index 16b77838..8df54a66 100644
--- a/src/source/routes/business.clj
+++ b/src/source/routes/business.clj
@@ -7,6 +7,7 @@
   {:summary "insert a business"
    :parameters {:body [:map
                        [:name :string]
+                       [:address {:optional true} :string]
                        [:url {:optional true} :string]
                        [:linkedin {:optional true} :string]
                        [:twitter {:optional true} :string]]}
@@ -27,6 +28,7 @@
                                   :description "business id"} :int]]
                 :body [:map
                        [:name :string]
+                       [:address {:optional true} :string]
                        [:url {:optional true} :string]
                        [:linkedin {:optional true} :string]
                        [:twitter {:optional true} :string]]}
diff --git a/src/source/routes/businesses.clj b/src/source/routes/businesses.clj
index 4d06fc14..cec00eb2 100644
--- a/src/source/routes/businesses.clj
+++ b/src/source/routes/businesses.clj
@@ -9,6 +9,7 @@
                             [:map
                              [:id :int]
                              [:name :string]
+                             [:address [:maybe :string]]
                              [:url [:maybe :string]]
                              [:linkedin [:maybe :string]]
                              [:twitter [:maybe :string]]]]]}}}
diff --git a/src/source/routes/me_business.clj b/src/source/routes/me_business.clj
index 5ebba417..347b0e8f 100644
--- a/src/source/routes/me_business.clj
+++ b/src/source/routes/me_business.clj
@@ -7,6 +7,7 @@
   {:summary "add business for logged-in user"
    :parameters {:body [:map
                        [:name {:optional true} :string]
+                       [:address {:optional true} :string]
                        [:url {:optional true} :string]
                        [:linkedin {:optional true} :string]
                        [:twitter {:optional true} :string]]}
@@ -19,7 +20,8 @@
       (-> (res/response {:message error})
           (res/status 400))
 
-      (let [business (services/insert-business! ds data)]
+      (let [business (services/insert-business! ds {:data data
+                                                    :ret :1})]
         (services/update-user! ds {:id (:id user)
                                    :data {:business-id (:id business)}})
         (res/response {:message "successfully added business"})))))

From 3befccf5eda348c93d78405958b3867386516426 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 15 Sep 2025 15:12:25 +0200
Subject: [PATCH 093/391] added services and endpoints for working with user
 sectors

---
 src/source/routes/me.clj             | 20 +++++++------
 src/source/routes/me_sectors.clj     | 18 +++++++++++
 src/source/routes/reitit.clj         |  4 ++-
 src/source/routes/sectors.clj        | 11 ++++---
 src/source/services/interface.clj    | 20 ++++++++++++-
 src/source/services/user_sectors.clj | 45 ++++++++++++++++++++++++++++
 6 files changed, 101 insertions(+), 17 deletions(-)
 create mode 100644 src/source/routes/me_sectors.clj
 create mode 100644 src/source/services/user_sectors.clj

diff --git a/src/source/routes/me.clj b/src/source/routes/me.clj
index a1ad22e1..0f654a0a 100644
--- a/src/source/routes/me.clj
+++ b/src/source/routes/me.clj
@@ -28,13 +28,13 @@
 (defn post
   {:summary "update logged-in user by access token"
    :parameters {:body [:map
-                       [:address [:maybe :string]]
-                       [:profile-image [:maybe :string]]
-                       [:firstname [:maybe :string]]
-                       [:lastname [:maybe :string]]
-                       [:email-verified [:maybe :int]]
-                       [:onboarded [:maybe :int]]
-                       [:mobile [:maybe :string]]]}
+                       [:address {:optional true} :string]
+                       [:profile-image {:optional true} [:maybe :string]]
+                       [:firstname {:optional true} :string]
+                       [:lastname {:optional true} :string]
+                       [:email-verified {:optional true} :int]
+                       [:onboarded {:optional true} :int]
+                       [:mobile {:optional true} :string]]}
    :responses {200 {:body [:map [:message :string]]}
                400 {:body [:map [:message :string]]}}}
 
@@ -43,5 +43,7 @@
     (if (not success)
       (-> (res/response {:message error})
           (res/status 400))
-      (services/update-user! ds {:id (:id user)
-                                 :data data}))))
+      (do (services/update-user! ds {:id (:id user)
+                                     :data data})
+          (res/response {:message "successfully updated user"})))))
+
diff --git a/src/source/routes/me_sectors.clj b/src/source/routes/me_sectors.clj
new file mode 100644
index 00000000..baaa1627
--- /dev/null
+++ b/src/source/routes/me_sectors.clj
@@ -0,0 +1,18 @@
+(ns source.routes.me-sectors 
+  (:require [source.services.interface :as services]
+            [ring.util.response :as res]))
+
+(defn post
+  {:summary "update sectors for the logged-in user"
+   :parameters {:body [:vector
+                       [:map
+                        [:id :int]
+                        [:name :string]]]}
+   :responses {200 {:body [:map [:message :string]]}}}
+  [{:keys [ds user body] :as _request}]
+  (let [update-data (reduce (fn [acc {:keys [id]}]
+                              (conj acc {:user-id (:id user)
+                                         :sector-id id})) [] body)]
+    (services/delete-user-sector! ds {:where [:= :user-id (:id user)]})
+    (services/insert-user-sector! ds {:data update-data})
+    (res/response {:message "successfully updated user sectors"})))
diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index d8dbd257..2ee68189 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -12,6 +12,7 @@
             [source.routes.users :as users]
             [source.routes.me :as me]
             [source.routes.me-business :as me-business]
+            [source.routes.me-sectors :as me-sectors]
             [source.routes.login :as login]
             [source.routes.register :as register]
             [source.routes.google-launch :as google-launch]
@@ -96,7 +97,8 @@
                          :openapi {:security [{:bearerAuth []}]}}
       [""               (route {:get me/get
                                 :post me/post})]
-      ["/business"      (route {:post me-business/post})]]
+      ["/business"      (route {:post me-business/post})]
+      ["/sectors"       (route {:post me-sectors/post})]]
 
      ["/mail"             {:middleware [[mw/apply-auth]]
                            :tags #{"mail"}
diff --git a/src/source/routes/sectors.clj b/src/source/routes/sectors.clj
index 1e22d4b0..0f8fa450 100644
--- a/src/source/routes/sectors.clj
+++ b/src/source/routes/sectors.clj
@@ -4,14 +4,13 @@
 
 (defn get
   {:summary "get all sectors"
-   :responses {200 {:body [:map
-                           [:sectors
-                            [:map
-                             [:id :int]
-                             [:name :string]]]]}}}
+   :responses {200 {:body [:vector
+                           [:map
+                            [:id :int]
+                            [:name :string]]]}}}
 
   [{:keys [ds] :as _request}]
-  (res/response {:sectors (sectors/sectors ds)}))
+  (res/response (sectors/sectors ds)))
 
 (comment
   (require '[source.db.util :as db.util])
diff --git a/src/source/services/interface.clj b/src/source/services/interface.clj
index 24a20bb7..5e377113 100644
--- a/src/source/services/interface.clj
+++ b/src/source/services/interface.clj
@@ -13,7 +13,8 @@
              [source.services.categories :as categories]
              [source.services.feed-categories :as feed-categories]
              [source.services.jobs :as jobs]
-             [source.services.businesses :as businesses]))
+             [source.services.businesses :as businesses]
+             [source.services.user-sectors :as user-sectors]))
 
 (defn users
   [& args]
@@ -181,6 +182,23 @@
 (defn category-id [ds {:keys [_feed-id _where] :as opts}]
   (feed-categories/category-id ds opts))
 
+(defn user-sectors
+  ([ds] (user-sectors ds {}))
+  ([ds {:keys [_where] :as opts}]
+   (user-sectors/user-sectors ds opts)))
+
+(defn insert-user-sector! [ds {:keys [_data _ret] :as opts}]
+  (user-sectors/insert-user-sector! ds opts))
+
+(defn delete-user-sector! [ds {:keys [_id _where] :as opts}]
+  (user-sectors/delete-user-sector! ds opts))
+
+(defn sectors-by-user [ds {:keys [_sector-id _where] :as opts}]
+  (user-sectors/sectors-by-user ds opts))
+
+(defn sector-id [ds {:keys [_user-id _where] :as opts}]
+  (user-sectors/sector-id ds opts))
+
 (defn insert-job! [ds {:keys [_data _ret] :as opts}]
   (jobs/insert-job! ds opts))
 
diff --git a/src/source/services/user_sectors.clj b/src/source/services/user_sectors.clj
new file mode 100644
index 00000000..3ac971b9
--- /dev/null
+++ b/src/source/services/user_sectors.clj
@@ -0,0 +1,45 @@
+(ns source.services.user-sectors
+  (:require [source.db.interface :as db]
+            [source.db.honey :as hon]))
+
+(defn user-sectors
+  ([ds] (user-sectors ds {}))
+  ([ds {:keys [where] :as opts}]
+   (->> {:tname :user-sectors
+         :where where
+         :ret :*}
+        (merge opts)
+        (db/find ds))))
+
+(defn insert-user-sector! [ds {:keys [_data _ret] :as opts}]
+  (->> {:tname :user-sectors}
+       (merge opts)
+       (db/insert! ds)))
+
+(defn delete-user-sector! [ds {:keys [id where] :as opts}]
+  (->> {:tname :user-sectors
+        :where (if (some? id)
+                 [:= :id id]
+                 where)
+        :ret :1}
+       (merge opts)
+       (db/delete! ds)))
+
+(defn sectors-by-user [ds {:keys [sector-id where] :as _opts}]
+  (hon/execute! ds
+                {:select [[:user-sectors.sector-id :id] :name]
+                 :from :sectors
+                 :join [:user-sectors [:= :user-sectors.sector-id :sectors.id]]
+                 :where (if (some? sector-id)
+                          [:= :sector-id sector-id]
+                          where)}
+                {:ret :*}))
+
+(defn sector-id [ds {:keys [user-id where] :as opts}]
+  (->> {:tname :user-sectors
+        :where (if (some? user-id)
+                 [:= :user-id user-id]
+                 where)
+        :ret :1}
+       (merge opts)
+       (db/find ds)))

From b19993b48ff44fc48ad1d1405596d97ea3cba71f Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Tue, 16 Sep 2025 16:54:29 +0200
Subject: [PATCH 094/391] implemented table setup and migrations for bundle
 databases

---
 .../bundle_migrations/001_init_bundle_db.clj  | 17 ++++++
 src/source/db/bundle.clj                      | 61 ++++++++++++-------
 src/source/migrate.clj                        | 29 ++++++++-
 3 files changed, 82 insertions(+), 25 deletions(-)
 create mode 100644 src/source/bundle_migrations/001_init_bundle_db.clj

diff --git a/src/source/bundle_migrations/001_init_bundle_db.clj b/src/source/bundle_migrations/001_init_bundle_db.clj
new file mode 100644
index 00000000..1988589a
--- /dev/null
+++ b/src/source/bundle_migrations/001_init_bundle_db.clj
@@ -0,0 +1,17 @@
+(ns source.bundle-migrations.001-init-bundle-db
+  (:require [source.db.bundle]
+            [source.db.tables :as tables]))
+
+(defn run-up! [context]
+  (let [ds-bundle (:db-bundle context)]
+    (tables/create-tables!
+     ds-bundle
+     :source.db.bundle
+     [:outgoing-posts
+      :post-heuristics
+      :analytics
+      :event-categories])))
+
+(defn run-down! [context]
+  (let [ds-bundle (:db-bundle context)]
+    (tables/drop-all-tables! ds-bundle)))
diff --git a/src/source/db/bundle.clj b/src/source/db/bundle.clj
index 620415ea..a0b1811f 100644
--- a/src/source/db/bundle.clj
+++ b/src/source/db/bundle.clj
@@ -1,39 +1,54 @@
-(ns source.db.bundle 
+(ns source.db.bundle
   (:require [source.db.tables :as tables]
             [honey.sql :as sql]))
 
 (def event-categories
   (tables/create-table-sql
-    :event-categories
-    (tables/table-id)
-    [:event-id :integer :not nil]
-    [:category-id :text :not nil]
-    (tables/foreign-key :event-id :analytics :id)))
+   :event-categories
+   (tables/table-id)
+   [:event-id :integer :not nil]
+   [:category-id :text :not nil]
+   (tables/foreign-key :event-id :analytics :id)))
 
-(def outgoing-posts 
+(def outgoing-posts
   (tables/create-table-sql
-    :outgoing-posts
-    (tables/table-id)
-    [:title :text]
-    [:subtitle :text]
-    [:stream-url :text [:default nil]]
-    [:content-type :text [:default nil]]
-    [:feed-id :integer]
-    [:creator-id :integer]
-    (tables/foreign-key :creator-id :users :id)
-    (tables/foreign-key :feed-id :feeds :id)))
+   :outgoing-posts
+   (tables/table-id)
+   [:post-id :text :not nil]
+   [:feed-id :integer :not nil]
+   [:title :text :not nil]
+   [:info :text]
+   [:url :text]
+   [:stream-url :text]
+   [:creator-id :integer :not nil]
+   [:season :integer]
+   [:episode :integer]
+   [:content-type-id :integer :not nil]
+   [:posted-at :datetime]
+   (tables/foreign-key :feed-id :feeds :id)
+   (tables/foreign-key :creator-id :users :id)
+   (tables/foreign-key :content-type-id :content-types :id)))
 
-(def analytics 
+(def post-heuristics
   (tables/create-table-sql
-    :analytics
-    (tables/table-id)
-    [:post-id :integer :not nil]
-    [:event-type :text :not nil]
-    [:timestamp :text :not nil]))
+   :post-heuristics
+   (tables/table-id)
+   [:post-id :integer :not nil]
+   [:long-heuristic :integer]
+   [:short-heuristic :integer]))
+
+(def analytics
+  (tables/create-table-sql
+   :analytics
+   (tables/table-id)
+   [:post-id :integer :not nil]
+   [:event-type :text :not nil]
+   [:timestamp :text :not nil]))
 
 (comment
   (sql/format event-categories)
   (sql/format outgoing-posts)
+  (sql/format post-heuristics)
   (sql/format analytics)
   ())
 
diff --git a/src/source/migrate.clj b/src/source/migrate.clj
index 92856bb4..c1bca5c0 100644
--- a/src/source/migrate.clj
+++ b/src/source/migrate.clj
@@ -3,7 +3,8 @@
             [k16.mallard.store.sqlite :as store]
             [k16.mallard.loader.fs :as loader.fs]
             [next.jdbc :as jdbc]
-            [source.db.util :as db.util]))
+            [source.db.util :as db.util]
+            [source.db.honey :as db]))
 
 ;; This is our interface for running migrations.
 ;;
@@ -16,6 +17,9 @@
 (def ^:private migrations
   (loader.fs/load! "src/source/migrations"))
 
+(def ^:private bundle-migrations
+  (loader.fs/load! "src/source/bundle_migrations"))
+
 (defn run-migrations [args]
   (let [context {:db-master (jdbc/get-datasource {:dbname (db.util/db-path "master") :dbtype "sqlite"})}
         db-migrate (jdbc/get-datasource {:dbname (db.util/db-path "migrate") :dbtype "sqlite"})
@@ -27,6 +31,27 @@
                   :operations migrations}
                  args)))
 
+(defn run-bundle-migrations [args]
+  (let [db-migrate (jdbc/get-datasource {:dbname (db.util/db-path "migrate") :dbtype "sqlite"})
+        datastore (store/create-datastore
+                   {:db db-migrate
+                    :table-name "bundle_migrations"})
+        ds-master (db.util/conn :master)
+        bundles (try
+                  (db/find ds-master {:tname :bundles
+                                      :ret :*})
+                  (catch Exception _ []))]
+    (run!
+     (fn [{:keys [id]}]
+       (let [db-name (db.util/db-name :bundle id)
+             context {:db-bundle (jdbc/get-datasource {:dbname (db.util/db-path db-name)
+                                                       :dbtype "sqlite"})}]
+         (mallard/run {:context context
+                       :store datastore
+                       :operations bundle-migrations}
+                      args)))
+     bundles)))
+
 (defn -main [& args]
+  (run-bundle-migrations args)
   (run-migrations args))
-

From ae7a8b963c26015a577546d9136a735c922e2090 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Tue, 16 Sep 2025 16:55:04 +0200
Subject: [PATCH 095/391] added integrations and integration-categories tables
 and services therefore

---
 src/source/db/master.clj                      | 27 ++++++++++-
 src/source/migrations/001_init_master_db.clj  |  2 +
 .../services/integration_categories.clj       | 46 +++++++++++++++++++
 src/source/services/integrations.clj          | 35 ++++++++++++++
 4 files changed, 109 insertions(+), 1 deletion(-)
 create mode 100644 src/source/services/integration_categories.clj
 create mode 100644 src/source/services/integrations.clj

diff --git a/src/source/db/master.clj b/src/source/db/master.clj
index 850b5eee..e9ac4f41 100644
--- a/src/source/db/master.clj
+++ b/src/source/db/master.clj
@@ -156,7 +156,30 @@
    [:episode :integer]
    [:content-type-id :integer :not nil]
    [:redacted :integer]
-   [:posted-at :datetime]))
+   [:posted-at :datetime]
+   (tables/foreign-key :feed-id :feeds :id)
+   (tables/foreign-key :creator-id :users :id)
+   (tables/foreign-key :content-type-id :content-types :id)))
+
+(def integrations
+  (tables/create-table-sql
+   :integrations
+   (tables/table-id)
+   [:name :text :not nil]
+   [:content-type-id :integer :not nil]
+   [:bundle-id :integer :not nil]
+   [:ts-and-cs :integer]
+   (tables/foreign-key :content-type-id :content-types :id)
+   (tables/foreign-key :bundle-id :bundles :id)))
+
+(def integration-categories
+  (tables/create-table-sql
+   :integration-categories
+   (tables/table-id)
+   [:integration-id :int :not nil]
+   [:category-id :int :not nil]
+   (tables/foreign-key :integration-id :integrations :id)
+   (tables/foreign-key :category-id :categories :id)))
 
 (def jobs
   (tables/create-table-sql
@@ -197,6 +220,8 @@
   (sql/format bundles)
   (sql/format feeds)
   (sql/format feed-categories)
+  (sql/format integrations)
+  (sql/format integration-categories)
   (sql/format providers)
   (sql/format businesses)
   (sql/format user-sectors)
diff --git a/src/source/migrations/001_init_master_db.clj b/src/source/migrations/001_init_master_db.clj
index 68ea8a2e..358a72a6 100644
--- a/src/source/migrations/001_init_master_db.clj
+++ b/src/source/migrations/001_init_master_db.clj
@@ -80,6 +80,8 @@
       :bundles
       :feeds
       :feed-categories
+      :integrations
+      :integration-categories
       :providers
       :businesses
       :user-sectors
diff --git a/src/source/services/integration_categories.clj b/src/source/services/integration_categories.clj
new file mode 100644
index 00000000..09776a3d
--- /dev/null
+++ b/src/source/services/integration_categories.clj
@@ -0,0 +1,46 @@
+(ns source.services.integration-categories
+  (:require [source.db.interface :as db]
+            [source.db.honey :as hon]
+            [honey.sql.helpers :as hsql]))
+
+(defn integration-categories
+  ([ds] (integration-categories ds {}))
+  ([ds {:keys [where] :as opts}]
+   (->> {:tname :integration-categories
+         :where where
+         :ret :*}
+        (merge opts)
+        (db/find ds))))
+
+(defn insert-integration-category! [ds {:keys [_data _ret] :as opts}]
+  (->> {:tname :integration-categories}
+       (merge opts)
+       (db/insert! ds)))
+
+(defn delete-integration-category! [ds {:keys [id where] :as opts}]
+  (->> {:tname :integration-categories
+        :where (if (some? id)
+                 [:= :id id]
+                 where)
+        :ret :1}
+       (merge opts)
+       (db/delete! ds)))
+
+(defn categories-by-integration [ds {:keys [integration-id where] :as _opts}]
+  (hon/execute! ds
+                {:select [[:integration-categories.category-id :id] :name]
+                 :from :categories
+                 :join [:integration-categories [:= :integration-categories.category-id :categories.id]]
+                 :where (if (some? integration-id)
+                          [:= :integration-id integration-id]
+                          where)}
+                {:ret :*}))
+
+(defn category-id [ds {:keys [integration-id where] :as opts}]
+  (->> {:tname :integration-categories
+        :where (if (some? integration-id)
+                 [:= :integration-id integration-id]
+                 where)
+        :ret :1}
+       (merge opts)
+       (db/find ds)))
diff --git a/src/source/services/integrations.clj b/src/source/services/integrations.clj
new file mode 100644
index 00000000..b5f935ad
--- /dev/null
+++ b/src/source/services/integrations.clj
@@ -0,0 +1,35 @@
+(ns source.services.integrations
+  (:require [source.db.interface :as db]))
+
+(defn insert-integration! [ds {:keys [data] :as opts}]
+  (->> {:tname :integrations
+        :data data
+        :ret :1}
+       (merge opts)
+       (db/insert! ds)))
+
+(defn update-integration! [ds {:keys [id data where] :as opts}]
+  (->> {:tname :integrations
+        :values data
+        :where (if (some? id) [:= :id id] where)
+        :ret :1}
+       (merge opts)
+       (db/update! ds)))
+
+(defn integrations
+  ([ds] (integrations ds {}))
+  ([ds {:keys [where] :as opts}]
+   (->> {:tname :feeds
+         :where where
+         :ret :*}
+        (merge opts)
+        (db/find ds))))
+
+(defn integration [ds {:keys [id where] :as opts}]
+  (->> {:tname :integrations
+        :where (if (some? id)
+                 [:= :id id]
+                 where)
+        :ret :1}
+       (merge opts)
+       (db/find ds)))

From 1386514ca238f15f025972edcfabd35ca3fc8147 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 17 Sep 2025 10:52:08 +0200
Subject: [PATCH 096/391] remove integrations table and rely on existing
 bundles table instead

---
 src/source/db/master.clj                      | 36 ++++++---------
 src/source/migrate.clj                        |  7 +--
 src/source/migrations/001_init_master_db.clj  |  3 +-
 src/source/services/bundle_categories.clj     | 45 ++++++++++++++++++
 src/source/services/bundles.clj               | 32 +++++++++----
 .../services/integration_categories.clj       | 46 -------------------
 src/source/services/integrations.clj          | 35 --------------
 7 files changed, 86 insertions(+), 118 deletions(-)
 create mode 100644 src/source/services/bundle_categories.clj
 delete mode 100644 src/source/services/integration_categories.clj
 delete mode 100644 src/source/services/integrations.clj

diff --git a/src/source/db/master.clj b/src/source/db/master.clj
index e9ac4f41..71300bda 100644
--- a/src/source/db/master.clj
+++ b/src/source/db/master.clj
@@ -55,14 +55,27 @@
   (tables/create-table-sql
    :bundles
    (tables/table-id)
+   [:name :text :not nil]
    [:uuid :text :not nil :unique]
    [:user-id :integer]
    [:video :integer :not nil [:default 0]]
    [:podcast :integer :not nil [:default 0]]
    [:blog :integer :not nil [:default 0]]
    [:hash :text]
+   [:content-type-id :integer :not nil]
+   [:ts-and-cs :integer]
+   (tables/foreign-key :content-type-id :content-types :id)
    (tables/foreign-key :user-id :users :id)))
 
+(def bundle-categories
+  (tables/create-table-sql
+   :bundle-categories
+   (tables/table-id)
+   [:bundle-id :int :not nil]
+   [:category-id :int :not nil]
+   (tables/foreign-key :bundle-id :bundles :id)
+   (tables/foreign-key :category-id :categories :id)))
+
 (def feeds
   (tables/create-table-sql
    :feeds
@@ -161,26 +174,6 @@
    (tables/foreign-key :creator-id :users :id)
    (tables/foreign-key :content-type-id :content-types :id)))
 
-(def integrations
-  (tables/create-table-sql
-   :integrations
-   (tables/table-id)
-   [:name :text :not nil]
-   [:content-type-id :integer :not nil]
-   [:bundle-id :integer :not nil]
-   [:ts-and-cs :integer]
-   (tables/foreign-key :content-type-id :content-types :id)
-   (tables/foreign-key :bundle-id :bundles :id)))
-
-(def integration-categories
-  (tables/create-table-sql
-   :integration-categories
-   (tables/table-id)
-   [:integration-id :int :not nil]
-   [:category-id :int :not nil]
-   (tables/foreign-key :integration-id :integrations :id)
-   (tables/foreign-key :category-id :categories :id)))
-
 (def jobs
   (tables/create-table-sql
    :jobs
@@ -218,10 +211,9 @@
   (sql/format categories)
   (sql/format baselines)
   (sql/format bundles)
+  (sql/format bundle-categories)
   (sql/format feeds)
   (sql/format feed-categories)
-  (sql/format integrations)
-  (sql/format integration-categories)
   (sql/format providers)
   (sql/format businesses)
   (sql/format user-sectors)
diff --git a/src/source/migrate.clj b/src/source/migrate.clj
index c1bca5c0..34eec135 100644
--- a/src/source/migrate.clj
+++ b/src/source/migrate.clj
@@ -4,7 +4,8 @@
             [k16.mallard.loader.fs :as loader.fs]
             [next.jdbc :as jdbc]
             [source.db.util :as db.util]
-            [source.db.honey :as db]))
+            [source.db.honey :as db]
+            [source.db.tables :as tables]))
 
 ;; This is our interface for running migrations.
 ;;
@@ -37,10 +38,10 @@
                    {:db db-migrate
                     :table-name "bundle_migrations"})
         ds-master (db.util/conn :master)
-        bundles (try
+        bundles (if (some #(= % "bundles") (tables/table-names ds-master))
                   (db/find ds-master {:tname :bundles
                                       :ret :*})
-                  (catch Exception _ []))]
+                  [])]
     (run!
      (fn [{:keys [id]}]
        (let [db-name (db.util/db-name :bundle id)
diff --git a/src/source/migrations/001_init_master_db.clj b/src/source/migrations/001_init_master_db.clj
index 358a72a6..80f53073 100644
--- a/src/source/migrations/001_init_master_db.clj
+++ b/src/source/migrations/001_init_master_db.clj
@@ -78,10 +78,9 @@
       :cadences
       :baselines
       :bundles
+      :bundle-categories
       :feeds
       :feed-categories
-      :integrations
-      :integration-categories
       :providers
       :businesses
       :user-sectors
diff --git a/src/source/services/bundle_categories.clj b/src/source/services/bundle_categories.clj
new file mode 100644
index 00000000..183aacc2
--- /dev/null
+++ b/src/source/services/bundle_categories.clj
@@ -0,0 +1,45 @@
+(ns source.services.bundle-categories
+  (:require [source.db.interface :as db]
+            [source.db.honey :as hon]))
+
+(defn bundle-categories
+  ([ds] (bundle-categories ds {}))
+  ([ds {:keys [where] :as opts}]
+   (->> {:tname :bundle-categories
+         :where where
+         :ret :*}
+        (merge opts)
+        (db/find ds))))
+
+(defn insert-bundle-category! [ds {:keys [_data _ret] :as opts}]
+  (->> {:tname :bundle-categories}
+       (merge opts)
+       (db/insert! ds)))
+
+(defn delete-bundle-category! [ds {:keys [id where] :as opts}]
+  (->> {:tname :bundle-categories
+        :where (if (some? id)
+                 [:= :id id]
+                 where)
+        :ret :1}
+       (merge opts)
+       (db/delete! ds)))
+
+(defn categories-by-bundle [ds {:keys [bundle-id where] :as _opts}]
+  (hon/execute! ds
+                {:select [[:bundle-categories.category-id :id] :name]
+                 :from :categories
+                 :join [:bundle-categories [:= :bundle-categories.category-id :categories.id]]
+                 :where (if (some? bundle-id)
+                          [:= :bundle-id bundle-id]
+                          where)}
+                {:ret :*}))
+
+(defn category-id [ds {:keys [bundle-id where] :as opts}]
+  (->> {:tname :bundle-categories
+        :where (if (some? bundle-id)
+                 [:= :bundle-id bundle-id]
+                 where)
+        :ret :1}
+       (merge opts)
+       (db/find ds)))
diff --git a/src/source/services/bundles.clj b/src/source/services/bundles.clj
index 27d7f946..52df888f 100644
--- a/src/source/services/bundles.clj
+++ b/src/source/services/bundles.clj
@@ -1,6 +1,28 @@
 (ns source.services.bundles
   (:require [source.db.interface :as db]))
 
+(defn insert-bundle! [ds {:keys [_values _ret] :as opts}]
+  (->> {:tname :bundles}
+       (merge opts)
+       (db/insert! ds)))
+
+(defn update-bundle! [ds {:keys [id data where] :as opts}]
+  (->> {:tname :bundles
+        :values data
+        :where (if (some? id) [:= :id id] where)
+        :ret :1}
+       (merge opts)
+       (db/update! ds)))
+
+(defn bundles
+  ([ds] (bundles ds {}))
+  ([ds {:keys [where] :as opts}]
+   (->> {:tname :bundles
+         :where where
+         :ret :*}
+        (merge opts)
+        (db/find ds))))
+
 (defn bundle [ds {:keys [id where] :as opts}]
   (->> {:tname :bundles
         :where (if (some? id)
@@ -9,13 +31,3 @@
         :ret :1}
        (merge opts)
        (db/find ds)))
-
-(defn insert-bundle! [ds {:keys [_values _ret] :as opts}]
-  (->> {:tname :bundles}
-       (merge opts)
-       (db/insert! ds)))
-
-(comment 
-  (require '[source.db.util :as db.util])
-  (db/find (db.util/conn :master) {:tname ""})
-  ())
diff --git a/src/source/services/integration_categories.clj b/src/source/services/integration_categories.clj
deleted file mode 100644
index 09776a3d..00000000
--- a/src/source/services/integration_categories.clj
+++ /dev/null
@@ -1,46 +0,0 @@
-(ns source.services.integration-categories
-  (:require [source.db.interface :as db]
-            [source.db.honey :as hon]
-            [honey.sql.helpers :as hsql]))
-
-(defn integration-categories
-  ([ds] (integration-categories ds {}))
-  ([ds {:keys [where] :as opts}]
-   (->> {:tname :integration-categories
-         :where where
-         :ret :*}
-        (merge opts)
-        (db/find ds))))
-
-(defn insert-integration-category! [ds {:keys [_data _ret] :as opts}]
-  (->> {:tname :integration-categories}
-       (merge opts)
-       (db/insert! ds)))
-
-(defn delete-integration-category! [ds {:keys [id where] :as opts}]
-  (->> {:tname :integration-categories
-        :where (if (some? id)
-                 [:= :id id]
-                 where)
-        :ret :1}
-       (merge opts)
-       (db/delete! ds)))
-
-(defn categories-by-integration [ds {:keys [integration-id where] :as _opts}]
-  (hon/execute! ds
-                {:select [[:integration-categories.category-id :id] :name]
-                 :from :categories
-                 :join [:integration-categories [:= :integration-categories.category-id :categories.id]]
-                 :where (if (some? integration-id)
-                          [:= :integration-id integration-id]
-                          where)}
-                {:ret :*}))
-
-(defn category-id [ds {:keys [integration-id where] :as opts}]
-  (->> {:tname :integration-categories
-        :where (if (some? integration-id)
-                 [:= :integration-id integration-id]
-                 where)
-        :ret :1}
-       (merge opts)
-       (db/find ds)))
diff --git a/src/source/services/integrations.clj b/src/source/services/integrations.clj
deleted file mode 100644
index b5f935ad..00000000
--- a/src/source/services/integrations.clj
+++ /dev/null
@@ -1,35 +0,0 @@
-(ns source.services.integrations
-  (:require [source.db.interface :as db]))
-
-(defn insert-integration! [ds {:keys [data] :as opts}]
-  (->> {:tname :integrations
-        :data data
-        :ret :1}
-       (merge opts)
-       (db/insert! ds)))
-
-(defn update-integration! [ds {:keys [id data where] :as opts}]
-  (->> {:tname :integrations
-        :values data
-        :where (if (some? id) [:= :id id] where)
-        :ret :1}
-       (merge opts)
-       (db/update! ds)))
-
-(defn integrations
-  ([ds] (integrations ds {}))
-  ([ds {:keys [where] :as opts}]
-   (->> {:tname :feeds
-         :where where
-         :ret :*}
-        (merge opts)
-        (db/find ds))))
-
-(defn integration [ds {:keys [id where] :as opts}]
-  (->> {:tname :integrations
-        :where (if (some? id)
-                 [:= :id id]
-                 where)
-        :ret :1}
-       (merge opts)
-       (db/find ds)))

From d36f85de944d302d42df54a97f2147d8108f0894 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 17 Sep 2025 11:04:13 +0200
Subject: [PATCH 097/391] added services to interface

---
 src/source/services/interface.clj | 31 ++++++++++++++++++++++++++++++-
 1 file changed, 30 insertions(+), 1 deletion(-)

diff --git a/src/source/services/interface.clj b/src/source/services/interface.clj
index 5e377113..9ef6a73b 100644
--- a/src/source/services/interface.clj
+++ b/src/source/services/interface.clj
@@ -4,6 +4,7 @@
              [source.services.auth :as auth]
              [source.services.xml-schemas :as xml]
              [source.services.bundles :as bundles]
+             [source.services.bundle-categories :as bundle-categories]
              [source.services.providers :as providers]
              [source.services.feeds :as feeds]
              [source.services.incoming-posts :as incoming-posts]
@@ -52,9 +53,37 @@
 (defn selection-schema [ds {:keys [_id] :as opts}]
   (xml/selection-schema ds opts))
 
+(defn insert-bundle! [ds {:keys [_values _ret] :as opts}]
+  (bundles/insert-bundle! ds opts))
+
+(defn update-bundle! [ds {:keys [_id _data _where] :as opts}]
+  (bundles/update-bundle! ds opts))
+
+(defn bundles
+  ([ds] (bundles ds {}))
+  ([ds {:keys [_where] :as opts}]
+   (bundles/bundles opts ds)))
+
 (defn bundle [ds {:keys [_id _where] :as opts}]
   (bundles/bundle ds opts))
 
+(defn bundle-categories
+  ([ds] (bundle-categories ds {}))
+  ([ds {:keys [_where] :as opts}]
+   (bundle-categories/bundle-categories ds opts)))
+
+(defn insert-bundle-category! [ds {:keys [_data _ret] :as opts}]
+  (bundle-categories/insert-bundle-category! ds opts))
+
+(defn delete-bundle-category! [ds {:keys [_id _where] :as opts}]
+  (delete-bundle-category! ds opts))
+
+(defn categories-by-bundle [ds {:keys [_bundle-id _where] :as opts}]
+  (bundle-categories/categories-by-bundle ds opts))
+
+(defn category-id-by-bundle [ds {:keys [_bundle-id _where] :as opts}]
+  (bundle-categories/category-id ds opts))
+
 (defn selection-schemas
   ([ds]
    (selection-schemas ds {}))
@@ -179,7 +208,7 @@
 (defn categories-by-feed [ds {:keys [_feed-id _where] :as opts}]
   (feed-categories/categories-by-feed ds opts))
 
-(defn category-id [ds {:keys [_feed-id _where] :as opts}]
+(defn category-id-by-feed [ds {:keys [_feed-id _where] :as opts}]
   (feed-categories/category-id ds opts))
 
 (defn user-sectors

From 4dd6628401320f777389997ff552924f068d6396 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 17 Sep 2025 12:28:48 +0200
Subject: [PATCH 098/391] fixed incorrect argument order in bundles call

---
 src/source/services/interface.clj | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/source/services/interface.clj b/src/source/services/interface.clj
index 9ef6a73b..b648a5d4 100644
--- a/src/source/services/interface.clj
+++ b/src/source/services/interface.clj
@@ -62,7 +62,7 @@
 (defn bundles
   ([ds] (bundles ds {}))
   ([ds {:keys [_where] :as opts}]
-   (bundles/bundles opts ds)))
+   (bundles/bundles ds opts)))
 
 (defn bundle [ds {:keys [_id _where] :as opts}]
   (bundles/bundle ds opts))

From 5100019158b17c1fca1d3eebbdb7ebd06fae3b25 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Thu, 18 Sep 2025 16:34:02 +0200
Subject: [PATCH 099/391] fixed bundle migrations

---
 .../bundle_migrations/001_init_bundle_db.clj  |  7 ++++-
 src/source/migrate.clj                        | 29 +++++++++----------
 2 files changed, 20 insertions(+), 16 deletions(-)

diff --git a/src/source/bundle_migrations/001_init_bundle_db.clj b/src/source/bundle_migrations/001_init_bundle_db.clj
index 1988589a..48bca5fb 100644
--- a/src/source/bundle_migrations/001_init_bundle_db.clj
+++ b/src/source/bundle_migrations/001_init_bundle_db.clj
@@ -14,4 +14,9 @@
 
 (defn run-down! [context]
   (let [ds-bundle (:db-bundle context)]
-    (tables/drop-all-tables! ds-bundle)))
+    (tables/drop-tables!
+     ds-bundle
+     [:outgoing-posts
+      :post-heuristics
+      :analytics
+      :event-categories])))
diff --git a/src/source/migrate.clj b/src/source/migrate.clj
index 34eec135..09134c9a 100644
--- a/src/source/migrate.clj
+++ b/src/source/migrate.clj
@@ -32,26 +32,25 @@
                   :operations migrations}
                  args)))
 
-(defn run-bundle-migrations [args]
-  (let [db-migrate (jdbc/get-datasource {:dbname (db.util/db-path "migrate") :dbtype "sqlite"})
+(defn migrate-bundle [bundle-id args]
+  (let [db-name (db.util/db-name :bundle bundle-id)
+        context {:db-bundle (jdbc/get-datasource {:dbname (db.util/db-path db-name)
+                                                  :dbtype "sqlite"})}
         datastore (store/create-datastore
-                   {:db db-migrate
-                    :table-name "bundle_migrations"})
-        ds-master (db.util/conn :master)
+                   {:db (:db-bundle context)
+                    :table-name "migrations"})]
+    (mallard/run {:context context
+                  :store datastore
+                  :operations bundle-migrations}
+                 args)))
+
+(defn run-bundle-migrations [args]
+  (let [ds-master (db.util/conn :master)
         bundles (if (some #(= % "bundles") (tables/table-names ds-master))
                   (db/find ds-master {:tname :bundles
                                       :ret :*})
                   [])]
-    (run!
-     (fn [{:keys [id]}]
-       (let [db-name (db.util/db-name :bundle id)
-             context {:db-bundle (jdbc/get-datasource {:dbname (db.util/db-path db-name)
-                                                       :dbtype "sqlite"})}]
-         (mallard/run {:context context
-                       :store datastore
-                       :operations bundle-migrations}
-                      args)))
-     bundles)))
+    (run! #(migrate-bundle (:id %) args) bundles)))
 
 (defn -main [& args]
   (run-bundle-migrations args)

From ba6ed50563fae8549400392b6614b5353720ac77 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Thu, 18 Sep 2025 16:34:51 +0200
Subject: [PATCH 100/391] made schema changes to bundle db

---
 src/source/db/bundle.clj | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/source/db/bundle.clj b/src/source/db/bundle.clj
index a0b1811f..3aff342e 100644
--- a/src/source/db/bundle.clj
+++ b/src/source/db/bundle.clj
@@ -27,7 +27,8 @@
    [:posted-at :datetime]
    (tables/foreign-key :feed-id :feeds :id)
    (tables/foreign-key :creator-id :users :id)
-   (tables/foreign-key :content-type-id :content-types :id)))
+   (tables/foreign-key :content-type-id :content-types :id)
+   [[:unique [:composite :post-id]]]))
 
 (def post-heuristics
   (tables/create-table-sql
@@ -35,7 +36,8 @@
    (tables/table-id)
    [:post-id :integer :not nil]
    [:long-heuristic :integer]
-   [:short-heuristic :integer]))
+   [:short-heuristic :integer]
+   [[:unique [:composite :post-id]]]))
 
 (def analytics
   (tables/create-table-sql

From b7f23e91c257b6305529f0ce8ab639e1256798b8 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Thu, 18 Sep 2025 16:36:27 +0200
Subject: [PATCH 101/391] added and updated services

---
 src/source/services/incoming_posts.clj  | 24 ++++++++++-
 src/source/services/interface.clj       | 31 ++++++++++++++-
 src/source/services/outgoing_posts.clj  | 19 ++++++++-
 src/source/services/post_heuristics.clj | 53 +++++++++++++++++++++++++
 4 files changed, 124 insertions(+), 3 deletions(-)
 create mode 100644 src/source/services/post_heuristics.clj

diff --git a/src/source/services/incoming_posts.clj b/src/source/services/incoming_posts.clj
index cfa76bc5..0578d65c 100644
--- a/src/source/services/incoming_posts.clj
+++ b/src/source/services/incoming_posts.clj
@@ -1,5 +1,6 @@
 (ns source.services.incoming-posts
-  (:require [source.db.interface :as db]))
+  (:require [source.db.interface :as db]
+            [source.db.honey :as hon]))
 
 (defn insert-incoming-post! [ds {:keys [data ret] :as opts}]
   (->> {:tname :incoming-posts
@@ -25,6 +26,16 @@
         (merge opts)
         (db/find ds))))
 
+(defn incoming-posts-with-feeds
+  [ds {:keys [_where] :as opts}]
+  (hon/execute! ds
+                (merge
+                 {:select [[:incoming-posts.id :id] :incoming-posts.post-id :feed-id]
+                  :from :incoming-posts
+                  :join [:feeds [:= :incoming-posts.feed-id :feeds.id]]}
+                 opts)
+                {:ret :*}))
+
 (defn incoming-post [ds {:keys [id where] :as opts}]
   (->> {:tname :incoming-posts
         :where (if (some? id)
@@ -33,3 +44,14 @@
         :ret :1}
        (merge opts)
        (db/find ds)))
+
+(defn categories-by-posts [ds {:keys [_where] :as opts}]
+  (hon/execute! ds
+                (merge
+                 {:select [[:incoming-posts.id :post-id] [:categories.id :id] :categories.name]
+                  :from :incoming-posts
+                  :join [:feeds [:= :incoming-posts.feed-id :feeds.id]]
+                  :left-join [:feed-categories [:= :feed-categories.feed-id :feeds.id]]
+                  :right-join [:categories [:= :categories.id :feed-categories.category-id]]}
+                 opts)
+                {:ret :*}))
diff --git a/src/source/services/interface.clj b/src/source/services/interface.clj
index 9ef6a73b..1db10ec2 100644
--- a/src/source/services/interface.clj
+++ b/src/source/services/interface.clj
@@ -5,9 +5,11 @@
              [source.services.xml-schemas :as xml]
              [source.services.bundles :as bundles]
              [source.services.bundle-categories :as bundle-categories]
+             [source.services.post-heuristics :as post-heuristics]
              [source.services.providers :as providers]
              [source.services.feeds :as feeds]
              [source.services.incoming-posts :as incoming-posts]
+             [source.services.outgoing-posts :as outgoing-posts]
              [source.services.cadences :as cadences]
              [source.services.baselines :as baselines]
              [source.services.content-types :as content-types]
@@ -62,7 +64,7 @@
 (defn bundles
   ([ds] (bundles ds {}))
   ([ds {:keys [_where] :as opts}]
-   (bundles/bundles opts ds)))
+   (bundles/bundles ds opts)))
 
 (defn bundle [ds {:keys [_id _where] :as opts}]
   (bundles/bundle ds opts))
@@ -84,6 +86,29 @@
 (defn category-id-by-bundle [ds {:keys [_bundle-id _where] :as opts}]
   (bundle-categories/category-id ds opts))
 
+(defn insert-post-heuristics! [ds {:keys [_data] :as opts}]
+  (post-heuristics/insert-post-heuristics! ds opts))
+
+(defn update-post-heuristics! [ds {:keys [_id _data _where] :as opts}]
+  (post-heuristics/update-post-heuristics! ds opts))
+
+(defn post-heuristics
+  ([ds] (post-heuristics ds {}))
+  ([ds {:keys [_where] :as opts}]
+   (post-heuristics/post-heuristics ds opts)))
+
+(defn upsert-post-heuristics! [ds {:keys [_data] :as opts}]
+  (post-heuristics/upsert-post-heuristics! ds opts))
+
+(defn post-heuristic [ds {:keys [_id _where] :as opts}]
+  (post-heuristics/post-heuristic ds opts))
+
+(defn top-posts-by-heuristic [ds {:keys [_select _limit _heuristic] :as opts}]
+  (post-heuristics/top-posts-by-heuristic ds opts))
+
+(defn upsert-outgoing-posts! [ds {:keys [_data] :as opts}]
+  (outgoing-posts/upsert-outgoing-posts! ds opts))
+
 (defn selection-schemas
   ([ds]
    (selection-schemas ds {}))
@@ -156,6 +181,10 @@
   ([ds {:keys [_where] :as opts}]
    (incoming-posts/incoming-posts ds opts)))
 
+(defn incoming-posts-with-feeds
+  [ds {:keys [_where] :as opts}]
+  (incoming-posts/incoming-posts-with-feeds ds opts))
+
 (defn incoming-post [ds id]
   (incoming-posts/incoming-post ds id))
 
diff --git a/src/source/services/outgoing_posts.clj b/src/source/services/outgoing_posts.clj
index 6ca27f7c..1a02f9d1 100644
--- a/src/source/services/outgoing_posts.clj
+++ b/src/source/services/outgoing_posts.clj
@@ -1,6 +1,7 @@
 (ns source.services.outgoing-posts
   (:require [source.db.interface :as db]
-            [source.db.util :as db.util]))
+            [source.db.honey :as hon]
+            [honey.sql.helpers :as hsql]))
 
 (defn outgoing-post [ds {:keys [id where] :as opts}]
   (->> {:tname :outgoing-posts
@@ -11,3 +12,19 @@
        (merge opts)
        (db/find ds)))
 
+(defn upsert-outgoing-posts! [ds {:keys [data]}]
+  (hon/execute!
+   ds
+   (-> (hsql/insert-into :outgoing-posts)
+       (hsql/values data)
+       (assoc :on-conflict [:post-id])
+       (assoc :do-update-set {:feed-id          :excluded.feed-id
+                              :title            :excluded.title
+                              :info             :excluded.info
+                              :url              :excluded.url
+                              :stream-url       :excluded.stream-url
+                              :creator-id       :excluded.creator-id
+                              :season           :excluded.season
+                              :episode          :excluded.episode
+                              :content-type-id  :excluded.content-type-id
+                              :posted-at        :excluded.posted-at}))))
diff --git a/src/source/services/post_heuristics.clj b/src/source/services/post_heuristics.clj
new file mode 100644
index 00000000..30c93c57
--- /dev/null
+++ b/src/source/services/post_heuristics.clj
@@ -0,0 +1,53 @@
+(ns source.services.post-heuristics
+  (:require [source.db.interface :as db]
+            [source.db.honey :as hon]
+            [honey.sql.helpers :as hsql]))
+
+(defn insert-post-heuristics! [ds {:keys [data] :as opts}]
+  (->> {:tname :post-heuristics
+        :data data
+        :ret :1}
+       (merge opts)
+       (db/insert! ds)))
+
+(defn update-post-heuristics! [ds {:keys [id _data where] :as opts}]
+  (->> {:tname :post-heuristics
+        :where (if (some? id) [:= :id id] where)
+        :ret :1}
+       (merge opts)
+       (db/update! ds)))
+
+(defn upsert-post-heuristics! [ds {:keys [data]}]
+  (hon/execute!
+   ds
+   (-> (hsql/insert-into :post-heuristics)
+       (hsql/values data)
+       (assoc :on-conflict [:post-id])
+       (assoc :do-update-set {:long-heuristic :excluded.long-heuristic
+                              :short-heuristic :excluded.short-heuristic}))))
+
+(defn post-heuristics
+  ([ds] (post-heuristics ds {}))
+  ([ds {:keys [where] :as opts}]
+   (->> {:tname :post-heuristics
+         :where where
+         :ret :*}
+        (merge opts)
+        (db/find ds))))
+
+(defn top-posts-by-heuristic [ds {:keys [select limit heuristic] :as _opts}]
+  (hon/execute! ds
+                (merge {:select (or select :*)
+                        :from :post-heuristics
+                        :order-by [[heuristic :desc]]
+                        :limit limit})
+                {:ret :*}))
+
+(defn post-heuristic [ds {:keys [id where] :as opts}]
+  (->> {:tname :post-heuristics
+        :where (if (some? id)
+                 [:= :id id]
+                 where)
+        :ret :1}
+       (merge opts)
+       (db/find ds)))

From 46844bc2b7787db122fa6ae9348a5cedc4d97e44 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Thu, 18 Sep 2025 16:38:05 +0200
Subject: [PATCH 102/391] added endpoints for getting and adding integrations
 and integration categories, also a job to handle rudimentary long heuristics
 and pulling incoming posts into outgoing posts

---
 src/source/jobs/handlers.clj                 | 53 +++++++++++++-
 src/source/routes/integration.clj            | 23 ++++++
 src/source/routes/integration_categories.clj | 36 ++++++++++
 src/source/routes/integrations.clj           | 75 ++++++++++++++++++++
 src/source/routes/reitit.clj                 | 12 ++++
 5 files changed, 198 insertions(+), 1 deletion(-)
 create mode 100644 src/source/routes/integration.clj
 create mode 100644 src/source/routes/integration_categories.clj
 create mode 100644 src/source/routes/integrations.clj

diff --git a/src/source/jobs/handlers.clj b/src/source/jobs/handlers.clj
index a56bedcb..96287f36 100644
--- a/src/source/jobs/handlers.clj
+++ b/src/source/jobs/handlers.clj
@@ -1,6 +1,9 @@
 (ns source.jobs.handlers
   (:require [source.services.interface :as services]
-            [source.util :as util]))
+            [source.util :as util]
+            [source.services.incoming-posts :as incoming-posts]
+            [source.db.util :as db.util]
+            [clojure.set :as set]))
 
 (defmulti handler
   (fn [opts]
@@ -46,3 +49,51 @@
          extended-posts))
       (catch Exception _ :fail))))
 
+; run long heuristics and pull the highest scoring incoming posts into the bundle's outgoing posts
+(defmethod handler :update-bundle [_]
+  (fn [{:keys [args ds]}]
+    (println "hello" (get args :bundle-id) args)
+    (let [{:keys [bundle-id categories]} args
+          ds-bundle (db.util/conn :bundle bundle-id)
+          incoming-posts (services/incoming-posts-with-feeds (db.util/conn) {:where [:= :feeds.state "live"]})
+          posts-categories (incoming-posts/categories-by-posts ds {:where [:= :state "live"]})]
+      (run!
+       (fn [post]
+          ; calculate score for post
+          ; determine number of categories matched
+         (let [; get vector of category ids in the given post, e.g. [1 3]
+               post-categories-vec (reduce (fn [acc {:keys [post-id id]}]
+                                             (if (= post-id (:id post))
+                                               (conj acc id)
+                                               acc)) [] posts-categories)
+               ; get vector of category ids in categories to match, e.g. [1 2 3 4]
+               match-categories-vec (reduce (fn [acc {:keys [id]}]
+                                              (conj acc id)) [] categories)
+               ; get number of matches between the 2 vectors, e.g. #{1 3} intersect #{1 2 3 4} -> (count #{1 3}) -> 2
+               matches (count (set/intersection (set post-categories-vec)
+                                                (set match-categories-vec)))]
+            ; use matches as a score and upsert long-heuristic for this post
+           (services/upsert-post-heuristics! ds-bundle {:data [{:post-id (:id post)
+                                                                :long-heuristic matches}]})))
+       incoming-posts)
+
+      ; pull highest scored posts by long heuristics into outgoing posts
+      (let [; top 30 post-heuristics records ordered by long heuristic in descending order
+            top-by-long-heuristics (services/top-posts-by-heuristic ds-bundle
+                                                                    {:heuristic :long-heuristic
+                                                                     :limit 30})
+            ; convert into a vector of id numbers
+            ids (reduce (fn [acc {:keys [id]}]
+                          (conj acc id)) [] top-by-long-heuristics)
+
+            ; get all incoming posts with the above id numbers
+            posts-in (services/incoming-posts ds {:where [:in :id ids]})
+            ; remove redacted posts
+            outgoing-posts (reduce (fn [acc {:keys [redacted] :as post}]
+                                     (if (:= redacted 0)
+                                       (conj acc (dissoc post :redacted))
+                                       acc))
+                                   [] posts-in)]
+        (when (seq posts-in)
+          (services/upsert-outgoing-posts! ds-bundle {:data outgoing-posts}))))))
+
diff --git a/src/source/routes/integration.clj b/src/source/routes/integration.clj
new file mode 100644
index 00000000..e5a6a752
--- /dev/null
+++ b/src/source/routes/integration.clj
@@ -0,0 +1,23 @@
+(ns source.routes.integration
+  (:require [ring.util.response :as res]
+            [source.services.interface :as services]))
+
+(defn get
+  {:summary "get integration by id"
+   :parameters {:path [:map [:id {:title "id"
+                                  :description "integration id"} :int]]}
+   :responses {200 {:body [:map
+                           [:name :string]
+                           [:uuid :string]
+                           [:user-id :int]
+                           [:video :int]
+                           [:podcast :int]
+                           [:blog :int]
+                           [:hash [:maybe :string]]
+                           [:content-type-id :int]
+                           [:ts-and-cs [:maybe :int]]]}
+               401 {:body [:map [:message :string]]}
+               403 {:body [:map [:message :string]]}}}
+
+  [{:keys [ds path-params] :as _request}]
+  (res/response (services/bundle ds {:id (:id path-params)})))
diff --git a/src/source/routes/integration_categories.clj b/src/source/routes/integration_categories.clj
new file mode 100644
index 00000000..183d0727
--- /dev/null
+++ b/src/source/routes/integration_categories.clj
@@ -0,0 +1,36 @@
+(ns source.routes.integration-categories
+  (:require [source.services.interface :as services]
+            [ring.util.response :as res]))
+
+(defn get
+  {:summary "get all categories belonging to the integration with the given id"
+   :parameters {:path [:map [:id {:title "id"
+                                  :description "integration id"} :int]]}
+   :responses {200 {:body [:vector
+                           [:map
+                            [:id :int]
+                            [:name :string]
+                            [:bundle-id :int]
+                            [:category-id :int]]]}}}
+
+  [{:keys [ds path-params] :as _request}]
+  (->> (services/categories-by-bundle ds {:bundle-id (:id path-params)})
+       (res/response)))
+
+(defn post
+  {:summary "update categories belonging to the integration with the given id"
+   :parameters {:path [:map [:id {:title "id"
+                                  :description "integration id"} :int]]
+                :body [:vector
+                       [:map
+                        [:id :int]
+                        [:name :string]]]}
+   :responses {200 {:body [:map [:message :string]]}}}
+
+  [{:keys [ds path-params body] :as _request}]
+  (let [update-data (reduce (fn [acc {:keys [id]}]
+                              (conj acc {:bundle-id (:id path-params)
+                                         :category-id id})) [] body)]
+    (services/delete-bundle-category! ds {:where [:= :integration-id (:id path-params)]})
+    (services/insert-bundle-category! ds {:data update-data})
+    (res/response {:message "successfully updated integration categories"})))
diff --git a/src/source/routes/integrations.clj b/src/source/routes/integrations.clj
new file mode 100644
index 00000000..02a35811
--- /dev/null
+++ b/src/source/routes/integrations.clj
@@ -0,0 +1,75 @@
+(ns source.routes.integrations
+  (:require [source.services.interface :as services]
+            [ring.util.response :as res]
+            [source.util :as utils]
+            [source.migrate :as migrate]
+            [congest.jobs :as congest]
+            [source.jobs.core :as jobs]))
+
+(defn get
+  {:summary "get all integrations"
+   :responses {200 {:body [:vector
+                           [:map
+                            [:name :string]
+                            [:uuid :string]
+                            [:user-id :int]
+                            [:video :int]
+                            [:podcast :int]
+                            [:blog :int]
+                            [:hash [:maybe :string]]
+                            [:content-type-id :int]
+                            [:ts-and-cs [:maybe :int]]]]}
+               401 {:body [:map [:message :string]]}
+               403 {:body [:map [:message :string]]}}}
+
+  [{:keys [ds] :as _request}]
+  (res/response (services/bundles ds)))
+
+(defn post
+  {:summary "add an integration"
+   :parameters {:body [:map
+                       [:name :string]
+                       [:content-type-id :int]
+                       [:ts-and-cs {:optional true} :int]
+                       [:categories [:vector
+                                     [:map
+                                      [:id :int]
+                                      [:name :string]]]]]}
+   :responses {201 {:body [:map [:message :string]]}
+               401 {:body [:map [:message :string]]}
+               403 {:body [:map [:message :string]]}}}
+
+  [{:keys [js ds store user body] :as _request}]
+  (let [new-bundle (services/insert-bundle! ds {:data (merge
+                                                       (dissoc body :categories)
+                                                       {:user-id (:id user)
+                                                        :uuid (utils/uuid)})
+                                                :ret :1})
+        update-data (reduce (fn [acc {:keys [id]}]
+                              (conj acc {:bundle-id (:id new-bundle)
+                                         :category-id id})) [] (:categories body))
+        ; insert bundle categories
+        _ (services/insert-bundle-category! ds {:data update-data})
+        categories-by-bundle (services/categories-by-bundle ds {:bundle-id (:id new-bundle)})]
+
+    (migrate/migrate-bundle (:id new-bundle) ["up"])
+
+    (->> (jobs/prepare-congest-metadata
+          ds
+          store
+          {:id (str "bundle_" (:id new-bundle))
+           :initial-delay #_(* 1000 60 60 24) 0
+           :auto-start true
+           :stop-after-fail false,
+           :interval #_(* 1000 60 60 24) (* 1000 60)
+           :recurring? true
+           :ds ds
+           :args {:bundle-id (:id new-bundle)
+                  :categories categories-by-bundle}
+           :handler :update-bundle
+           :created-at (utils/get-utc-timestamp-string)
+           :sleep false})
+         (congest/register! js))
+
+    (res/response {:message "successfully added integration"})))
+
diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index 2ee68189..edb3690a 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -38,6 +38,9 @@
             [source.routes.feeds :as feeds]
             [source.routes.feed :as feed]
             [source.routes.feed-categories :as feed-categories]
+            [source.routes.integrations :as integrations]
+            [source.routes.integration :as integration]
+            [source.routes.integration-categories :as integration-categories]
             [source.routes.posts :as posts]
             [source.routes.admin-feeds :as admin-feeds]
             [source.routes.xml :as xml]
@@ -153,6 +156,15 @@
       [""               {:get content-types/get}]
       ["/:id"           {:get content-type/get}]]
 
+     ["/integrations"   {:middleware [[mw/apply-auth]]
+                         :tags #{"integrations"}}
+      [""               (route {:get integrations/get
+                                :post integrations/post})]
+      ["/:id"
+       [""              (route {:get integration/get})]
+       ["/categories"   (route {:get integration-categories/get
+                                :post integration-categories/post})]]]
+
      ["/feeds"          {:middleware [[mw/apply-auth]]
                          :tags #{"feeds"}}
       [""               (route {:get feeds/get

From a34745ec3075f0714043646db4243706132a718c Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Thu, 18 Sep 2025 19:52:00 +0200
Subject: [PATCH 103/391] moved bundle-categories to bundle database and
 refactored services and endpoints as such

---
 .../bundle_migrations/001_init_bundle_db.clj  |  2 ++
 src/source/db/bundle.clj                      | 10 ++++++++++
 src/source/db/master.clj                      | 10 ----------
 src/source/migrations/001_init_master_db.clj  |  1 -
 src/source/routes/integration_categories.clj  | 14 +++++++------
 src/source/routes/integrations.clj            | 20 ++++++++++---------
 src/source/services/bundle_categories.clj     |  2 +-
 7 files changed, 32 insertions(+), 27 deletions(-)

diff --git a/src/source/bundle_migrations/001_init_bundle_db.clj b/src/source/bundle_migrations/001_init_bundle_db.clj
index 48bca5fb..d3bcf91c 100644
--- a/src/source/bundle_migrations/001_init_bundle_db.clj
+++ b/src/source/bundle_migrations/001_init_bundle_db.clj
@@ -8,6 +8,7 @@
      ds-bundle
      :source.db.bundle
      [:outgoing-posts
+      :bundle-categories
       :post-heuristics
       :analytics
       :event-categories])))
@@ -17,6 +18,7 @@
     (tables/drop-tables!
      ds-bundle
      [:outgoing-posts
+      :bundle-categories
       :post-heuristics
       :analytics
       :event-categories])))
diff --git a/src/source/db/bundle.clj b/src/source/db/bundle.clj
index 3aff342e..e1a9f720 100644
--- a/src/source/db/bundle.clj
+++ b/src/source/db/bundle.clj
@@ -30,6 +30,15 @@
    (tables/foreign-key :content-type-id :content-types :id)
    [[:unique [:composite :post-id]]]))
 
+(def bundle-categories
+  (tables/create-table-sql
+   :bundle-categories
+   (tables/table-id)
+   [:bundle-id :int :not nil]
+   [:category-id :int :not nil]
+   (tables/foreign-key :bundle-id :bundles :id)
+   (tables/foreign-key :category-id :categories :id)))
+
 (def post-heuristics
   (tables/create-table-sql
    :post-heuristics
@@ -50,6 +59,7 @@
 (comment
   (sql/format event-categories)
   (sql/format outgoing-posts)
+  (sql/format bundle-categories)
   (sql/format post-heuristics)
   (sql/format analytics)
   ())
diff --git a/src/source/db/master.clj b/src/source/db/master.clj
index 71300bda..d5c0690f 100644
--- a/src/source/db/master.clj
+++ b/src/source/db/master.clj
@@ -67,15 +67,6 @@
    (tables/foreign-key :content-type-id :content-types :id)
    (tables/foreign-key :user-id :users :id)))
 
-(def bundle-categories
-  (tables/create-table-sql
-   :bundle-categories
-   (tables/table-id)
-   [:bundle-id :int :not nil]
-   [:category-id :int :not nil]
-   (tables/foreign-key :bundle-id :bundles :id)
-   (tables/foreign-key :category-id :categories :id)))
-
 (def feeds
   (tables/create-table-sql
    :feeds
@@ -211,7 +202,6 @@
   (sql/format categories)
   (sql/format baselines)
   (sql/format bundles)
-  (sql/format bundle-categories)
   (sql/format feeds)
   (sql/format feed-categories)
   (sql/format providers)
diff --git a/src/source/migrations/001_init_master_db.clj b/src/source/migrations/001_init_master_db.clj
index 80f53073..68ea8a2e 100644
--- a/src/source/migrations/001_init_master_db.clj
+++ b/src/source/migrations/001_init_master_db.clj
@@ -78,7 +78,6 @@
       :cadences
       :baselines
       :bundles
-      :bundle-categories
       :feeds
       :feed-categories
       :providers
diff --git a/src/source/routes/integration_categories.clj b/src/source/routes/integration_categories.clj
index 183d0727..80cd6c5e 100644
--- a/src/source/routes/integration_categories.clj
+++ b/src/source/routes/integration_categories.clj
@@ -1,6 +1,7 @@
 (ns source.routes.integration-categories
   (:require [source.services.interface :as services]
-            [ring.util.response :as res]))
+            [ring.util.response :as res]
+            [source.db.util :as db.util]))
 
 (defn get
   {:summary "get all categories belonging to the integration with the given id"
@@ -9,13 +10,14 @@
    :responses {200 {:body [:vector
                            [:map
                             [:id :int]
-                            [:name :string]
-                            [:bundle-id :int]
-                            [:category-id :int]]]}}}
+                            [:name :string]]]}}}
 
   [{:keys [ds path-params] :as _request}]
-  (->> (services/categories-by-bundle ds {:bundle-id (:id path-params)})
-       (res/response)))
+  (let [bundle-ds (db.util/conn :bundle (:id path-params))
+        category-ids (services/category-id-by-bundle bundle-ds {:bundle-id (:id path-params)})
+        id-vec (mapv (fn [{:keys [category-id]}] category-id) category-ids)
+        categories (services/categories ds {:where [:in :id id-vec]})]
+    (res/response categories)))
 
 (defn post
   {:summary "update categories belonging to the integration with the given id"
diff --git a/src/source/routes/integrations.clj b/src/source/routes/integrations.clj
index 02a35811..bb6af66a 100644
--- a/src/source/routes/integrations.clj
+++ b/src/source/routes/integrations.clj
@@ -4,7 +4,8 @@
             [source.util :as utils]
             [source.migrate :as migrate]
             [congest.jobs :as congest]
-            [source.jobs.core :as jobs]))
+            [source.jobs.core :as jobs]
+            [source.db.util :as db.util]))
 
 (defn get
   {:summary "get all integrations"
@@ -45,14 +46,16 @@
                                                        {:user-id (:id user)
                                                         :uuid (utils/uuid)})
                                                 :ret :1})
-        update-data (reduce (fn [acc {:keys [id]}]
-                              (conj acc {:bundle-id (:id new-bundle)
-                                         :category-id id})) [] (:categories body))
+        bundle-categories (reduce (fn [acc {:keys [id]}]
+                                    (conj acc {:bundle-id (:id new-bundle)
+                                               :category-id id})) [] (:categories body))
+        _ (migrate/migrate-bundle (:id new-bundle) ["up"])
         ; insert bundle categories
-        _ (services/insert-bundle-category! ds {:data update-data})
-        categories-by-bundle (services/categories-by-bundle ds {:bundle-id (:id new-bundle)})]
-
-    (migrate/migrate-bundle (:id new-bundle) ["up"])
+        bundle-ds (db.util/conn :bundle (:id new-bundle))
+        _ (services/insert-bundle-category! bundle-ds {:data bundle-categories})
+        category-ids (services/category-id-by-bundle bundle-ds {:bundle-id (:id new-bundle)})
+        id-vec (mapv (fn [{:keys [category-id]}] category-id) category-ids)
+        categories-by-bundle (services/categories ds {:where [:in :id id-vec]})]
 
     (->> (jobs/prepare-congest-metadata
           ds
@@ -72,4 +75,3 @@
          (congest/register! js))
 
     (res/response {:message "successfully added integration"})))
-
diff --git a/src/source/services/bundle_categories.clj b/src/source/services/bundle_categories.clj
index 183aacc2..6f4b9168 100644
--- a/src/source/services/bundle_categories.clj
+++ b/src/source/services/bundle_categories.clj
@@ -40,6 +40,6 @@
         :where (if (some? bundle-id)
                  [:= :bundle-id bundle-id]
                  where)
-        :ret :1}
+        :ret :*}
        (merge opts)
        (db/find ds)))

From 16aaaaaff78da25ced52801ae4cc62f81435ba23 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Thu, 18 Sep 2025 20:37:17 +0200
Subject: [PATCH 104/391] updated schemas to include id

---
 src/source/routes/integration.clj  | 1 +
 src/source/routes/integrations.clj | 1 +
 2 files changed, 2 insertions(+)

diff --git a/src/source/routes/integration.clj b/src/source/routes/integration.clj
index e5a6a752..fe5a2685 100644
--- a/src/source/routes/integration.clj
+++ b/src/source/routes/integration.clj
@@ -7,6 +7,7 @@
    :parameters {:path [:map [:id {:title "id"
                                   :description "integration id"} :int]]}
    :responses {200 {:body [:map
+                           [:id :int]
                            [:name :string]
                            [:uuid :string]
                            [:user-id :int]
diff --git a/src/source/routes/integrations.clj b/src/source/routes/integrations.clj
index bb6af66a..58dbfd2a 100644
--- a/src/source/routes/integrations.clj
+++ b/src/source/routes/integrations.clj
@@ -11,6 +11,7 @@
   {:summary "get all integrations"
    :responses {200 {:body [:vector
                            [:map
+                            [:id :int]
                             [:name :string]
                             [:uuid :string]
                             [:user-id :int]

From ca52090da337502cec7b0eb7a419ff97d1e1d767 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Thu, 18 Sep 2025 22:04:46 +0200
Subject: [PATCH 105/391] added route metadata for content types endpoints

---
 src/source/routes/content_type.clj  | 10 +++++++++-
 src/source/routes/content_types.clj |  9 ++++++++-
 src/source/routes/reitit.clj        |  6 +++---
 3 files changed, 20 insertions(+), 5 deletions(-)

diff --git a/src/source/routes/content_type.clj b/src/source/routes/content_type.clj
index 0eb0b039..521c2fb8 100644
--- a/src/source/routes/content_type.clj
+++ b/src/source/routes/content_type.clj
@@ -2,7 +2,15 @@
   (:require [source.services.interface :as services]
             [ring.util.response :as res]))
 
-(defn get [{:keys [ds path-params] :as _request}]
+(defn get
+  {:summary "get content type by id"
+   :parameters {:path [:map [:id {:title "id"
+                                  :description "content type id"} :int]]}
+   :responses {200 {:body [:map
+                           [:id :int]
+                           [:name :string]]}}}
+
+  [{:keys [ds path-params] :as _request}]
   (->> path-params
        (services/content-type ds)
        (res/response)))
diff --git a/src/source/routes/content_types.clj b/src/source/routes/content_types.clj
index 9b226e1c..c7c2808d 100644
--- a/src/source/routes/content_types.clj
+++ b/src/source/routes/content_types.clj
@@ -2,6 +2,13 @@
   (:require [source.services.interface :as services]
             [ring.util.response :as res]))
 
-(defn get [{:keys [ds] :as _request}]
+(defn get
+  {:summary "get all content types"
+   :responses {200 {:body [:vector
+                           [:map
+                            [:id :int]
+                            [:name :string]]]}}}
+
+  [{:keys [ds] :as _request}]
   (-> (services/content-types ds)
       (res/response)))
diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index edb3690a..ea5ca925 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -152,9 +152,9 @@
      ["/baselines"      {:tags #{"baselines"}}
       [""               (route {:get baselines/get})]]
 
-     ["/content-types"  {:tags #{"content types"}}
-      [""               {:get content-types/get}]
-      ["/:id"           {:get content-type/get}]]
+     ["/contentTypes"  {:tags #{"content types"}}
+      [""               (route {:get content-types/get})]
+      ["/:id"           (route {:get content-type/get})]]
 
      ["/integrations"   {:middleware [[mw/apply-auth]]
                          :tags #{"integrations"}}

From a35bc469ae6b1fb204c0dcf0c8d78de9a8f63b71 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Fri, 19 Sep 2025 14:08:43 +0200
Subject: [PATCH 106/391] addressed comments

---
 src/source/jobs/handlers.clj       | 1 -
 src/source/routes/integrations.clj | 8 ++++----
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/src/source/jobs/handlers.clj b/src/source/jobs/handlers.clj
index 96287f36..4ccde137 100644
--- a/src/source/jobs/handlers.clj
+++ b/src/source/jobs/handlers.clj
@@ -52,7 +52,6 @@
 ; run long heuristics and pull the highest scoring incoming posts into the bundle's outgoing posts
 (defmethod handler :update-bundle [_]
   (fn [{:keys [args ds]}]
-    (println "hello" (get args :bundle-id) args)
     (let [{:keys [bundle-id categories]} args
           ds-bundle (db.util/conn :bundle bundle-id)
           incoming-posts (services/incoming-posts-with-feeds (db.util/conn) {:where [:= :feeds.state "live"]})
diff --git a/src/source/routes/integrations.clj b/src/source/routes/integrations.clj
index 58dbfd2a..5b24f265 100644
--- a/src/source/routes/integrations.clj
+++ b/src/source/routes/integrations.clj
@@ -24,8 +24,8 @@
                401 {:body [:map [:message :string]]}
                403 {:body [:map [:message :string]]}}}
 
-  [{:keys [ds] :as _request}]
-  (res/response (services/bundles ds)))
+  [{:keys [ds user] :as _request}]
+  (res/response (services/bundles ds {:where [:= :user-id (:id user)]})))
 
 (defn post
   {:summary "add an integration"
@@ -62,10 +62,10 @@
           ds
           store
           {:id (str "bundle_" (:id new-bundle))
-           :initial-delay #_(* 1000 60 60 24) 0
+           :initial-delay (* 1000 60 60 24)
            :auto-start true
            :stop-after-fail false,
-           :interval #_(* 1000 60 60 24) (* 1000 60)
+           :interval (* 1000 60 60 24)
            :recurring? true
            :ds ds
            :args {:bundle-id (:id new-bundle)

From 0ba17b9d5186463b20426b31f9c26f15ad7c7349 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 22 Sep 2025 14:23:33 +0200
Subject: [PATCH 107/391] added update endpoint

---
 src/source/email/templates.clj               |  2 +-
 src/source/routes/integration.clj            | 59 +++++++++++++++++++-
 src/source/routes/integration_categories.clj |  9 +--
 src/source/routes/integrations.clj           | 16 +++++-
 src/source/routes/reitit.clj                 |  3 +-
 src/source/services/interface.clj            |  2 +-
 6 files changed, 80 insertions(+), 11 deletions(-)

diff --git a/src/source/email/templates.clj b/src/source/email/templates.clj
index 093ce228..8806d079 100644
--- a/src/source/email/templates.clj
+++ b/src/source/email/templates.clj
@@ -94,7 +94,7 @@
           [:br] [:br]
           "Click on the link below to go to your dashboard and view your feed."]]
         (button {:text "View your feed"
-                 :redirect (str (conf/read-value :cors-origin) "/dashboard/feeds/" feed-id)})
+                 :redirect (str (conf/read-value :cors-origin) "/dashboard/feed/" feed-id)})
         [:tr
          [:td {:class "body"
                :style "padding: 40px; text-align: left; font-size: 16px; line-height: 1.6;"}
diff --git a/src/source/routes/integration.clj b/src/source/routes/integration.clj
index fe5a2685..ed76ee19 100644
--- a/src/source/routes/integration.clj
+++ b/src/source/routes/integration.clj
@@ -1,6 +1,10 @@
 (ns source.routes.integration
   (:require [ring.util.response :as res]
-            [source.services.interface :as services]))
+            [source.services.interface :as services]
+            [source.db.util :as db.util]
+            [congest.jobs :as congest]
+            [source.util :as utils]
+            [source.jobs.core :as jobs]))
 
 (defn get
   {:summary "get integration by id"
@@ -22,3 +26,56 @@
 
   [{:keys [ds path-params] :as _request}]
   (res/response (services/bundle ds {:id (:id path-params)})))
+
+(defn post
+  {:summary "update an integration by id"
+   :parameters {:path [:map [:id {:title "id"
+                                  :description "integration id"} :int]]
+                :body [:map
+                       [:name :string]
+                       [:content-type-id :int]
+                       [:categories [:vector
+                                     [:map
+                                      [:id :int]
+                                      [:name :string]]]]]}
+   :responses {200 {:body [:map [:message :string]]}
+               401 {:body [:map [:message :string]]}
+               403 {:body [:map [:message :string]]}}}
+
+  [{:keys [js ds store path-params body] :as _request}]
+  (let [_ (services/update-bundle! ds {:id (:id path-params)
+                                       :data (dissoc body :categories)})
+
+        bundle-categories (mapv (fn [{:keys [id]}]
+                                  {:bundle-id (:id path-params)
+                                   :category-id id}) (:categories body))
+        job-id (str "bundle_" (:id path-params))
+
+        ; update bundle categories
+        bundle-ds (db.util/conn :bundle (:id path-params))
+        _ (services/delete-bundle-category! bundle-ds {:where [:= :bundle-id (:id path-params)]})
+        _ (services/insert-bundle-category! bundle-ds {:data bundle-categories})
+
+        category-ids (services/category-id-by-bundle bundle-ds {:bundle-id (:id path-params)})
+        id-vec (mapv (fn [{:keys [category-id]}] category-id) category-ids)
+        categories-by-bundle (services/categories ds {:where [:in :id id-vec]})]
+
+    (congest/deregister! js job-id)
+    (->> (jobs/prepare-congest-metadata
+          ds
+          store
+          {:id job-id
+           :initial-delay (* 1000 60 60 24)
+           :auto-start true
+           :stop-after-fail false,
+           :interval (* 1000 60 60 24)
+           :recurring? true
+           :ds ds
+           :args {:bundle-id (:id path-params)
+                  :categories categories-by-bundle}
+           :handler :update-bundle
+           :created-at (utils/get-utc-timestamp-string)
+           :sleep false})
+         (congest/register! js))
+
+    (res/response {:message "successfully updated integration"})))
diff --git a/src/source/routes/integration_categories.clj b/src/source/routes/integration_categories.clj
index 80cd6c5e..9e2f0d16 100644
--- a/src/source/routes/integration_categories.clj
+++ b/src/source/routes/integration_categories.clj
@@ -29,10 +29,11 @@
                         [:name :string]]]}
    :responses {200 {:body [:map [:message :string]]}}}
 
-  [{:keys [ds path-params body] :as _request}]
+  [{:keys [path-params body] :as _request}]
   (let [update-data (reduce (fn [acc {:keys [id]}]
                               (conj acc {:bundle-id (:id path-params)
-                                         :category-id id})) [] body)]
-    (services/delete-bundle-category! ds {:where [:= :integration-id (:id path-params)]})
-    (services/insert-bundle-category! ds {:data update-data})
+                                         :category-id id})) [] body)
+        bundle-ds (db.util/conn :bundle (:id path-params))]
+    (services/delete-bundle-category! bundle-ds {:where [:= :bundle-id (:id path-params)]})
+    (services/insert-bundle-category! bundle-ds {:data update-data})
     (res/response {:message "successfully updated integration categories"})))
diff --git a/src/source/routes/integrations.clj b/src/source/routes/integrations.clj
index 5b24f265..c6f25f1c 100644
--- a/src/source/routes/integrations.clj
+++ b/src/source/routes/integrations.clj
@@ -37,7 +37,17 @@
                                      [:map
                                       [:id :int]
                                       [:name :string]]]]]}
-   :responses {201 {:body [:map [:message :string]]}
+   :responses {201 {:body [:map
+                           [:id :int]
+                           [:name :string]
+                           [:uuid :string]
+                           [:user-id :int]
+                           [:video :int]
+                           [:podcast :int]
+                           [:blog :int]
+                           [:hash {:optional true} [:maybe :string]]
+                           [:content-type-id :int]
+                           [:ts-and-cs {:optional true} :int]]}
                401 {:body [:map [:message :string]]}
                403 {:body [:map [:message :string]]}}}
 
@@ -62,7 +72,7 @@
           ds
           store
           {:id (str "bundle_" (:id new-bundle))
-           :initial-delay (* 1000 60 60 24)
+           :initial-delay 0
            :auto-start true
            :stop-after-fail false,
            :interval (* 1000 60 60 24)
@@ -75,4 +85,4 @@
            :sleep false})
          (congest/register! js))
 
-    (res/response {:message "successfully added integration"})))
+    (res/response new-bundle)))
diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index ea5ca925..78023a43 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -161,7 +161,8 @@
       [""               (route {:get integrations/get
                                 :post integrations/post})]
       ["/:id"
-       [""              (route {:get integration/get})]
+       [""              (route {:get integration/get
+                                :post integration/post})]
        ["/categories"   (route {:get integration-categories/get
                                 :post integration-categories/post})]]]
 
diff --git a/src/source/services/interface.clj b/src/source/services/interface.clj
index 1db10ec2..4a8e347f 100644
--- a/src/source/services/interface.clj
+++ b/src/source/services/interface.clj
@@ -78,7 +78,7 @@
   (bundle-categories/insert-bundle-category! ds opts))
 
 (defn delete-bundle-category! [ds {:keys [_id _where] :as opts}]
-  (delete-bundle-category! ds opts))
+  (bundle-categories/delete-bundle-category! ds opts))
 
 (defn categories-by-bundle [ds {:keys [_bundle-id _where] :as opts}]
   (bundle-categories/categories-by-bundle ds opts))

From 1f1cf83aeb843414b39c86df5f4fa4bb8e68926d Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 22 Sep 2025 14:28:51 +0200
Subject: [PATCH 108/391] updated find function in honey helpers to allow for
 limit and order by

---
 src/source/db/honey.clj | 20 ++++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/src/source/db/honey.clj b/src/source/db/honey.clj
index f8dec2e1..51d94181 100644
--- a/src/source/db/honey.clj
+++ b/src/source/db/honey.clj
@@ -29,10 +29,12 @@
   clause follows the same data DSL as honeysql. Automatically transforms kebab
   case keys into snake case for sql. e.g. :provider-id becomes \"provider_id\"
   when honey sql prepares the statement in execute!"
-  [ds {:keys [tname where ret]}]
+  [ds {:keys [tname where order-by limit ret]}]
   (execute! ds
             (-> (hsql/select :*)
                 (hsql/from (csk/->snake_case_keyword tname))
+                (merge (if (some? order-by) {:order-by order-by} {}))
+                (merge (if (some? limit) (hsql/limit limit) {}))
                 (hsql/where
                  (or (cske/transform-keys
                       csk/->snake_case_keyword where)
@@ -98,21 +100,23 @@
 
   (def ds (db.util/conn :master))
 
-  (find ds {:tname :users
-            :ret :1})
+  (find ds {:tname :incoming-posts
+            :limit 5
+            :order-by [[:id :asc]]
+            :ret :*})
 
   (insert! ds {:tname :sectors
                :values {:name "something"}
                :ret :*})
 
   (delete! ds
-           {:tname :sectors
-            :where [:> :id 3]
+           {:tname :feeds
+            :where [:= :id 6]
             :ret :*})
 
   (update! ds
-           {:tname :sectors
-            :where [:= :id 7]
-            :values {:name "something else"}})
+           {:tname :users
+            :where [:= :id 3]
+            :values {:type "creator"}})
 
   ())

From e9875be3616fbe4c706c951be3682581f1145126 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 22 Sep 2025 15:29:17 +0200
Subject: [PATCH 109/391] added function in data extraction to parse xml cdata

---
 src/source/rss/core.clj | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/src/source/rss/core.clj b/src/source/rss/core.clj
index e311d3e8..d02e50c7 100644
--- a/src/source/rss/core.clj
+++ b/src/source/rss/core.clj
@@ -7,6 +7,18 @@
   [xml]
   (-> xml h/parse h/as-hickory))
 
+(defn cdata-content [cdata-node]
+  (let [front-removed (subs cdata-node 9)]
+    (subs front-removed 0 (- (count front-removed) 3))))
+
+(defn unwrap-cdata [maybe-cdata-node]
+  (if (and
+       (string? maybe-cdata-node)
+       (> (count maybe-cdata-node) 9)
+       (= (subs maybe-cdata-node 0 9) "<![CDATA["))
+    (cdata-content maybe-cdata-node)
+    maybe-cdata-node))
+
 (defn collect-leaf-paths
   "This function does a DFS on a hickory tree and assigns paths to each node relative to the root and
   then returns the ast with paths.
@@ -67,8 +79,8 @@
       (-> (:attrs node)
           (get seg-val))
 
-      (-> (:content node)
-          (get (Integer/parseInt (name seg-val)))))))
+      (unwrap-cdata (-> (:content node)
+                        (get (Integer/parseInt (name seg-val))))))))
 
 (defn extract-data
   "Recursively extracts data from a hickory xml tree according to the input selection schema.

From e4c4a189e1f45a738d5890f0040d9945169fb7ea Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 22 Sep 2025 15:40:20 +0200
Subject: [PATCH 110/391] replaced hardcoded numbers with defs

---
 src/source/rss/core.clj | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/source/rss/core.clj b/src/source/rss/core.clj
index d02e50c7..5ca8a7dd 100644
--- a/src/source/rss/core.clj
+++ b/src/source/rss/core.clj
@@ -7,9 +7,12 @@
   [xml]
   (-> xml h/parse h/as-hickory))
 
+(def cdata-prefix (count "<![CDATA["))
+(def cdata-postfix (count "]]>"))
+
 (defn cdata-content [cdata-node]
-  (let [front-removed (subs cdata-node 9)]
-    (subs front-removed 0 (- (count front-removed) 3))))
+  (let [front-removed (subs cdata-node cdata-prefix)]
+    (subs front-removed 0 (- (count front-removed) cdata-postfix))))
 
 (defn unwrap-cdata [maybe-cdata-node]
   (if (and

From 30da9e3ec3125aaf269f463e96d579655e89b37d Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 22 Sep 2025 16:22:18 +0200
Subject: [PATCH 111/391] updated bundle auth middleware

---
 src/source/middleware/auth/core.clj | 33 +++++++++++++++++------------
 1 file changed, 20 insertions(+), 13 deletions(-)

diff --git a/src/source/middleware/auth/core.clj b/src/source/middleware/auth/core.clj
index 32ddd064..9be392fb 100644
--- a/src/source/middleware/auth/core.clj
+++ b/src/source/middleware/auth/core.clj
@@ -49,15 +49,16 @@
   [handler]
   (fn [request]
     (let [ds (db.util/conn :master)
-          bundle-uuid (get-in request [:query-params "uuid"])]
-
-      (if (db/exists? ds {:tname :bundles
-                          :where [:= :uuid bundle-uuid]})
-        (handler request)
-
+          bundle-uuid (get-in request [:query-params "uuid"])
+          {:keys [id]} (db/find-one ds {:tname :bundles
+                                        :where [:= :uuid bundle-uuid]})]
+      (if (some? id)
+        (-> request
+            (assoc :bundle-id id)
+            (handler))
         (->
-         (res/response {:message "Unauthorized"})
-         (res/status 403))))))
+         (res/response {:message "The bundle you are looking for does not exist"})
+         (res/status 404))))))
 
 (comment
   (let [authed-request {:headers {"Authorization"
@@ -89,18 +90,20 @@
 
   (require '[source.util :as utils])
   (let [garbage-request {:query-params {"uuid" "garbage"}}
+        ds (db.util/conn)
         uuid (utils/uuid)
         bundle-request {:query-params {"uuid" uuid}}
         test-handler (-> (fn [request]
                            request)
                          (wrap-bundle-id))]
-
-    (bundles/insert-bundle! (db.util/conn :master) {:data {:uuid uuid
+    (bundles/insert-bundle! (db.util/conn :master) {:data {:name (str "test-bundle-" uuid)
+                                                           :uuid uuid
+                                                           :content-type-id 1
                                                            :video 0
                                                            :podcast 0
                                                            :blog 0}})
     (assert (=
-             403
+             404
              (-> garbage-request
                  (test-handler)
                  (:status))))
@@ -110,6 +113,10 @@
              (-> bundle-request
                  (test-handler)
                  (:bundle-id))))
-    (println "tests passed"))
-  ())
+    (println "tests passed")
+    (db/delete! ds
+             {:tname :bundles
+              :where [:like :name "test-bundle-%"]
+              :ret :*}))
 
+  ())

From c754e96afbaa1603890f9ece428c0e67e42cd872 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 29 Sep 2025 15:32:34 +0200
Subject: [PATCH 112/391] added routes for getting all and one outgoing posts
 from a bundle

---
 src/source/routes/bundle_post.clj      | 31 +++++++++++++++++++++
 src/source/routes/bundle_posts.clj     | 37 ++++++++++++++++++++++++++
 src/source/routes/reitit.clj           |  8 ++++++
 src/source/services/interface.clj      |  8 ++++++
 src/source/services/outgoing_posts.clj | 10 +++++++
 5 files changed, 94 insertions(+)
 create mode 100644 src/source/routes/bundle_post.clj
 create mode 100644 src/source/routes/bundle_posts.clj

diff --git a/src/source/routes/bundle_post.clj b/src/source/routes/bundle_post.clj
new file mode 100644
index 00000000..13056374
--- /dev/null
+++ b/src/source/routes/bundle_post.clj
@@ -0,0 +1,31 @@
+(ns source.routes.bundle-post
+  (:require [source.services.interface :as services]
+            [source.db.util :as db.util]
+            [ring.util.response :as res]))
+
+(defn get
+  {:summary "get a single outgoing post in the uuid-authorized bundle by post id"
+   :parameters {:path [:map [:id {:title "id"
+                                  :description "post id"} :int]]}
+   :responses {200 {:body [:map
+                           [:id :int]
+                           [:post-id :string]
+                           [:feed-id :int]
+                           [:creator-id :int]
+                           [:content-type-id :int]
+                           [:title :string]
+                           [:thumbnail [:maybe :string]]
+                           [:info [:maybe :string]]
+                           [:url [:maybe :string]]
+                           [:stream-url [:maybe :string]]
+                           [:season [:maybe :int]]
+                           [:episode [:maybe :int]]
+                           [:redacted [:maybe :int]]
+                           [:posted-at [:maybe :string]]]}
+               404 {:body [:map [:message :string]]}}}
+
+  [{:keys [bundle-id path-params] :as _request}]
+  (let [bundle-ds (db.util/conn :bundle bundle-id)
+        id (:id path-params)]
+    (res/response (services/outgoing-post bundle-ds {:id id}))))
+
diff --git a/src/source/routes/bundle_posts.clj b/src/source/routes/bundle_posts.clj
new file mode 100644
index 00000000..62434f39
--- /dev/null
+++ b/src/source/routes/bundle_posts.clj
@@ -0,0 +1,37 @@
+(ns source.routes.bundle-posts
+  (:require [source.services.interface :as services]
+            [source.db.util :as db.util]
+            [clojure.walk :as walk]
+            [ring.util.response :as res]))
+
+(defn get
+  {:summary "get all outgoing posts in the uuid-authorized bundle"
+   :parameters {:query [:map
+                        [:uuid :string]
+                        [:limit :int]
+                        [:start :int]]}
+   :responses {200 {:body [:vector
+                           [:map
+                            [:id :int]
+                            [:post-id :string]
+                            [:feed-id :int]
+                            [:creator-id :int]
+                            [:content-type-id :int]
+                            [:title :string]
+                            [:thumbnail [:maybe :string]]
+                            [:info [:maybe :string]]
+                            [:url [:maybe :string]]
+                            [:stream-url [:maybe :string]]
+                            [:season [:maybe :int]]
+                            [:episode [:maybe :int]]
+                            [:redacted [:maybe :int]]
+                            [:posted-at [:maybe :string]]]]}
+               404 {:body [:map [:message :string]]}}}
+
+  [{:keys [bundle-id query-params] :as _request}]
+  (let [bundle-ds (db.util/conn :bundle bundle-id)
+        {:keys [limit start]} (walk/keywordize-keys query-params)]
+    (res/response (services/outgoing-posts bundle-ds {:where (when start [:>= :id start])
+                                                      :limit limit
+                                                      :order-by [[:id :asc]]}))))
+
diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index 78023a43..81d156a8 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -41,6 +41,8 @@
             [source.routes.integrations :as integrations]
             [source.routes.integration :as integration]
             [source.routes.integration-categories :as integration-categories]
+            [source.routes.bundle-posts :as bundle-posts]
+            [source.routes.bundle-post :as bundle-post]
             [source.routes.posts :as posts]
             [source.routes.admin-feeds :as admin-feeds]
             [source.routes.xml :as xml]
@@ -177,6 +179,12 @@
        ["/categories"   (route {:get feed-categories/get
                                 :post feed-categories/post})]]]
 
+     ["/bundles"        {:middleware [[mw/apply-bundle]]
+                         :tags #{"bundles"}}
+      ["/posts"
+       [""              (route {:get bundle-posts/get})]
+       ["/:id"          (route {:get bundle-post/get})]]]
+
      ["/admin"                  {:middleware [[mw/apply-auth {:required-type :admin}]]
                                  :tags #{"admin"}
                                  :swagger {:security [{"auth" []}]}
diff --git a/src/source/services/interface.clj b/src/source/services/interface.clj
index 4a8e347f..a32b59a8 100644
--- a/src/source/services/interface.clj
+++ b/src/source/services/interface.clj
@@ -106,6 +106,14 @@
 (defn top-posts-by-heuristic [ds {:keys [_select _limit _heuristic] :as opts}]
   (post-heuristics/top-posts-by-heuristic ds opts))
 
+(defn outgoing-posts
+  ([ds] (outgoing-posts ds {}))
+  ([ds {:keys [_where] :as opts}]
+   (outgoing-posts/outgoing-posts ds opts)))
+
+(defn outgoing-post [ds {:keys [_id _where] :as opts}]
+  (outgoing-posts/outgoing-post ds opts))
+
 (defn upsert-outgoing-posts! [ds {:keys [_data] :as opts}]
   (outgoing-posts/upsert-outgoing-posts! ds opts))
 
diff --git a/src/source/services/outgoing_posts.clj b/src/source/services/outgoing_posts.clj
index 1a02f9d1..0e676c62 100644
--- a/src/source/services/outgoing_posts.clj
+++ b/src/source/services/outgoing_posts.clj
@@ -3,6 +3,15 @@
             [source.db.honey :as hon]
             [honey.sql.helpers :as hsql]))
 
+(defn outgoing-posts
+  ([ds] (outgoing-posts ds {}))
+  ([ds {:keys [where] :as opts}]
+   (->> {:tname :outgoing-posts
+         :where where
+         :ret :*}
+        (merge opts)
+        (db/find ds))))
+
 (defn outgoing-post [ds {:keys [id where] :as opts}]
   (->> {:tname :outgoing-posts
         :where (if (some? id)
@@ -20,6 +29,7 @@
        (assoc :on-conflict [:post-id])
        (assoc :do-update-set {:feed-id          :excluded.feed-id
                               :title            :excluded.title
+                              :thumbnail        :excluded.thumbnail
                               :info             :excluded.info
                               :url              :excluded.url
                               :stream-url       :excluded.stream-url

From 322236854430be28bff89a7a9dc155e52f036bc9 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 29 Sep 2025 15:33:18 +0200
Subject: [PATCH 113/391] fixed wrong keyword used in bundle handler

---
 src/source/jobs/handlers.clj | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/src/source/jobs/handlers.clj b/src/source/jobs/handlers.clj
index 4ccde137..631866e6 100644
--- a/src/source/jobs/handlers.clj
+++ b/src/source/jobs/handlers.clj
@@ -60,8 +60,8 @@
        (fn [post]
           ; calculate score for post
           ; determine number of categories matched
-         (let [; get vector of category ids in the given post, e.g. [1 3]
-               post-categories-vec (reduce (fn [acc {:keys [post-id id]}]
+              ; get vector of category ids in the given post, e.g. [1 3]
+         (let [post-categories-vec (reduce (fn [acc {:keys [post-id id]}]
                                              (if (= post-id (:id post))
                                                (conj acc id)
                                                acc)) [] posts-categories)
@@ -77,13 +77,12 @@
        incoming-posts)
 
       ; pull highest scored posts by long heuristics into outgoing posts
-      (let [; top 30 post-heuristics records ordered by long heuristic in descending order
-            top-by-long-heuristics (services/top-posts-by-heuristic ds-bundle
+            ; top 30 post-heuristics records ordered by long heuristic in descending order
+      (let [top-by-long-heuristics (services/top-posts-by-heuristic ds-bundle
                                                                     {:heuristic :long-heuristic
-                                                                     :limit 30})
+                                                                     :limit 100})
             ; convert into a vector of id numbers
-            ids (reduce (fn [acc {:keys [id]}]
-                          (conj acc id)) [] top-by-long-heuristics)
+            ids (mapv (fn [{:keys [post-id]}] post-id) top-by-long-heuristics)
 
             ; get all incoming posts with the above id numbers
             posts-in (services/incoming-posts ds {:where [:in :id ids]})
@@ -95,4 +94,3 @@
                                    [] posts-in)]
         (when (seq posts-in)
           (services/upsert-outgoing-posts! ds-bundle {:data outgoing-posts}))))))
-

From 4cf4d936e79ebd1440e22034acc1d5049596d037 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 29 Sep 2025 15:34:04 +0200
Subject: [PATCH 114/391] added thumbnail to post schemas

---
 src/source/db/bundle.clj    | 1 +
 src/source/db/master.clj    | 1 +
 src/source/routes/posts.clj | 1 +
 3 files changed, 3 insertions(+)

diff --git a/src/source/db/bundle.clj b/src/source/db/bundle.clj
index e1a9f720..2de9db30 100644
--- a/src/source/db/bundle.clj
+++ b/src/source/db/bundle.clj
@@ -17,6 +17,7 @@
    [:post-id :text :not nil]
    [:feed-id :integer :not nil]
    [:title :text :not nil]
+   [:thumbnail :text]
    [:info :text]
    [:url :text]
    [:stream-url :text]
diff --git a/src/source/db/master.clj b/src/source/db/master.clj
index d5c0690f..3355e6a9 100644
--- a/src/source/db/master.clj
+++ b/src/source/db/master.clj
@@ -152,6 +152,7 @@
    [:post-id :text :not nil]
    [:feed-id :integer :not nil]
    [:title :text :not nil]
+   [:thumbnail :text]
    [:info :text]
    [:url :text]
    [:stream-url :text]
diff --git a/src/source/routes/posts.clj b/src/source/routes/posts.clj
index 327662e1..b7308196 100644
--- a/src/source/routes/posts.clj
+++ b/src/source/routes/posts.clj
@@ -14,6 +14,7 @@
                             [:creator-id :int]
                             [:content-type-id :int]
                             [:title :string]
+                            [:thumbnail [:maybe :string]]
                             [:info [:maybe :string]]
                             [:url [:maybe :string]]
                             [:stream-url [:maybe :string]]

From e7ee67db2f373aa5b560e2d847d9c743d1537e5f Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 29 Sep 2025 15:45:59 +0200
Subject: [PATCH 115/391] updated post number in bundle job

---
 src/source/jobs/handlers.clj | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/source/jobs/handlers.clj b/src/source/jobs/handlers.clj
index 631866e6..255d8974 100644
--- a/src/source/jobs/handlers.clj
+++ b/src/source/jobs/handlers.clj
@@ -77,7 +77,7 @@
        incoming-posts)
 
       ; pull highest scored posts by long heuristics into outgoing posts
-            ; top 30 post-heuristics records ordered by long heuristic in descending order
+            ; top 100 post-heuristics records ordered by long heuristic in descending order
       (let [top-by-long-heuristics (services/top-posts-by-heuristic ds-bundle
                                                                     {:heuristic :long-heuristic
                                                                      :limit 100})

From a6595e018cc6f413c118e694f98a4c7f42913b6e Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Tue, 30 Sep 2025 10:30:50 +0200
Subject: [PATCH 116/391] addressed comments

---
 src/source/jobs/handlers.clj | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/source/jobs/handlers.clj b/src/source/jobs/handlers.clj
index 255d8974..1ef17bfc 100644
--- a/src/source/jobs/handlers.clj
+++ b/src/source/jobs/handlers.clj
@@ -54,17 +54,17 @@
   (fn [{:keys [args ds]}]
     (let [{:keys [bundle-id categories]} args
           ds-bundle (db.util/conn :bundle bundle-id)
-          incoming-posts (services/incoming-posts-with-feeds (db.util/conn) {:where [:= :feeds.state "live"]})
+          incoming-posts (services/incoming-posts-with-feeds ds {:where [:= :feeds.state "live"]})
           posts-categories (incoming-posts/categories-by-posts ds {:where [:= :state "live"]})]
       (run!
        (fn [post]
           ; calculate score for post
           ; determine number of categories matched
               ; get vector of category ids in the given post, e.g. [1 3]
-         (let [post-categories-vec (reduce (fn [acc {:keys [post-id id]}]
-                                             (if (= post-id (:id post))
-                                               (conj acc id)
-                                               acc)) [] posts-categories)
+         (let [post-categories-vec (->> posts-categories
+                                        (mapv (fn [{:keys [post-id id]}]
+                                                (when (= post-id (:id post)) id)))
+                                        (filterv identity))
                ; get vector of category ids in categories to match, e.g. [1 2 3 4]
                match-categories-vec (reduce (fn [acc {:keys [id]}]
                                               (conj acc id)) [] categories)
@@ -82,7 +82,7 @@
                                                                     {:heuristic :long-heuristic
                                                                      :limit 100})
             ; convert into a vector of id numbers
-            ids (mapv (fn [{:keys [post-id]}] post-id) top-by-long-heuristics)
+            ids (mapv :post-id top-by-long-heuristics)
 
             ; get all incoming posts with the above id numbers
             posts-in (services/incoming-posts ds {:where [:in :id ids]})

From 245f10dae650d534688257e033618a45fbb3d2ca Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Tue, 30 Sep 2025 10:31:40 +0200
Subject: [PATCH 117/391] added v2 migrations for posts

---
 .../bundle_migrations/002_outgoing_posts.clj     | 16 ++++++++++++++++
 src/source/migrations/002_incoming_posts.clj     | 14 ++++++++++++++
 2 files changed, 30 insertions(+)
 create mode 100644 src/source/bundle_migrations/002_outgoing_posts.clj
 create mode 100644 src/source/migrations/002_incoming_posts.clj

diff --git a/src/source/bundle_migrations/002_outgoing_posts.clj b/src/source/bundle_migrations/002_outgoing_posts.clj
new file mode 100644
index 00000000..fce2ecd3
--- /dev/null
+++ b/src/source/bundle_migrations/002_outgoing_posts.clj
@@ -0,0 +1,16 @@
+(ns source.bundle-migrations.002-outgoing-posts
+  (:require [source.db.bundle]
+            [source.db.tables :as tables]))
+
+(defn run-up! [context]
+  (let [ds-bundle (:db-bundle context)]
+    (tables/create-tables!
+     ds-bundle
+     :source.db.bundle
+     [:outgoing-posts])))
+
+(defn run-down! [context]
+  (let [ds-bundle (:db-bundle context)]
+    (tables/drop-tables!
+     ds-bundle
+     [:outgoing-posts])))
diff --git a/src/source/migrations/002_incoming_posts.clj b/src/source/migrations/002_incoming_posts.clj
new file mode 100644
index 00000000..5f7787bc
--- /dev/null
+++ b/src/source/migrations/002_incoming_posts.clj
@@ -0,0 +1,14 @@
+(ns source.migrations.002-incoming-posts
+  (:require [source.db.master]
+            [source.db.tables :as tables]))
+
+(defn run-up! [context]
+  (let [ds-master (:db-master context)]
+    (tables/create-tables!
+     ds-master
+     :source.db.master
+     [:incoming-posts])))
+
+(defn run-down! [context]
+  (let [ds-master (:db-master context)]
+    (tables/drop-table! ds-master :incoming-posts)))

From be9ed2b4281d801fa3673f88e9801a866329162b Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 1 Oct 2025 14:14:13 +0200
Subject: [PATCH 118/391] added endpoints to get bundle metadata and feeds
 associated with the bundle

---
 src/source/routes/bundle.clj       | 22 ++++++++++++++++++++++
 src/source/routes/bundle_feeds.clj | 30 ++++++++++++++++++++++++++++++
 src/source/routes/reitit.clj       |  6 +++++-
 3 files changed, 57 insertions(+), 1 deletion(-)
 create mode 100644 src/source/routes/bundle.clj
 create mode 100644 src/source/routes/bundle_feeds.clj

diff --git a/src/source/routes/bundle.clj b/src/source/routes/bundle.clj
new file mode 100644
index 00000000..ac922c95
--- /dev/null
+++ b/src/source/routes/bundle.clj
@@ -0,0 +1,22 @@
+(ns source.routes.bundle
+  (:require [ring.util.response :as res]
+            [source.services.interface :as services]))
+
+(defn get
+  {:summary "get bundle metadata by authorized uuid"
+   :parameters {:query [:map [:uuid :string]]}
+   :responses {200 {:body [:map
+                           [:id :int]
+                           [:name :string]
+                           [:uuid :string]
+                           [:user-id :int]
+                           [:video :int]
+                           [:podcast :int]
+                           [:blog :int]
+                           [:hash [:maybe :string]]
+                           [:content-type-id :int]
+                           [:ts-and-cs [:maybe :int]]]}
+               404 {:body [:map [:message :string]]}}}
+
+  [{:keys [ds bundle-id] :as _request}]
+  (res/response (services/bundle ds {:id bundle-id})))
diff --git a/src/source/routes/bundle_feeds.clj b/src/source/routes/bundle_feeds.clj
new file mode 100644
index 00000000..db30a0a7
--- /dev/null
+++ b/src/source/routes/bundle_feeds.clj
@@ -0,0 +1,30 @@
+(ns source.routes.bundle-feeds
+  (:require [ring.util.response :as res]
+            [source.services.interface :as services]
+            [source.db.util :as db.util]))
+
+(defn get
+  {:summary "get all feeds present in the bundle authorised by uuid"
+   :parameters {:query [:map [:uuid :string]]}
+   :responses {200 {:body [:vector
+                           [:map
+                            [:id :int]
+                            [:title :string]
+                            [:display-picture [:maybe :string]]
+                            [:url [:maybe :string]]
+                            [:rss-url :string]
+                            [:user-id :int]
+                            [:provider-id [:maybe :int]]
+                            [:created-at :string]
+                            [:updated-at [:maybe :string]]
+                            [:content-type-id :int]
+                            [:cadence-id :int]
+                            [:baseline-id :int]
+                            [:ts-and-cs [:maybe :int]]
+                            [:state [:enum "live" "not live" "pending"]]]]}
+               404 {:body [:map [:message :string]]}}}
+
+  [{:keys [ds bundle-id] :as _request}]
+  (let [bundle-ds (db.util/conn :bundle bundle-id)
+        feed-ids (mapv :feed-id (services/outgoing-posts bundle-ds))]
+    (res/response (services/feeds ds {:where [:in :id feed-ids]}))))
diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index 81d156a8..fc88594c 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -41,6 +41,8 @@
             [source.routes.integrations :as integrations]
             [source.routes.integration :as integration]
             [source.routes.integration-categories :as integration-categories]
+            [source.routes.bundle :as bundle]
+            [source.routes.bundle-feeds :as bundle-feeds]
             [source.routes.bundle-posts :as bundle-posts]
             [source.routes.bundle-post :as bundle-post]
             [source.routes.posts :as posts]
@@ -179,8 +181,10 @@
        ["/categories"   (route {:get feed-categories/get
                                 :post feed-categories/post})]]]
 
-     ["/bundles"        {:middleware [[mw/apply-bundle]]
+     ["/bundle"        {:middleware [[mw/apply-bundle]]
                          :tags #{"bundles"}}
+      [""               (route {:get bundle/get})]
+      ["/feeds"         (route {:get bundle-feeds/get})]
       ["/posts"
        [""              (route {:get bundle-posts/get})]
        ["/:id"          (route {:get bundle-post/get})]]]

From 51eda8f2f9676f5645540e2760f783baed31ae4d Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 1 Oct 2025 14:16:36 +0200
Subject: [PATCH 119/391] updated malli schemas for posts

---
 src/source/routes/bundle_post.clj  | 5 +++--
 src/source/routes/bundle_posts.clj | 9 ++++-----
 src/source/routes/posts.clj        | 2 +-
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/source/routes/bundle_post.clj b/src/source/routes/bundle_post.clj
index 13056374..28265226 100644
--- a/src/source/routes/bundle_post.clj
+++ b/src/source/routes/bundle_post.clj
@@ -5,7 +5,8 @@
 
 (defn get
   {:summary "get a single outgoing post in the uuid-authorized bundle by post id"
-   :parameters {:path [:map [:id {:title "id"
+   :parameters {:query [:map [:uuid :string]]
+                :path [:map [:id {:title "id"
                                   :description "post id"} :int]]}
    :responses {200 {:body [:map
                            [:id :int]
@@ -20,7 +21,7 @@
                            [:stream-url [:maybe :string]]
                            [:season [:maybe :int]]
                            [:episode [:maybe :int]]
-                           [:redacted [:maybe :int]]
+                           [:redacted {:optional true} [:maybe :int]]
                            [:posted-at [:maybe :string]]]}
                404 {:body [:map [:message :string]]}}}
 
diff --git a/src/source/routes/bundle_posts.clj b/src/source/routes/bundle_posts.clj
index 62434f39..aa65437a 100644
--- a/src/source/routes/bundle_posts.clj
+++ b/src/source/routes/bundle_posts.clj
@@ -8,8 +8,8 @@
   {:summary "get all outgoing posts in the uuid-authorized bundle"
    :parameters {:query [:map
                         [:uuid :string]
-                        [:limit :int]
-                        [:start :int]]}
+                        [:limit {:optional true} :int]
+                        [:start {:optional true} :int]]}
    :responses {200 {:body [:vector
                            [:map
                             [:id :int]
@@ -24,9 +24,9 @@
                             [:stream-url [:maybe :string]]
                             [:season [:maybe :int]]
                             [:episode [:maybe :int]]
-                            [:redacted [:maybe :int]]
+                            [:redacted {:optional true} [:maybe :int]]
                             [:posted-at [:maybe :string]]]]}
-               404 {:body [:map [:message :string]]}}}
+               404 {:boy [:map [:message :string]]}}}
 
   [{:keys [bundle-id query-params] :as _request}]
   (let [bundle-ds (db.util/conn :bundle bundle-id)
@@ -34,4 +34,3 @@
     (res/response (services/outgoing-posts bundle-ds {:where (when start [:>= :id start])
                                                       :limit limit
                                                       :order-by [[:id :asc]]}))))
-
diff --git a/src/source/routes/posts.clj b/src/source/routes/posts.clj
index b7308196..e3d93cfe 100644
--- a/src/source/routes/posts.clj
+++ b/src/source/routes/posts.clj
@@ -20,7 +20,7 @@
                             [:stream-url [:maybe :string]]
                             [:season [:maybe :int]]
                             [:episode [:maybe :int]]
-                            [:redacted [:maybe :int]]
+                            [:redacted {:optional true} [:maybe :int]]
                             [:posted-at [:maybe :string]]]]}
                401 {:body [:map [:message :string]]}
                403 {:body [:map [:message :string]]}}}

From 6bdc360b7f75f7811eaeee58c4f2fcf2023b138d Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 6 Oct 2025 11:41:32 +0200
Subject: [PATCH 120/391] added middleware to route metadata and updated feed
 authorization

---
 src/source/routes/feed.clj            |  6 ++++--
 src/source/routes/feed_categories.clj |  6 ++++--
 src/source/routes/feeds.clj           | 10 ++++++----
 src/source/routes/reitit.clj          | 10 +++++-----
 4 files changed, 19 insertions(+), 13 deletions(-)

diff --git a/src/source/routes/feed.clj b/src/source/routes/feed.clj
index 077e28b4..62349fb0 100644
--- a/src/source/routes/feed.clj
+++ b/src/source/routes/feed.clj
@@ -1,6 +1,7 @@
 (ns source.routes.feed
   (:require [source.services.interface :as services]
-            [ring.util.response :as res]))
+            [ring.util.response :as res]
+            [source.middleware.interface :as mw]))
 
 (defn get
   {:summary "get feed by id"
@@ -27,7 +28,8 @@
       (res/response)))
 
 (defn post
-  {:summary "update feed by id"
+  {:middleware [[mw/apply-auth]]
+   :summary "update feed by id"
    :parameters {:path [:map [:id {:title "id"
                                   :description "feed id"} :int]]
                 :body [:map
diff --git a/src/source/routes/feed_categories.clj b/src/source/routes/feed_categories.clj
index 6be698b9..9a66cd46 100644
--- a/src/source/routes/feed_categories.clj
+++ b/src/source/routes/feed_categories.clj
@@ -1,6 +1,7 @@
 (ns source.routes.feed-categories
   (:require [source.services.interface :as services]
-            [ring.util.response :as res]))
+            [ring.util.response :as res]
+            [source.middleware.interface :as mw]))
 
 (defn get
   {:summary "get all categories belonging to the feed with the given id"
@@ -17,7 +18,8 @@
        (res/response)))
 
 (defn post
-  {:summary "update categories belonging to the feed with the given id"
+  {:middleware [[mw/apply-auth]]
+   :summary "update categories belonging to the feed with the given id"
    :parameters {:path [:map [:id {:title "id"
                                   :description "feed id"} :int]]
                 :body [:vector
diff --git a/src/source/routes/feeds.clj b/src/source/routes/feeds.clj
index 1394db06..ad001533 100644
--- a/src/source/routes/feeds.clj
+++ b/src/source/routes/feeds.clj
@@ -3,7 +3,8 @@
             [source.util :as utils]
             [congest.jobs :as congest]
             [source.jobs.core :as jobs]
-            [ring.util.response :as res]))
+            [ring.util.response :as res]
+            [source.middleware.interface :as mw]))
 
 (defn get
   {:summary "get all feeds"
@@ -24,12 +25,13 @@
                              [:ts-and-cs [:maybe :int]]
                              [:state [:enum "live" "not live" "pending"]]]]}}}
 
-  [{:keys [ds user] :as _request}]
-  (-> (services/feeds ds {:where [:= :user-id (:id user)]})
+  [{:keys [ds] :as _request}]
+  (-> (services/feeds ds)
       (res/response)))
 
 (defn post
-  {:summary "adds a feed and extracts data from RSS feed URL to create incoming posts and schedules a job to keep them updated"
+  {:middleware [[mw/apply-auth]]
+   :summary "adds a feed and extracts data from RSS feed URL to create incoming posts and schedules a job to keep them updated"
    :parameters {:body [:map
                        [:display-picture {:optional true} :string]
                        [:url {:optional true} :string]
diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index fc88594c..79702a9b 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -61,8 +61,9 @@
 
 (defn route [handlers]
   (reduce (fn [acc [k v]]
-            (let [{:keys [summary parameters responses]} (util/metadata v)]
-              (merge acc {k {:summary summary
+            (let [{:keys [middleware summary parameters responses]} (util/metadata v)]
+              (merge acc {k {:middleware middleware
+                             :summary summary
                              :parameters parameters
                              :responses responses
                              :handler v}})))
@@ -170,8 +171,7 @@
        ["/categories"   (route {:get integration-categories/get
                                 :post integration-categories/post})]]]
 
-     ["/feeds"          {:middleware [[mw/apply-auth]]
-                         :tags #{"feeds"}}
+     ["/feeds"          {:tags #{"feeds"}}
       [""               (route {:get feeds/get
                                 :post feeds/post})]
       ["/:id"
@@ -182,7 +182,7 @@
                                 :post feed-categories/post})]]]
 
      ["/bundle"        {:middleware [[mw/apply-bundle]]
-                         :tags #{"bundles"}}
+                        :tags #{"bundles"}}
       [""               (route {:get bundle/get})]
       ["/feeds"         (route {:get bundle-feeds/get})]
       ["/posts"

From 6a7e66025d09dbf66684db551b1f1cefa2a0644b Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Tue, 7 Oct 2025 11:59:45 +0200
Subject: [PATCH 121/391] removed public endpoints and made new ones for
 bundles

---
 src/source/routes/bundle_feed.clj       | 29 +++++++++++++++++++++++
 src/source/routes/bundle_feed_posts.clj | 31 +++++++++++++++++++++++++
 src/source/routes/feed.clj              |  6 ++---
 src/source/routes/feed_categories.clj   |  6 ++---
 src/source/routes/feeds.clj             |  6 ++---
 src/source/routes/reitit.clj            | 11 +++++++--
 6 files changed, 75 insertions(+), 14 deletions(-)
 create mode 100644 src/source/routes/bundle_feed.clj
 create mode 100644 src/source/routes/bundle_feed_posts.clj

diff --git a/src/source/routes/bundle_feed.clj b/src/source/routes/bundle_feed.clj
new file mode 100644
index 00000000..229e0a70
--- /dev/null
+++ b/src/source/routes/bundle_feed.clj
@@ -0,0 +1,29 @@
+(ns source.routes.bundle-feed
+  (:require [source.services.interface :as services]
+            [ring.util.response :as res]))
+
+(defn get
+  {:summary "get feed by id"
+   :parameters {:query [:map [:uuid :string]]
+                :path [:map [:id {:title "id"
+                                  :description "feed id"} :int]]}
+   :responses  {200 {:body [:map
+                            [:id :int]
+                            [:title :string]
+                            [:display-picture [:maybe :string]]
+                            [:url [:maybe :string]]
+                            [:rss-url :string]
+                            [:user-id :int]
+                            [:provider-id [:maybe :int]]
+                            [:created-at :string]
+                            [:updated-at [:maybe :string]]
+                            [:content-type-id :int]
+                            [:cadence-id :int]
+                            [:baseline-id :int]
+                            [:ts-and-cs [:maybe :int]]
+                            [:state [:enum "live" "not live" "pending"]]]}
+                404 {:body [:map [:message :string]]}}}
+
+  [{:keys [ds path-params] :as _request}]
+  (-> (services/feed ds path-params)
+      (res/response)))
diff --git a/src/source/routes/bundle_feed_posts.clj b/src/source/routes/bundle_feed_posts.clj
new file mode 100644
index 00000000..49120ded
--- /dev/null
+++ b/src/source/routes/bundle_feed_posts.clj
@@ -0,0 +1,31 @@
+(ns source.routes.bundle-feed-posts
+  (:require [source.services.interface :as services]
+            [ring.util.response :as res]))
+
+(defn get
+  {:summary "get all posts by feed id"
+   :parameters {:query [:map [:uuid :string]]
+                :path [:map [:id {:title "id"
+                                  :description "feed id"} :int]]}
+   :responses {200 {:body [:vector
+                           [:map
+                            [:id :int]
+                            [:post-id :string]
+                            [:feed-id :int]
+                            [:creator-id :int]
+                            [:content-type-id :int]
+                            [:title :string]
+                            [:thumbnail [:maybe :string]]
+                            [:info [:maybe :string]]
+                            [:url [:maybe :string]]
+                            [:stream-url [:maybe :string]]
+                            [:season [:maybe :int]]
+                            [:episode [:maybe :int]]
+                            [:redacted {:optional true} [:maybe :int]]
+                            [:posted-at [:maybe :string]]]]}
+               401 {:body [:map [:message :string]]}
+               403 {:body [:map [:message :string]]}}}
+
+  [{:keys [ds path-params] :as _request}]
+  (-> (services/incoming-posts ds {:where [:= :feed-id (:id path-params)]})
+      (res/response)))
diff --git a/src/source/routes/feed.clj b/src/source/routes/feed.clj
index 62349fb0..077e28b4 100644
--- a/src/source/routes/feed.clj
+++ b/src/source/routes/feed.clj
@@ -1,7 +1,6 @@
 (ns source.routes.feed
   (:require [source.services.interface :as services]
-            [ring.util.response :as res]
-            [source.middleware.interface :as mw]))
+            [ring.util.response :as res]))
 
 (defn get
   {:summary "get feed by id"
@@ -28,8 +27,7 @@
       (res/response)))
 
 (defn post
-  {:middleware [[mw/apply-auth]]
-   :summary "update feed by id"
+  {:summary "update feed by id"
    :parameters {:path [:map [:id {:title "id"
                                   :description "feed id"} :int]]
                 :body [:map
diff --git a/src/source/routes/feed_categories.clj b/src/source/routes/feed_categories.clj
index 9a66cd46..6be698b9 100644
--- a/src/source/routes/feed_categories.clj
+++ b/src/source/routes/feed_categories.clj
@@ -1,7 +1,6 @@
 (ns source.routes.feed-categories
   (:require [source.services.interface :as services]
-            [ring.util.response :as res]
-            [source.middleware.interface :as mw]))
+            [ring.util.response :as res]))
 
 (defn get
   {:summary "get all categories belonging to the feed with the given id"
@@ -18,8 +17,7 @@
        (res/response)))
 
 (defn post
-  {:middleware [[mw/apply-auth]]
-   :summary "update categories belonging to the feed with the given id"
+  {:summary "update categories belonging to the feed with the given id"
    :parameters {:path [:map [:id {:title "id"
                                   :description "feed id"} :int]]
                 :body [:vector
diff --git a/src/source/routes/feeds.clj b/src/source/routes/feeds.clj
index ad001533..e305a7f1 100644
--- a/src/source/routes/feeds.clj
+++ b/src/source/routes/feeds.clj
@@ -3,8 +3,7 @@
             [source.util :as utils]
             [congest.jobs :as congest]
             [source.jobs.core :as jobs]
-            [ring.util.response :as res]
-            [source.middleware.interface :as mw]))
+            [ring.util.response :as res]))
 
 (defn get
   {:summary "get all feeds"
@@ -30,8 +29,7 @@
       (res/response)))
 
 (defn post
-  {:middleware [[mw/apply-auth]]
-   :summary "adds a feed and extracts data from RSS feed URL to create incoming posts and schedules a job to keep them updated"
+  {:summary "adds a feed and extracts data from RSS feed URL to create incoming posts and schedules a job to keep them updated"
    :parameters {:body [:map
                        [:display-picture {:optional true} :string]
                        [:url {:optional true} :string]
diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index 79702a9b..8ab3ac98 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -43,6 +43,8 @@
             [source.routes.integration-categories :as integration-categories]
             [source.routes.bundle :as bundle]
             [source.routes.bundle-feeds :as bundle-feeds]
+            [source.routes.bundle-feed :as bundle-feed]
+            [source.routes.bundle-feed-posts :as bundle-feed-posts]
             [source.routes.bundle-posts :as bundle-posts]
             [source.routes.bundle-post :as bundle-post]
             [source.routes.posts :as posts]
@@ -171,7 +173,8 @@
        ["/categories"   (route {:get integration-categories/get
                                 :post integration-categories/post})]]]
 
-     ["/feeds"          {:tags #{"feeds"}}
+     ["/feeds"          {:middleware [[mw/apply-auth]]
+                         :tags #{"feeds"}}
       [""               (route {:get feeds/get
                                 :post feeds/post})]
       ["/:id"
@@ -184,7 +187,11 @@
      ["/bundle"        {:middleware [[mw/apply-bundle]]
                         :tags #{"bundles"}}
       [""               (route {:get bundle/get})]
-      ["/feeds"         (route {:get bundle-feeds/get})]
+      ["/feeds"
+       [""              (route {:get bundle-feeds/get})]
+       ["/:id"
+        [""             (route {:get bundle-feed/get})]
+        ["/posts"       (route {:get bundle-feed-posts/get})]]]
       ["/posts"
        [""              (route {:get bundle-posts/get})]
        ["/:id"          (route {:get bundle-post/get})]]]

From 012efbbfce88e7974f1b4970d39e5940ca60b390 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Tue, 7 Oct 2025 13:59:15 +0200
Subject: [PATCH 122/391] updated feed insertion and update job to pull display
 pictures from RSS feeds and use them as a backup for thumbnails

---
 src/source/jobs/handlers.clj          | 24 ++++++++++++++++++------
 src/source/routes/feed_categories.clj |  5 +++--
 src/source/routes/feeds.clj           |  5 ++++-
 3 files changed, 25 insertions(+), 9 deletions(-)

diff --git a/src/source/jobs/handlers.clj b/src/source/jobs/handlers.clj
index 1ef17bfc..8279fc0c 100644
--- a/src/source/jobs/handlers.clj
+++ b/src/source/jobs/handlers.clj
@@ -31,14 +31,25 @@
             extracted (services/extract-data store {:schema-id latest-ss
                                                     :url url})
             extracted-posts (get-in extracted [:feed :posts])
-            extended-posts (mapv (fn [post]
+            extracted-display (get-in extracted [:feed :display-picture])
+            extended-posts (mapv (fn [{:keys [thumbnail] :as post}]
                                    (merge post
                                           {:feed-id feed-id
                                            :creator-id creator-id
-                                           :content-type-id content-type-id})) extracted-posts)
-            existing-posts (services/incoming-posts ds {:where [:= :creator-id creator-id]})]
+                                           :content-type-id content-type-id
+                                           :thumbnail (if (and thumbnail
+                                                               (seq thumbnail))
+                                                        thumbnail
+                                                        extracted-display)}))
+                                 extracted-posts)
+            existing-posts (services/incoming-posts ds {:where [:= :creator-id creator-id]})
+            existing-feed (services/feed ds {:id feed-id})]
         (services/update-feed! ds {:id feed-id
                                    :data {:title (get-in extracted [:feed :title])
+                                          :display-picture (if (and (:display-picture existing-feed)
+                                                                    (seq (:display-picture existing-feed)))
+                                                             (:display-picture existing-feed)
+                                                             extracted-display)
                                           :updated-at (util/get-utc-timestamp-string)}})
         (run!
          (fn [post]
@@ -77,10 +88,10 @@
        incoming-posts)
 
       ; pull highest scored posts by long heuristics into outgoing posts
-            ; top 100 post-heuristics records ordered by long heuristic in descending order
+            ; top 1000 post-heuristics records ordered by long heuristic in descending order
       (let [top-by-long-heuristics (services/top-posts-by-heuristic ds-bundle
                                                                     {:heuristic :long-heuristic
-                                                                     :limit 100})
+                                                                     :limit 1000})
             ; convert into a vector of id numbers
             ids (mapv :post-id top-by-long-heuristics)
 
@@ -93,4 +104,5 @@
                                        acc))
                                    [] posts-in)]
         (when (seq posts-in)
-          (services/upsert-outgoing-posts! ds-bundle {:data outgoing-posts}))))))
+          (services/delete-outgoing-post! ds-bundle {})
+          (services/insert-outgoing-post! ds-bundle {:data outgoing-posts}))))))
diff --git a/src/source/routes/feed_categories.clj b/src/source/routes/feed_categories.clj
index 6be698b9..b3d994b1 100644
--- a/src/source/routes/feed_categories.clj
+++ b/src/source/routes/feed_categories.clj
@@ -29,6 +29,7 @@
   (let [update-data (reduce (fn [acc {:keys [id]}]
                               (conj acc {:feed-id (:id path-params)
                                          :category-id id})) [] body)]
-    (services/delete-feed-category! ds {:where [:= :feed-id (:id path-params)]})
-    (services/insert-feed-category! ds {:data update-data})
+    (when (seq update-data)
+      (services/delete-feed-category! ds {:where [:= :feed-id (:id path-params)]})
+      (services/insert-feed-category! ds {:data update-data}))
     (res/response {:message "successfully updated feed categories"})))
diff --git a/src/source/routes/feeds.clj b/src/source/routes/feeds.clj
index e305a7f1..f079cbf8 100644
--- a/src/source/routes/feeds.clj
+++ b/src/source/routes/feeds.clj
@@ -72,6 +72,7 @@
         new-feed (services/insert-feed!
                   ds
                   {:data (merge body {:title (get-in extracted [:feed :title])
+                                      :display-picture (get-in extracted [:feed :display-picture])
                                       :user-id (:id user)
                                       :created-at datetime
                                       :state "pending"})})
@@ -79,7 +80,9 @@
                                (merge post
                                       {:feed-id (:id new-feed)
                                        :creator-id (:id user)
-                                       :content-type-id content-type-id})) extracted-posts)
+                                       :content-type-id content-type-id
+                                       :thumbnail (or (:thumbnail post) (:display-picture new-feed))}))
+                             extracted-posts)
         {:keys [email]} (services/user ds {:id (:id user)})]
 
     (if (some? extracted-posts)

From 368fa3fc858c5c20590d0036c2f65b8490ce898b Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Tue, 7 Oct 2025 13:59:44 +0200
Subject: [PATCH 123/391] updated outgoing post services

---
 src/source/services/interface.clj      |  6 ++++++
 src/source/services/outgoing_posts.clj | 14 ++++++++++++++
 2 files changed, 20 insertions(+)

diff --git a/src/source/services/interface.clj b/src/source/services/interface.clj
index a32b59a8..1fd0cedc 100644
--- a/src/source/services/interface.clj
+++ b/src/source/services/interface.clj
@@ -114,6 +114,12 @@
 (defn outgoing-post [ds {:keys [_id _where] :as opts}]
   (outgoing-posts/outgoing-post ds opts))
 
+(defn insert-outgoing-post! [ds {:keys [_values _ret] :as opts}]
+  (outgoing-posts/insert-outgoing-post! ds opts))
+
+(defn delete-outgoing-post! [ds {:keys [_id _where] :as opts}]
+  (outgoing-posts/delete-outgoing-post! ds opts))
+
 (defn upsert-outgoing-posts! [ds {:keys [_data] :as opts}]
   (outgoing-posts/upsert-outgoing-posts! ds opts))
 
diff --git a/src/source/services/outgoing_posts.clj b/src/source/services/outgoing_posts.clj
index 0e676c62..99c8ef6b 100644
--- a/src/source/services/outgoing_posts.clj
+++ b/src/source/services/outgoing_posts.clj
@@ -21,6 +21,20 @@
        (merge opts)
        (db/find ds)))
 
+(defn insert-outgoing-post! [ds {:keys [_values _ret] :as opts}]
+  (->> {:tname :outgoing-posts}
+       (merge opts)
+       (db/insert! ds)))
+
+(defn delete-outgoing-post! [ds {:keys [id where] :as opts}]
+  (->> {:tname :outgoing-posts
+        :where (if (some? id)
+                 [:= :id id]
+                 where)
+        :ret :1}
+       (merge opts)
+       (db/delete! ds)))
+
 (defn upsert-outgoing-posts! [ds {:keys [data]}]
   (hon/execute!
    ds

From f69a40fb5aed103e1ef3dabf94a6065704b2be57 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Thu, 9 Oct 2025 11:00:04 +0200
Subject: [PATCH 124/391] added endpoint for bundle to get single post from
 feed

---
 src/source/routes/bundle_feed_post.clj | 30 ++++++++++++++++++++++++++
 src/source/routes/reitit.clj           |  5 ++++-
 src/source/services/interface.clj      |  4 ++--
 3 files changed, 36 insertions(+), 3 deletions(-)
 create mode 100644 src/source/routes/bundle_feed_post.clj

diff --git a/src/source/routes/bundle_feed_post.clj b/src/source/routes/bundle_feed_post.clj
new file mode 100644
index 00000000..f4ab98f9
--- /dev/null
+++ b/src/source/routes/bundle_feed_post.clj
@@ -0,0 +1,30 @@
+(ns source.routes.bundle-feed-post
+  (:require [source.services.interface :as services]
+            [ring.util.response :as res]))
+
+(defn get
+  {:summary "get post by post id"
+   :parameters {:query [:map [:uuid :string]]
+                :path [:map [:post-id {:title "post-id"
+                                       :description "post id"} :int]]}
+   :responses {200 {:body
+                    [:map
+                     [:id :int]
+                     [:post-id :string]
+                     [:feed-id :int]
+                     [:creator-id :int]
+                     [:content-type-id :int]
+                     [:title :string]
+                     [:thumbnail [:maybe :string]]
+                     [:info [:maybe :string]]
+                     [:url [:maybe :string]]
+                     [:stream-url [:maybe :string]]
+                     [:season [:maybe :int]]
+                     [:episode [:maybe :int]]
+                     [:redacted {:optional true} [:maybe :int]]
+                     [:posted-at [:maybe :string]]]}
+               404 {:body [:map [:message :string]]}}}
+
+  [{:keys [ds path-params] :as _request}]
+  (-> (services/incoming-post ds {:id (:post-id path-params)})
+      (res/response)))
diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index 8ab3ac98..d0ebbd90 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -45,6 +45,7 @@
             [source.routes.bundle-feeds :as bundle-feeds]
             [source.routes.bundle-feed :as bundle-feed]
             [source.routes.bundle-feed-posts :as bundle-feed-posts]
+            [source.routes.bundle-feed-post :as bundle-feed-post]
             [source.routes.bundle-posts :as bundle-posts]
             [source.routes.bundle-post :as bundle-post]
             [source.routes.posts :as posts]
@@ -191,7 +192,9 @@
        [""              (route {:get bundle-feeds/get})]
        ["/:id"
         [""             (route {:get bundle-feed/get})]
-        ["/posts"       (route {:get bundle-feed-posts/get})]]]
+        ["/posts"
+         [""            (route {:get bundle-feed-posts/get})]
+         ["/:post-id"   (route {:get bundle-feed-post/get})]]]]
       ["/posts"
        [""              (route {:get bundle-posts/get})]
        ["/:id"          (route {:get bundle-post/get})]]]
diff --git a/src/source/services/interface.clj b/src/source/services/interface.clj
index 1fd0cedc..fabf6cfe 100644
--- a/src/source/services/interface.clj
+++ b/src/source/services/interface.clj
@@ -199,8 +199,8 @@
   [ds {:keys [_where] :as opts}]
   (incoming-posts/incoming-posts-with-feeds ds opts))
 
-(defn incoming-post [ds id]
-  (incoming-posts/incoming-post ds id))
+(defn incoming-post [ds opts]
+  (incoming-posts/incoming-post ds opts))
 
 (defn insert-cadence! [ds {:keys [_values _ret] :as opts}]
   (cadences/insert-cadence! ds opts))

From 63791861a4caa0429d6a5c18505d08cc1386da2b Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Fri, 10 Oct 2025 13:59:08 +0200
Subject: [PATCH 125/391] updated bundle posts endpoint to allow for filtering
 by content type and categories

---
 src/source/routes/bundle_posts.clj     | 44 +++++++++++++++++++++-----
 src/source/routes/reitit.clj           |  2 +-
 src/source/services/outgoing_posts.clj |  4 +--
 3 files changed, 39 insertions(+), 11 deletions(-)

diff --git a/src/source/routes/bundle_posts.clj b/src/source/routes/bundle_posts.clj
index aa65437a..d446773e 100644
--- a/src/source/routes/bundle_posts.clj
+++ b/src/source/routes/bundle_posts.clj
@@ -4,12 +4,14 @@
             [clojure.walk :as walk]
             [ring.util.response :as res]))
 
-(defn get
+(defn post
   {:summary "get all outgoing posts in the uuid-authorized bundle"
-   :parameters {:query [:map
+   :parameters {:body [:map [:category-ids [:vector :int]]]
+                :query [:map
                         [:uuid :string]
                         [:limit {:optional true} :int]
-                        [:start {:optional true} :int]]}
+                        [:start {:optional true} :int]
+                        [:type {:optional true} :int]]}
    :responses {200 {:body [:vector
                            [:map
                             [:id :int]
@@ -28,9 +30,35 @@
                             [:posted-at [:maybe :string]]]]}
                404 {:boy [:map [:message :string]]}}}
 
-  [{:keys [bundle-id query-params] :as _request}]
+  [{:keys [ds bundle-id query-params body] :as _request}]
   (let [bundle-ds (db.util/conn :bundle bundle-id)
-        {:keys [limit start]} (walk/keywordize-keys query-params)]
-    (res/response (services/outgoing-posts bundle-ds {:where (when start [:>= :id start])
-                                                      :limit limit
-                                                      :order-by [[:id :asc]]}))))
+        {:keys [limit start type]} (walk/keywordize-keys query-params)
+        {:keys [category-ids]} body
+
+        content-type-comp (when type [:= :content-type-id type])
+        start (when start (try (Integer/parseInt start) (catch Exception _)))
+        limit (when limit (try (Integer/parseInt limit) (catch Exception _)))
+
+        filtered-posts (services/outgoing-posts bundle-ds {:where content-type-comp})
+
+        categorised-posts (vec (if (seq category-ids)
+                                 (->> filtered-posts
+                                      (mapv
+                                       (fn [post]
+                                         (when (some (set category-ids) (->> {:feed-id (:feed-id post)}
+                                                                             (services/categories-by-feed ds)
+                                                                             (mapv :id)))
+                                           post)))
+                                      (remove nil?))
+                                 filtered-posts))
+
+        valid-start? (and (some? start) (>= start 0) (< start (count categorised-posts)))
+        started-posts (if valid-start?
+                        (subvec categorised-posts start)
+                        categorised-posts)
+
+        limited-posts (if (and (some? limit) (> (count started-posts) limit))
+                        (subvec started-posts 0 limit)
+                        started-posts)]
+
+    (res/response limited-posts)))
diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index d0ebbd90..4b93c5b3 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -196,7 +196,7 @@
          [""            (route {:get bundle-feed-posts/get})]
          ["/:post-id"   (route {:get bundle-feed-post/get})]]]]
       ["/posts"
-       [""              (route {:get bundle-posts/get})]
+       [""              (route {:post bundle-posts/post})]
        ["/:id"          (route {:get bundle-post/get})]]]
 
      ["/admin"                  {:middleware [[mw/apply-auth {:required-type :admin}]]
diff --git a/src/source/services/outgoing_posts.clj b/src/source/services/outgoing_posts.clj
index 99c8ef6b..611fd0c5 100644
--- a/src/source/services/outgoing_posts.clj
+++ b/src/source/services/outgoing_posts.clj
@@ -5,10 +5,10 @@
 
 (defn outgoing-posts
   ([ds] (outgoing-posts ds {}))
-  ([ds {:keys [where] :as opts}]
+  ([ds {:keys [where ret] :as opts}]
    (->> {:tname :outgoing-posts
          :where where
-         :ret :*}
+         :ret (or ret :*)}
         (merge opts)
         (db/find ds))))
 

From 5ba69ee995756a86c111fda4a08773f3d03cf83c Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Fri, 10 Oct 2025 15:42:44 +0200
Subject: [PATCH 126/391] added post shuffling for randomness

---
 src/source/routes/bundle_posts.clj | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/source/routes/bundle_posts.clj b/src/source/routes/bundle_posts.clj
index d446773e..67ab4aca 100644
--- a/src/source/routes/bundle_posts.clj
+++ b/src/source/routes/bundle_posts.clj
@@ -39,7 +39,7 @@
         start (when start (try (Integer/parseInt start) (catch Exception _)))
         limit (when limit (try (Integer/parseInt limit) (catch Exception _)))
 
-        filtered-posts (services/outgoing-posts bundle-ds {:where content-type-comp})
+        filtered-posts (shuffle (services/outgoing-posts bundle-ds {:where content-type-comp}))
 
         categorised-posts (vec (if (seq category-ids)
                                  (->> filtered-posts

From abe3ef58509744bf71884d4f410a164c434a3cac Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 13 Oct 2025 14:10:18 +0200
Subject: [PATCH 127/391] changed category filtering to use set intersection

---
 src/source/routes/bundle_posts.clj | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/src/source/routes/bundle_posts.clj b/src/source/routes/bundle_posts.clj
index 67ab4aca..c2d2e05e 100644
--- a/src/source/routes/bundle_posts.clj
+++ b/src/source/routes/bundle_posts.clj
@@ -2,7 +2,8 @@
   (:require [source.services.interface :as services]
             [source.db.util :as db.util]
             [clojure.walk :as walk]
-            [ring.util.response :as res]))
+            [ring.util.response :as res]
+            [clojure.set :as set]))
 
 (defn post
   {:summary "get all outgoing posts in the uuid-authorized bundle"
@@ -45,9 +46,12 @@
                                  (->> filtered-posts
                                       (mapv
                                        (fn [post]
-                                         (when (some (set category-ids) (->> {:feed-id (:feed-id post)}
-                                                                             (services/categories-by-feed ds)
-                                                                             (mapv :id)))
+                                         (when (seq (set/intersection
+                                                     (set category-ids)
+                                                     (->> {:feed-id (:feed-id post)}
+                                                          (services/categories-by-feed ds)
+                                                          (mapv :id)
+                                                          (set))))
                                            post)))
                                       (remove nil?))
                                  filtered-posts))

From 0dfccc5767a6582175eead2e92bdc079285154fd Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Tue, 14 Oct 2025 15:04:29 +0200
Subject: [PATCH 128/391] added route to generate an api key

---
 src/source/routes/integration_key.clj | 16 ++++++++++++++++
 src/source/routes/reitit.clj          |  2 ++
 2 files changed, 18 insertions(+)
 create mode 100644 src/source/routes/integration_key.clj

diff --git a/src/source/routes/integration_key.clj b/src/source/routes/integration_key.clj
new file mode 100644
index 00000000..0620fc90
--- /dev/null
+++ b/src/source/routes/integration_key.clj
@@ -0,0 +1,16 @@
+(ns source.routes.integration-key
+  (:require [source.middleware.auth.util :as auth.util]
+            [ring.util.response :as res]))
+
+(defn post
+  {:summary "generate an API key for the integration with the given id"
+   :parameters {:path [:map [:id {:title "id"
+                                  :description "integration id"} :int]]}
+   :responses {200 {:body [:map [:key :string]]}
+               401 {:body [:map [:message :string]]}
+               403 {:body [:map [:message :string]]}}}
+
+  [{:keys [user path-params] :as _request}]
+  (let [api-key (auth.util/sign-jwt {:user-id (:id user)
+                                     :bundle-id (:id path-params)})]
+    (res/response {:key api-key})))
diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index 4b93c5b3..c6c548cf 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -40,6 +40,7 @@
             [source.routes.feed-categories :as feed-categories]
             [source.routes.integrations :as integrations]
             [source.routes.integration :as integration]
+            [source.routes.integration-key :as integration-key]
             [source.routes.integration-categories :as integration-categories]
             [source.routes.bundle :as bundle]
             [source.routes.bundle-feeds :as bundle-feeds]
@@ -171,6 +172,7 @@
       ["/:id"
        [""              (route {:get integration/get
                                 :post integration/post})]
+       ["/key"          (route {:post integration-key/post})]
        ["/categories"   (route {:get integration-categories/get
                                 :post integration-categories/post})]]]
 

From e8d971982e814f29ea1291014b90be31379e711f Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Tue, 14 Oct 2025 15:05:49 +0200
Subject: [PATCH 129/391] added middleware to authorise api key

---
 src/source/middleware/auth/core.clj | 28 +++++++++++++++++++++++-----
 1 file changed, 23 insertions(+), 5 deletions(-)

diff --git a/src/source/middleware/auth/core.clj b/src/source/middleware/auth/core.clj
index 9be392fb..5d7f432b 100644
--- a/src/source/middleware/auth/core.clj
+++ b/src/source/middleware/auth/core.clj
@@ -4,7 +4,8 @@
             [ring.util.response :as res]
             [source.services.users :as users]
             [source.services.bundles :as bundles]
-            [source.db.honey :as db]))
+            [source.db.honey :as db]
+            [source.services.interface :as services]))
 
 (defn create-session [user]
   (let [payload {:id (:id user)
@@ -57,9 +58,26 @@
             (assoc :bundle-id id)
             (handler))
         (->
-         (res/response {:message "The bundle you are looking for does not exist"})
+         (res/response {:message "The bundle you are looking for does not exist."})
          (res/status 404))))))
 
+(defn wrap-auth-api-key
+  "validates the api key from the Authorization header for unauthenticated 
+  users and attaches the bundle-id to the request"
+  [handler]
+  (fn [request]
+    (let [ds (db.util/conn :master)
+          {:keys [bundle-id user-id]} (validate-request request)
+          existing-bundle (services/bundle ds {:where [:and
+                                                       [:= :id bundle-id]
+                                                       [:= :user-id user-id]]})]
+      (if (some? existing-bundle)
+        (-> request
+            (assoc :bundle-id bundle-id)
+            (handler))
+        (-> (res/response {:message "The bundle you are looking for does not exist."})
+            (res/status 404))))))
+
 (comment
   (let [authed-request {:headers {"Authorization"
                                   (str
@@ -115,8 +133,8 @@
                  (:bundle-id))))
     (println "tests passed")
     (db/delete! ds
-             {:tname :bundles
-              :where [:like :name "test-bundle-%"]
-              :ret :*}))
+                {:tname :bundles
+                 :where [:like :name "test-bundle-%"]
+                 :ret :*}))
 
   ())

From 06586af08828f1fd3be0a24613c115190804707a Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 15 Oct 2025 10:05:04 +0200
Subject: [PATCH 130/391] fixed cyclic dependency

---
 src/source/middleware/auth/core.clj | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/src/source/middleware/auth/core.clj b/src/source/middleware/auth/core.clj
index 5d7f432b..53afec49 100644
--- a/src/source/middleware/auth/core.clj
+++ b/src/source/middleware/auth/core.clj
@@ -4,8 +4,7 @@
             [ring.util.response :as res]
             [source.services.users :as users]
             [source.services.bundles :as bundles]
-            [source.db.honey :as db]
-            [source.services.interface :as services]))
+            [source.db.honey :as db]))
 
 (defn create-session [user]
   (let [payload {:id (:id user)
@@ -68,9 +67,9 @@
   (fn [request]
     (let [ds (db.util/conn :master)
           {:keys [bundle-id user-id]} (validate-request request)
-          existing-bundle (services/bundle ds {:where [:and
-                                                       [:= :id bundle-id]
-                                                       [:= :user-id user-id]]})]
+          existing-bundle (bundles/bundle ds {:where [:and
+                                                      [:= :id bundle-id]
+                                                      [:= :user-id user-id]]})]
       (if (some? existing-bundle)
         (-> request
             (assoc :bundle-id bundle-id)

From 1aa9afc6ad76981032e9233f22b63627fdf731e5 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 15 Oct 2025 10:13:48 +0200
Subject: [PATCH 131/391] added middleware to interface

---
 src/source/middleware/core.clj      | 8 ++++++--
 src/source/middleware/interface.clj | 3 +++
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/src/source/middleware/core.clj b/src/source/middleware/core.clj
index df88af5f..af6c7fa3 100644
--- a/src/source/middleware/core.clj
+++ b/src/source/middleware/core.clj
@@ -23,7 +23,7 @@
         (assoc :store store)
         (handler))))
 
-(defn wrap-js 
+(defn wrap-js
   "attaches the provided job service to the handler's request"
   [handler js]
   (fn [request]
@@ -39,7 +39,7 @@
   (-> app
       (wrap-store store)))
 
-(defn apply-js 
+(defn apply-js
   "middleware for attaching the job service to the request"
   [app js]
   (-> app
@@ -96,3 +96,7 @@
 (defn apply-bundle [app]
   (-> app
       (auth/wrap-bundle-id)))
+
+(defn apply-api-key [app]
+  (-> app
+      (auth/wrap-auth-api-key)))
diff --git a/src/source/middleware/interface.clj b/src/source/middleware/interface.clj
index 27b1db52..065e6d9f 100644
--- a/src/source/middleware/interface.clj
+++ b/src/source/middleware/interface.clj
@@ -11,3 +11,6 @@
 
 (defn apply-bundle [app]
   (mw/apply-bundle app))
+
+(defn apply-api-key [app]
+  (mw/apply-api-key app))

From 5921660a838e421c51293108472402a6baf3d244 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Fri, 17 Oct 2025 10:35:51 +0200
Subject: [PATCH 132/391] removed unneeded unique and upsert service in
 outgoing posts

---
 src/source/db/bundle.clj               |  3 +--
 src/source/services/interface.clj      |  3 ---
 src/source/services/outgoing_posts.clj | 18 ------------------
 3 files changed, 1 insertion(+), 23 deletions(-)

diff --git a/src/source/db/bundle.clj b/src/source/db/bundle.clj
index 2de9db30..3f930a37 100644
--- a/src/source/db/bundle.clj
+++ b/src/source/db/bundle.clj
@@ -28,8 +28,7 @@
    [:posted-at :datetime]
    (tables/foreign-key :feed-id :feeds :id)
    (tables/foreign-key :creator-id :users :id)
-   (tables/foreign-key :content-type-id :content-types :id)
-   [[:unique [:composite :post-id]]]))
+   (tables/foreign-key :content-type-id :content-types :id)))
 
 (def bundle-categories
   (tables/create-table-sql
diff --git a/src/source/services/interface.clj b/src/source/services/interface.clj
index fabf6cfe..bc477661 100644
--- a/src/source/services/interface.clj
+++ b/src/source/services/interface.clj
@@ -120,9 +120,6 @@
 (defn delete-outgoing-post! [ds {:keys [_id _where] :as opts}]
   (outgoing-posts/delete-outgoing-post! ds opts))
 
-(defn upsert-outgoing-posts! [ds {:keys [_data] :as opts}]
-  (outgoing-posts/upsert-outgoing-posts! ds opts))
-
 (defn selection-schemas
   ([ds]
    (selection-schemas ds {}))
diff --git a/src/source/services/outgoing_posts.clj b/src/source/services/outgoing_posts.clj
index 611fd0c5..fcd13789 100644
--- a/src/source/services/outgoing_posts.clj
+++ b/src/source/services/outgoing_posts.clj
@@ -34,21 +34,3 @@
         :ret :1}
        (merge opts)
        (db/delete! ds)))
-
-(defn upsert-outgoing-posts! [ds {:keys [data]}]
-  (hon/execute!
-   ds
-   (-> (hsql/insert-into :outgoing-posts)
-       (hsql/values data)
-       (assoc :on-conflict [:post-id])
-       (assoc :do-update-set {:feed-id          :excluded.feed-id
-                              :title            :excluded.title
-                              :thumbnail        :excluded.thumbnail
-                              :info             :excluded.info
-                              :url              :excluded.url
-                              :stream-url       :excluded.stream-url
-                              :creator-id       :excluded.creator-id
-                              :season           :excluded.season
-                              :episode          :excluded.episode
-                              :content-type-id  :excluded.content-type-id
-                              :posted-at        :excluded.posted-at}))))

From 144d54e0623eb5a52f0e022670d581be458d0e07 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Fri, 17 Oct 2025 15:29:52 +0200
Subject: [PATCH 133/391] added table services and updated endpoints to use
 them

---
 src/source/db/master.clj                      | 10 +++++
 .../migrations/003_bundle_content_types.clj   | 14 ++++++
 src/source/routes/integration.clj             | 25 +++++++++--
 src/source/routes/integrations.clj            | 21 ++++++---
 src/source/services/bundle_content_types.clj  | 45 +++++++++++++++++++
 src/source/services/interface.clj             | 18 ++++++++
 6 files changed, 124 insertions(+), 9 deletions(-)
 create mode 100644 src/source/migrations/003_bundle_content_types.clj
 create mode 100644 src/source/services/bundle_content_types.clj

diff --git a/src/source/db/master.clj b/src/source/db/master.clj
index 3355e6a9..13f00ced 100644
--- a/src/source/db/master.clj
+++ b/src/source/db/master.clj
@@ -67,6 +67,15 @@
    (tables/foreign-key :content-type-id :content-types :id)
    (tables/foreign-key :user-id :users :id)))
 
+(def bundle-content-types
+  (tables/create-table-sql
+   :bundle-content-types
+   (tables/table-id)
+   [:bundle-id :integer :not nil]
+   [:content-type-id :integer :not nil]
+   (tables/foreign-key :bundle-id :bundles :id)
+   (tables/foreign-key :content-type-id :content-types :id)))
+
 (def feeds
   (tables/create-table-sql
    :feeds
@@ -203,6 +212,7 @@
   (sql/format categories)
   (sql/format baselines)
   (sql/format bundles)
+  (sql/format bundle-content-types)
   (sql/format feeds)
   (sql/format feed-categories)
   (sql/format providers)
diff --git a/src/source/migrations/003_bundle_content_types.clj b/src/source/migrations/003_bundle_content_types.clj
new file mode 100644
index 00000000..c52930d5
--- /dev/null
+++ b/src/source/migrations/003_bundle_content_types.clj
@@ -0,0 +1,14 @@
+(ns source.migrations.003-bundle-content-types
+  (:require [source.db.master]
+            [source.db.tables :as tables]))
+
+(defn run-up! [context]
+  (let [ds-master (:db-master context)]
+    (tables/create-tables!
+     ds-master
+     :source.db.master
+     [:bundle-content-types])))
+
+(defn run-down! [context]
+  (let [ds-master (:db-master context)]
+    (tables/drop-table! ds-master :bundle-content-types)))
diff --git a/src/source/routes/integration.clj b/src/source/routes/integration.clj
index ed76ee19..43f46182 100644
--- a/src/source/routes/integration.clj
+++ b/src/source/routes/integration.clj
@@ -20,12 +20,18 @@
                            [:blog :int]
                            [:hash [:maybe :string]]
                            [:content-type-id :int]
-                           [:ts-and-cs [:maybe :int]]]}
+                           [:ts-and-cs [:maybe :int]]
+                           [:content-types [:vector
+                                            [:map
+                                             [:id :int]
+                                             [:name :string]]]]]}
                401 {:body [:map [:message :string]]}
                403 {:body [:map [:message :string]]}}}
 
   [{:keys [ds path-params] :as _request}]
-  (res/response (services/bundle ds {:id (:id path-params)})))
+  (let [integration (services/bundle ds {:id (:id path-params)})
+        content-types (services/content-types-by-bundle ds {:bundle-id (:id path-params)})]
+    (res/response (assoc integration :content-types content-types))))
 
 (defn post
   {:summary "update an integration by id"
@@ -33,7 +39,10 @@
                                   :description "integration id"} :int]]
                 :body [:map
                        [:name :string]
-                       [:content-type-id :int]
+                       [:content-types [:vector
+                                        [:map
+                                         [:id :int]
+                                         [:name :string]]]]
                        [:categories [:vector
                                      [:map
                                       [:id :int]
@@ -44,11 +53,15 @@
 
   [{:keys [js ds store path-params body] :as _request}]
   (let [_ (services/update-bundle! ds {:id (:id path-params)
-                                       :data (dissoc body :categories)})
+                                       :data (dissoc body :categories :content-types)})
 
         bundle-categories (mapv (fn [{:keys [id]}]
                                   {:bundle-id (:id path-params)
                                    :category-id id}) (:categories body))
+        bundle-content-types (mapv (fn [{:keys [id]}]
+                                     {:bundle-id (:id path-params)
+                                      :content-type-id id}) (:content-types body))
+
         job-id (str "bundle_" (:id path-params))
 
         ; update bundle categories
@@ -56,6 +69,10 @@
         _ (services/delete-bundle-category! bundle-ds {:where [:= :bundle-id (:id path-params)]})
         _ (services/insert-bundle-category! bundle-ds {:data bundle-categories})
 
+        ; update bundle content types
+        _ (services/delete-bundle-content-types! ds {:where [:= :bundle-id (:id path-params)]})
+        _ (services/insert-bundle-content-types! ds {:data bundle-content-types})
+
         category-ids (services/category-id-by-bundle bundle-ds {:bundle-id (:id path-params)})
         id-vec (mapv (fn [{:keys [category-id]}] category-id) category-ids)
         categories-by-bundle (services/categories ds {:where [:in :id id-vec]})]
diff --git a/src/source/routes/integrations.clj b/src/source/routes/integrations.clj
index c6f25f1c..34a34f38 100644
--- a/src/source/routes/integrations.clj
+++ b/src/source/routes/integrations.clj
@@ -31,8 +31,11 @@
   {:summary "add an integration"
    :parameters {:body [:map
                        [:name :string]
-                       [:content-type-id :int]
                        [:ts-and-cs {:optional true} :int]
+                       [:content-types [:vector
+                                        [:map
+                                         [:id :int]
+                                         [:name :string]]]]
                        [:categories [:vector
                                      [:map
                                       [:id :int]
@@ -53,17 +56,25 @@
 
   [{:keys [js ds store user body] :as _request}]
   (let [new-bundle (services/insert-bundle! ds {:data (merge
-                                                       (dissoc body :categories)
+                                                       (dissoc body :categories :content-types)
                                                        {:user-id (:id user)
+                                                        :content-type-id 1 ; temporarily assign garbo id
                                                         :uuid (utils/uuid)})
                                                 :ret :1})
-        bundle-categories (reduce (fn [acc {:keys [id]}]
-                                    (conj acc {:bundle-id (:id new-bundle)
-                                               :category-id id})) [] (:categories body))
+        bundle-categories (mapv (fn [{:keys [id]}]
+                                  {:bundle-id (:id new-bundle)
+                                   :category-id id}) (:categories body))
+        bundle-content-types (mapv (fn [{:keys [id]}]
+                                     {:bundle-id (:id new-bundle)
+                                      :content-type-id id}) (:content-types body))
+
         _ (migrate/migrate-bundle (:id new-bundle) ["up"])
         ; insert bundle categories
         bundle-ds (db.util/conn :bundle (:id new-bundle))
+
         _ (services/insert-bundle-category! bundle-ds {:data bundle-categories})
+        _ (services/insert-bundle-content-types! ds {:data bundle-content-types})
+
         category-ids (services/category-id-by-bundle bundle-ds {:bundle-id (:id new-bundle)})
         id-vec (mapv (fn [{:keys [category-id]}] category-id) category-ids)
         categories-by-bundle (services/categories ds {:where [:in :id id-vec]})]
diff --git a/src/source/services/bundle_content_types.clj b/src/source/services/bundle_content_types.clj
new file mode 100644
index 00000000..a0b4562f
--- /dev/null
+++ b/src/source/services/bundle_content_types.clj
@@ -0,0 +1,45 @@
+(ns source.services.bundle-content-types
+  (:require [source.db.interface :as db]
+            [source.db.honey :as hon]))
+
+(defn bundle-content-types
+  ([ds] (bundle-content-types ds {}))
+  ([ds {:keys [where] :as opts}]
+   (->> {:tname :bundle-content-types
+         :where where
+         :ret :*}
+        (merge opts)
+        (db/find ds))))
+
+(defn insert-bundle-content-types! [ds {:keys [_data _ret] :as opts}]
+  (->> {:tname :bundle-content-types}
+       (merge opts)
+       (db/insert! ds)))
+
+(defn delete-bundle-content-types! [ds {:keys [id where] :as opts}]
+  (->> {:tname :bundle-content-types
+        :where (if (some? id)
+                 [:= :id id]
+                 where)
+        :ret :1}
+       (merge opts)
+       (db/delete! ds)))
+
+(defn content-types-by-bundle [ds {:keys [bundle-id where] :as _opts}]
+  (hon/execute! ds
+                {:select [[:bundle-content-types.content-type-id :id] :name]
+                 :from :content-types
+                 :join [:bundle-content-types [:= :bundle-content-types.content-type-id :content-types.id]]
+                 :where (if (some? bundle-id)
+                          [:= :bundle-id bundle-id]
+                          where)}
+                {:ret :*}))
+
+(defn content-type-id [ds {:keys [bundle-id where] :as opts}]
+  (->> {:tname :bundle-content-types
+        :where (if (some? bundle-id)
+                 [:= :bundle-id bundle-id]
+                 where)
+        :ret :*}
+       (merge opts)
+       (db/find ds)))
diff --git a/src/source/services/interface.clj b/src/source/services/interface.clj
index bc477661..b3b949af 100644
--- a/src/source/services/interface.clj
+++ b/src/source/services/interface.clj
@@ -5,6 +5,7 @@
              [source.services.xml-schemas :as xml]
              [source.services.bundles :as bundles]
              [source.services.bundle-categories :as bundle-categories]
+             [source.services.bundle-content-types :as bundle-content-types]
              [source.services.post-heuristics :as post-heuristics]
              [source.services.providers :as providers]
              [source.services.feeds :as feeds]
@@ -86,6 +87,23 @@
 (defn category-id-by-bundle [ds {:keys [_bundle-id _where] :as opts}]
   (bundle-categories/category-id ds opts))
 
+(defn bundle-content-types
+  ([ds] (bundle-content-types ds {}))
+  ([ds {:keys [_where] :as opts}]
+   (bundle-content-types/bundle-content-types ds opts)))
+
+(defn insert-bundle-content-types! [ds {:keys [_data _ret] :as opts}]
+  (bundle-content-types/insert-bundle-content-types! ds opts))
+
+(defn delete-bundle-content-types! [ds {:keys [_id _where] :as opts}]
+  (bundle-content-types/delete-bundle-content-types! ds opts))
+
+(defn content-types-by-bundle [ds {:keys [_bundle-id _where] :as opts}]
+  (bundle-content-types/content-types-by-bundle ds opts))
+
+(defn content-type-id [ds {:keys [_bundle-id _where] :as opts}]
+  (bundle-content-types/content-type-id ds opts))
+
 (defn insert-post-heuristics! [ds {:keys [_data] :as opts}]
   (post-heuristics/insert-post-heuristics! ds opts))
 

From 2946287438aaf65e1a4f37daf93a1a2649e294bf Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Fri, 17 Oct 2025 15:58:02 +0200
Subject: [PATCH 134/391] fixed migrations to either move single content type
 to table or bring the latest from table

---
 .../migrations/003_bundle_content_types.clj   | 24 +++++++++++++++----
 1 file changed, 20 insertions(+), 4 deletions(-)

diff --git a/src/source/migrations/003_bundle_content_types.clj b/src/source/migrations/003_bundle_content_types.clj
index c52930d5..0ac8785f 100644
--- a/src/source/migrations/003_bundle_content_types.clj
+++ b/src/source/migrations/003_bundle_content_types.clj
@@ -1,14 +1,30 @@
 (ns source.migrations.003-bundle-content-types
   (:require [source.db.master]
-            [source.db.tables :as tables]))
+            [source.db.tables :as tables]
+            [source.services.interface :as services]))
 
 (defn run-up! [context]
-  (let [ds-master (:db-master context)]
+  (let [ds-master (:db-master context)
+        bundles (services/bundles ds-master)]
+
     (tables/create-tables!
      ds-master
      :source.db.master
-     [:bundle-content-types])))
+     [:bundle-content-types])
+
+    (run! (fn [{:keys [id content-type-id]}]
+            (when (some? content-type-id)
+              (services/insert-bundle-content-types! ds-master {:data {:bundle-id id
+                                                                       :content-type-id content-type-id}})))
+          bundles)))
 
 (defn run-down! [context]
-  (let [ds-master (:db-master context)]
+  (let [ds-master (:db-master context)
+        bundle-content-types (services/bundles ds-master)]
+
+    (run! (fn [{:keys [bundle-id content-type-id]}]
+            (services/update-bundle! ds-master {:id bundle-id
+                                                :data {:content-type-id content-type-id}}))
+          bundle-content-types)
+
     (tables/drop-table! ds-master :bundle-content-types)))

From 1708625629d4b07f8d5c25f2aa40fb8f95e04d90 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 20 Oct 2025 11:26:43 +0200
Subject: [PATCH 135/391] updated api key to be sha256 key stored in table

---
 src/source/middleware/auth/core.clj   |  8 +++-----
 src/source/routes/integration_key.clj | 13 ++++++++-----
 src/source/util.clj                   | 16 +++++++++++++---
 3 files changed, 24 insertions(+), 13 deletions(-)

diff --git a/src/source/middleware/auth/core.clj b/src/source/middleware/auth/core.clj
index 53afec49..6248aa7c 100644
--- a/src/source/middleware/auth/core.clj
+++ b/src/source/middleware/auth/core.clj
@@ -66,13 +66,11 @@
   [handler]
   (fn [request]
     (let [ds (db.util/conn :master)
-          {:keys [bundle-id user-id]} (validate-request request)
-          existing-bundle (bundles/bundle ds {:where [:and
-                                                      [:= :id bundle-id]
-                                                      [:= :user-id user-id]]})]
+          token (util/auth-token request)
+          existing-bundle (bundles/bundle ds {:where [:= :hash token]})]
       (if (some? existing-bundle)
         (-> request
-            (assoc :bundle-id bundle-id)
+            (assoc :bundle-id (:id existing-bundle))
             (handler))
         (-> (res/response {:message "The bundle you are looking for does not exist."})
             (res/status 404))))))
diff --git a/src/source/routes/integration_key.clj b/src/source/routes/integration_key.clj
index 0620fc90..6a282688 100644
--- a/src/source/routes/integration_key.clj
+++ b/src/source/routes/integration_key.clj
@@ -1,6 +1,7 @@
 (ns source.routes.integration-key
-  (:require [source.middleware.auth.util :as auth.util]
-            [ring.util.response :as res]))
+  (:require [source.util :as util]
+            [ring.util.response :as res]
+            [source.services.interface :as services]))
 
 (defn post
   {:summary "generate an API key for the integration with the given id"
@@ -10,7 +11,9 @@
                401 {:body [:map [:message :string]]}
                403 {:body [:map [:message :string]]}}}
 
-  [{:keys [user path-params] :as _request}]
-  (let [api-key (auth.util/sign-jwt {:user-id (:id user)
-                                     :bundle-id (:id path-params)})]
+  [{:keys [ds user path-params] :as _request}]
+  (let [uuid (util/uuid)
+        api-key (util/sha256 (str (:id user) (:id path-params) uuid))]
+    (services/update-bundle! ds {:id (:id path-params)
+                                 :data {:hash api-key}})
     (res/response {:key api-key})))
diff --git a/src/source/util.clj b/src/source/util.clj
index e5ad60dc..fea080d5 100644
--- a/src/source/util.clj
+++ b/src/source/util.clj
@@ -3,8 +3,9 @@
             [buddy.core.nonce :as nonce]
             [clojure.main :refer [demunge]]
             [malli.core :as m]
-            [malli.transform :as mt]
-            [malli.error :as me]))
+            [malli.error :as me])
+  (:import (java.math BigInteger)
+           (java.security MessageDigest)))
 
 (defn vectors?
   "Returns true if coll is a 2d vector"
@@ -44,6 +45,14 @@
       (find-var)
       (meta)))
 
+(defn sha256
+  "Computes SHA256 hash of given string and returns it as a hex string"
+  [s]
+  (let [digest (MessageDigest/getInstance "SHA-256")
+        bytes (.getBytes s "UTF-8")
+        hash-bytes (.digest digest bytes)]
+    (format "%064x" (BigInteger. 1 hash-bytes))))
+
 (defn validate [handler data]
   (let [schema (get-in (metadata handler) [:parameters :body])
         success (m/validate schema data)]
@@ -53,8 +62,9 @@
                                    (m/explain schema)
                                    (me/humanize)))}))
 
-(comment 
+(comment
   (require '[source.routes.business :as business])
   (validate business/post {:cheese "modulr"})
+  (sha256 "1")
   ())
 

From 49e5737d992ce78d0e8b00a733ae971135d7bac2 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 20 Oct 2025 19:45:32 +0200
Subject: [PATCH 136/391] updated categories table with display picture field

---
 src/source/db/master.clj                      |  3 ++-
 src/source/migrations/004_category_images.clj | 15 +++++++++++++++
 src/source/routes/categories.clj              |  6 ++++--
 3 files changed, 21 insertions(+), 3 deletions(-)
 create mode 100644 src/source/migrations/004_category_images.clj

diff --git a/src/source/db/master.clj b/src/source/db/master.clj
index 13f00ced..15dc2913 100644
--- a/src/source/db/master.clj
+++ b/src/source/db/master.clj
@@ -28,7 +28,8 @@
   (tables/create-table-sql
    :categories
    (tables/table-id)
-   [:name :text]))
+   [:name :text]
+   [:display-picture :string]))
 
 (def content-types
   (tables/create-table-sql
diff --git a/src/source/migrations/004_category_images.clj b/src/source/migrations/004_category_images.clj
new file mode 100644
index 00000000..8f90b2d8
--- /dev/null
+++ b/src/source/migrations/004_category_images.clj
@@ -0,0 +1,15 @@
+(ns source.migrations.004-category-images
+  (:require [source.db.master]
+            [source.db.honey :as hon]))
+
+(defn run-up! [context]
+  (let [ds-master (:db-master context)]
+    (->> {:alter-table :categories
+          :add-column [:display-picture :text]}
+         (hon/execute! ds-master))))
+
+(defn run-down! [context]
+  (let [ds-master (:db-master context)]
+    (->> {:alter-table :categories
+          :drop-column :display-picture}
+         (hon/execute! ds-master))))
diff --git a/src/source/routes/categories.clj b/src/source/routes/categories.clj
index cddbb67b..57b6b286 100644
--- a/src/source/routes/categories.clj
+++ b/src/source/routes/categories.clj
@@ -1,13 +1,15 @@
 (ns source.routes.categories
   (:require [source.services.interface :as services]
-            [ring.util.response :as res]))
+            [ring.util.response :as res]
+            [source.db.util :as db.util]))
 
 (defn get
   {:summary "get all categories"
    :responses {200 {:body [:vector
                            [:map
                             [:id :int]
-                            [:name :string]]]}}}
+                            [:name :string]
+                            [:display-picture {:optional true} [:maybe :string]]]]}}}
   [{:keys [ds] :as _request}]
   (->> (services/categories ds)
        (res/response)))

From 5d9f352ffa5c9ce323edf245fe12471a9aa65e38 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 22 Oct 2025 15:27:57 +0200
Subject: [PATCH 137/391] updated handler to fallback to feed display picture
 if the image pulled is an MP3

---
 src/source/jobs/handlers.clj | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/source/jobs/handlers.clj b/src/source/jobs/handlers.clj
index 8279fc0c..b189438a 100644
--- a/src/source/jobs/handlers.clj
+++ b/src/source/jobs/handlers.clj
@@ -3,7 +3,8 @@
             [source.util :as util]
             [source.services.incoming-posts :as incoming-posts]
             [source.db.util :as db.util]
-            [clojure.set :as set]))
+            [clojure.set :as set]
+            [clojure.string :as string]))
 
 (defmulti handler
   (fn [opts]
@@ -38,7 +39,8 @@
                                            :creator-id creator-id
                                            :content-type-id content-type-id
                                            :thumbnail (if (and thumbnail
-                                                               (seq thumbnail))
+                                                               (seq thumbnail)
+                                                               (not (string/includes? thumbnail ".mp3")))
                                                         thumbnail
                                                         extracted-display)}))
                                  extracted-posts)

From c4f0e3669e2ce6528708e1a0c7007a32abffd3b1 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Thu, 23 Oct 2025 15:02:24 +0200
Subject: [PATCH 138/391] updated bundle feeds route to filter by content type
 and categories

---
 src/source/routes/bundle_feeds.clj | 31 ++++++++++++++++++++++++------
 src/source/routes/reitit.clj       |  2 +-
 2 files changed, 26 insertions(+), 7 deletions(-)

diff --git a/src/source/routes/bundle_feeds.clj b/src/source/routes/bundle_feeds.clj
index db30a0a7..fee3baae 100644
--- a/src/source/routes/bundle_feeds.clj
+++ b/src/source/routes/bundle_feeds.clj
@@ -1,11 +1,15 @@
 (ns source.routes.bundle-feeds
   (:require [ring.util.response :as res]
             [source.services.interface :as services]
-            [source.db.util :as db.util]))
+            [source.db.util :as db.util]
+            [clojure.walk :as walk]))
 
-(defn get
+(defn post
   {:summary "get all feeds present in the bundle authorised by uuid"
-   :parameters {:query [:map [:uuid :string]]}
+   :parameters {:query [:map
+                        [:uuid :string]
+                        [:type {:optional true} :int]]
+                :body [:map [:category-ids [:vector :int]]]}
    :responses {200 {:body [:vector
                            [:map
                             [:id :int]
@@ -24,7 +28,22 @@
                             [:state [:enum "live" "not live" "pending"]]]]}
                404 {:body [:map [:message :string]]}}}
 
-  [{:keys [ds bundle-id] :as _request}]
+  [{:keys [ds bundle-id query-params body] :as _request}]
   (let [bundle-ds (db.util/conn :bundle bundle-id)
-        feed-ids (mapv :feed-id (services/outgoing-posts bundle-ds))]
-    (res/response (services/feeds ds {:where [:in :id feed-ids]}))))
+        {:keys [category-ids]} body
+        {:keys [type]} (walk/keywordize-keys query-params)
+        feed-ids (mapv :feed-id (services/outgoing-posts bundle-ds))
+        category-filtered-feed-ids (if (empty? category-ids)
+                                     feed-ids
+                                     (->> {:where [:and
+                                                   [:in :feed-id feed-ids]
+                                                   [:in :category-id category-ids]]}
+                                          (services/feed-categories ds)
+                                          (mapv :feed-id)))
+        id-comp [:in :id category-filtered-feed-ids]
+        where-clause (if (some? type)
+                       [:and id-comp [:= :content-type-id type]]
+                       id-comp)
+        type-filtered (services/feeds ds {:where where-clause})]
+
+    (res/response type-filtered)))
diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index c6c548cf..81e08594 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -191,7 +191,7 @@
                         :tags #{"bundles"}}
       [""               (route {:get bundle/get})]
       ["/feeds"
-       [""              (route {:get bundle-feeds/get})]
+       [""              (route {:post bundle-feeds/post})]
        ["/:id"
         [""             (route {:get bundle-feed/get})]
         ["/posts"

From b82f6cfc5a38846f920d5bff3c9f74cf6c39c5ec Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Fri, 24 Oct 2025 09:56:45 +0200
Subject: [PATCH 139/391] addressed comments

---
 src/source/routes/bundle_feeds.clj | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/src/source/routes/bundle_feeds.clj b/src/source/routes/bundle_feeds.clj
index fee3baae..146aa1b6 100644
--- a/src/source/routes/bundle_feeds.clj
+++ b/src/source/routes/bundle_feeds.clj
@@ -2,7 +2,8 @@
   (:require [ring.util.response :as res]
             [source.services.interface :as services]
             [source.db.util :as db.util]
-            [clojure.walk :as walk]))
+            [clojure.walk :as walk]
+            [honey.sql.helpers :as hsql]))
 
 (defn post
   {:summary "get all feeds present in the bundle authorised by uuid"
@@ -35,15 +36,13 @@
         feed-ids (mapv :feed-id (services/outgoing-posts bundle-ds))
         category-filtered-feed-ids (if (empty? category-ids)
                                      feed-ids
-                                     (->> {:where [:and
-                                                   [:in :feed-id feed-ids]
-                                                   [:in :category-id category-ids]]}
+                                     (->> (hsql/where
+                                           [:in :feed-id feed-ids]
+                                           [:in :category-id category-ids])
                                           (services/feed-categories ds)
                                           (mapv :feed-id)))
-        id-comp [:in :id category-filtered-feed-ids]
-        where-clause (if (some? type)
-                       [:and id-comp [:= :content-type-id type]]
-                       id-comp)
-        type-filtered (services/feeds ds {:where where-clause})]
+        type-filtered (->> (when type [:= :content-type-id type])
+                           (hsql/where [:in :id category-filtered-feed-ids])
+                           (services/feeds ds))]
 
     (res/response type-filtered)))

From 27459005c76b1deb6c95f2184fc1183b0225040b Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Fri, 24 Oct 2025 12:39:20 +0200
Subject: [PATCH 140/391] removed erroneous function from bundle-categories
 services

---
 src/source/services/bundle_categories.clj | 13 +------------
 src/source/services/interface.clj         |  3 ---
 2 files changed, 1 insertion(+), 15 deletions(-)

diff --git a/src/source/services/bundle_categories.clj b/src/source/services/bundle_categories.clj
index 6f4b9168..c63b4aa0 100644
--- a/src/source/services/bundle_categories.clj
+++ b/src/source/services/bundle_categories.clj
@@ -1,6 +1,5 @@
 (ns source.services.bundle-categories
-  (:require [source.db.interface :as db]
-            [source.db.honey :as hon]))
+  (:require [source.db.interface :as db]))
 
 (defn bundle-categories
   ([ds] (bundle-categories ds {}))
@@ -25,16 +24,6 @@
        (merge opts)
        (db/delete! ds)))
 
-(defn categories-by-bundle [ds {:keys [bundle-id where] :as _opts}]
-  (hon/execute! ds
-                {:select [[:bundle-categories.category-id :id] :name]
-                 :from :categories
-                 :join [:bundle-categories [:= :bundle-categories.category-id :categories.id]]
-                 :where (if (some? bundle-id)
-                          [:= :bundle-id bundle-id]
-                          where)}
-                {:ret :*}))
-
 (defn category-id [ds {:keys [bundle-id where] :as opts}]
   (->> {:tname :bundle-categories
         :where (if (some? bundle-id)
diff --git a/src/source/services/interface.clj b/src/source/services/interface.clj
index b3b949af..9597fef1 100644
--- a/src/source/services/interface.clj
+++ b/src/source/services/interface.clj
@@ -81,9 +81,6 @@
 (defn delete-bundle-category! [ds {:keys [_id _where] :as opts}]
   (bundle-categories/delete-bundle-category! ds opts))
 
-(defn categories-by-bundle [ds {:keys [_bundle-id _where] :as opts}]
-  (bundle-categories/categories-by-bundle ds opts))
-
 (defn category-id-by-bundle [ds {:keys [_bundle-id _where] :as opts}]
   (bundle-categories/category-id ds opts))
 

From c37572dfe0ea864a882c614f0a2d60e7f9a16c45 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Fri, 24 Oct 2025 12:41:39 +0200
Subject: [PATCH 141/391] added bundle categories endpoint

---
 src/source/routes/bundle_categories.clj | 22 ++++++++++++++++++++++
 src/source/routes/reitit.clj            | 25 ++++++++++++++++++-------
 2 files changed, 40 insertions(+), 7 deletions(-)
 create mode 100644 src/source/routes/bundle_categories.clj

diff --git a/src/source/routes/bundle_categories.clj b/src/source/routes/bundle_categories.clj
new file mode 100644
index 00000000..ac31df95
--- /dev/null
+++ b/src/source/routes/bundle_categories.clj
@@ -0,0 +1,22 @@
+(ns source.routes.bundle-categories
+  (:require [source.services.interface :as services]
+            [source.db.util :as db.util]
+            [ring.util.response :as res]))
+
+(defn get
+  {:summary "get categories in the uuid-authorized bundle"
+   :parameters {:query [:map [:uuid :string]]}
+   :responses {200 {:body [:vector
+                           [:map
+                            [:id :int]
+                            [:name :string]
+                            [:display-picture {:optional true} [:maybe :string]]]]}
+               404 {:body [:map [:message :string]]}}}
+
+  [{:keys [bundle-id ds] :as _request}]
+  (let [bundle-ds (db.util/conn :bundle bundle-id)
+        feed-ids (->> (services/outgoing-posts bundle-ds)
+                      (mapv :feed-id))
+        category-ids (->> (services/feed-categories ds {:where [:in :feed-id feed-ids]})
+                          (mapv :category-id))]
+    (res/response (services/categories ds {:where [:in :id category-ids]}))))
diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index 81e08594..d6f71868 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -8,6 +8,7 @@
             [malli.util :as mu]
             [source.middleware.interface :as mw]
             [clojure.data.json :as json]
+            [source.util :as util]
             [source.routes.user :as user]
             [source.routes.users :as users]
             [source.routes.me :as me]
@@ -32,6 +33,7 @@
             [source.routes.provider :as provider]
             [source.routes.cadences :as cadences]
             [source.routes.categories :as categories]
+            [source.routes.category :as category]
             [source.routes.baselines :as baselines]
             [source.routes.content-types :as content-types]
             [source.routes.content-type :as content-type]
@@ -43,6 +45,7 @@
             [source.routes.integration-key :as integration-key]
             [source.routes.integration-categories :as integration-categories]
             [source.routes.bundle :as bundle]
+            [source.routes.bundle-categories :as bundle-categories]
             [source.routes.bundle-feeds :as bundle-feeds]
             [source.routes.bundle-feed :as bundle-feed]
             [source.routes.bundle-feed-posts :as bundle-feed-posts]
@@ -60,8 +63,7 @@
             [source.routes.job-stop :as job-stop]
             [source.routes.report :as report]
             [source.routes.approve-feed :as approve-feed]
-            [source.routes.reject-feed :as reject-feed]
-            [source.util :as util]))
+            [source.routes.reject-feed :as reject-feed]))
 
 (defn route [handlers]
   (reduce (fn [acc [k v]]
@@ -82,7 +84,10 @@
                                                 :version "0.0.1"}
                                          :securityDefinitions {"auth" {:type :apiKey
                                                                        :in :header
-                                                                       :name "Authorization"}}}
+                                                                       :name "Authorization"}
+                                                               "apiKey" {:type :apiKey
+                                                                         :in :header
+                                                                         :name "Authorization"}}}
                                :handler (swagger/create-swagger-handler)}}]
 
      ["/openapi.json"   {:get {:no-doc true
@@ -92,7 +97,10 @@
                                          :components {:securitySchemes {"bearerAuth" {:type :http
                                                                                       :scheme :bearer
                                                                                       :bearerFormat "JWT"
-                                                                                      :description "JWT Authorization using the Bearer scheme"}}}}
+                                                                                      :description "JWT Authorization using the Bearer scheme"}
+                                                                        "apiKey" {:type :http
+                                                                                  :scheme :bearer
+                                                                                  :description "API Key authorization using the Bearer scheme"}}}}
                                :handler (openapi/create-openapi-handler)}}]
 
      ["/users"          {:middleware [[mw/apply-auth {:required-type :admin}]]
@@ -152,11 +160,12 @@
       [""               (route {:get providers/get})]
       ["/:id"           (route {:get provider/get})]]
 
-     ["/cadences"      {:tags #{"cadences"}}
+     ["/cadences"       {:tags #{"cadences"}}
       [""               (route {:get cadences/get})]]
 
-     ["/categories"      {:tags #{"categories"}}
-      [""               (route {:get categories/get})]]
+     ["/categories"     {:tags #{"categories"}}
+      [""               (route {:get categories/get})]
+      ["/:id"           (route {:get category/get})]]
 
      ["/baselines"      {:tags #{"baselines"}}
       [""               (route {:get baselines/get})]]
@@ -190,6 +199,8 @@
      ["/bundle"        {:middleware [[mw/apply-bundle]]
                         :tags #{"bundles"}}
       [""               (route {:get bundle/get})]
+      ["/categories"
+       [""              (route {:get bundle-categories/get})]]
       ["/feeds"
        [""              (route {:post bundle-feeds/post})]
        ["/:id"

From 611f8fc2faaa9f47949399eecffe5e8529de85fe Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Fri, 24 Oct 2025 16:12:15 +0200
Subject: [PATCH 142/391] added missing file

---
 src/source/routes/category.clj | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
 create mode 100644 src/source/routes/category.clj

diff --git a/src/source/routes/category.clj b/src/source/routes/category.clj
new file mode 100644
index 00000000..3ffe6d9b
--- /dev/null
+++ b/src/source/routes/category.clj
@@ -0,0 +1,14 @@
+(ns source.routes.category
+  (:require [ring.util.response :as res]
+            [source.services.interface :as services]))
+
+(defn get
+  {:summary "get category by id"
+   :parameters {:path [:map [:id {:title "id"
+                                  :description "category id"} :int]]}
+   :responses {200 {:body [:map
+                           [:id :int]
+                           [:name :string]]}}}
+
+  [{:keys [ds path-params] :as _request}]
+  (res/response (services/category ds path-params)))

From fff553795221e2bf5ba00d182a01bc0463c3652f Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 27 Oct 2025 15:09:51 +0200
Subject: [PATCH 143/391] updated feed/posts job to change date format to
 ISO8601

---
 src/source/jobs/handlers.clj |  3 ++-
 src/source/util.clj          | 11 +++++++++++
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/src/source/jobs/handlers.clj b/src/source/jobs/handlers.clj
index b189438a..ae03c189 100644
--- a/src/source/jobs/handlers.clj
+++ b/src/source/jobs/handlers.clj
@@ -33,11 +33,12 @@
                                                     :url url})
             extracted-posts (get-in extracted [:feed :posts])
             extracted-display (get-in extracted [:feed :display-picture])
-            extended-posts (mapv (fn [{:keys [thumbnail] :as post}]
+            extended-posts (mapv (fn [{:keys [posted-at thumbnail] :as post}]
                                    (merge post
                                           {:feed-id feed-id
                                            :creator-id creator-id
                                            :content-type-id content-type-id
+                                           :posted-at (util/format-rfc-1123-date posted-at)
                                            :thumbnail (if (and thumbnail
                                                                (seq thumbnail)
                                                                (not (string/includes? thumbnail ".mp3")))
diff --git a/src/source/util.clj b/src/source/util.clj
index fea080d5..68900d3f 100644
--- a/src/source/util.clj
+++ b/src/source/util.clj
@@ -62,6 +62,17 @@
                                    (m/explain schema)
                                    (me/humanize)))}))
 
+(defn format-rfc-1123-date
+  "Takes a date as a string in RFC 1123 format and returns it in a format that meets ISO 8601 standards for SQLite. 
+  Returns the original string if it is not in this format."
+  [s]
+  (try
+    (let [zdt (java.time.ZonedDateTime/parse s java.time.format.DateTimeFormatter/RFC_1123_DATE_TIME)
+          zdt-utc (.withZoneSameInstant zdt (java.time.ZoneId/of "UTC"))
+          out (.format zdt-utc (java.time.format.DateTimeFormatter/ofPattern "yyyy-MM-dd HH:mm:ss"))]
+      out)
+    (catch Exception _ s)))
+
 (comment
   (require '[source.routes.business :as business])
   (validate business/post {:cheese "modulr"})

From a5da4bbb4b609009040a0b2ae1c682c04d359316 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 27 Oct 2025 15:11:05 +0200
Subject: [PATCH 144/391] added latest filter to bundle feeds and posts
 endpoints

---
 src/source/routes/bundle_feeds.clj | 3 ++-
 src/source/routes/bundle_posts.clj | 8 +++++---
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/src/source/routes/bundle_feeds.clj b/src/source/routes/bundle_feeds.clj
index 146aa1b6..9bb9c06c 100644
--- a/src/source/routes/bundle_feeds.clj
+++ b/src/source/routes/bundle_feeds.clj
@@ -32,13 +32,14 @@
   [{:keys [ds bundle-id query-params body] :as _request}]
   (let [bundle-ds (db.util/conn :bundle bundle-id)
         {:keys [category-ids]} body
-        {:keys [type]} (walk/keywordize-keys query-params)
+        {:keys [type latest]} (walk/keywordize-keys query-params)
         feed-ids (mapv :feed-id (services/outgoing-posts bundle-ds))
         category-filtered-feed-ids (if (empty? category-ids)
                                      feed-ids
                                      (->> (hsql/where
                                            [:in :feed-id feed-ids]
                                            [:in :category-id category-ids])
+                                          (hsql/order-by (when latest [:created-at :desc]))
                                           (services/feed-categories ds)
                                           (mapv :feed-id)))
         type-filtered (->> (when type [:= :content-type-id type])
diff --git a/src/source/routes/bundle_posts.clj b/src/source/routes/bundle_posts.clj
index c2d2e05e..fe3ff5d2 100644
--- a/src/source/routes/bundle_posts.clj
+++ b/src/source/routes/bundle_posts.clj
@@ -12,7 +12,8 @@
                         [:uuid :string]
                         [:limit {:optional true} :int]
                         [:start {:optional true} :int]
-                        [:type {:optional true} :int]]}
+                        [:type {:optional true} :int]
+                        [:latest {:optional true} [:enum "true" "false"]]]}
    :responses {200 {:body [:vector
                            [:map
                             [:id :int]
@@ -33,14 +34,15 @@
 
   [{:keys [ds bundle-id query-params body] :as _request}]
   (let [bundle-ds (db.util/conn :bundle bundle-id)
-        {:keys [limit start type]} (walk/keywordize-keys query-params)
+        {:keys [limit start type latest]} (walk/keywordize-keys query-params)
         {:keys [category-ids]} body
 
         content-type-comp (when type [:= :content-type-id type])
         start (when start (try (Integer/parseInt start) (catch Exception _)))
         limit (when limit (try (Integer/parseInt limit) (catch Exception _)))
 
-        filtered-posts (shuffle (services/outgoing-posts bundle-ds {:where content-type-comp}))
+        filtered-posts (services/outgoing-posts bundle-ds {:where content-type-comp
+                                                           :order-by (when (= latest "true") [[:posted-at :desc]])})
 
         categorised-posts (vec (if (seq category-ids)
                                  (->> filtered-posts

From 30c8e1d44165b4b27bdb40807779aa2318602149 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 29 Oct 2025 15:22:07 +0200
Subject: [PATCH 145/391] addressed requested changes

---
 src/source/util.clj | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/source/util.clj b/src/source/util.clj
index 68900d3f..c8bbb7f3 100644
--- a/src/source/util.clj
+++ b/src/source/util.clj
@@ -62,8 +62,9 @@
                                    (m/explain schema)
                                    (me/humanize)))}))
 
-(defn format-rfc-1123-date
-  "Takes a date as a string in RFC 1123 format and returns it in a format that meets ISO 8601 standards for SQLite. 
+(defn format-rss-date
+  "Takes a date as a string in RFC 1123 format and returns it in a format that meets ISO 8601 standards for SQLite.
+  This is necessary because some RSS feeds use a different date than what is accepted by SQLite.
   Returns the original string if it is not in this format."
   [s]
   (try

From 24b090b97195a63165410d4c75f874909b545be9 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 29 Oct 2025 16:06:50 +0200
Subject: [PATCH 146/391] added outgoing endpoints api in reitit pointing to
 the same functions as used in other routes

---
 src/source/routes/reitit.clj | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index d6f71868..69ae8414 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -212,6 +212,28 @@
        [""              (route {:post bundle-posts/post})]
        ["/:id"          (route {:get bundle-post/get})]]]
 
+     ["/api"             {:middleware [[mw/apply-api-key]]
+                          :tags #{"api"}
+                          :swagger {:security [{"apiKey" []}]}
+                          :openapi {:security [{:apiKey []}]}}
+      ["/bundle"         {:middleware [[mw/apply-bundle]]}
+       [""               (route {:get bundle/get})]
+       ["/categories"
+        [""              (route {:get bundle-categories/get})]]
+       ["/feeds"
+        [""              (route {:post bundle-feeds/post})]
+        ["/:id"
+         [""             (route {:get bundle-feed/get})]
+         ["/posts"
+          [""            (route {:get bundle-feed-posts/get})]
+          ["/:post-id"   (route {:get bundle-feed-post/get})]]]]
+       ["/posts"
+        [""              (route {:post bundle-posts/post})]
+        ["/:id"          (route {:get bundle-post/get})]]]
+      ["/categories"
+       [""               (route {:get categories/get})]
+       ["/:id"           (route {:get category/get})]]]
+
      ["/admin"                  {:middleware [[mw/apply-auth {:required-type :admin}]]
                                  :tags #{"admin"}
                                  :swagger {:security [{"auth" []}]}

From fc8da5514af21fd0956114b7a8e008cd77a958f8 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 29 Oct 2025 16:07:17 +0200
Subject: [PATCH 147/391] fixed bugs from earlier

---
 src/source/jobs/handlers.clj | 2 +-
 src/source/routes/feeds.clj  | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/source/jobs/handlers.clj b/src/source/jobs/handlers.clj
index ae03c189..bc734883 100644
--- a/src/source/jobs/handlers.clj
+++ b/src/source/jobs/handlers.clj
@@ -38,7 +38,7 @@
                                           {:feed-id feed-id
                                            :creator-id creator-id
                                            :content-type-id content-type-id
-                                           :posted-at (util/format-rfc-1123-date posted-at)
+                                           :posted-at (util/format-rss-date posted-at)
                                            :thumbnail (if (and thumbnail
                                                                (seq thumbnail)
                                                                (not (string/includes? thumbnail ".mp3")))
diff --git a/src/source/routes/feeds.clj b/src/source/routes/feeds.clj
index f079cbf8..c7b35bc2 100644
--- a/src/source/routes/feeds.clj
+++ b/src/source/routes/feeds.clj
@@ -24,8 +24,8 @@
                              [:ts-and-cs [:maybe :int]]
                              [:state [:enum "live" "not live" "pending"]]]]}}}
 
-  [{:keys [ds] :as _request}]
-  (-> (services/feeds ds)
+  [{:keys [ds user] :as _request}]
+  (-> (services/feeds ds {:where [:= :user-id (:id user)]})
       (res/response)))
 
 (defn post

From db79f54fb7640ff71d137c6f66d467e5260d30c0 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Thu, 30 Oct 2025 10:46:40 +0200
Subject: [PATCH 148/391] fixed category filtering bug

---
 src/source/routes/bundle_feeds.clj | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/source/routes/bundle_feeds.clj b/src/source/routes/bundle_feeds.clj
index 9bb9c06c..2f85d4b7 100644
--- a/src/source/routes/bundle_feeds.clj
+++ b/src/source/routes/bundle_feeds.clj
@@ -39,11 +39,11 @@
                                      (->> (hsql/where
                                            [:in :feed-id feed-ids]
                                            [:in :category-id category-ids])
-                                          (hsql/order-by (when latest [:created-at :desc]))
                                           (services/feed-categories ds)
                                           (mapv :feed-id)))
-        type-filtered (->> (when type [:= :content-type-id type])
-                           (hsql/where [:in :id category-filtered-feed-ids])
-                           (services/feeds ds))]
+        query (-> (when type [:= :content-type-id type])
+                  (hsql/where [:in :id category-filtered-feed-ids])
+                  (hsql/order-by (when latest [:created-at :desc])))
+        type-filtered (services/feeds ds query)]
 
     (res/response type-filtered)))

From 53abc7092dd6d8c7b6e7902fa11be1f29ee1406d Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 3 Nov 2025 14:44:09 +0200
Subject: [PATCH 149/391] added endpoint to allow creators to prune or unprune
 incoming post

---
 src/source/routes/post.clj   | 46 ++++++++++++++++++++++++++++++++++++
 src/source/routes/reitit.clj |  7 +++++-
 2 files changed, 52 insertions(+), 1 deletion(-)
 create mode 100644 src/source/routes/post.clj

diff --git a/src/source/routes/post.clj b/src/source/routes/post.clj
new file mode 100644
index 00000000..cf34a733
--- /dev/null
+++ b/src/source/routes/post.clj
@@ -0,0 +1,46 @@
+(ns source.routes.post
+  (:require [source.services.interface :as services]
+            [ring.util.response :as res]))
+
+(defn get
+  {:summary "get post by id"
+   :parameters {:path [:map [:post-id {:title "post-id"
+                                       :description "post id"} :int]]}
+   :responses  {200 {:body [:map
+                            [:id :int]
+                            [:post-id :string]
+                            [:feed-id :int]
+                            [:creator-id :int]
+                            [:content-type-id :int]
+                            [:title :string]
+                            [:thumbnail [:maybe :string]]
+                            [:info [:maybe :string]]
+                            [:url [:maybe :string]]
+                            [:stream-url [:maybe :string]]
+                            [:season [:maybe :int]]
+                            [:episode [:maybe :int]]
+                            [:redacted {:optional true} [:maybe :int]]
+                            [:posted-at [:maybe :string]]]}}}
+
+  [{:keys [ds user path-params] :as _request}]
+  (-> (services/incoming-post ds {:where [:and
+                                          [:= :id (:post-id path-params)]
+                                          [:= :creator-id (:id user)]]})
+      (res/response)))
+
+(defn post
+  {:summary "Update redacted status of post with the given id"
+   :parameters {:path [:map [:post-id {:title "post-id"
+                                       :description "post id"} :int]]
+                :body [:map
+                       [:redacted :boolean]]}
+   :responses  {200 {:body [:map [:message :string]]}
+                401 {:body [:map [:message :string]]}
+                403 {:body [:map [:message :string]]}}}
+
+  [{:keys [ds path-params user body] :as _request}]
+  (services/update-incoming-post! ds {:where [:and
+                                              [:= :id (:post-id path-params)]
+                                              [:= :creator-id (:id user)]]
+                                      :data {:redacted (if (:redacted body) 1 0)}})
+  (res/response {:message "successfully updated post"}))
diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index 69ae8414..31865600 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -53,6 +53,7 @@
             [source.routes.bundle-posts :as bundle-posts]
             [source.routes.bundle-post :as bundle-post]
             [source.routes.posts :as posts]
+            [source.routes.post :as post]
             [source.routes.admin-feeds :as admin-feeds]
             [source.routes.xml :as xml]
             [source.routes.data :as data]
@@ -192,7 +193,11 @@
       ["/:id"
        [""              (route {:get feed/get
                                 :post feed/post})]
-       ["/posts"        (route {:get posts/get})]
+       ["/posts"
+        [""             (route {:get posts/get})]
+        ["/:post-id"
+         [""            (route {:get post/get})]
+         ["/prune"      (route {:post post/post})]]]
        ["/categories"   (route {:get feed-categories/get
                                 :post feed-categories/post})]]]
 

From 5d975ca261efcf751c68bd58bc9cde90636a8671 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 3 Nov 2025 16:17:58 +0200
Subject: [PATCH 150/391] addressed requested changes

---
 src/source/routes/post.clj       | 17 -----------------
 src/source/routes/post_prune.clj | 20 ++++++++++++++++++++
 src/source/routes/reitit.clj     |  3 ++-
 3 files changed, 22 insertions(+), 18 deletions(-)
 create mode 100644 src/source/routes/post_prune.clj

diff --git a/src/source/routes/post.clj b/src/source/routes/post.clj
index cf34a733..832529a9 100644
--- a/src/source/routes/post.clj
+++ b/src/source/routes/post.clj
@@ -27,20 +27,3 @@
                                           [:= :id (:post-id path-params)]
                                           [:= :creator-id (:id user)]]})
       (res/response)))
-
-(defn post
-  {:summary "Update redacted status of post with the given id"
-   :parameters {:path [:map [:post-id {:title "post-id"
-                                       :description "post id"} :int]]
-                :body [:map
-                       [:redacted :boolean]]}
-   :responses  {200 {:body [:map [:message :string]]}
-                401 {:body [:map [:message :string]]}
-                403 {:body [:map [:message :string]]}}}
-
-  [{:keys [ds path-params user body] :as _request}]
-  (services/update-incoming-post! ds {:where [:and
-                                              [:= :id (:post-id path-params)]
-                                              [:= :creator-id (:id user)]]
-                                      :data {:redacted (if (:redacted body) 1 0)}})
-  (res/response {:message "successfully updated post"}))
diff --git a/src/source/routes/post_prune.clj b/src/source/routes/post_prune.clj
new file mode 100644
index 00000000..4b252ea2
--- /dev/null
+++ b/src/source/routes/post_prune.clj
@@ -0,0 +1,20 @@
+(ns source.routes.post-prune
+  (:require [source.services.interface :as services]
+            [ring.util.response :as res]))
+
+(defn post
+  {:summary "Update redacted status of post with the given id"
+   :parameters {:path [:map [:post-id {:title "post-id"
+                                       :description "post id"} :int]]
+                :body [:map
+                       [:redacted :boolean]]}
+   :responses  {200 {:body [:map [:message :string]]}
+                401 {:body [:map [:message :string]]}
+                403 {:body [:map [:message :string]]}}}
+
+  [{:keys [ds path-params user body] :as _request}]
+  (services/update-incoming-post! ds {:where [:and
+                                              [:= :id (:post-id path-params)]
+                                              [:= :creator-id (:id user)]]
+                                      :data {:redacted (if (:redacted body) 1 0)}})
+  (res/response {:message "successfully updated post"}))
diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index 31865600..798a39e9 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -54,6 +54,7 @@
             [source.routes.bundle-post :as bundle-post]
             [source.routes.posts :as posts]
             [source.routes.post :as post]
+            [source.routes.post-prune :as post-prune]
             [source.routes.admin-feeds :as admin-feeds]
             [source.routes.xml :as xml]
             [source.routes.data :as data]
@@ -197,7 +198,7 @@
         [""             (route {:get posts/get})]
         ["/:post-id"
          [""            (route {:get post/get})]
-         ["/prune"      (route {:post post/post})]]]
+         ["/prune"      (route {:post post-prune/post})]]]
        ["/categories"   (route {:get feed-categories/get
                                 :post feed-categories/post})]]]
 

From 9b6132dd1bb2ca19ee7e49b30b0ea2a073f154da Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 12 Nov 2025 14:48:43 +0200
Subject: [PATCH 151/391] added filter tables and migrations for said tables

---
 src/source/db/master.clj                       | 18 ++++++++++++++++++
 .../migrations/005_filtered_feeds_posts.clj    | 18 ++++++++++++++++++
 2 files changed, 36 insertions(+)
 create mode 100644 src/source/migrations/005_filtered_feeds_posts.clj

diff --git a/src/source/db/master.clj b/src/source/db/master.clj
index 15dc2913..cab9ef8a 100644
--- a/src/source/db/master.clj
+++ b/src/source/db/master.clj
@@ -77,6 +77,24 @@
    (tables/foreign-key :bundle-id :bundles :id)
    (tables/foreign-key :content-type-id :content-types :id)))
 
+(def filtered-feeds
+  (tables/create-table-sql
+    :filtered-feeds
+    (tables/table-id)
+    [:feed-id :integer :not nil]
+    [:bundle-id :integer :not nil]
+    (tables/foreign-key :feed-id :feeds :id)
+    (tables/foreign-key :bundle-id :bundles :id)))
+
+(def filtered-posts
+  (tables/create-table-sql
+    :filtered-posts
+    (tables/table-id)
+    [:post-id :integer :not nil]
+    [:bundle-id :integer :not nil]
+    (tables/foreign-key :post-id :incoming-posts :id)
+    (tables/foreign-key :bundle-id :bundles :id)))
+
 (def feeds
   (tables/create-table-sql
    :feeds
diff --git a/src/source/migrations/005_filtered_feeds_posts.clj b/src/source/migrations/005_filtered_feeds_posts.clj
new file mode 100644
index 00000000..ab626a1b
--- /dev/null
+++ b/src/source/migrations/005_filtered_feeds_posts.clj
@@ -0,0 +1,18 @@
+(ns source.migrations.005-filtered-feeds-posts
+  (:require [source.db.master]
+            [source.db.tables :as tables]))
+
+(defn run-up! [context]
+  (let [ds-master (:db-master context)]
+    (tables/create-tables!
+     ds-master
+     :source.db.master
+     [:filtered-feeds
+      :filtered-posts])))
+
+(defn run-down! [context]
+  (let [ds-master (:db-master context)]
+    (tables/drop-tables! 
+      ds-master
+      [:filtered-feeds
+       :filtered-posts])))

From a53277e8c8ac86a1172e8cd2f8e025317c3d569c Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 12 Nov 2025 14:49:37 +0200
Subject: [PATCH 152/391] added services for working with filter tables

---
 src/source/services/filtered_feeds.clj | 35 ++++++++++++++++++++++++++
 src/source/services/filtered_posts.clj | 35 ++++++++++++++++++++++++++
 src/source/services/interface.clj      | 32 ++++++++++++++++++++++-
 3 files changed, 101 insertions(+), 1 deletion(-)
 create mode 100644 src/source/services/filtered_feeds.clj
 create mode 100644 src/source/services/filtered_posts.clj

diff --git a/src/source/services/filtered_feeds.clj b/src/source/services/filtered_feeds.clj
new file mode 100644
index 00000000..87da7ea0
--- /dev/null
+++ b/src/source/services/filtered_feeds.clj
@@ -0,0 +1,35 @@
+(ns source.services.filtered-feeds
+  (:require [source.db.interface :as db]))
+
+(defn insert-filtered-feeds! [ds {:keys [data ret] :as opts}]
+  (->> {:tname :filtered-feeds
+        :data data
+        :ret ret}
+       (merge opts)
+       (db/insert! ds)))
+
+(defn update-filtered-feeds! [ds {:keys [id data where] :as opts}]
+  (->> {:tname :filtered-feeds
+        :values data
+        :where (if (some? id) [:= :id id] where)
+        :ret :1}
+       (merge opts)
+       (db/update! ds)))
+
+(defn filtered-feeds
+  ([ds] (filtered-feeds ds {}))
+  ([ds {:keys [where] :as opts}]
+   (->> {:tname :filtered-feeds
+         :where where
+         :ret :*}
+        (merge opts)
+        (db/find ds))))
+
+(defn delete-filtered-feed! [ds {:keys [id where] :as opts}]
+  (->> {:tname :filtered-feeds
+        :where (if (some? id)
+                 [:= :id id]
+                 where)
+        :ret :1}
+       (merge opts)
+       (db/delete! ds)))
diff --git a/src/source/services/filtered_posts.clj b/src/source/services/filtered_posts.clj
new file mode 100644
index 00000000..a7f6cddc
--- /dev/null
+++ b/src/source/services/filtered_posts.clj
@@ -0,0 +1,35 @@
+(ns source.services.filtered-posts
+  (:require [source.db.interface :as db]))
+
+(defn insert-filtered-posts! [ds {:keys [data ret] :as opts}]
+  (->> {:tname :filtered-posts
+        :data data
+        :ret ret}
+       (merge opts)
+       (db/insert! ds)))
+
+(defn update-filtered-posts! [ds {:keys [id data where] :as opts}]
+  (->> {:tname :filtered-posts
+        :values data
+        :where (if (some? id) [:= :id id] where)
+        :ret :1}
+       (merge opts)
+       (db/update! ds)))
+
+(defn filtered-posts
+  ([ds] (filtered-posts ds {}))
+  ([ds {:keys [where] :as opts}]
+   (->> {:tname :filtered-posts
+         :where where
+         :ret :*}
+        (merge opts)
+        (db/find ds))))
+
+(defn delete-filtered-post! [ds {:keys [id where] :as opts}]
+  (->> {:tname :filtered-posts
+        :where (if (some? id)
+                 [:= :id id]
+                 where)
+        :ret :1}
+       (merge opts)
+       (db/delete! ds)))
diff --git a/src/source/services/interface.clj b/src/source/services/interface.clj
index 9597fef1..612b7cc2 100644
--- a/src/source/services/interface.clj
+++ b/src/source/services/interface.clj
@@ -18,7 +18,9 @@
              [source.services.feed-categories :as feed-categories]
              [source.services.jobs :as jobs]
              [source.services.businesses :as businesses]
-             [source.services.user-sectors :as user-sectors]))
+             [source.services.user-sectors :as user-sectors]
+             [source.services.filtered-feeds :as filtered-feeds]
+             [source.services.filtered-posts :as filtered-posts]))
 
 (defn users
   [& args]
@@ -312,6 +314,34 @@
 (defn job-metadata [ds {:keys [_id _where] :as opts}]
   (jobs/job-metadata ds opts))
 
+(defn insert-filtered-feeds! [ds {:keys [_data _ret] :as opts}]
+  (filtered-feeds/insert-filtered-feeds! ds opts))
+
+(defn update-filtered-feeds! [ds {:keys [_id _data _where] :as opts}]
+  (filtered-feeds/update-filtered-feeds! ds opts))
+
+(defn filtered-feeds
+  ([ds] (filtered-feeds ds {}))
+  ([ds {:keys [_where] :as opts}]
+   (filtered-feeds/filtered-feeds ds opts)))
+
+(defn delete-filtered-feed! [ds {:keys [_id _where] :as opts}]
+  (filtered-feeds/delete-filtered-feed! ds opts))
+
+(defn insert-filtered-posts! [ds {:keys [_data _ret] :as opts}]
+  (filtered-posts/insert-filtered-posts! ds opts))
+
+(defn update-filtered-posts! [ds {:keys [_id _data _where] :as opts}]
+  (filtered-posts/update-filtered-posts! ds opts))
+
+(defn filtered-posts
+  ([ds] (filtered-posts ds {}))
+  ([ds {:keys [_where] :as opts}]
+   (filtered-posts/filtered-posts ds opts)))
+
+(defn delete-filtered-post! [ds {:keys [_id _where] :as opts}]
+  (filtered-posts/delete-filtered-post! ds opts))
+
 (comment
   (users (db/ds :master))
   (user (db/ds :master) {:id 2})

From e3c4e2439c6a3ae1d035f7331c1a5794d4119044 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 12 Nov 2025 14:58:24 +0200
Subject: [PATCH 153/391] updated feeds and posts endpoints to exclude filtered
 content

---
 src/source/routes/bundle_feeds.clj | 12 +++++++++---
 src/source/routes/bundle_posts.clj | 12 +++++++++---
 2 files changed, 18 insertions(+), 6 deletions(-)

diff --git a/src/source/routes/bundle_feeds.clj b/src/source/routes/bundle_feeds.clj
index 2f85d4b7..50b88340 100644
--- a/src/source/routes/bundle_feeds.clj
+++ b/src/source/routes/bundle_feeds.clj
@@ -9,7 +9,9 @@
   {:summary "get all feeds present in the bundle authorised by uuid"
    :parameters {:query [:map
                         [:uuid :string]
-                        [:type {:optional true} :int]]
+                        [:type {:optional true} :int]
+                        [:latest {:optional true} :boolean]
+                        [:nonfiltered {:optional true} :boolean]]
                 :body [:map [:category-ids [:vector :int]]]}
    :responses {200 {:body [:vector
                            [:map
@@ -32,7 +34,7 @@
   [{:keys [ds bundle-id query-params body] :as _request}]
   (let [bundle-ds (db.util/conn :bundle bundle-id)
         {:keys [category-ids]} body
-        {:keys [type latest]} (walk/keywordize-keys query-params)
+        {:keys [type latest nonfiltered]} (walk/keywordize-keys query-params)
         feed-ids (mapv :feed-id (services/outgoing-posts bundle-ds))
         category-filtered-feed-ids (if (empty? category-ids)
                                      feed-ids
@@ -41,8 +43,12 @@
                                            [:in :category-id category-ids])
                                           (services/feed-categories ds)
                                           (mapv :feed-id)))
+        blocked-feed-ids (if (some? nonfiltered)
+                           []
+                           (mapv :feed-id (services/filtered-feeds ds {:where [:= :bundle-id bundle-id]})))
         query (-> (when type [:= :content-type-id type])
-                  (hsql/where [:in :id category-filtered-feed-ids])
+                  (hsql/where [:in :id category-filtered-feed-ids]
+                              [:not [:in :id blocked-feed-ids]])
                   (hsql/order-by (when latest [:created-at :desc])))
         type-filtered (services/feeds ds query)]
 
diff --git a/src/source/routes/bundle_posts.clj b/src/source/routes/bundle_posts.clj
index fe3ff5d2..f5e72642 100644
--- a/src/source/routes/bundle_posts.clj
+++ b/src/source/routes/bundle_posts.clj
@@ -3,7 +3,8 @@
             [source.db.util :as db.util]
             [clojure.walk :as walk]
             [ring.util.response :as res]
-            [clojure.set :as set]))
+            [clojure.set :as set]
+            [honey.sql.helpers :as hsql]))
 
 (defn post
   {:summary "get all outgoing posts in the uuid-authorized bundle"
@@ -41,8 +42,13 @@
         start (when start (try (Integer/parseInt start) (catch Exception _)))
         limit (when limit (try (Integer/parseInt limit) (catch Exception _)))
 
-        filtered-posts (services/outgoing-posts bundle-ds {:where content-type-comp
-                                                           :order-by (when (= latest "true") [[:posted-at :desc]])})
+        blocked-feed-ids (mapv :feed-id (services/filtered-feeds ds {:where [:= :bundle-id bundle-id]}))
+        blocked-post-ids (mapv :post-id (services/filtered-posts ds {:where [:= :bundle-id bundle-id]}))
+
+        filtered-posts (services/outgoing-posts bundle-ds (-> (hsql/where content-type-comp
+                                                                          [:not [:in :id blocked-post-ids]]
+                                                                          [:not [:in :feed-id blocked-feed-ids]])
+                                                              (hsql/order-by (when (= latest "true") [[:posted-at :desc]]))))
 
         categorised-posts (vec (if (seq category-ids)
                                  (->> filtered-posts

From 683f4b2c38b3326d59ef37bad79dd9365c9d3d24 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 12 Nov 2025 14:59:18 +0200
Subject: [PATCH 154/391] added endpoints to allow the distributor to filter
 and get filtered feeds and posts

---
 src/source/routes/integration_filter_feed.clj | 50 +++++++++++++++++++
 .../routes/integration_filter_feeds.clj       | 18 +++++++
 src/source/routes/integration_filter_post.clj | 47 +++++++++++++++++
 .../routes/integration_filter_posts.clj       | 18 +++++++
 src/source/routes/reitit.clj                  | 15 +++++-
 5 files changed, 147 insertions(+), 1 deletion(-)
 create mode 100644 src/source/routes/integration_filter_feed.clj
 create mode 100644 src/source/routes/integration_filter_feeds.clj
 create mode 100644 src/source/routes/integration_filter_post.clj
 create mode 100644 src/source/routes/integration_filter_posts.clj

diff --git a/src/source/routes/integration_filter_feed.clj b/src/source/routes/integration_filter_feed.clj
new file mode 100644
index 00000000..1af65f99
--- /dev/null
+++ b/src/source/routes/integration_filter_feed.clj
@@ -0,0 +1,50 @@
+(ns source.routes.integration-filter-feed
+  (:require [source.services.interface :as services]
+            [ring.util.response :as res]))
+
+(defn get
+  {:summary "Returns true if the feed with the given id is filtered out by the integration with the given id"
+   :parameters {:path [:map
+                       [:id {:title "id"
+                             :description "integration id"} :int]
+                       [:feed-id {:title "feed-id"
+                                  :description "feed id"} :int]]}
+   :responses {200 {:body [:map [:filtered :boolean]]}
+               401 {:body [:map [:message :string]]}
+               403 {:body [:map [:message :string]]}}}
+
+  [{:keys [ds path-params] :as _request}]
+
+  (let [returned (services/filtered-feeds ds {:where [:and
+                                                      [:= :bundle-id (:id path-params)]
+                                                      [:= :feed-id (:feed-id path-params)]]})
+        blocked (if (seq returned) true false)]
+    (res/response {:filtered blocked})))
+
+(defn post
+  {:summary "filters out the feed with the given id from the bundle with the given bundle id"
+   :parameters {:path [:map
+                       [:id {:title "id"
+                             :description "bundle id"} :int]
+                       [:feed-id {:title "feed-id"
+                                  :description "feed id"} :int]]
+                :body [:map
+                       [:filtered :boolean]]}
+   :responses {:body {200 [:map [:message :string]]
+                      401 [:map [:message :string]]
+                      403 [:map [:message :string]]}}}
+
+  [{:keys [ds path-params body] :as _request}]
+
+  (let [{:keys [filtered]} body
+        {:keys [id feed-id]} path-params]
+
+    (if filtered
+      (services/insert-filtered-feeds! ds {:data {:feed-id feed-id
+                                                  :bundle-id id}})
+
+      (services/delete-filtered-feed! ds {:where [:and
+                                                  [:= :feed-id feed-id]
+                                                  [:= :bundle-id id]]}))
+
+    (res/response {:message "Successfully updated feed filtering."})))
diff --git a/src/source/routes/integration_filter_feeds.clj b/src/source/routes/integration_filter_feeds.clj
new file mode 100644
index 00000000..1442680f
--- /dev/null
+++ b/src/source/routes/integration_filter_feeds.clj
@@ -0,0 +1,18 @@
+(ns source.routes.integration-filter-feeds
+  (:require [source.services.interface :as services]
+            [ring.util.response :as res]))
+
+(defn get
+  {:summary "gets all filtered feed ids by integration id"
+   :parameters {:path [:map [:id {:title "id"
+                                  :description "integration id"} :int]]}
+   :responses {200 {:body [:vector
+                           [:map
+                            [:feed-id :int]
+                            [:bundle-id :int]]]}
+               401 {:body [:map [:message :string]]}
+               403 {:body [:map [:message :string]]}}}
+
+  [{:keys [ds path-params] :as _request}]
+
+  (res/response (services/filtered-feeds ds {:where [:= :bundle-id (:id path-params)]})))
diff --git a/src/source/routes/integration_filter_post.clj b/src/source/routes/integration_filter_post.clj
new file mode 100644
index 00000000..35be03e6
--- /dev/null
+++ b/src/source/routes/integration_filter_post.clj
@@ -0,0 +1,47 @@
+(ns source.routes.integration-filter-post
+  (:require [source.services.interface :as services]
+            [ring.util.response :as res]))
+
+(defn get
+  {:summary "Returns true if the post with the given id is filtered out by the integration with the given id"
+   :parameters {:path [:map
+                       [:id {:title "id"
+                             :description "integration id"} :int]
+                       [:post-id {:title "post-id"
+                                  :description "post id"} :int]]}
+   :responses {200 {:body [:map [:filtered :boolean]]}
+               401 {:body [:map [:message :string]]}
+               403 {:body [:map [:message :string]]}}}
+
+  [{:keys [ds path-params] :as _request}]
+
+  (let [returned (services/filtered-feeds ds {:where [:and
+                                                      [:= :bundle-id (:id path-params)]
+                                                      [:= :post-id (:post-id path-params)]]})
+        blocked (if (seq returned) true false)]
+    (res/response {:filtered blocked})))
+
+(defn post
+  {:summary "filters out the post with the given id from the bundle with the given bundle id"
+   :parameters {:path [:map
+                       [:id {:title "id"
+                             :description "bundle id"} :int]
+                       [:post-id {:title "post-id"
+                                  :description "post id"} :int]]
+                :body [:map
+                       [:filtered :boolean]]}
+   :responses {:body {200 [:map [:message :string]]
+                      401 [:map [:message :string]]
+                      403 [:map [:message :string]]}}}
+
+  [{:keys [ds path-params body] :as _request}]
+
+  (let [{:keys [filtered]} body
+        {:keys [id post-id]} path-params]
+    (if filtered
+      (services/insert-filtered-posts! ds {:data {:post-id post-id
+                                                  :bundle-id id}})
+      (services/delete-filtered-post! ds {:where [:and
+                                                  [:= :post-id post-id]
+                                                  [:= :bundle-id id]]}))
+    (res/response {:message "successfully updated post filtering"})))
diff --git a/src/source/routes/integration_filter_posts.clj b/src/source/routes/integration_filter_posts.clj
new file mode 100644
index 00000000..3844de41
--- /dev/null
+++ b/src/source/routes/integration_filter_posts.clj
@@ -0,0 +1,18 @@
+(ns source.routes.integration-filter-posts
+  (:require [source.services.interface :as services]
+            [ring.util.response :as res]))
+
+(defn get
+  {:summary "gets all filtered post ids by integration id"
+   :parameters {:path [:map [:id {:title "id"
+                                  :description "integration id"} :int]]}
+   :responses {200 {:body [:vector
+                           [:map
+                            [:post-id :int]
+                            [:bundle-id :int]]]}
+               401 {:body [:map [:message :string]]}
+               403 {:body [:map [:message :string]]}}}
+
+  [{:keys [ds path-params] :as _request}]
+
+  (res/response (services/filtered-posts ds {:where [:= :bundle-id (:id path-params)]})))
diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index 798a39e9..6062648b 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -44,6 +44,10 @@
             [source.routes.integration :as integration]
             [source.routes.integration-key :as integration-key]
             [source.routes.integration-categories :as integration-categories]
+            [source.routes.integration-filter-feeds :as integration-filter-feeds]
+            [source.routes.integration-filter-feed :as integration-filter-feed]
+            [source.routes.integration-filter-posts :as integration-filter-posts]
+            [source.routes.integration-filter-post :as integration-filter-post]
             [source.routes.bundle :as bundle]
             [source.routes.bundle-categories :as bundle-categories]
             [source.routes.bundle-feeds :as bundle-feeds]
@@ -185,7 +189,16 @@
                                 :post integration/post})]
        ["/key"          (route {:post integration-key/post})]
        ["/categories"   (route {:get integration-categories/get
-                                :post integration-categories/post})]]]
+                                :post integration-categories/post})]
+       ["/filter"
+        ["/feeds"
+         [""            (route {:get integration-filter-feeds/get})]
+         ["/:feed-id"   (route {:get integration-filter-feed/get
+                                :post integration-filter-feed/post})]]
+        ["/posts"
+         [""            (route {:get integration-filter-posts/get})]
+         ["/:post-id"   (route {:get integration-filter-post/get
+                                :post integration-filter-post/post})]]]]]
 
      ["/feeds"          {:middleware [[mw/apply-auth]]
                          :tags #{"feeds"}}

From 04753eddcbb99c87ae965074f1f4fd2ee51464a5 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 19 Nov 2025 15:10:46 +0200
Subject: [PATCH 155/391] updated db dependencies and added pragma instructions
 to db conn

---
 deps.edn               |  6 +++---
 src/source/db/util.clj | 20 +++++++++++---------
 2 files changed, 14 insertions(+), 12 deletions(-)

diff --git a/deps.edn b/deps.edn
index 2a5d95f2..50288a0e 100644
--- a/deps.edn
+++ b/deps.edn
@@ -14,9 +14,9 @@
         ring/ring-json {:mvn/version "0.5.1"}
         ring/ring-defaults {:mvn/version "0.6.0"}
         ring-cors/ring-cors {:mvn/version "0.1.13"}
-        io.github.modulr-software/congest {:mvn/version "0.1.6"}
-        com.github.seancorfield/next.jdbc {:mvn/version "1.3.994"}
-        org.xerial/sqlite-jdbc {:mvn/version "3.49.1.0"}
+        io.github.modulr-software/congest {:mvn/version "0.1.7"}
+        com.github.seancorfield/next.jdbc {:mvn/version "1.3.1070"}
+        org.xerial/sqlite-jdbc {:mvn/version "3.51.0.0"}
         buddy/buddy-core {:mvn/version "1.12.0-430"}
         buddy/buddy-sign {:mvn/version "3.5.351"}
         aero/aero {:mvn/version "1.1.6"}
diff --git a/src/source/db/util.clj b/src/source/db/util.clj
index 7161b18f..7d55805c 100644
--- a/src/source/db/util.clj
+++ b/src/source/db/util.clj
@@ -6,10 +6,10 @@
 (defn db-path [dbname]
   (let [db-dir (conf/read-value :database :dir)]
     (str
-      db-dir
-      (when (not (= (last db-dir) \/))
-        "/")
-      dbname)))
+     db-dir
+     (when (not (= (last db-dir) \/))
+       "/")
+     dbname)))
 
 (defn db-name
   ([type]
@@ -18,10 +18,13 @@
    (str (name type) "_" id)))
 
 (defn- -conn [dbname]
-  (-> {:dbtype (conf/read-value :database :type)}
-       (merge {:dbname (db-path dbname)})
-       (jdbc/get-connection)
-       (jdbc/with-options {:builder-fn rs/as-unqualified-lower-maps})))
+  (let [conn (-> {:dbtype (conf/read-value :database :type)}
+                 (merge {:dbname (db-path dbname)})
+                 (jdbc/get-connection))]
+    (jdbc/execute! conn ["PRAGMA journal_mode = WAL;"])
+    (jdbc/execute! conn ["PRAGMA synchronous = NORMAL;"])
+    (jdbc/with-options conn {:builder-fn rs/as-unqualified-lower-maps})
+    conn))
 
 (defn conn
   ([]
@@ -32,4 +35,3 @@
   ([db-type id]
    (assert (or (= db-type :bundle) (= db-type :creator)))
    (-conn (db-name db-type id))))
-

From 5afdbc0213027096b6f67fa746d44c7bb9f81b0f Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 19 Nov 2025 15:15:07 +0200
Subject: [PATCH 156/391] updated endpoints to use with-open when creating
 connections to prevent memory leaks

---
 src/source/routes/bundle_categories.clj      | 12 ++--
 src/source/routes/bundle_feeds.clj           | 40 ++++++------
 src/source/routes/bundle_post.clj            |  6 +-
 src/source/routes/bundle_posts.clj           | 67 ++++++++++----------
 src/source/routes/integration_categories.clj | 24 +++----
 5 files changed, 75 insertions(+), 74 deletions(-)

diff --git a/src/source/routes/bundle_categories.clj b/src/source/routes/bundle_categories.clj
index ac31df95..cbf9ef60 100644
--- a/src/source/routes/bundle_categories.clj
+++ b/src/source/routes/bundle_categories.clj
@@ -14,9 +14,9 @@
                404 {:body [:map [:message :string]]}}}
 
   [{:keys [bundle-id ds] :as _request}]
-  (let [bundle-ds (db.util/conn :bundle bundle-id)
-        feed-ids (->> (services/outgoing-posts bundle-ds)
-                      (mapv :feed-id))
-        category-ids (->> (services/feed-categories ds {:where [:in :feed-id feed-ids]})
-                          (mapv :category-id))]
-    (res/response (services/categories ds {:where [:in :id category-ids]}))))
+  (with-open [bundle-ds (db.util/conn :bundle bundle-id)]
+    (let [feed-ids (->> (services/outgoing-posts bundle-ds)
+                        (mapv :feed-id))
+          category-ids (->> (services/feed-categories ds {:where [:in :feed-id feed-ids]})
+                            (mapv :category-id))]
+      (res/response (services/categories ds {:where [:in :id category-ids]})))))
diff --git a/src/source/routes/bundle_feeds.clj b/src/source/routes/bundle_feeds.clj
index 50b88340..9f643760 100644
--- a/src/source/routes/bundle_feeds.clj
+++ b/src/source/routes/bundle_feeds.clj
@@ -32,24 +32,24 @@
                404 {:body [:map [:message :string]]}}}
 
   [{:keys [ds bundle-id query-params body] :as _request}]
-  (let [bundle-ds (db.util/conn :bundle bundle-id)
-        {:keys [category-ids]} body
-        {:keys [type latest nonfiltered]} (walk/keywordize-keys query-params)
-        feed-ids (mapv :feed-id (services/outgoing-posts bundle-ds))
-        category-filtered-feed-ids (if (empty? category-ids)
-                                     feed-ids
-                                     (->> (hsql/where
-                                           [:in :feed-id feed-ids]
-                                           [:in :category-id category-ids])
-                                          (services/feed-categories ds)
-                                          (mapv :feed-id)))
-        blocked-feed-ids (if (some? nonfiltered)
-                           []
-                           (mapv :feed-id (services/filtered-feeds ds {:where [:= :bundle-id bundle-id]})))
-        query (-> (when type [:= :content-type-id type])
-                  (hsql/where [:in :id category-filtered-feed-ids]
-                              [:not [:in :id blocked-feed-ids]])
-                  (hsql/order-by (when latest [:created-at :desc])))
-        type-filtered (services/feeds ds query)]
+  (with-open [bundle-ds (db.util/conn :bundle bundle-id)]
+    (let [{:keys [category-ids]} body
+          {:keys [type latest nonfiltered]} (walk/keywordize-keys query-params)
+          feed-ids (mapv :feed-id (services/outgoing-posts bundle-ds))
+          category-filtered-feed-ids (if (empty? category-ids)
+                                       feed-ids
+                                       (->> (hsql/where
+                                             [:in :feed-id feed-ids]
+                                             [:in :category-id category-ids])
+                                            (services/feed-categories ds)
+                                            (mapv :feed-id)))
+          blocked-feed-ids (if (some? nonfiltered)
+                             []
+                             (mapv :feed-id (services/filtered-feeds ds {:where [:= :bundle-id bundle-id]})))
+          query (-> (when type [:= :content-type-id type])
+                    (hsql/where [:in :id category-filtered-feed-ids]
+                                [:not [:in :id blocked-feed-ids]])
+                    (hsql/order-by (when latest [:created-at :desc])))
+          type-filtered (services/feeds ds query)]
 
-    (res/response type-filtered)))
+      (res/response type-filtered))))
diff --git a/src/source/routes/bundle_post.clj b/src/source/routes/bundle_post.clj
index 28265226..01549ae3 100644
--- a/src/source/routes/bundle_post.clj
+++ b/src/source/routes/bundle_post.clj
@@ -26,7 +26,7 @@
                404 {:body [:map [:message :string]]}}}
 
   [{:keys [bundle-id path-params] :as _request}]
-  (let [bundle-ds (db.util/conn :bundle bundle-id)
-        id (:id path-params)]
-    (res/response (services/outgoing-post bundle-ds {:id id}))))
+  (with-open [bundle-ds (db.util/conn :bundle bundle-id)]
+    (let [id (:id path-params)]
+      (res/response (services/outgoing-post bundle-ds {:id id})))))
 
diff --git a/src/source/routes/bundle_posts.clj b/src/source/routes/bundle_posts.clj
index f5e72642..3930c32f 100644
--- a/src/source/routes/bundle_posts.clj
+++ b/src/source/routes/bundle_posts.clj
@@ -34,43 +34,44 @@
                404 {:boy [:map [:message :string]]}}}
 
   [{:keys [ds bundle-id query-params body] :as _request}]
-  (let [bundle-ds (db.util/conn :bundle bundle-id)
-        {:keys [limit start type latest]} (walk/keywordize-keys query-params)
-        {:keys [category-ids]} body
+  (with-open [bundle-ds (db.util/conn :bundle bundle-id)]
+    (let [{:keys [limit start type latest]} (walk/keywordize-keys query-params)
+          {:keys [category-ids]} body
 
-        content-type-comp (when type [:= :content-type-id type])
-        start (when start (try (Integer/parseInt start) (catch Exception _)))
-        limit (when limit (try (Integer/parseInt limit) (catch Exception _)))
+          content-type-comp (when type [:= :content-type-id type])
+          start (when start (try (Integer/parseInt start) (catch Exception _)))
+          limit (when limit (try (Integer/parseInt limit) (catch Exception _)))
 
-        blocked-feed-ids (mapv :feed-id (services/filtered-feeds ds {:where [:= :bundle-id bundle-id]}))
-        blocked-post-ids (mapv :post-id (services/filtered-posts ds {:where [:= :bundle-id bundle-id]}))
+          blocked-feed-ids (mapv :feed-id (services/filtered-feeds ds {:where [:= :bundle-id bundle-id]}))
+          blocked-post-ids (mapv :post-id (services/filtered-posts ds {:where [:= :bundle-id bundle-id]}))
 
-        filtered-posts (services/outgoing-posts bundle-ds (-> (hsql/where content-type-comp
-                                                                          [:not [:in :id blocked-post-ids]]
-                                                                          [:not [:in :feed-id blocked-feed-ids]])
-                                                              (hsql/order-by (when (= latest "true") [[:posted-at :desc]]))))
+          filtered-posts (services/outgoing-posts bundle-ds (-> (hsql/where content-type-comp
+                                                                            [:not [:in :id blocked-post-ids]]
+                                                                            [:not [:in :feed-id blocked-feed-ids]])
+                                                                (hsql/order-by (when (= latest "true") [[:posted-at :desc]]))))
 
-        categorised-posts (vec (if (seq category-ids)
-                                 (->> filtered-posts
-                                      (mapv
-                                       (fn [post]
-                                         (when (seq (set/intersection
-                                                     (set category-ids)
-                                                     (->> {:feed-id (:feed-id post)}
-                                                          (services/categories-by-feed ds)
-                                                          (mapv :id)
-                                                          (set))))
-                                           post)))
-                                      (remove nil?))
-                                 filtered-posts))
+          categorised-posts (vec
+                             (if (seq category-ids)
+                               (->> filtered-posts
+                                    (mapv
+                                     (fn [post]
+                                       (when (seq (set/intersection
+                                                   (set category-ids)
+                                                   (->> {:feed-id (:feed-id post)}
+                                                        (services/categories-by-feed ds)
+                                                        (mapv :id)
+                                                        (set))))
+                                         post)))
+                                    (remove nil?))
+                               filtered-posts))
 
-        valid-start? (and (some? start) (>= start 0) (< start (count categorised-posts)))
-        started-posts (if valid-start?
-                        (subvec categorised-posts start)
-                        categorised-posts)
+          valid-start? (and (some? start) (>= start 0) (< start (count categorised-posts)))
+          started-posts (if valid-start?
+                          (subvec categorised-posts start)
+                          categorised-posts)
 
-        limited-posts (if (and (some? limit) (> (count started-posts) limit))
-                        (subvec started-posts 0 limit)
-                        started-posts)]
+          limited-posts (if (and (some? limit) (> (count started-posts) limit))
+                          (subvec started-posts 0 limit)
+                          started-posts)]
 
-    (res/response limited-posts)))
+      (res/response limited-posts))))
diff --git a/src/source/routes/integration_categories.clj b/src/source/routes/integration_categories.clj
index 9e2f0d16..2b600a15 100644
--- a/src/source/routes/integration_categories.clj
+++ b/src/source/routes/integration_categories.clj
@@ -13,11 +13,11 @@
                             [:name :string]]]}}}
 
   [{:keys [ds path-params] :as _request}]
-  (let [bundle-ds (db.util/conn :bundle (:id path-params))
-        category-ids (services/category-id-by-bundle bundle-ds {:bundle-id (:id path-params)})
-        id-vec (mapv (fn [{:keys [category-id]}] category-id) category-ids)
-        categories (services/categories ds {:where [:in :id id-vec]})]
-    (res/response categories)))
+  (with-open [bundle-ds (db.util/conn :bundle (:id path-params))]
+    (let [category-ids (services/category-id-by-bundle bundle-ds {:bundle-id (:id path-params)})
+          id-vec (mapv (fn [{:keys [category-id]}] category-id) category-ids)
+          categories (services/categories ds {:where [:in :id id-vec]})]
+      (res/response categories))))
 
 (defn post
   {:summary "update categories belonging to the integration with the given id"
@@ -30,10 +30,10 @@
    :responses {200 {:body [:map [:message :string]]}}}
 
   [{:keys [path-params body] :as _request}]
-  (let [update-data (reduce (fn [acc {:keys [id]}]
-                              (conj acc {:bundle-id (:id path-params)
-                                         :category-id id})) [] body)
-        bundle-ds (db.util/conn :bundle (:id path-params))]
-    (services/delete-bundle-category! bundle-ds {:where [:= :bundle-id (:id path-params)]})
-    (services/insert-bundle-category! bundle-ds {:data update-data})
-    (res/response {:message "successfully updated integration categories"})))
+  (with-open [bundle-ds (db.util/conn :bundle (:id path-params))]
+    (let [update-data (reduce (fn [acc {:keys [id]}]
+                                (conj acc {:bundle-id (:id path-params)
+                                           :category-id id})) [] body)]
+      (services/delete-bundle-category! bundle-ds {:where [:= :bundle-id (:id path-params)]})
+      (services/insert-bundle-category! bundle-ds {:data update-data})
+      (res/response {:message "successfully updated integration categories"}))))

From 909320198272f90f1a5a7481c5557f333840f500 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 19 Nov 2025 15:16:22 +0200
Subject: [PATCH 157/391] added admin debugging endpoint to view all job data
 in congest

---
 src/source/routes/jobs_view.clj | 23 +++++++++++++++++++++++
 src/source/routes/reitit.clj    |  2 ++
 2 files changed, 25 insertions(+)
 create mode 100644 src/source/routes/jobs_view.clj

diff --git a/src/source/routes/jobs_view.clj b/src/source/routes/jobs_view.clj
new file mode 100644
index 00000000..4f0d26cb
--- /dev/null
+++ b/src/source/routes/jobs_view.clj
@@ -0,0 +1,23 @@
+(ns source.routes.jobs-view
+  (:require [ring.util.response :as res]
+            [congest.jobs :as congest]
+            [clojure.walk :as walk]))
+
+(defn stringify-unknowns [x]
+  (walk/postwalk
+   (fn [v]
+     (if (or (map? v)
+             (sequential? v)
+             (string? v)
+             (number? v)
+             (boolean? v)
+             (keyword? v)
+             (nil? v))
+       v
+       (str v))) x))
+
+(defn get [{:keys [js]}]
+ ;(let [raw-jobs (congest/view js)
+ ;      formatted (stringify-unknowns raw-jobs)]
+ ;  (res/response formatted))
+  (res/response {:message "Hello, world!"}))
diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index 6062648b..c757eb99 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -63,6 +63,7 @@
             [source.routes.xml :as xml]
             [source.routes.data :as data]
             [source.routes.jobs :as jobs]
+            [source.routes.jobs-view :as jobs-view]
             [source.routes.job :as job]
             [source.routes.job-deregister :as job-deregister]
             [source.routes.job-start :as job-start]
@@ -265,6 +266,7 @@
       ["/jobs"
        [""                      {:get jobs/get}]
        ["/manage"
+        ["/view"                {:get jobs-view/get}]
         ["/register"            {:post jobs/post}]]
        ["/:id"
         [""                     {:get job/get}]

From e7b5e42a1f7980b04de7156591d74b5fc0370211 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 19 Nov 2025 15:26:48 +0200
Subject: [PATCH 158/391] uncommented endpoint

---
 src/source/routes/jobs_view.clj | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/src/source/routes/jobs_view.clj b/src/source/routes/jobs_view.clj
index 4f0d26cb..30ee5d1e 100644
--- a/src/source/routes/jobs_view.clj
+++ b/src/source/routes/jobs_view.clj
@@ -17,7 +17,6 @@
        (str v))) x))
 
 (defn get [{:keys [js]}]
- ;(let [raw-jobs (congest/view js)
- ;      formatted (stringify-unknowns raw-jobs)]
- ;  (res/response formatted))
-  (res/response {:message "Hello, world!"}))
+  (let [raw-jobs (congest/view js)
+        formatted (stringify-unknowns raw-jobs)]
+    (res/response formatted)))

From e21cfdb965f14dc05b6d84ac92e282e24c99498b Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 26 Nov 2025 11:57:06 +0200
Subject: [PATCH 159/391] added event tables

---
 src/source/db/master.clj             | 31 ++++++++++++++++++++++++++++
 src/source/migrations/006_events.clj | 18 ++++++++++++++++
 2 files changed, 49 insertions(+)
 create mode 100644 src/source/migrations/006_events.clj

diff --git a/src/source/db/master.clj b/src/source/db/master.clj
index cab9ef8a..63e108dc 100644
--- a/src/source/db/master.clj
+++ b/src/source/db/master.clj
@@ -221,6 +221,34 @@
    [:created-at :datetime]
    [:sleep :integer]))
 
+(def events
+  (tables/create-table-sql
+   :events
+   (tables/table-id)
+   [:timestamp :datetime]
+   [:event :text [:check [:in :event ["impression" "click" "view"]]]]
+   [:feed-id :integer :not nil]
+   [:post-id :integer]
+   [:content-type-id :not nil]
+   [:creator-id :not nil]
+   [:bundle-id :integer :not nil]
+   [:distributor-id :integer :not nil]
+   (tables/foreign-key :feed-id :feeds :id)
+   (tables/foreign-key :post-id :incoming-posts :id)
+   (tables/foreign-key :content-type-id :content-types :id)
+   (tables/foreign-key :creator-id :users :id)
+   (tables/foreign-key :bundle-id :bundles :id)
+   (tables/foreign-key :distributor-id :users :id)))
+
+(def event-categories
+  (tables/create-table-sql
+   :event-categories
+   (tables/table-id)
+   [:event-id :integer :not nil]
+   [:category-id :integer :not nil]
+   (tables/foreign-key :event-id :events :id)
+   (tables/foreign-key :category-id :categories :id)))
+
 (comment
   (require '[honey.sql :as sql])
 
@@ -242,4 +270,7 @@
   (sql/format incoming-posts)
   (sql/format jobs)
   (sql/format job-metadata)
+
+  (sql/format events)
+  (sql/format event-categories)
   ())
diff --git a/src/source/migrations/006_events.clj b/src/source/migrations/006_events.clj
new file mode 100644
index 00000000..567319d6
--- /dev/null
+++ b/src/source/migrations/006_events.clj
@@ -0,0 +1,18 @@
+(ns source.migrations.006-events
+  (:require [source.db.master]
+            [source.db.tables :as tables]))
+
+(defn run-up! [context]
+  (let [ds-master (:db-master context)]
+    (tables/create-tables!
+     ds-master
+     :source.db.master
+     [:events
+      :event-categories])))
+
+(defn run-down! [context]
+  (let [ds-master (:db-master context)]
+    (tables/drop-tables!
+     ds-master
+     [:events
+      :event-categories])))

From 9ec208cef5a7480f45c07012594226f7c84ffb12 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 26 Nov 2025 11:58:12 +0200
Subject: [PATCH 160/391] implemented analytics functional api

---
 src/source/services/analytics.clj           |   7 -
 src/source/services/analytics/core.clj      | 367 ++++++++++++++++++++
 src/source/services/analytics/interface.clj |  88 +++++
 3 files changed, 455 insertions(+), 7 deletions(-)
 delete mode 100644 src/source/services/analytics.clj
 create mode 100644 src/source/services/analytics/core.clj
 create mode 100644 src/source/services/analytics/interface.clj

diff --git a/src/source/services/analytics.clj b/src/source/services/analytics.clj
deleted file mode 100644
index fc21e704..00000000
--- a/src/source/services/analytics.clj
+++ /dev/null
@@ -1,7 +0,0 @@
-(ns source.services.analytics
-  (:require [source.db.interface :as db]))
-
-(defn insert-event! [ds {:keys [_values _ret] :as opts}]
-  (->> {:tname :analytics}
-       (merge opts)
-       (db/insert! ds)))
diff --git a/src/source/services/analytics/core.clj b/src/source/services/analytics/core.clj
new file mode 100644
index 00000000..4e536f6e
--- /dev/null
+++ b/src/source/services/analytics/core.clj
@@ -0,0 +1,367 @@
+(ns source.services.analytics.core
+  (:require [honey.sql.helpers :as hsql]
+            [source.db.honey :as hon]
+            [source.services.bundles :as bundles]
+            [source.util :as util]
+            [source.services.feed-categories :as feed-categories]))
+
+(defn metric-query
+  "Generic select query function for returning analytics data from the events table"
+  [ds {:keys [select order-by group-by metric feed-id post-id content-type-id bundle-id creator-id distributor-id min-date max-date category-ids ret]}]
+  (let [clauses (cond-> {}
+                  (some? metric) (hsql/where [:= :event metric])
+                  (some? feed-id) (hsql/where [:= :feed-id feed-id])
+                  (some? post-id) (hsql/where [:= :post-id post-id])
+                  (some? content-type-id) (hsql/where [:= :content-type-id content-type-id])
+                  (some? bundle-id) (hsql/where [:= :bundle-id bundle-id])
+                  (some? creator-id) (hsql/where [:= :creator-id creator-id])
+                  (some? distributor-id) (hsql/where [:= :distributor-id distributor-id])
+                  (and (some? min-date) (nil? max-date)) (hsql/where [:>= :timestamp min-date])
+                  (and (some? max-date) (nil? min-date)) (hsql/where [:<= :timestamp max-date])
+                  (and (some? min-date) (some? max-date)) (hsql/where [:between :timestamp min-date max-date])
+                  (some? select) (merge select)
+                  (some? order-by) (merge order-by)
+                  (some? group-by) (merge group-by)
+                  (seq category-ids) (-> (hsql/join [:event-categories :ec] [:= :events.id :ec.event-id])
+                                         (hsql/where [:in :ec.category-id category-ids])))]
+    (hon/execute!
+     ds
+     (merge {:select [[[:count :*] :total]]
+             :from [:events]}
+            clauses)
+     {:ret (if ret ret :*)})))
+
+(defn statistics-query
+  "returns the number of impressions, clicks and views, filtered by any other arguments accepted by metric-query"
+  [ds opts]
+  (metric-query ds (merge {:select (hsql/select [(hsql/filter :%count.* (hsql/where := :event "impression")) :impressions]
+                                                [(hsql/filter :%count.* (hsql/where := :event "click")) :clicks]
+                                                [(hsql/filter :%count.* (hsql/where := :event "view")) :views])
+                           :ret :1}
+                          opts)))
+
+(defn interval-statistics-query
+  "returns the number of impressions, clicks and views per interval (:daily, :weekly, :monthly or :yearly) over the given time period, 
+  filtered by any other arguments accepted by metric-query.
+  
+  Date parameters must be in the format YYYY-mm-dd."
+  [ds interval min-date max-date opts]
+  (let [select (cond
+                 (= interval :daily) [[:date :timestamp] :day]
+                 (= interval :weekly) [[:strftime "%W" :timestamp] :week]
+                 (= interval :monthly) [[:strftime "%m" :timestamp] :month]
+                 (= interval :yearly) [[:strftime "%Y" :timestamp] :year]
+                 :else [[:date :timestamp] :day])
+        column (cond
+                 (= interval :daily) :day
+                 (= interval :weekly) :week
+                 (= interval :monthly) :month
+                 (= interval :yearly) :year
+                 :else :day)]
+
+    (metric-query ds (merge {:select (hsql/select select
+                                                  [(hsql/filter :%count.* (hsql/where := :event "impression")) :impressions]
+                                                  [(hsql/filter :%count.* (hsql/where := :event "click")) :clicks]
+                                                  [(hsql/filter :%count.* (hsql/where := :event "view")) :views])
+                             :min-date (str min-date " 00:00:00")
+                             :max-date (str max-date " 23:59:59")
+                             :group-by (hsql/group-by column)
+                             :order-by (hsql/order-by column)}
+                            opts))))
+
+(defn weekly-growth-averages
+  "Returns the percentage of growth in impressions, clicks and views per week, over the given time period. 
+  Uses the first week as a basis for comparison, not included in results.
+  Can be filtered by any other arguments accepted by metric-query."
+  [ds min-date max-date opts]
+  (let [weeks (interval-statistics-query ds :weekly min-date max-date opts)
+        {:keys [impressions clicks views]} (first weeks)]
+    (mapv (fn [w]
+            {:week (:week w)
+             :impressions (float (* (/ (- (:impressions w) impressions) impressions) 100))
+             :clicks (float (* (/ (- (:clicks w) clicks) clicks) 100))
+             :views (float (* (/ (- (:views w) views) views) 100))})
+          weeks)))
+
+(defn average-engagement
+  "Returns the average engagement (average clicks and views) over the given time period.
+  Can be filtered by any other arguments accepted by metric-query."
+  [ds min-date max-date opts]
+  (let [{:keys [clicks views]} (statistics-query ds (merge {:min-date min-date
+                                                            :max-date max-date}
+                                                           opts))]
+    (float (/ (+ clicks views) 2))))
+
+(defn click-through-rate
+  "Returns the click-through rate based on impressions and clicks filtered by any arguments accepted by metric-query"
+  [ds opts]
+  (let [{:keys [impressions clicks]} (statistics-query ds opts)]
+    (float (* (/ clicks impressions) 100))))
+
+(defn insert-event [ds {:keys [data ret] :as opts}]
+  (->> {:tname :events
+        :data data
+        :ret ret}
+       (merge opts)
+       (hon/insert! ds)))
+
+(defn insert-event-categories [ds {:keys [data ret] :as opts}]
+  (->> {:tname :event-categories
+        :data data
+        :ret ret}
+       (merge opts)
+       (hon/insert! ds)))
+
+(defn insert-feed-event-categories
+  "Given a list of events and a list of feeds (or a single event/feed), 
+  inserts an event category record for each event and each category 
+  associated with the given feeds."
+  [ds events feeds]
+  (let [multi-events? (vector? events)
+        multi-feeds? (vector? feeds)
+        events' (if multi-events? events [events])
+        feeds' (if multi-feeds? feeds [feeds])
+        category-ids (->> {:where [:in :feed-id (mapv :id feeds')]}
+                          (feed-categories/feed-categories ds)
+                          (mapv :category-id))
+        event-categories (->> events'
+                              (mapv (fn [{:keys [id]}]
+                                      (mapv (fn [c]
+                                              {:event-id id
+                                               :category-id c})
+                                            category-ids)))
+                              (flatten)
+                              (vec))]
+    (insert-event-categories ds {:data event-categories})))
+
+(defn insert-post-event-categories
+  "Given a list of events and a list of posts (or a single event/post),
+  inserts an event category record for each event and each category 
+  associated with the given posts"
+  [ds events posts]
+  (let [multi-events? (vector? events)
+        multi-posts? (vector? posts)
+        events' (if multi-events? events [events])
+        posts' (if multi-posts? posts [posts])
+        feed-ids (mapv :feed-id posts')
+        category-ids (->> {:where [:in :feed-id feed-ids]}
+                          (feed-categories/feed-categories ds)
+                          (mapv :category-id))
+        event-categories (->> events'
+                              (mapv (fn [{:keys [id]}]
+                                      (mapv (fn [c]
+                                              {:event-id id
+                                               :category-id c})
+                                            category-ids)))
+                              (flatten)
+                              (vec))]
+    (insert-event-categories ds {:data event-categories})))
+
+(defn insert-feed-impressions
+  "Given a list of feeds and a bundle id, inserts impression event reconds 
+  for each given feed. Inserts event categories for each feed."
+  [ds feeds bundle-id]
+  (let [bundle (bundles/bundle ds {:id bundle-id})
+        events (mapv (fn [{:keys [id content-type-id user-id]}]
+                       {:timestamp (util/get-utc-timestamp-string)
+                        :event "impression"
+                        :feed-id id
+                        :content-type-id content-type-id
+                        :creator-id user-id
+                        :bundle-id bundle-id
+                        :distributor-id (:user-id bundle)}) feeds)
+        events' (insert-event ds {:data events
+                                  :ret :*})]
+    (insert-feed-event-categories ds events' feeds)))
+
+(defn insert-post-impressions
+  "Given a list of posts and a bundle id, inserts impression event reconds 
+  for each given post. Inserts event categories for each post."
+  [ds posts bundle-id]
+  (let [bundle (bundles/bundle ds {:id bundle-id})
+        events (mapv (fn [{:keys [id feed-id content-type-id creator-id]}]
+                       {:timestamp (util/get-utc-timestamp-string)
+                        :event "impression"
+                        :feed-id feed-id
+                        :post-id id
+                        :content-type-id content-type-id
+                        :creator-id creator-id
+                        :bundle-id bundle-id
+                        :distributor-id (:user-id bundle)}) posts)
+        events' (insert-event ds {:data events
+                                  :ret :*})]
+    (insert-post-event-categories ds events' posts)))
+
+(defn insert-feed-click
+  "Given a feed and a bundle id, inserts a click event record 
+  for the given feed"
+  [ds {:keys [id content-type-id user-id] :as feed} bundle-id]
+  (let [bundle (bundles/bundle ds {:id bundle-id})
+        event {:timestamp (util/get-utc-timestamp-string)
+               :event "click"
+               :feed-id id
+               :content-type-id content-type-id
+               :creator-id user-id
+               :bundle-id bundle-id
+               :distributor-id (:user-id bundle)}
+        event' (insert-event ds {:data event
+                                 :ret :*})]
+    (insert-feed-event-categories ds event' feed)))
+
+(defn insert-post-click
+  "Given a post and a bundle id, inserts a click event record 
+  for the given post"
+  [ds {:keys [id feed-id content-type-id creator-id] :as post} bundle-id]
+  (let [bundle (bundles/bundle ds {:id bundle-id})
+        event {:timestamp (util/get-utc-timestamp-string)
+               :event "click"
+               :feed-id feed-id
+               :post-id id
+               :content-type-id content-type-id
+               :creator-id creator-id
+               :bundle-id bundle-id
+               :distributor-id (:user-id bundle)}
+        event' (insert-event ds {:data event
+                                 :ret :*})]
+    (insert-post-event-categories ds event' post)))
+
+(defn insert-post-view
+  "Given a post and a bundle id, inserts a view event record 
+  for the given post"
+  [ds {:keys [id feed-id content-type-id creator-id] :as post} bundle-id]
+  (let [bundle (bundles/bundle ds {:id bundle-id})
+        event {:timestamp (util/get-utc-timestamp-string)
+               :event "view"
+               :feed-id feed-id
+               :post-id id
+               :content-type-id content-type-id
+               :creator-id creator-id
+               :bundle-id bundle-id
+               :distributor-id (:user-id bundle)}
+        event' (insert-event ds {:data event
+                                 :ret :*})]
+    (insert-post-event-categories ds event' post)))
+
+(comment
+  (require '[source.db.util :as db.util]
+           '[honey.sql :as sql])
+
+  (defonce ds (db.util/conn))
+
+  (def maximums {:creators 100
+                 :distributors 50
+                 :feeds 150
+                 :posts 6000
+                 :bundles 75})
+
+  (defn seed-event! [{:keys [creators distributors feeds posts bundles]}]
+    (let [creator-id (inc (rand-int creators))
+          distributor-id (inc (rand-int distributors))
+          feed-id (inc (rand-int feeds))
+          post-id (inc (rand-int posts))
+          bundle-id (inc (rand-int bundles))
+          event-type (rand-int 3)
+          category-ids [(inc (rand-int 50)) (inc (rand-int 50))]
+
+          new-event (hon/insert! ds {:tname :events
+                                     :ret :1
+                                     :data {:timestamp (util/get-utc-timestamp-string)
+                                            :event (cond
+                                                     (= event-type 0) "impression"
+                                                     (= event-type 1) "click"
+                                                     (= event-type 2) "view")
+                                            :feed-id feed-id
+                                            :post-id (when (> (rand-int 10) 3) post-id)
+                                            :content-type-id (inc (rand-int 3))
+                                            :creator-id creator-id
+                                            :bundle-id bundle-id
+                                            :distributor-id distributor-id}})
+
+          event-categories (mapv (fn [x] {:event-id (:id new-event)
+                                          :category-id x}) category-ids)]
+
+      (hon/insert! ds {:tname :event-categories
+                       :data event-categories})))
+
+  (defn seed-events! [num-records]
+    (dotimes [_ num-records]
+      (seed-event! maximums)))
+
+  (time (metric-query ds {:min-date "2025-11-25 15:00:00"
+                          :feed-id "1"}))
+
+  (time (statistics-query ds {:content-type 1
+                              :creator-id 1
+                              :bundle-id 1
+                              :ret :1}))
+
+  (time (hon/find ds {:tname :events
+                      :where [:< :id 500]
+                      :ret :*}))
+
+  (time (hon/update! ds {:tname :events
+                         :where [:between :id 5000000 5500000]
+                         :data {:timestamp (str "2025-11-22" " 13:00:00")}
+                         :ret :*}))
+
+  (time (hon/execute! ds (- (hsql/select [[:date :timestamp] :day]
+                                         [(hsql/filter :%count.* (hsql/where := :event "impression")) :impressions]
+                                         [(hsql/filter :%count.* (hsql/where := :event "click")) :clicks]
+                                         [(hsql/filter :%count.* (hsql/where := :event "view")) :views])
+                            (hsql/from :events)
+                            (hsql/where [:between :timestamp "2025-11-17 00:00:00" "2025-11-24 23:59:59"])
+                            (hsql/group-by :day)
+                            (hsql/order-by :day)) {:ret :*}))
+
+  (time (interval-statistics-query ds :daily "2025-11-17" "2025-11-24" {:feed-id 4}))
+
+  (time (hon/execute! ds (-> (hsql/select [[:strftime "%W" :timestamp] :week]
+                                          [(hsql/filter :%count.* (hsql/where := :event "impression")) :impressions]
+                                          [(hsql/filter :%count.* (hsql/where := :event "click")) :clicks]
+                                          [(hsql/filter :%count.* (hsql/where := :event "view")) :views])
+                             (hsql/from :events)
+                             (hsql/where [:between :timestamp "2025-11-01 00:00:00" "2025-11-30 23:59:00"])
+                             (hsql/group-by :week)
+                             (hsql/order-by :week))
+                      {:ret :*}))
+
+  (time (interval-statistics-query ds :weekly "2025-11-01" "2025-11-30" {:feed-id 4}))
+  (time (interval-statistics-query ds :monthly "2025-10-01" "2025-12-01" {:feed-id 4}))
+  (time (interval-statistics-query ds :yearly "2024-01-01" "2026-01-01" {:feed-id 4}))
+
+  (time (weekly-growth-averages ds "2025-11-01" "2025-11-30" {:feed-id 4}))
+
+  (time (average-engagement ds "2025-11-24 00:00:00" "2025-11-24 23:59:59" {:feed-id 4}))
+
+  (time (click-through-rate ds {:min-date "2025-11-24 00:00:00"
+                                :max-date "2025-11-24 23:59:59"
+                                :feed-id 4}))
+
+  (time
+   (hon/execute!
+    ds
+    {:select [[[:count :*] :total]]
+     :from [:events]}
+    {:ret :*}))
+
+  (time (hon/find ds {:tname :event-categories
+                      :ret :*}))
+
+  (time (seed-event! maximums))
+
+  (time (seed-events! 3000000))
+
+  (time (sql/format (hsql/select [(hsql/filter :%count.* (hsql/where := :event "impression")) :impressions]
+                                 [(hsql/filter :%count.* (hsql/where := :event "click")) :clicks]
+                                 [(hsql/filter :%count.* (hsql/where := :event "view")) :views])))
+
+  (time (insert-event ds {:data {:timestamp (util/get-utc-timestamp-string)
+                                 :event "impression"
+                                 :feed-id 1
+                                 :content-type-id 1
+                                 :creator-id 1
+                                 :bundle-id 1
+                                 :distributor-id 1}}))
+
+  ())
+
diff --git a/src/source/services/analytics/interface.clj b/src/source/services/analytics/interface.clj
new file mode 100644
index 00000000..cb2b55e6
--- /dev/null
+++ b/src/source/services/analytics/interface.clj
@@ -0,0 +1,88 @@
+(ns source.services.analytics.interface
+  (:require [source.services.analytics.core :as core]))
+
+(defn metric-query
+  "Generic select query function for returning analytics data from the events table"
+  [ds {:keys [_select _order-by _group-by _metric _feed-id _post-id _content-type-id _bundle-id _creator-id _distributor-id _min-date _max-date _category-ids _ret] :as opts}]
+  (core/metric-query ds opts))
+
+(defn statistics-query
+  "returns the number of impressions, clicks and views, filtered by any other arguments accepted by metric-query"
+  [ds opts]
+  (core/statistics-query ds opts))
+
+(defn interval-statistics-query
+  "returns the number of impressions, clicks and views per interval (:daily, :weekly, :monthly or :yearly) over the given time period, 
+  filtered by any other arguments accepted by metric-query.
+  
+  Date parameters must be in the format YYYY-mm-dd."
+  [ds interval min-date max-date opts]
+  (core/interval-statistics-query ds interval min-date max-date opts))
+
+(defn weekly-growth-averages
+  "Returns the percentage of growth in impressions, clicks and views per week, over the given time period. 
+  Uses the first week as a basis for comparison, not included in results.
+  Can be filtered by any other arguments accepted by metric-query."
+  [ds min-date max-date opts]
+  (core/weekly-growth-averages ds min-date max-date opts))
+
+(defn average-engagement
+  "Returns the average engagement (average clicks and views) over the given time period.
+  Can be filtered by any other arguments accepted by metric-query."
+  [ds min-date max-date opts]
+  (core/average-engagement ds min-date max-date opts))
+
+(defn click-through-rate
+  "Returns the click-through rate based on impressions and clicks filtered by any arguments accepted by metric-query"
+  [ds opts]
+  (core/click-through-rate ds opts))
+
+(defn insert-event [ds {:keys [_data _ret] :as opts}]
+  (core/insert-event ds opts))
+
+(defn insert-event-categories [ds {:keys [_data _ret] :as opts}]
+  (core/insert-event-categories ds opts))
+
+(defn insert-feed-event-categories
+  "Given a list of events and a list of feeds (or a single event/feed), 
+  inserts an event category record for each event and each category 
+  associated with the given feeds."
+  [ds events feeds]
+  (core/insert-feed-event-categories ds events feeds))
+
+(defn insert-post-event-categories
+  "Given a list of events and a list of posts (or a single event/post),
+  inserts an event category record for each event and each category 
+  associated with the given posts"
+  [ds events posts]
+  (core/insert-post-event-categories ds events posts))
+
+(defn insert-feed-impressions
+  "Given a list of feeds and a bundle id, inserts impression event reconds 
+  for each given feed. Inserts event categories for each feed."
+  [ds feeds bundle-id]
+  (core/insert-feed-impressions ds feeds bundle-id))
+
+(defn insert-post-impressions
+  "Given a list of posts and a bundle id, inserts impression event reconds 
+  for each given post. Inserts event categories for each post."
+  [ds posts bundle-id]
+  (core/insert-post-impressions ds posts bundle-id))
+
+(defn insert-feed-click
+  "Given a feed and a bundle id, inserts a click event record 
+  for the given feed"
+  [ds {:keys [_id _content-type-id _user-id] :as feed} bundle-id]
+  (core/insert-feed-click ds feed bundle-id))
+
+(defn insert-post-click
+  "Given a post and a bundle id, inserts a click event record 
+  for the given post"
+  [ds {:keys [_id _feed-id _content-type-id _creator-id] :as post} bundle-id]
+  (core/insert-post-click ds post bundle-id))
+
+(defn insert-post-view
+  "Given a post and a bundle id, inserts a view event record 
+  for the given post"
+  [ds {:keys [_id _feed-id _content-type-id _creator-id] :as post} bundle-id]
+  (core/insert-post-view ds post bundle-id))

From 217d24bd21fe8579fa034a801141841be45a0cc1 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 26 Nov 2025 12:00:36 +0200
Subject: [PATCH 161/391] updated bundle endpoints to update analytics events
 implicitly

---
 src/source/routes/bundle_feed.clj       | 10 ++++++----
 src/source/routes/bundle_feed_post.clj  | 10 ++++++----
 src/source/routes/bundle_feed_posts.clj | 10 ++++++----
 src/source/routes/bundle_feeds.clj      |  4 +++-
 src/source/routes/bundle_post.clj       | 11 +++++++----
 src/source/routes/bundle_posts.clj      |  4 +++-
 6 files changed, 31 insertions(+), 18 deletions(-)

diff --git a/src/source/routes/bundle_feed.clj b/src/source/routes/bundle_feed.clj
index 229e0a70..a9990cf5 100644
--- a/src/source/routes/bundle_feed.clj
+++ b/src/source/routes/bundle_feed.clj
@@ -1,6 +1,7 @@
 (ns source.routes.bundle-feed
   (:require [source.services.interface :as services]
-            [ring.util.response :as res]))
+            [ring.util.response :as res]
+            [source.services.analytics.interface :as analytics]))
 
 (defn get
   {:summary "get feed by id"
@@ -24,6 +25,7 @@
                             [:state [:enum "live" "not live" "pending"]]]}
                 404 {:body [:map [:message :string]]}}}
 
-  [{:keys [ds path-params] :as _request}]
-  (-> (services/feed ds path-params)
-      (res/response)))
+  [{:keys [ds bundle-id path-params] :as _request}]
+  (let [feed (services/feed ds path-params)]
+    (analytics/insert-feed-click ds feed bundle-id)
+    (res/response feed)))
diff --git a/src/source/routes/bundle_feed_post.clj b/src/source/routes/bundle_feed_post.clj
index f4ab98f9..d7f61b1c 100644
--- a/src/source/routes/bundle_feed_post.clj
+++ b/src/source/routes/bundle_feed_post.clj
@@ -1,6 +1,7 @@
 (ns source.routes.bundle-feed-post
   (:require [source.services.interface :as services]
-            [ring.util.response :as res]))
+            [ring.util.response :as res]
+            [source.services.analytics.interface :as analytics]))
 
 (defn get
   {:summary "get post by post id"
@@ -25,6 +26,7 @@
                      [:posted-at [:maybe :string]]]}
                404 {:body [:map [:message :string]]}}}
 
-  [{:keys [ds path-params] :as _request}]
-  (-> (services/incoming-post ds {:id (:post-id path-params)})
-      (res/response)))
+  [{:keys [ds bundle-id path-params] :as _request}]
+  (let [post (services/incoming-post ds {:id (:post-id path-params)})]
+    (analytics/insert-post-click ds post bundle-id)
+    (res/response post)))
diff --git a/src/source/routes/bundle_feed_posts.clj b/src/source/routes/bundle_feed_posts.clj
index 49120ded..9c677c79 100644
--- a/src/source/routes/bundle_feed_posts.clj
+++ b/src/source/routes/bundle_feed_posts.clj
@@ -1,6 +1,7 @@
 (ns source.routes.bundle-feed-posts
   (:require [source.services.interface :as services]
-            [ring.util.response :as res]))
+            [ring.util.response :as res]
+            [source.services.analytics.interface :as analytics]))
 
 (defn get
   {:summary "get all posts by feed id"
@@ -26,6 +27,7 @@
                401 {:body [:map [:message :string]]}
                403 {:body [:map [:message :string]]}}}
 
-  [{:keys [ds path-params] :as _request}]
-  (-> (services/incoming-posts ds {:where [:= :feed-id (:id path-params)]})
-      (res/response)))
+  [{:keys [ds bundle-id path-params] :as _request}]
+  (let [posts (services/incoming-posts ds {:where [:= :feed-id (:id path-params)]})]
+    (analytics/insert-post-impressions ds posts bundle-id)
+    (res/response posts)))
diff --git a/src/source/routes/bundle_feeds.clj b/src/source/routes/bundle_feeds.clj
index 9f643760..bb197b93 100644
--- a/src/source/routes/bundle_feeds.clj
+++ b/src/source/routes/bundle_feeds.clj
@@ -3,7 +3,8 @@
             [source.services.interface :as services]
             [source.db.util :as db.util]
             [clojure.walk :as walk]
-            [honey.sql.helpers :as hsql]))
+            [honey.sql.helpers :as hsql]
+            [source.services.analytics.interface :as analytics]))
 
 (defn post
   {:summary "get all feeds present in the bundle authorised by uuid"
@@ -52,4 +53,5 @@
                     (hsql/order-by (when latest [:created-at :desc])))
           type-filtered (services/feeds ds query)]
 
+      (analytics/insert-feed-impressions ds type-filtered bundle-id)
       (res/response type-filtered))))
diff --git a/src/source/routes/bundle_post.clj b/src/source/routes/bundle_post.clj
index 01549ae3..c3848ba9 100644
--- a/src/source/routes/bundle_post.clj
+++ b/src/source/routes/bundle_post.clj
@@ -1,7 +1,8 @@
 (ns source.routes.bundle-post
   (:require [source.services.interface :as services]
             [source.db.util :as db.util]
-            [ring.util.response :as res]))
+            [ring.util.response :as res]
+            [source.services.analytics.interface :as analytics]))
 
 (defn get
   {:summary "get a single outgoing post in the uuid-authorized bundle by post id"
@@ -25,8 +26,10 @@
                            [:posted-at [:maybe :string]]]}
                404 {:body [:map [:message :string]]}}}
 
-  [{:keys [bundle-id path-params] :as _request}]
+  [{:keys [ds bundle-id path-params] :as _request}]
   (with-open [bundle-ds (db.util/conn :bundle bundle-id)]
-    (let [id (:id path-params)]
-      (res/response (services/outgoing-post bundle-ds {:id id})))))
+    (let [id (:id path-params)
+          post (services/outgoing-post bundle-ds {:id id})]
+      (analytics/insert-post-click ds post bundle-id)
+      (res/response post))))
 
diff --git a/src/source/routes/bundle_posts.clj b/src/source/routes/bundle_posts.clj
index 3930c32f..5f5e41f6 100644
--- a/src/source/routes/bundle_posts.clj
+++ b/src/source/routes/bundle_posts.clj
@@ -4,7 +4,8 @@
             [clojure.walk :as walk]
             [ring.util.response :as res]
             [clojure.set :as set]
-            [honey.sql.helpers :as hsql]))
+            [honey.sql.helpers :as hsql]
+            [source.services.analytics.interface :as analytics]))
 
 (defn post
   {:summary "get all outgoing posts in the uuid-authorized bundle"
@@ -74,4 +75,5 @@
                           (subvec started-posts 0 limit)
                           started-posts)]
 
+      (analytics/insert-post-impressions ds limited-posts bundle-id)
       (res/response limited-posts))))

From b2845f8c1762514e4a64b32ead1c6afa1369f352 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 26 Nov 2025 12:01:41 +0200
Subject: [PATCH 162/391] added base for analytics http endpoints

---
 .../routes/analytics/creator/general.clj      | 23 +++++++++++++++++++
 src/source/routes/reitit.clj                  |  9 ++++++++
 2 files changed, 32 insertions(+)
 create mode 100644 src/source/routes/analytics/creator/general.clj

diff --git a/src/source/routes/analytics/creator/general.clj b/src/source/routes/analytics/creator/general.clj
new file mode 100644
index 00000000..b31fe0dc
--- /dev/null
+++ b/src/source/routes/analytics/creator/general.clj
@@ -0,0 +1,23 @@
+(ns source.routes.analytics.creator.general
+  (:require [source.services.analytics.interface :as analytics]
+            [ring.util.response :as res]
+            [clojure.walk :as w]))
+
+(defn get
+  {:summary "Gets the number of impressions, clicks and views per day over the given time period. Optionally filtered by feed."
+   :parameters {:query [:map
+                        [:mindate :string]
+                        [:maxdate :string]
+                        [:creator :int]
+                        [:feed {:optional true} [:maybe :int]]]}
+   :responses {200 {:body [:vector
+                           [:map
+                            [:day :string]
+                            [:impressions :int]
+                            [:clicks :int]
+                            [:views :int]]]}}}
+
+  [{:keys [ds query-params] :as _request}]
+  (let [{:keys [mindate maxdate creator feed]} (w/keywordize-keys query-params)]
+    (res/response (analytics/interval-statistics-query ds :daily mindate maxdate {:creator-id creator
+                                                                                  :feed-id feed}))))
diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index c757eb99..efcebe94 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -40,6 +40,7 @@
             [source.routes.feeds :as feeds]
             [source.routes.feed :as feed]
             [source.routes.feed-categories :as feed-categories]
+            [source.routes.analytics.creator.general :as analytics-creator-general]
             [source.routes.integrations :as integrations]
             [source.routes.integration :as integration]
             [source.routes.integration-key :as integration-key]
@@ -216,6 +217,14 @@
        ["/categories"   (route {:get feed-categories/get
                                 :post feed-categories/post})]]]
 
+     ["/analytics"
+      ["/creator"
+       ["/general"      (route {:get analytics-creator-general/get})]
+       ["/deltas"]
+       ["/top"]]
+      ["/bundle"]
+      ["admin"]]
+
      ["/bundle"        {:middleware [[mw/apply-bundle]]
                         :tags #{"bundles"}}
       [""               (route {:get bundle/get})]

From f325cc8a9a62a6e0eaf7330a2393d74e100b0447 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 26 Nov 2025 13:49:14 +0200
Subject: [PATCH 163/391] added exclamation marks to side-effectful functions

---
 src/source/routes/bundle_feed.clj           |  2 +-
 src/source/routes/bundle_feed_post.clj      |  2 +-
 src/source/routes/bundle_feed_posts.clj     |  2 +-
 src/source/routes/bundle_feeds.clj          |  2 +-
 src/source/routes/bundle_post.clj           |  2 +-
 src/source/routes/bundle_posts.clj          |  2 +-
 src/source/services/analytics/core.clj      | 44 ++++++++++-----------
 src/source/services/analytics/interface.clj | 36 ++++++++---------
 8 files changed, 46 insertions(+), 46 deletions(-)

diff --git a/src/source/routes/bundle_feed.clj b/src/source/routes/bundle_feed.clj
index a9990cf5..714c6e6c 100644
--- a/src/source/routes/bundle_feed.clj
+++ b/src/source/routes/bundle_feed.clj
@@ -27,5 +27,5 @@
 
   [{:keys [ds bundle-id path-params] :as _request}]
   (let [feed (services/feed ds path-params)]
-    (analytics/insert-feed-click ds feed bundle-id)
+    (analytics/insert-feed-click! ds feed bundle-id)
     (res/response feed)))
diff --git a/src/source/routes/bundle_feed_post.clj b/src/source/routes/bundle_feed_post.clj
index d7f61b1c..87d01f5b 100644
--- a/src/source/routes/bundle_feed_post.clj
+++ b/src/source/routes/bundle_feed_post.clj
@@ -28,5 +28,5 @@
 
   [{:keys [ds bundle-id path-params] :as _request}]
   (let [post (services/incoming-post ds {:id (:post-id path-params)})]
-    (analytics/insert-post-click ds post bundle-id)
+    (analytics/insert-post-click! ds post bundle-id)
     (res/response post)))
diff --git a/src/source/routes/bundle_feed_posts.clj b/src/source/routes/bundle_feed_posts.clj
index 9c677c79..f36bf94b 100644
--- a/src/source/routes/bundle_feed_posts.clj
+++ b/src/source/routes/bundle_feed_posts.clj
@@ -29,5 +29,5 @@
 
   [{:keys [ds bundle-id path-params] :as _request}]
   (let [posts (services/incoming-posts ds {:where [:= :feed-id (:id path-params)]})]
-    (analytics/insert-post-impressions ds posts bundle-id)
+    (analytics/insert-post-impressions! ds posts bundle-id)
     (res/response posts)))
diff --git a/src/source/routes/bundle_feeds.clj b/src/source/routes/bundle_feeds.clj
index bb197b93..722227f6 100644
--- a/src/source/routes/bundle_feeds.clj
+++ b/src/source/routes/bundle_feeds.clj
@@ -53,5 +53,5 @@
                     (hsql/order-by (when latest [:created-at :desc])))
           type-filtered (services/feeds ds query)]
 
-      (analytics/insert-feed-impressions ds type-filtered bundle-id)
+      (analytics/insert-feed-impressions! ds type-filtered bundle-id)
       (res/response type-filtered))))
diff --git a/src/source/routes/bundle_post.clj b/src/source/routes/bundle_post.clj
index c3848ba9..259607d7 100644
--- a/src/source/routes/bundle_post.clj
+++ b/src/source/routes/bundle_post.clj
@@ -30,6 +30,6 @@
   (with-open [bundle-ds (db.util/conn :bundle bundle-id)]
     (let [id (:id path-params)
           post (services/outgoing-post bundle-ds {:id id})]
-      (analytics/insert-post-click ds post bundle-id)
+      (analytics/insert-post-click! ds post bundle-id)
       (res/response post))))
 
diff --git a/src/source/routes/bundle_posts.clj b/src/source/routes/bundle_posts.clj
index 5f5e41f6..3b75381c 100644
--- a/src/source/routes/bundle_posts.clj
+++ b/src/source/routes/bundle_posts.clj
@@ -75,5 +75,5 @@
                           (subvec started-posts 0 limit)
                           started-posts)]
 
-      (analytics/insert-post-impressions ds limited-posts bundle-id)
+      (analytics/insert-post-impressions! ds limited-posts bundle-id)
       (res/response limited-posts))))
diff --git a/src/source/services/analytics/core.clj b/src/source/services/analytics/core.clj
index 4e536f6e..39221453 100644
--- a/src/source/services/analytics/core.clj
+++ b/src/source/services/analytics/core.clj
@@ -98,21 +98,21 @@
   (let [{:keys [impressions clicks]} (statistics-query ds opts)]
     (float (* (/ clicks impressions) 100))))
 
-(defn insert-event [ds {:keys [data ret] :as opts}]
+(defn insert-event! [ds {:keys [data ret] :as opts}]
   (->> {:tname :events
         :data data
         :ret ret}
        (merge opts)
        (hon/insert! ds)))
 
-(defn insert-event-categories [ds {:keys [data ret] :as opts}]
+(defn insert-event-categories! [ds {:keys [data ret] :as opts}]
   (->> {:tname :event-categories
         :data data
         :ret ret}
        (merge opts)
        (hon/insert! ds)))
 
-(defn insert-feed-event-categories
+(defn insert-feed-event-categories!
   "Given a list of events and a list of feeds (or a single event/feed), 
   inserts an event category record for each event and each category 
   associated with the given feeds."
@@ -132,9 +132,9 @@
                                             category-ids)))
                               (flatten)
                               (vec))]
-    (insert-event-categories ds {:data event-categories})))
+    (insert-event-categories! ds {:data event-categories})))
 
-(defn insert-post-event-categories
+(defn insert-post-event-categories!
   "Given a list of events and a list of posts (or a single event/post),
   inserts an event category record for each event and each category 
   associated with the given posts"
@@ -155,9 +155,9 @@
                                             category-ids)))
                               (flatten)
                               (vec))]
-    (insert-event-categories ds {:data event-categories})))
+    (insert-event-categories! ds {:data event-categories})))
 
-(defn insert-feed-impressions
+(defn insert-feed-impressions!
   "Given a list of feeds and a bundle id, inserts impression event reconds 
   for each given feed. Inserts event categories for each feed."
   [ds feeds bundle-id]
@@ -170,11 +170,11 @@
                         :creator-id user-id
                         :bundle-id bundle-id
                         :distributor-id (:user-id bundle)}) feeds)
-        events' (insert-event ds {:data events
+        events' (insert-event! ds {:data events
                                   :ret :*})]
-    (insert-feed-event-categories ds events' feeds)))
+    (insert-feed-event-categories! ds events' feeds)))
 
-(defn insert-post-impressions
+(defn insert-post-impressions!
   "Given a list of posts and a bundle id, inserts impression event reconds 
   for each given post. Inserts event categories for each post."
   [ds posts bundle-id]
@@ -188,11 +188,11 @@
                         :creator-id creator-id
                         :bundle-id bundle-id
                         :distributor-id (:user-id bundle)}) posts)
-        events' (insert-event ds {:data events
+        events' (insert-event! ds {:data events
                                   :ret :*})]
-    (insert-post-event-categories ds events' posts)))
+    (insert-post-event-categories! ds events' posts)))
 
-(defn insert-feed-click
+(defn insert-feed-click!
   "Given a feed and a bundle id, inserts a click event record 
   for the given feed"
   [ds {:keys [id content-type-id user-id] :as feed} bundle-id]
@@ -204,11 +204,11 @@
                :creator-id user-id
                :bundle-id bundle-id
                :distributor-id (:user-id bundle)}
-        event' (insert-event ds {:data event
+        event' (insert-event! ds {:data event
                                  :ret :*})]
-    (insert-feed-event-categories ds event' feed)))
+    (insert-feed-event-categories! ds event' feed)))
 
-(defn insert-post-click
+(defn insert-post-click!
   "Given a post and a bundle id, inserts a click event record 
   for the given post"
   [ds {:keys [id feed-id content-type-id creator-id] :as post} bundle-id]
@@ -221,11 +221,11 @@
                :creator-id creator-id
                :bundle-id bundle-id
                :distributor-id (:user-id bundle)}
-        event' (insert-event ds {:data event
+        event' (insert-event! ds {:data event
                                  :ret :*})]
-    (insert-post-event-categories ds event' post)))
+    (insert-post-event-categories! ds event' post)))
 
-(defn insert-post-view
+(defn insert-post-view!
   "Given a post and a bundle id, inserts a view event record 
   for the given post"
   [ds {:keys [id feed-id content-type-id creator-id] :as post} bundle-id]
@@ -238,9 +238,9 @@
                :creator-id creator-id
                :bundle-id bundle-id
                :distributor-id (:user-id bundle)}
-        event' (insert-event ds {:data event
+        event' (insert-event! ds {:data event
                                  :ret :*})]
-    (insert-post-event-categories ds event' post)))
+    (insert-post-event-categories! ds event' post)))
 
 (comment
   (require '[source.db.util :as db.util]
@@ -355,7 +355,7 @@
                                  [(hsql/filter :%count.* (hsql/where := :event "click")) :clicks]
                                  [(hsql/filter :%count.* (hsql/where := :event "view")) :views])))
 
-  (time (insert-event ds {:data {:timestamp (util/get-utc-timestamp-string)
+  (time (insert-event! ds {:data {:timestamp (util/get-utc-timestamp-string)
                                  :event "impression"
                                  :feed-id 1
                                  :content-type-id 1
diff --git a/src/source/services/analytics/interface.clj b/src/source/services/analytics/interface.clj
index cb2b55e6..bf390959 100644
--- a/src/source/services/analytics/interface.clj
+++ b/src/source/services/analytics/interface.clj
@@ -37,52 +37,52 @@
   [ds opts]
   (core/click-through-rate ds opts))
 
-(defn insert-event [ds {:keys [_data _ret] :as opts}]
-  (core/insert-event ds opts))
+(defn insert-event! [ds {:keys [_data _ret] :as opts}]
+  (core/insert-event! ds opts))
 
-(defn insert-event-categories [ds {:keys [_data _ret] :as opts}]
-  (core/insert-event-categories ds opts))
+(defn insert-event-categories! [ds {:keys [_data _ret] :as opts}]
+  (core/insert-event-categories! ds opts))
 
-(defn insert-feed-event-categories
+(defn insert-feed-event-categories!
   "Given a list of events and a list of feeds (or a single event/feed), 
   inserts an event category record for each event and each category 
   associated with the given feeds."
   [ds events feeds]
-  (core/insert-feed-event-categories ds events feeds))
+  (core/insert-feed-event-categories! ds events feeds))
 
-(defn insert-post-event-categories
+(defn insert-post-event-categories!
   "Given a list of events and a list of posts (or a single event/post),
   inserts an event category record for each event and each category 
   associated with the given posts"
   [ds events posts]
-  (core/insert-post-event-categories ds events posts))
+  (core/insert-post-event-categories! ds events posts))
 
-(defn insert-feed-impressions
+(defn insert-feed-impressions!
   "Given a list of feeds and a bundle id, inserts impression event reconds 
   for each given feed. Inserts event categories for each feed."
   [ds feeds bundle-id]
-  (core/insert-feed-impressions ds feeds bundle-id))
+  (core/insert-feed-impressions! ds feeds bundle-id))
 
-(defn insert-post-impressions
+(defn insert-post-impressions!
   "Given a list of posts and a bundle id, inserts impression event reconds 
   for each given post. Inserts event categories for each post."
   [ds posts bundle-id]
-  (core/insert-post-impressions ds posts bundle-id))
+  (core/insert-post-impressions! ds posts bundle-id))
 
-(defn insert-feed-click
+(defn insert-feed-click!
   "Given a feed and a bundle id, inserts a click event record 
   for the given feed"
   [ds {:keys [_id _content-type-id _user-id] :as feed} bundle-id]
-  (core/insert-feed-click ds feed bundle-id))
+  (core/insert-feed-click! ds feed bundle-id))
 
-(defn insert-post-click
+(defn insert-post-click!
   "Given a post and a bundle id, inserts a click event record 
   for the given post"
   [ds {:keys [_id _feed-id _content-type-id _creator-id] :as post} bundle-id]
-  (core/insert-post-click ds post bundle-id))
+  (core/insert-post-click! ds post bundle-id))
 
-(defn insert-post-view
+(defn insert-post-view!
   "Given a post and a bundle id, inserts a view event record 
   for the given post"
   [ds {:keys [_id _feed-id _content-type-id _creator-id] :as post} bundle-id]
-  (core/insert-post-view ds post bundle-id))
+  (core/insert-post-view! ds post bundle-id))

From 4052cfc2dbbf2eadd9e35b25a34c5beaaf594543 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 26 Nov 2025 15:53:56 +0200
Subject: [PATCH 164/391] added service function to get top analytic data

---
 src/source/services/analytics/core.clj      | 78 +++++++++++++--------
 src/source/services/analytics/interface.clj |  5 ++
 2 files changed, 53 insertions(+), 30 deletions(-)

diff --git a/src/source/services/analytics/core.clj b/src/source/services/analytics/core.clj
index 39221453..17f658dc 100644
--- a/src/source/services/analytics/core.clj
+++ b/src/source/services/analytics/core.clj
@@ -7,7 +7,7 @@
 
 (defn metric-query
   "Generic select query function for returning analytics data from the events table"
-  [ds {:keys [select order-by group-by metric feed-id post-id content-type-id bundle-id creator-id distributor-id min-date max-date category-ids ret]}]
+  [ds {:keys [select order-by group-by limit metric feed-id post-id content-type-id bundle-id creator-id distributor-id min-date max-date category-ids where ret]}]
   (let [clauses (cond-> {}
                   (some? metric) (hsql/where [:= :event metric])
                   (some? feed-id) (hsql/where [:= :feed-id feed-id])
@@ -16,28 +16,31 @@
                   (some? bundle-id) (hsql/where [:= :bundle-id bundle-id])
                   (some? creator-id) (hsql/where [:= :creator-id creator-id])
                   (some? distributor-id) (hsql/where [:= :distributor-id distributor-id])
+                  (some? where) (merge where)
                   (and (some? min-date) (nil? max-date)) (hsql/where [:>= :timestamp min-date])
                   (and (some? max-date) (nil? min-date)) (hsql/where [:<= :timestamp max-date])
                   (and (some? min-date) (some? max-date)) (hsql/where [:between :timestamp min-date max-date])
                   (some? select) (merge select)
+                  (nil? select) (merge {:select [[[:count :*] :total]]})
+                  (some? limit) (hsql/limit limit)
                   (some? order-by) (merge order-by)
                   (some? group-by) (merge group-by)
                   (seq category-ids) (-> (hsql/join [:event-categories :ec] [:= :events.id :ec.event-id])
                                          (hsql/where [:in :ec.category-id category-ids])))]
     (hon/execute!
      ds
-     (merge {:select [[[:count :*] :total]]
-             :from [:events]}
+     (merge {:from [:events]}
             clauses)
      {:ret (if ret ret :*)})))
 
 (defn statistics-query
-  "returns the number of impressions, clicks and views, filtered by any other arguments accepted by metric-query"
-  [ds opts]
+  "Returns the number of impressions, clicks and views, filtered by any other arguments accepted by metric-query.
+  If ret is not given, returns a single record."
+  [ds {:keys [ret] :as opts}]
   (metric-query ds (merge {:select (hsql/select [(hsql/filter :%count.* (hsql/where := :event "impression")) :impressions]
                                                 [(hsql/filter :%count.* (hsql/where := :event "click")) :clicks]
                                                 [(hsql/filter :%count.* (hsql/where := :event "view")) :views])
-                           :ret :1}
+                           :ret (if ret ret :1)}
                           opts)))
 
 (defn interval-statistics-query
@@ -69,9 +72,25 @@
                              :order-by (hsql/order-by column)}
                             opts))))
 
+(defn top-statistics-query
+  "Returns the top n of the given top field in order of the number of their impressions, clicks and views within the given time period"
+  [ds min-date max-date n top-field opts]
+  (metric-query ds (merge {:select (hsql/select top-field
+                                                [(hsql/filter :%count.* (hsql/where := :event "impression")) :impressions]
+                                                [(hsql/filter :%count.* (hsql/where := :event "click")) :clicks]
+                                                [(hsql/filter :%count.* (hsql/where := :event "view")) :views])
+                           :min-date min-date
+                           :max-date max-date
+                           :where (hsql/where [:!= top-field nil])
+                           :group-by (hsql/group-by top-field)
+                           :order-by (hsql/order-by [:impressions :desc] [:clicks :desc] [:views :desc])
+                           :limit n
+                           :ret :*}
+                          opts)))
+
 (defn weekly-growth-averages
   "Returns the percentage of growth in impressions, clicks and views per week, over the given time period. 
-  Uses the first week as a basis for comparison, not included in results.
+  Uses the first week as a basis for comparison.
   Can be filtered by any other arguments accepted by metric-query."
   [ds min-date max-date opts]
   (let [weeks (interval-statistics-query ds :weekly min-date max-date opts)
@@ -171,7 +190,7 @@
                         :bundle-id bundle-id
                         :distributor-id (:user-id bundle)}) feeds)
         events' (insert-event! ds {:data events
-                                  :ret :*})]
+                                   :ret :*})]
     (insert-feed-event-categories! ds events' feeds)))
 
 (defn insert-post-impressions!
@@ -189,7 +208,7 @@
                         :bundle-id bundle-id
                         :distributor-id (:user-id bundle)}) posts)
         events' (insert-event! ds {:data events
-                                  :ret :*})]
+                                   :ret :*})]
     (insert-post-event-categories! ds events' posts)))
 
 (defn insert-feed-click!
@@ -205,7 +224,7 @@
                :bundle-id bundle-id
                :distributor-id (:user-id bundle)}
         event' (insert-event! ds {:data event
-                                 :ret :*})]
+                                  :ret :*})]
     (insert-feed-event-categories! ds event' feed)))
 
 (defn insert-post-click!
@@ -222,7 +241,7 @@
                :bundle-id bundle-id
                :distributor-id (:user-id bundle)}
         event' (insert-event! ds {:data event
-                                 :ret :*})]
+                                  :ret :*})]
     (insert-post-event-categories! ds event' post)))
 
 (defn insert-post-view!
@@ -239,7 +258,7 @@
                :bundle-id bundle-id
                :distributor-id (:user-id bundle)}
         event' (insert-event! ds {:data event
-                                 :ret :*})]
+                                  :ret :*})]
     (insert-post-event-categories! ds event' post)))
 
 (comment
@@ -290,10 +309,7 @@
   (time (metric-query ds {:min-date "2025-11-25 15:00:00"
                           :feed-id "1"}))
 
-  (time (statistics-query ds {:content-type 1
-                              :creator-id 1
-                              :bundle-id 1
-                              :ret :1}))
+  (time (statistics-query ds {:ret :*}))
 
   (time (hon/find ds {:tname :events
                       :where [:< :id 500]
@@ -304,14 +320,14 @@
                          :data {:timestamp (str "2025-11-22" " 13:00:00")}
                          :ret :*}))
 
-  (time (hon/execute! ds (- (hsql/select [[:date :timestamp] :day]
-                                         [(hsql/filter :%count.* (hsql/where := :event "impression")) :impressions]
-                                         [(hsql/filter :%count.* (hsql/where := :event "click")) :clicks]
-                                         [(hsql/filter :%count.* (hsql/where := :event "view")) :views])
-                            (hsql/from :events)
-                            (hsql/where [:between :timestamp "2025-11-17 00:00:00" "2025-11-24 23:59:59"])
-                            (hsql/group-by :day)
-                            (hsql/order-by :day)) {:ret :*}))
+  (time (hon/execute! ds (-> (hsql/select [[:date :timestamp] :day]
+                                          [(hsql/filter :%count.* (hsql/where := :event "impression")) :impressions]
+                                          [(hsql/filter :%count.* (hsql/where := :event "click")) :clicks]
+                                          [(hsql/filter :%count.* (hsql/where := :event "view")) :views])
+                             (hsql/from :events)
+                             (hsql/where [:between :timestamp "2025-11-17 00:00:00" "2025-11-24 23:59:59"])
+                             (hsql/group-by :day)
+                             (hsql/order-by :day)) {:ret :*}))
 
   (time (interval-statistics-query ds :daily "2025-11-17" "2025-11-24" {:feed-id 4}))
 
@@ -356,12 +372,14 @@
                                  [(hsql/filter :%count.* (hsql/where := :event "view")) :views])))
 
   (time (insert-event! ds {:data {:timestamp (util/get-utc-timestamp-string)
-                                 :event "impression"
-                                 :feed-id 1
-                                 :content-type-id 1
-                                 :creator-id 1
-                                 :bundle-id 1
-                                 :distributor-id 1}}))
+                                  :event "impression"
+                                  :feed-id 1
+                                  :content-type-id 1
+                                  :creator-id 1
+                                  :bundle-id 1
+                                  :distributor-id 1}}))
+
+  (time (top-statistics-query ds "2025-11-17" "2025-11-24" 10 :post-id {}))
 
   ())
 
diff --git a/src/source/services/analytics/interface.clj b/src/source/services/analytics/interface.clj
index bf390959..6de4478a 100644
--- a/src/source/services/analytics/interface.clj
+++ b/src/source/services/analytics/interface.clj
@@ -26,6 +26,11 @@
   [ds min-date max-date opts]
   (core/weekly-growth-averages ds min-date max-date opts))
 
+(defn top-statistics-query
+  "Returns the top n of the given top field in order of the number of their impressions, clicks and views within the given time period."
+  [ds min-date max-date n top-field opts]
+  (core/top-statistics-query ds min-date max-date n top-field opts))
+
 (defn average-engagement
   "Returns the average engagement (average clicks and views) over the given time period.
   Can be filtered by any other arguments accepted by metric-query."

From b7d50677c72acaf5e0cf8dc82f84b98aa9b8d67e Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 26 Nov 2025 15:54:31 +0200
Subject: [PATCH 165/391] fixed bug in authz middleware

---
 src/source/middleware/auth/core.clj | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/source/middleware/auth/core.clj b/src/source/middleware/auth/core.clj
index 6248aa7c..9a679696 100644
--- a/src/source/middleware/auth/core.clj
+++ b/src/source/middleware/auth/core.clj
@@ -28,7 +28,7 @@
        (res/status 401)))))
 
 (defn wrap-auth-user-type
-  "returns an unauthorized response if the user's type is not the required user type (provider | distributor | admin)"
+  "returns an unauthorized response if the user's type is not the required user type (creator | distributor | admin)"
   [handler & {:keys [required-type]}]
   (fn [request]
     (let [ds (db.util/conn :master)
@@ -38,7 +38,7 @@
                              (:type))]
       (cond
         (not (some? required-type)) (handler request)
-        (and (= user-type (name :admin)) (= user-type expected-type)) (handler request)
+        (and (= user-type (name required-type)) (= user-type expected-type)) (handler request)
         :else (->
                (res/response {:message "Unauthorized"})
                (res/status 403))))))

From b1a9d90c57f16b1fe8dde90b7a82be0e0d5aff90 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 26 Nov 2025 15:55:23 +0200
Subject: [PATCH 166/391] added http endpoints for analytics for creators,
 distributors and the bundle

---
 .../analytics/bundle/posts/_id_/views.clj     | 16 +++++++++++
 .../routes/analytics/creator/deltas.clj       | 22 +++++++++++++++
 .../routes/analytics/creator/general.clj      |  9 +++---
 src/source/routes/analytics/creator/top.clj   | 28 +++++++++++++++++++
 .../routes/analytics/distributor/general.clj  | 22 +++++++++++++++
 .../routes/analytics/distributor/top.clj      | 28 +++++++++++++++++++
 src/source/routes/reitit.clj                  | 21 ++++++++++----
 7 files changed, 136 insertions(+), 10 deletions(-)
 create mode 100644 src/source/routes/analytics/bundle/posts/_id_/views.clj
 create mode 100644 src/source/routes/analytics/creator/deltas.clj
 create mode 100644 src/source/routes/analytics/creator/top.clj
 create mode 100644 src/source/routes/analytics/distributor/general.clj
 create mode 100644 src/source/routes/analytics/distributor/top.clj

diff --git a/src/source/routes/analytics/bundle/posts/_id_/views.clj b/src/source/routes/analytics/bundle/posts/_id_/views.clj
new file mode 100644
index 00000000..a0993e8d
--- /dev/null
+++ b/src/source/routes/analytics/bundle/posts/_id_/views.clj
@@ -0,0 +1,16 @@
+(ns source.routes.analytics.bundle.posts.-id-.views
+  (:require [ring.util.response :as res]
+            [source.services.analytics.interface :as analytics]
+            [source.services.interface :as services]))
+
+(defn post
+  {:summary "Explicitly insert a view event for the post with the given id for the purpose of analytics"
+   :parameters {:query [:map [:uuid :string]]
+                :path [:map [:id {:title "id"
+                                  :description "post id"} :int]]}
+   :responses {200 {:body [:map [:message :string]]}}}
+
+  [{:keys [ds bundle-id path-params] :as _request}]
+  (let [post (services/incoming-post ds {:id (:id path-params)})]
+    (analytics/insert-post-view! ds post bundle-id)
+    (res/response {:message "Successfully inserted view event"})))
diff --git a/src/source/routes/analytics/creator/deltas.clj b/src/source/routes/analytics/creator/deltas.clj
new file mode 100644
index 00000000..d5f8f4d8
--- /dev/null
+++ b/src/source/routes/analytics/creator/deltas.clj
@@ -0,0 +1,22 @@
+(ns source.routes.analytics.creator.deltas
+  (:require [clojure.walk :as w]
+            [ring.util.response :as res]
+            [source.services.analytics.interface :as analytics]))
+
+(defn get
+  {:summary "Returns the percentage of growth in impressions, clicks and views per week, over the given time period. Optionally filtered by feed."
+   :parameters {:query [:map
+                        [:mindate :string]
+                        [:maxdate :string]
+                        [:feed {:optional true} [:maybe :int]]]}
+   :responses {200 {:body [:vector
+                           [:map
+                            [:week :string]
+                            [:impressions :int]
+                            [:clicks :int]
+                            [:views :int]]]}}}
+
+  [{:keys [ds user query-params] :as _request}]
+  (let [{:keys [mindate maxdate feed]} (w/keywordize-keys query-params)]
+    (res/response (analytics/weekly-growth-averages ds mindate maxdate {:creator-id (:id user)
+                                                                        :feed-id feed}))))
diff --git a/src/source/routes/analytics/creator/general.clj b/src/source/routes/analytics/creator/general.clj
index b31fe0dc..bba8f402 100644
--- a/src/source/routes/analytics/creator/general.clj
+++ b/src/source/routes/analytics/creator/general.clj
@@ -4,11 +4,10 @@
             [clojure.walk :as w]))
 
 (defn get
-  {:summary "Gets the number of impressions, clicks and views per day over the given time period. Optionally filtered by feed."
+  {:summary "Gets the number of impressions, clicks and views per day for a creator over the given time period. Optionally filtered by feed."
    :parameters {:query [:map
                         [:mindate :string]
                         [:maxdate :string]
-                        [:creator :int]
                         [:feed {:optional true} [:maybe :int]]]}
    :responses {200 {:body [:vector
                            [:map
@@ -17,7 +16,7 @@
                             [:clicks :int]
                             [:views :int]]]}}}
 
-  [{:keys [ds query-params] :as _request}]
-  (let [{:keys [mindate maxdate creator feed]} (w/keywordize-keys query-params)]
-    (res/response (analytics/interval-statistics-query ds :daily mindate maxdate {:creator-id creator
+  [{:keys [ds user query-params] :as _request}]
+  (let [{:keys [mindate maxdate feed]} (w/keywordize-keys query-params)]
+    (res/response (analytics/interval-statistics-query ds :daily mindate maxdate {:creator-id (:id user)
                                                                                   :feed-id feed}))))
diff --git a/src/source/routes/analytics/creator/top.clj b/src/source/routes/analytics/creator/top.clj
new file mode 100644
index 00000000..34ef4493
--- /dev/null
+++ b/src/source/routes/analytics/creator/top.clj
@@ -0,0 +1,28 @@
+(ns source.routes.analytics.creator.top
+  (:require [source.services.analytics.interface :as analytics]
+            [ring.util.response :as res]
+            [clojure.walk :as w]
+            [clojure.set :as set]))
+
+(defn get
+  {:summary "Get the top n records with the highest number of impressions, clicks and views, in terms of the given top field. Optionally filtered by content type."
+   :parameters {:query [:map
+                        [:n :int]
+                        [:mindate :string]
+                        [:maxdate :string]
+                        [:top [:enum "post" "bundle"]]
+                        [:contenttype {:optional true} [:maybe :int]]]}
+   :responses {200 {:body [:vector
+                           [:map
+                            [:top :string]
+                            [:impressions :int]
+                            [:clicks :int]
+                            [:views :int]]]}}}
+
+  [{:keys [ds user query-params] :as _request}]
+  (let [{:keys [n mindate maxdate top contenttype]} (w/keywordize-keys query-params)
+        top-field (if (= top "post") :post-id :bundle-id)]
+    (res/response (set/rename-keys
+                   (analytics/top-statistics-query ds mindate maxdate n top-field {:creator-id (:id user)
+                                                                                   :content-type-id contenttype})
+                   {top-field :top}))))
diff --git a/src/source/routes/analytics/distributor/general.clj b/src/source/routes/analytics/distributor/general.clj
new file mode 100644
index 00000000..b0b94535
--- /dev/null
+++ b/src/source/routes/analytics/distributor/general.clj
@@ -0,0 +1,22 @@
+(ns source.routes.analytics.distributor.general
+  (:require [source.services.analytics.interface :as analytics]
+            [ring.util.response :as res]
+            [clojure.walk :as w]))
+
+(defn get
+  {:summary "Gets the number of impressions, clicks and views per day for a distributor over the given time period. Optionally filtered by bundle."
+   :parameters {:query [:map
+                        [:mindate :string]
+                        [:maxdate :string]
+                        [:bundle {:optional true} [:maybe :int]]]}
+   :responses {200 {:body [:vector
+                           [:map
+                            [:day :string]
+                            [:impressions :int]
+                            [:clicks :int]
+                            [:views :int]]]}}}
+
+  [{:keys [ds user query-params] :as _request}]
+  (let [{:keys [mindate maxdate bundle]} (w/keywordize-keys query-params)]
+    (res/response (analytics/interval-statistics-query ds :daily mindate maxdate {:distributor-id (:id user)
+                                                                                  :bundle-id bundle}))))
diff --git a/src/source/routes/analytics/distributor/top.clj b/src/source/routes/analytics/distributor/top.clj
new file mode 100644
index 00000000..2105737a
--- /dev/null
+++ b/src/source/routes/analytics/distributor/top.clj
@@ -0,0 +1,28 @@
+(ns source.routes.analytics.distributor.top
+  (:require [source.services.analytics.interface :as analytics]
+            [ring.util.response :as res]
+            [clojure.walk :as w]
+            [clojure.set :as set]))
+
+(defn get
+  {:summary "Get the top n records with the highest number of impressions, clicks and views, in terms of the given top field. Optionally filtered by content type."
+   :parameters {:query [:map
+                        [:n :int]
+                        [:mindate :string]
+                        [:maxdate :string]
+                        [:top [:enum "post" "feed"]]
+                        [:contenttype {:optional true} [:maybe :int]]]}
+   :responses {200 {:body [:vector
+                           [:map
+                            [:top :string]
+                            [:impressions :int]
+                            [:clicks :int]
+                            [:views :int]]]}}}
+
+  [{:keys [ds user query-params] :as _request}]
+  (let [{:keys [n mindate maxdate top contenttype]} (w/keywordize-keys query-params)
+        top-field (if (= top "post") :post-id :feed-id)]
+    (res/response (set/rename-keys
+                   (analytics/top-statistics-query ds mindate maxdate n top-field {:distributor-id (:id user)
+                                                                                   :content-type-id contenttype})
+                   {top-field :top}))))
diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index efcebe94..b1f5813f 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -41,6 +41,11 @@
             [source.routes.feed :as feed]
             [source.routes.feed-categories :as feed-categories]
             [source.routes.analytics.creator.general :as analytics-creator-general]
+            [source.routes.analytics.creator.deltas :as analytics-creator-deltas]
+            [source.routes.analytics.creator.top :as analytics-creator-top]
+            [source.routes.analytics.distributor.general :as analytics-distributor-general]
+            [source.routes.analytics.distributor.top :as analytics-distributor-top]
+            [source.routes.analytics.bundle.posts.-id-.views :as analytics-bundle-posts-id-views]
             [source.routes.integrations :as integrations]
             [source.routes.integration :as integration]
             [source.routes.integration-key :as integration-key]
@@ -218,12 +223,18 @@
                                 :post feed-categories/post})]]]
 
      ["/analytics"
-      ["/creator"
+      ["/creator"       {:middleware [[mw/apply-auth {:required-type :creator}]]}
        ["/general"      (route {:get analytics-creator-general/get})]
-       ["/deltas"]
-       ["/top"]]
-      ["/bundle"]
-      ["admin"]]
+       ["/deltas"       (route {:get analytics-creator-deltas/get})]
+       ["/top"          (route {:get analytics-creator-top/get})]]
+      ["/distributor"   {:middleware [[mw/apply-auth {:required-type :distributor}]]}
+       ["/general"      (route {:get analytics-distributor-general/get})]
+       ["/top"          (route {:get analytics-distributor-top/get})]]
+      ["/bundle"        {:middleware [[mw/apply-bundle]]}
+       ["/posts"
+        ["/:id"
+         ["/views"       (route {:post analytics-bundle-posts-id-views/post})]]]]
+      ["admin"          {:middleware [[mw/apply-auth {:required-type :admin}]]}]]
 
      ["/bundle"        {:middleware [[mw/apply-bundle]]
                         :tags #{"bundles"}}

From b7a9fb006de331493650b2c29076736a6667c3b5 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 26 Nov 2025 16:21:55 +0200
Subject: [PATCH 167/391] fixed bug in top endpoints

---
 src/source/routes/analytics/creator/top.clj     | 10 +++++-----
 src/source/routes/analytics/distributor/top.clj | 10 +++++-----
 src/source/routes/reitit.clj                    |  6 ++++--
 3 files changed, 14 insertions(+), 12 deletions(-)

diff --git a/src/source/routes/analytics/creator/top.clj b/src/source/routes/analytics/creator/top.clj
index 34ef4493..ea0deb2b 100644
--- a/src/source/routes/analytics/creator/top.clj
+++ b/src/source/routes/analytics/creator/top.clj
@@ -21,8 +21,8 @@
 
   [{:keys [ds user query-params] :as _request}]
   (let [{:keys [n mindate maxdate top contenttype]} (w/keywordize-keys query-params)
-        top-field (if (= top "post") :post-id :bundle-id)]
-    (res/response (set/rename-keys
-                   (analytics/top-statistics-query ds mindate maxdate n top-field {:creator-id (:id user)
-                                                                                   :content-type-id contenttype})
-                   {top-field :top}))))
+        top-field (if (= top "post") :post-id :bundle-id)
+        results (analytics/top-statistics-query ds mindate maxdate n top-field {:creator-id (:id user)
+                                                                                :content-type-id contenttype})]
+    (res/response (mapv (fn [result]
+                          (set/rename-keys result {top-field :top})) results))))
diff --git a/src/source/routes/analytics/distributor/top.clj b/src/source/routes/analytics/distributor/top.clj
index 2105737a..23517b8b 100644
--- a/src/source/routes/analytics/distributor/top.clj
+++ b/src/source/routes/analytics/distributor/top.clj
@@ -21,8 +21,8 @@
 
   [{:keys [ds user query-params] :as _request}]
   (let [{:keys [n mindate maxdate top contenttype]} (w/keywordize-keys query-params)
-        top-field (if (= top "post") :post-id :feed-id)]
-    (res/response (set/rename-keys
-                   (analytics/top-statistics-query ds mindate maxdate n top-field {:distributor-id (:id user)
-                                                                                   :content-type-id contenttype})
-                   {top-field :top}))))
+        top-field (if (= top "post") :post-id :feed-id)
+        results (analytics/top-statistics-query ds mindate maxdate n top-field {:distributor-id (:id user)
+                                                                                :content-type-id contenttype})]
+    (res/response (mapv (fn [result]
+                          (set/rename-keys result {top-field :top})) results))))
diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index b1f5813f..38e92abe 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -233,8 +233,10 @@
       ["/bundle"        {:middleware [[mw/apply-bundle]]}
        ["/posts"
         ["/:id"
-         ["/views"       (route {:post analytics-bundle-posts-id-views/post})]]]]
-      ["admin"          {:middleware [[mw/apply-auth {:required-type :admin}]]}]]
+         ["/views"      (route {:post analytics-bundle-posts-id-views/post})]]]]
+      ["admin"          {:middleware [[mw/apply-auth {:required-type :admin}]]}
+       ["/general"]
+       ["/top"]]]
 
      ["/bundle"        {:middleware [[mw/apply-bundle]]
                         :tags #{"bundles"}}

From 48b8a4da7ad5ad045ca8257fe3ebfea27ca9b4f6 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 26 Nov 2025 16:32:37 +0200
Subject: [PATCH 168/391] updated namespace to use interface

---
 src/source/db/event.clj | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/source/db/event.clj b/src/source/db/event.clj
index 87919369..eb95bd1f 100644
--- a/src/source/db/event.clj
+++ b/src/source/db/event.clj
@@ -1,6 +1,6 @@
 (ns source.db.event
   (:require [source.services.event-categories :as ec]
-            [source.services.analytics :as analytics]
+            [source.services.analytics.interface :as analytics]
             [source.services.outgoing-posts :as outgoing-posts]
             [source.services.feed-categories :as feed-categories]
             [source.db.util :as db.util]

From c5a60ec63ff7d2731a60384de885299fcf8d8406 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Thu, 27 Nov 2025 15:56:36 +0200
Subject: [PATCH 169/391] fixed date format

---
 src/source/services/analytics/core.clj | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/src/source/services/analytics/core.clj b/src/source/services/analytics/core.clj
index 17f658dc..7aeadbd7 100644
--- a/src/source/services/analytics/core.clj
+++ b/src/source/services/analytics/core.clj
@@ -47,7 +47,7 @@
   "returns the number of impressions, clicks and views per interval (:daily, :weekly, :monthly or :yearly) over the given time period, 
   filtered by any other arguments accepted by metric-query.
   
-  Date parameters must be in the format YYYY-mm-dd."
+  Date parameters must be in the format YYYY-MM-DD."
   [ds interval min-date max-date opts]
   (let [select (cond
                  (= interval :daily) [[:date :timestamp] :day]
@@ -66,8 +66,8 @@
                                                   [(hsql/filter :%count.* (hsql/where := :event "impression")) :impressions]
                                                   [(hsql/filter :%count.* (hsql/where := :event "click")) :clicks]
                                                   [(hsql/filter :%count.* (hsql/where := :event "view")) :views])
-                             :min-date (str min-date " 00:00:00")
-                             :max-date (str max-date " 23:59:59")
+                             :min-date (str min-date "T00:00:00Z")
+                             :max-date (str max-date "T23:59:59Z")
                              :group-by (hsql/group-by column)
                              :order-by (hsql/order-by column)}
                             opts))))
@@ -306,7 +306,7 @@
     (dotimes [_ num-records]
       (seed-event! maximums)))
 
-  (time (metric-query ds {:min-date "2025-11-25 15:00:00"
+  (time (metric-query ds {:min-date "2025-11-25T15:00:00Z"
                           :feed-id "1"}))
 
   (time (statistics-query ds {:ret :*}))
@@ -317,7 +317,7 @@
 
   (time (hon/update! ds {:tname :events
                          :where [:between :id 5000000 5500000]
-                         :data {:timestamp (str "2025-11-22" " 13:00:00")}
+                         :data {:timestamp (str "2025-11-22" "T13:00:00Z")}
                          :ret :*}))
 
   (time (hon/execute! ds (-> (hsql/select [[:date :timestamp] :day]
@@ -325,7 +325,7 @@
                                           [(hsql/filter :%count.* (hsql/where := :event "click")) :clicks]
                                           [(hsql/filter :%count.* (hsql/where := :event "view")) :views])
                              (hsql/from :events)
-                             (hsql/where [:between :timestamp "2025-11-17 00:00:00" "2025-11-24 23:59:59"])
+                             (hsql/where [:between :timestamp "2025-11-17T00:00:00Z" "2025-11-24T23:59:59Z"])
                              (hsql/group-by :day)
                              (hsql/order-by :day)) {:ret :*}))
 
@@ -336,7 +336,7 @@
                                           [(hsql/filter :%count.* (hsql/where := :event "click")) :clicks]
                                           [(hsql/filter :%count.* (hsql/where := :event "view")) :views])
                              (hsql/from :events)
-                             (hsql/where [:between :timestamp "2025-11-01 00:00:00" "2025-11-30 23:59:00"])
+                             (hsql/where [:between :timestamp "2025-11-01T00:00:00Z" "2025-11-30T23:59:00Z"])
                              (hsql/group-by :week)
                              (hsql/order-by :week))
                       {:ret :*}))
@@ -347,10 +347,10 @@
 
   (time (weekly-growth-averages ds "2025-11-01" "2025-11-30" {:feed-id 4}))
 
-  (time (average-engagement ds "2025-11-24 00:00:00" "2025-11-24 23:59:59" {:feed-id 4}))
+  (time (average-engagement ds "2025-11-24T00:00:00Z" "2025-11-24T23:59:59Z" {:feed-id 4}))
 
-  (time (click-through-rate ds {:min-date "2025-11-24 00:00:00"
-                                :max-date "2025-11-24 23:59:59"
+  (time (click-through-rate ds {:min-date "2025-11-24T00:00:00Z"
+                                :max-date "2025-11-24T23:59:59Z"
                                 :feed-id 4}))
 
   (time

From 0aca15c4687777d32913d5fbe3445b99a613a111 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Tue, 2 Dec 2025 09:47:56 +0200
Subject: [PATCH 170/391] fixed schema in deltas endpoint

---
 src/source/routes/analytics/creator/deltas.clj | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/source/routes/analytics/creator/deltas.clj b/src/source/routes/analytics/creator/deltas.clj
index d5f8f4d8..69546bf1 100644
--- a/src/source/routes/analytics/creator/deltas.clj
+++ b/src/source/routes/analytics/creator/deltas.clj
@@ -12,9 +12,9 @@
    :responses {200 {:body [:vector
                            [:map
                             [:week :string]
-                            [:impressions :int]
-                            [:clicks :int]
-                            [:views :int]]]}}}
+                            [:impressions :float]
+                            [:clicks :float]
+                            [:views :float]]]}}}
 
   [{:keys [ds user query-params] :as _request}]
   (let [{:keys [mindate maxdate feed]} (w/keywordize-keys query-params)]

From 1fe1ce4b2f6c4c95fec1503cc9eaa87027f06627 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Tue, 2 Dec 2025 09:48:24 +0200
Subject: [PATCH 171/391] updated top endpoints to return names instead of ids

---
 src/source/events_test.clj                    | 106 ++++++++++++++++++
 src/source/routes/analytics/creator/top.clj   |  30 ++++-
 .../routes/analytics/distributor/top.clj      |  26 ++++-
 3 files changed, 152 insertions(+), 10 deletions(-)
 create mode 100644 src/source/events_test.clj

diff --git a/src/source/events_test.clj b/src/source/events_test.clj
new file mode 100644
index 00000000..fba1ea5f
--- /dev/null
+++ b/src/source/events_test.clj
@@ -0,0 +1,106 @@
+(ns source.events-test
+  (:require [source.db.honey :as hon]
+            [honey.sql.helpers :as hsql]
+            [source.util :as util]
+            [honey.sql :as sql]))
+
+(defn metric-query
+  "Generic select query function for returning analytics data from the events table"
+  [ds {:keys [select order-by group-by metric feed-id post-id content-type-id bundle-id creator-id distributor-id min-date max-date category-ids ret]}]
+  (let [clauses (cond-> {}
+                  (some? metric) (hsql/where [:= :event metric])
+                  (some? feed-id) (hsql/where [:= :feed-id feed-id])
+                  (some? post-id) (hsql/where [:= :post-id post-id])
+                  (some? content-type-id) (hsql/where [:= :content-type-id content-type-id])
+                  (some? bundle-id) (hsql/where [:= :bundle-id bundle-id])
+                  (some? creator-id) (hsql/where [:= :creator-id creator-id])
+                  (some? distributor-id) (hsql/where [:= :distributor-id distributor-id])
+                  (and (some? min-date) (nil? max-date)) (hsql/where [:>= :timestamp min-date])
+                  (and (some? max-date) (nil? min-date)) (hsql/where [:<= :timestamp max-date])
+                  (and (some? min-date) (some? max-date)) (hsql/where [:between :timestamp min-date max-date])
+                  (some? select) (merge select)
+                  (some? order-by) (merge order-by)
+                  (some? group-by) (merge group-by)
+                  (seq category-ids) (-> (hsql/join [:event-categories :ec] [:= :events.id :ec.event-id])
+                                         (hsql/where [:in :ec.category-id category-ids])))]
+    (hon/execute!
+     ds
+     (merge {:select [[[:count :*] :total]]
+             :from [:events]}
+            clauses)
+     {:ret (if ret ret :*)})))
+
+(defn statistics-query
+  "returns the number of impressions, clicks and views, filtered by any other arguments accepted by metric-query"
+  [ds opts]
+  (metric-query ds (merge {:select (hsql/select [(hsql/filter :%count.* (hsql/where := :event "impression")) :impressions]
+                                                [(hsql/filter :%count.* (hsql/where := :event "click")) :clicks]
+                                                [(hsql/filter :%count.* (hsql/where := :event "view")) :views])
+                           :ret :1}
+                          opts)))
+
+(defn interval-statistics-query
+  "returns the number of impressions, clicks and views per interval (:daily, :weekly, :monthly or :yearly) over the given time period, 
+  filtered by any other arguments accepted by metric-query.
+  
+  Date parameters must be in the format YYYY-mm-dd."
+  [ds interval min-date max-date opts]
+  (let [select (cond
+                 (= interval :daily) [[:date :timestamp] :day]
+                 (= interval :weekly) [[:strftime "%W" :timestamp] :week]
+                 (= interval :monthly) [[:strftime "%m" :timestamp] :month]
+                 (= interval :yearly) [[:strftime "%Y" :timestamp] :year]
+                 :else [[:date :timestamp] :day])
+        column (cond
+                 (= interval :daily) :day
+                 (= interval :weekly) :week
+                 (= interval :monthly) :month
+                 (= interval :yearly) :year
+                 :else :day)]
+
+    (metric-query ds (merge {:select (hsql/select select
+                                                  [(hsql/filter :%count.* (hsql/where := :event "impression")) :impressions]
+                                                  [(hsql/filter :%count.* (hsql/where := :event "click")) :clicks]
+                                                  [(hsql/filter :%count.* (hsql/where := :event "view")) :views])
+                             :min-date (str min-date " 00:00:00")
+                             :max-date (str max-date " 23:59:59")
+                             :group-by (hsql/group-by column)
+                             :order-by (hsql/order-by column)}
+                            opts))))
+
+(defn weekly-growth-averages
+  "Returns the percentage of growth in impressions, clicks and views per week, over the given time period. 
+  Uses the first week as a basis for comparison, not included in results.
+  Can be filtered by any other arguments accepted by metric-query."
+  [ds min-date max-date opts]
+  (let [weeks (interval-statistics-query ds :weekly min-date max-date opts)
+        {:keys [impressions clicks views]} (first weeks)]
+    (mapv (fn [w]
+            {:week (:week w)
+             :impressions (float (* (/ (- (:impressions w) impressions) impressions) 100))
+             :clicks (float (* (/ (- (:clicks w) clicks) clicks) 100))
+             :views (float (* (/ (- (:views w) views) views) 100))})
+          weeks)))
+
+(defn average-engagement
+  "Returns the average engagement (average clicks and views) over the given time period.
+  Can be filtered by any other arguments accepted by metric-query."
+  [ds min-date max-date opts]
+  (let [{:keys [clicks views]} (statistics-query ds (merge {:min-date min-date
+                                                            :max-date max-date}
+                                                           opts))]
+    (float (/ (+ clicks views) 2))))
+
+(defn click-through-rate
+  "Returns the click-through rate based on impressions and clicks filtered by any arguments accepted by metric-query"
+  [ds opts]
+  (let [{:keys [impressions clicks]} (statistics-query ds opts)]
+    (float (* (/ clicks impressions) 100))))
+
+(defn insert-event [ds {:keys [data ret] :as opts}]
+  (->> {:tname :events
+        :data data
+        :ret ret}
+       (merge opts)
+       (hon/insert! ds)))
+
diff --git a/src/source/routes/analytics/creator/top.clj b/src/source/routes/analytics/creator/top.clj
index ea0deb2b..b75a92c1 100644
--- a/src/source/routes/analytics/creator/top.clj
+++ b/src/source/routes/analytics/creator/top.clj
@@ -2,7 +2,8 @@
   (:require [source.services.analytics.interface :as analytics]
             [ring.util.response :as res]
             [clojure.walk :as w]
-            [clojure.set :as set]))
+            [clojure.set :as set]
+            [source.services.interface :as services]))
 
 (defn get
   {:summary "Get the top n records with the highest number of impressions, clicks and views, in terms of the given top field. Optionally filtered by content type."
@@ -22,7 +23,26 @@
   [{:keys [ds user query-params] :as _request}]
   (let [{:keys [n mindate maxdate top contenttype]} (w/keywordize-keys query-params)
         top-field (if (= top "post") :post-id :bundle-id)
-        results (analytics/top-statistics-query ds mindate maxdate n top-field {:creator-id (:id user)
-                                                                                :content-type-id contenttype})]
-    (res/response (mapv (fn [result]
-                          (set/rename-keys result {top-field :top})) results))))
+        results (->> {:creator-id (:id user)
+                      :content-type-id contenttype}
+                     (analytics/top-statistics-query ds mindate maxdate n top-field)
+                     (mapv (fn [result]
+                             (set/rename-keys result {top-field :top}))))
+        ids (mapv :top results)
+        names (if (= top-field :post-id)
+                (mapv (fn [{:keys [id title]}]
+                        {:id id
+                         :name title})
+                      (services/incoming-posts ds {:where [:in :id ids]}))
+                (mapv (fn [{:keys [id name]}]
+                        {:id id
+                         :name name})
+                      (services/bundles ds {:where [:in :id ids]})))
+        juxted (->> names
+                    (mapv (juxt :id :name))
+                    (into {}))
+        named-results (mapv (fn [{:keys [top] :as r}]
+                              (assoc r :top (clojure.core/get juxted top (str top))))
+                            results)]
+
+    (res/response named-results)))
diff --git a/src/source/routes/analytics/distributor/top.clj b/src/source/routes/analytics/distributor/top.clj
index 23517b8b..1568e6c8 100644
--- a/src/source/routes/analytics/distributor/top.clj
+++ b/src/source/routes/analytics/distributor/top.clj
@@ -2,7 +2,8 @@
   (:require [source.services.analytics.interface :as analytics]
             [ring.util.response :as res]
             [clojure.walk :as w]
-            [clojure.set :as set]))
+            [clojure.set :as set]
+            [source.services.interface :as services]))
 
 (defn get
   {:summary "Get the top n records with the highest number of impressions, clicks and views, in terms of the given top field. Optionally filtered by content type."
@@ -22,7 +23,22 @@
   [{:keys [ds user query-params] :as _request}]
   (let [{:keys [n mindate maxdate top contenttype]} (w/keywordize-keys query-params)
         top-field (if (= top "post") :post-id :feed-id)
-        results (analytics/top-statistics-query ds mindate maxdate n top-field {:distributor-id (:id user)
-                                                                                :content-type-id contenttype})]
-    (res/response (mapv (fn [result]
-                          (set/rename-keys result {top-field :top})) results))))
+        results (->> {:distributor-id (:id user)
+                      :content-type-id contenttype}
+                     (analytics/top-statistics-query ds mindate maxdate n top-field)
+                     (mapv (fn [result]
+                             (set/rename-keys result {top-field :top}))))
+        ids (mapv :top results)
+        names (if (= top-field :post-id)
+                (services/incoming-posts ds {:where [:in :id ids]})
+                (services/feeds ds {:where [:in :id ids]}))
+        juxted (->> names
+                    (mapv (fn [{:keys [id title]}]
+                            {:id id
+                             :name title}))
+                    (mapv (juxt :id :name))
+                    (into {}))
+        named-results (mapv (fn [{:keys [top] :as r}]
+                              (assoc r :top (clojure.core/get juxted top (str top))))
+                            results)]
+    (res/response named-results)))

From 077b7286b0ed0ede797320d39e26191325c8d7c2 Mon Sep 17 00:00:00 2001
From: Kaidan Theron <kaidantheron@Kaidans-MacBook-Air.local>
Date: Tue, 2 Dec 2025 11:02:17 +0200
Subject: [PATCH 172/391] remove display-picture from master/categories initial
 table schema

---
 src/source/db/master.clj | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/source/db/master.clj b/src/source/db/master.clj
index 63e108dc..8a2b5a42 100644
--- a/src/source/db/master.clj
+++ b/src/source/db/master.clj
@@ -28,8 +28,7 @@
   (tables/create-table-sql
    :categories
    (tables/table-id)
-   [:name :text]
-   [:display-picture :string]))
+   [:name :text]))
 
 (def content-types
   (tables/create-table-sql

From ad3d1b51047c19bf7d6519db481772d675024c12 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 3 Dec 2025 10:09:27 +0200
Subject: [PATCH 173/391] added middleware to convert all query param string
 keys into keywords

---
 src/source/middleware/core.clj | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/src/source/middleware/core.clj b/src/source/middleware/core.clj
index af6c7fa3..af0553b5 100644
--- a/src/source/middleware/core.clj
+++ b/src/source/middleware/core.clj
@@ -9,7 +9,8 @@
             [ring.middleware.params :refer [wrap-params]]
             [ring.middleware.defaults :refer [wrap-defaults site-defaults]]
             [ring.middleware.json :as ring]
-            [ring.middleware.cookies :as cookies]))
+            [ring.middleware.cookies :as cookies]
+            [clojure.walk :as walk]))
 
 (defn wrap-ds [handler ds]
   (fn [request]
@@ -72,6 +73,12 @@
         (handler)
         (process-body csk/->camelCaseKeyword))))
 
+(defn wrap-query [handler]
+  (fn [{:keys [query-params] :as request}]
+    (-> request
+        (assoc :query-params (walk/keywordize-keys query-params))
+        (handler))))
+
 (defn apply-generic [app & {:keys [ds store js]}]
   (-> app
       (wrap-exception-logger)
@@ -79,6 +86,7 @@
       (apply-store store)
       (apply-js js)
       (wrap-case-conversion)
+      (wrap-query)
       (content-type/wrap-content-type)
       (wrap-cors :access-control-allow-origin [(re-pattern (conf/read-value :cors-origin))]
                  :access-control-allow-methods [:get :put :post :delete])

From dff8f73a8c78e69f14a730668c3097066d52b219 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 3 Dec 2025 10:10:36 +0200
Subject: [PATCH 174/391] updated the validate function to also support query
 param validation and automatic string to int conversion

---
 src/source/util.clj | 25 ++++++++++++++++---------
 1 file changed, 16 insertions(+), 9 deletions(-)

diff --git a/src/source/util.clj b/src/source/util.clj
index c8bbb7f3..2240eb1d 100644
--- a/src/source/util.clj
+++ b/src/source/util.clj
@@ -3,7 +3,8 @@
             [buddy.core.nonce :as nonce]
             [clojure.main :refer [demunge]]
             [malli.core :as m]
-            [malli.error :as me])
+            [malli.error :as me]
+            [malli.transform :as mt])
   (:import (java.math BigInteger)
            (java.security MessageDigest)))
 
@@ -53,14 +54,18 @@
         hash-bytes (.digest digest bytes)]
     (format "%064x" (BigInteger. 1 hash-bytes))))
 
-(defn validate [handler data]
-  (let [schema (get-in (metadata handler) [:parameters :body])
-        success (m/validate schema data)]
-    {:data (when success data)
-     :success success
-     :error (when-not success (->> data
-                                   (m/explain schema)
-                                   (me/humanize)))}))
+(defn validate
+  ([handler data]
+   (validate handler data :body))
+  ([handler data schema-type]
+   (let [schema (get-in (metadata handler) [:parameters schema-type])
+         transformed (m/decode schema data mt/string-transformer)
+         success (m/validate schema transformed)]
+     {:data (when success transformed)
+      :success success
+      :error (when-not success (->> transformed
+                                    (m/explain schema)
+                                    (me/humanize)))})))
 
 (defn format-rss-date
   "Takes a date as a string in RFC 1123 format and returns it in a format that meets ISO 8601 standards for SQLite.
@@ -76,7 +81,9 @@
 
 (comment
   (require '[source.routes.business :as business])
+  (require '[source.routes.analytics.creator.top :as ctop])
   (validate business/post {:cheese "modulr"})
+  (validate ctop/get {:mindate "2025-12-02" :maxdate "2025-12-02" :n "10" :contenttype 1 :top "post"} :query)
   (sha256 "1")
   ())
 

From a67d91b3639a71ab77cc752b8af48882c8d6ee12 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 3 Dec 2025 10:12:29 +0200
Subject: [PATCH 175/391] updated top endpoints to make use of query param
 validation

---
 src/source/routes/analytics/creator/top.clj   | 60 +++++++++++--------
 .../routes/analytics/distributor/top.clj      | 55 ++++++++++-------
 2 files changed, 66 insertions(+), 49 deletions(-)

diff --git a/src/source/routes/analytics/creator/top.clj b/src/source/routes/analytics/creator/top.clj
index b75a92c1..9d0001d3 100644
--- a/src/source/routes/analytics/creator/top.clj
+++ b/src/source/routes/analytics/creator/top.clj
@@ -1,9 +1,20 @@
 (ns source.routes.analytics.creator.top
   (:require [source.services.analytics.interface :as analytics]
             [ring.util.response :as res]
-            [clojure.walk :as w]
             [clojure.set :as set]
-            [source.services.interface :as services]))
+            [source.services.interface :as services]
+            [source.util :as utils]))
+
+(defn record-names [ds top-field ids]
+  (if (= top-field :post-id)
+    (mapv (fn [{:keys [id title]}]
+            {:id id
+             :name title})
+          (services/incoming-posts ds {:where [:in :id ids]}))
+    (mapv (fn [{:keys [id name]}]
+            {:id id
+             :name name})
+          (services/bundles ds {:where [:in :id ids]}))))
 
 (defn get
   {:summary "Get the top n records with the highest number of impressions, clicks and views, in terms of the given top field. Optionally filtered by content type."
@@ -21,28 +32,25 @@
                             [:views :int]]]}}}
 
   [{:keys [ds user query-params] :as _request}]
-  (let [{:keys [n mindate maxdate top contenttype]} (w/keywordize-keys query-params)
-        top-field (if (= top "post") :post-id :bundle-id)
-        results (->> {:creator-id (:id user)
-                      :content-type-id contenttype}
-                     (analytics/top-statistics-query ds mindate maxdate n top-field)
-                     (mapv (fn [result]
-                             (set/rename-keys result {top-field :top}))))
-        ids (mapv :top results)
-        names (if (= top-field :post-id)
-                (mapv (fn [{:keys [id title]}]
-                        {:id id
-                         :name title})
-                      (services/incoming-posts ds {:where [:in :id ids]}))
-                (mapv (fn [{:keys [id name]}]
-                        {:id id
-                         :name name})
-                      (services/bundles ds {:where [:in :id ids]})))
-        juxted (->> names
-                    (mapv (juxt :id :name))
-                    (into {}))
-        named-results (mapv (fn [{:keys [top] :as r}]
-                              (assoc r :top (clojure.core/get juxted top (str top))))
-                            results)]
+  (let [{:keys [data success error]} (utils/validate get query-params :query)
+        {:keys [n mindate maxdate top contenttype]} data
+        top-field (if (= top "post") :post-id :bundle-id)]
+    (if success
+      (let [results (->> {:creator-id (:id user)
+                          :content-type-id contenttype}
+                         (analytics/top-statistics-query ds mindate maxdate n top-field)
+                         (mapv (fn [result]
+                                 (set/rename-keys result {top-field :top}))))
+            ids (mapv :top results)
+            names (record-names ds top-field ids)
+            juxted (->> names
+                        (mapv (juxt :id :name))
+                        (into {}))
+            named-results (mapv (fn [{:keys [top] :as r}]
+                                  (assoc r :top (clojure.core/get juxted top (str top))))
+                                results)]
+
+        (res/response named-results))
 
-    (res/response named-results)))
+      (-> (res/response error)
+          (res/status 400)))))
diff --git a/src/source/routes/analytics/distributor/top.clj b/src/source/routes/analytics/distributor/top.clj
index 1568e6c8..2da1298f 100644
--- a/src/source/routes/analytics/distributor/top.clj
+++ b/src/source/routes/analytics/distributor/top.clj
@@ -1,9 +1,15 @@
 (ns source.routes.analytics.distributor.top
   (:require [source.services.analytics.interface :as analytics]
             [ring.util.response :as res]
-            [clojure.walk :as w]
             [clojure.set :as set]
-            [source.services.interface :as services]))
+            [source.services.interface :as services]
+            [source.util :as utils]
+            [clojure.walk :as walk]))
+
+(defn record-names [ds top-field ids]
+  (if (= top-field :post-id)
+    (services/incoming-posts ds {:where [:in :id ids]})
+    (services/feeds ds {:where [:in :id ids]})))
 
 (defn get
   {:summary "Get the top n records with the highest number of impressions, clicks and views, in terms of the given top field. Optionally filtered by content type."
@@ -21,24 +27,27 @@
                             [:views :int]]]}}}
 
   [{:keys [ds user query-params] :as _request}]
-  (let [{:keys [n mindate maxdate top contenttype]} (w/keywordize-keys query-params)
-        top-field (if (= top "post") :post-id :feed-id)
-        results (->> {:distributor-id (:id user)
-                      :content-type-id contenttype}
-                     (analytics/top-statistics-query ds mindate maxdate n top-field)
-                     (mapv (fn [result]
-                             (set/rename-keys result {top-field :top}))))
-        ids (mapv :top results)
-        names (if (= top-field :post-id)
-                (services/incoming-posts ds {:where [:in :id ids]})
-                (services/feeds ds {:where [:in :id ids]}))
-        juxted (->> names
-                    (mapv (fn [{:keys [id title]}]
-                            {:id id
-                             :name title}))
-                    (mapv (juxt :id :name))
-                    (into {}))
-        named-results (mapv (fn [{:keys [top] :as r}]
-                              (assoc r :top (clojure.core/get juxted top (str top))))
-                            results)]
-    (res/response named-results)))
+  (let [{:keys [data success error]} (utils/validate get query-params :query)
+        {:keys [n mindate maxdate top contenttype]} data
+        top-field (if (= top "post") :post-id :feed-id)]
+    (if success
+      (let [results (->> {:distributor-id (:id user)
+                          :content-type-id contenttype}
+                         (analytics/top-statistics-query ds mindate maxdate n top-field)
+                         (mapv (fn [result]
+                                 (set/rename-keys result {top-field :top}))))
+            ids (mapv :top results)
+            names (record-names ds top-field ids)
+            juxted (->> names
+                        (mapv (fn [{:keys [id title]}]
+                                {:id id
+                                 :name title}))
+                        (mapv (juxt :id :name))
+                        (into {}))
+            named-results (mapv (fn [{:keys [top] :as r}]
+                                  (assoc r :top (clojure.core/get juxted top (str top))))
+                                results)]
+        (res/response named-results))
+
+      (-> (res/response error)
+          (res/status 400)))))

From 6dfb5c8c69ecb783b93577fc772316cb6074d379 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 3 Dec 2025 10:15:53 +0200
Subject: [PATCH 176/391] removed unused test file

---
 src/source/events_test.clj | 106 -------------------------------------
 1 file changed, 106 deletions(-)
 delete mode 100644 src/source/events_test.clj

diff --git a/src/source/events_test.clj b/src/source/events_test.clj
deleted file mode 100644
index fba1ea5f..00000000
--- a/src/source/events_test.clj
+++ /dev/null
@@ -1,106 +0,0 @@
-(ns source.events-test
-  (:require [source.db.honey :as hon]
-            [honey.sql.helpers :as hsql]
-            [source.util :as util]
-            [honey.sql :as sql]))
-
-(defn metric-query
-  "Generic select query function for returning analytics data from the events table"
-  [ds {:keys [select order-by group-by metric feed-id post-id content-type-id bundle-id creator-id distributor-id min-date max-date category-ids ret]}]
-  (let [clauses (cond-> {}
-                  (some? metric) (hsql/where [:= :event metric])
-                  (some? feed-id) (hsql/where [:= :feed-id feed-id])
-                  (some? post-id) (hsql/where [:= :post-id post-id])
-                  (some? content-type-id) (hsql/where [:= :content-type-id content-type-id])
-                  (some? bundle-id) (hsql/where [:= :bundle-id bundle-id])
-                  (some? creator-id) (hsql/where [:= :creator-id creator-id])
-                  (some? distributor-id) (hsql/where [:= :distributor-id distributor-id])
-                  (and (some? min-date) (nil? max-date)) (hsql/where [:>= :timestamp min-date])
-                  (and (some? max-date) (nil? min-date)) (hsql/where [:<= :timestamp max-date])
-                  (and (some? min-date) (some? max-date)) (hsql/where [:between :timestamp min-date max-date])
-                  (some? select) (merge select)
-                  (some? order-by) (merge order-by)
-                  (some? group-by) (merge group-by)
-                  (seq category-ids) (-> (hsql/join [:event-categories :ec] [:= :events.id :ec.event-id])
-                                         (hsql/where [:in :ec.category-id category-ids])))]
-    (hon/execute!
-     ds
-     (merge {:select [[[:count :*] :total]]
-             :from [:events]}
-            clauses)
-     {:ret (if ret ret :*)})))
-
-(defn statistics-query
-  "returns the number of impressions, clicks and views, filtered by any other arguments accepted by metric-query"
-  [ds opts]
-  (metric-query ds (merge {:select (hsql/select [(hsql/filter :%count.* (hsql/where := :event "impression")) :impressions]
-                                                [(hsql/filter :%count.* (hsql/where := :event "click")) :clicks]
-                                                [(hsql/filter :%count.* (hsql/where := :event "view")) :views])
-                           :ret :1}
-                          opts)))
-
-(defn interval-statistics-query
-  "returns the number of impressions, clicks and views per interval (:daily, :weekly, :monthly or :yearly) over the given time period, 
-  filtered by any other arguments accepted by metric-query.
-  
-  Date parameters must be in the format YYYY-mm-dd."
-  [ds interval min-date max-date opts]
-  (let [select (cond
-                 (= interval :daily) [[:date :timestamp] :day]
-                 (= interval :weekly) [[:strftime "%W" :timestamp] :week]
-                 (= interval :monthly) [[:strftime "%m" :timestamp] :month]
-                 (= interval :yearly) [[:strftime "%Y" :timestamp] :year]
-                 :else [[:date :timestamp] :day])
-        column (cond
-                 (= interval :daily) :day
-                 (= interval :weekly) :week
-                 (= interval :monthly) :month
-                 (= interval :yearly) :year
-                 :else :day)]
-
-    (metric-query ds (merge {:select (hsql/select select
-                                                  [(hsql/filter :%count.* (hsql/where := :event "impression")) :impressions]
-                                                  [(hsql/filter :%count.* (hsql/where := :event "click")) :clicks]
-                                                  [(hsql/filter :%count.* (hsql/where := :event "view")) :views])
-                             :min-date (str min-date " 00:00:00")
-                             :max-date (str max-date " 23:59:59")
-                             :group-by (hsql/group-by column)
-                             :order-by (hsql/order-by column)}
-                            opts))))
-
-(defn weekly-growth-averages
-  "Returns the percentage of growth in impressions, clicks and views per week, over the given time period. 
-  Uses the first week as a basis for comparison, not included in results.
-  Can be filtered by any other arguments accepted by metric-query."
-  [ds min-date max-date opts]
-  (let [weeks (interval-statistics-query ds :weekly min-date max-date opts)
-        {:keys [impressions clicks views]} (first weeks)]
-    (mapv (fn [w]
-            {:week (:week w)
-             :impressions (float (* (/ (- (:impressions w) impressions) impressions) 100))
-             :clicks (float (* (/ (- (:clicks w) clicks) clicks) 100))
-             :views (float (* (/ (- (:views w) views) views) 100))})
-          weeks)))
-
-(defn average-engagement
-  "Returns the average engagement (average clicks and views) over the given time period.
-  Can be filtered by any other arguments accepted by metric-query."
-  [ds min-date max-date opts]
-  (let [{:keys [clicks views]} (statistics-query ds (merge {:min-date min-date
-                                                            :max-date max-date}
-                                                           opts))]
-    (float (/ (+ clicks views) 2))))
-
-(defn click-through-rate
-  "Returns the click-through rate based on impressions and clicks filtered by any arguments accepted by metric-query"
-  [ds opts]
-  (let [{:keys [impressions clicks]} (statistics-query ds opts)]
-    (float (* (/ clicks impressions) 100))))
-
-(defn insert-event [ds {:keys [data ret] :as opts}]
-  (->> {:tname :events
-        :data data
-        :ret ret}
-       (merge opts)
-       (hon/insert! ds)))
-

From 9d597b8ff473985a5d567db8ceaf43ba3958c51d Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 3 Dec 2025 13:41:51 +0200
Subject: [PATCH 177/391] added average endpoints to get average engagement on
 creator and distributor side

---
 .../routes/analytics/creator/top_average.clj  | 23 +++++++++++++++++++
 .../analytics/distributor/top_average.clj     | 23 +++++++++++++++++++
 2 files changed, 46 insertions(+)
 create mode 100644 src/source/routes/analytics/creator/top_average.clj
 create mode 100644 src/source/routes/analytics/distributor/top_average.clj

diff --git a/src/source/routes/analytics/creator/top_average.clj b/src/source/routes/analytics/creator/top_average.clj
new file mode 100644
index 00000000..971ee6ca
--- /dev/null
+++ b/src/source/routes/analytics/creator/top_average.clj
@@ -0,0 +1,23 @@
+(ns source.routes.analytics.creator.top-average
+  (:require [source.services.analytics.interface :as analytics]
+            [ring.util.response :as res]
+            [source.util :as utils]))
+
+(defn get
+  {:summary "Get the average engagement (clicks and views) for a creator, optionally filtered by content type."
+   :parameters {:query [:map
+                        [:mindate :string]
+                        [:maxdate :string]
+                        [:contenttype {:optional true} [:maybe :int]]]}
+   :responses {200 {:body [:map [:average :float]]}}}
+
+  [{:keys [ds user query-params] :as _request}]
+  (let [{:keys [data success error]} (utils/validate get query-params :query)
+        {:keys [mindate maxdate contenttype]} data]
+    (if success
+      (let [result (analytics/average-engagement ds mindate maxdate {:creator-id (:id user)
+                                                                     :content-type-id contenttype})]
+        (res/response {:average result}))
+
+      (-> (res/response error)
+          (res/status 400)))))
diff --git a/src/source/routes/analytics/distributor/top_average.clj b/src/source/routes/analytics/distributor/top_average.clj
new file mode 100644
index 00000000..805b55b5
--- /dev/null
+++ b/src/source/routes/analytics/distributor/top_average.clj
@@ -0,0 +1,23 @@
+(ns source.routes.analytics.distributor.top-average
+  (:require [source.services.analytics.interface :as analytics]
+            [ring.util.response :as res]
+            [source.util :as utils]))
+
+(defn get
+  {:summary "Get the average engagement (clicks and views) for a distributor, optionally filtered by content type."
+   :parameters {:query [:map
+                        [:mindate :string]
+                        [:maxdate :string]
+                        [:contenttype {:optional true} [:maybe :int]]]}
+   :responses {200 {:body [:map [:average :float]]}}}
+
+  [{:keys [ds user query-params] :as _request}]
+  (let [{:keys [data success error]} (utils/validate get query-params :query)
+        {:keys [mindate maxdate contenttype]} data]
+    (if success
+      (let [result (analytics/average-engagement ds mindate maxdate {:distributor-id (:id user)
+                                                                     :content-type-id contenttype})]
+        (res/response {:average result}))
+
+      (-> (res/response error)
+          (res/status 400)))))

From 09c52969a136f88ed7cabcdf5be786893fdcdb5f Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 3 Dec 2025 13:42:02 +0200
Subject: [PATCH 178/391] updated reitit

---
 src/source/routes/reitit.clj | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index 38e92abe..020515be 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -43,8 +43,10 @@
             [source.routes.analytics.creator.general :as analytics-creator-general]
             [source.routes.analytics.creator.deltas :as analytics-creator-deltas]
             [source.routes.analytics.creator.top :as analytics-creator-top]
+            [source.routes.analytics.creator.top-average :as analytics-creator-top-average]
             [source.routes.analytics.distributor.general :as analytics-distributor-general]
             [source.routes.analytics.distributor.top :as analytics-distributor-top]
+            [source.routes.analytics.distributor.top-average :as analytics-distributor-top-average]
             [source.routes.analytics.bundle.posts.-id-.views :as analytics-bundle-posts-id-views]
             [source.routes.integrations :as integrations]
             [source.routes.integration :as integration]
@@ -222,14 +224,18 @@
        ["/categories"   (route {:get feed-categories/get
                                 :post feed-categories/post})]]]
 
-     ["/analytics"
+     ["/analytics"      {:tags #{"analytics"}}
       ["/creator"       {:middleware [[mw/apply-auth {:required-type :creator}]]}
        ["/general"      (route {:get analytics-creator-general/get})]
        ["/deltas"       (route {:get analytics-creator-deltas/get})]
-       ["/top"          (route {:get analytics-creator-top/get})]]
+       ["/top"
+        [""             (route {:get analytics-creator-top/get})]
+        ["/average"     (route {:get analytics-creator-top-average/get})]]]
       ["/distributor"   {:middleware [[mw/apply-auth {:required-type :distributor}]]}
        ["/general"      (route {:get analytics-distributor-general/get})]
-       ["/top"          (route {:get analytics-distributor-top/get})]]
+       ["/top"
+        [""             (route {:get analytics-distributor-top/get})]
+        ["/average"     (route {:get analytics-distributor-top-average/get})]]]
       ["/bundle"        {:middleware [[mw/apply-bundle]]}
        ["/posts"
         ["/:id"

From 79b25332e9af5db8568719f6b04ab577465cc744 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Thu, 4 Dec 2025 10:21:24 +0200
Subject: [PATCH 179/391] updated weekly growth endpoint to prevent
 divide-by-zero and fixed week indexing

---
 src/source/routes/analytics/creator/deltas.clj | 10 ++++++++--
 src/source/services/analytics/core.clj         |  8 +++++++-
 2 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/src/source/routes/analytics/creator/deltas.clj b/src/source/routes/analytics/creator/deltas.clj
index 69546bf1..3877bbb0 100644
--- a/src/source/routes/analytics/creator/deltas.clj
+++ b/src/source/routes/analytics/creator/deltas.clj
@@ -18,5 +18,11 @@
 
   [{:keys [ds user query-params] :as _request}]
   (let [{:keys [mindate maxdate feed]} (w/keywordize-keys query-params)]
-    (res/response (analytics/weekly-growth-averages ds mindate maxdate {:creator-id (:id user)
-                                                                        :feed-id feed}))))
+    (let [results (analytics/weekly-growth-averages ds mindate maxdate {:creator-id (:id user)
+                                                                        :feed-id feed})
+          indexed-results (mapv (fn [{:keys [impressions clicks views]} i]
+                                  {:week (str "week " i)
+                                   :impressions impressions
+                                   :clicks clicks
+                                   :views views}) results (range 1 (inc (count results))))]
+      (res/response indexed-results))))
diff --git a/src/source/services/analytics/core.clj b/src/source/services/analytics/core.clj
index 7aeadbd7..ac19f5cc 100644
--- a/src/source/services/analytics/core.clj
+++ b/src/source/services/analytics/core.clj
@@ -94,7 +94,13 @@
   Can be filtered by any other arguments accepted by metric-query."
   [ds min-date max-date opts]
   (let [weeks (interval-statistics-query ds :weekly min-date max-date opts)
-        {:keys [impressions clicks views]} (first weeks)]
+        {:keys [impressions clicks views]} (first weeks)
+        {:keys [impressions clicks views]} (cond-> {:impressions impressions
+                                                    :clicks clicks
+                                                    :views views}
+                                             (= impressions 0) (assoc :impressions 1)
+                                             (= clicks 0) (assoc :clicks 1)
+                                             (= views 0) (assoc :views 1))]
     (mapv (fn [w]
             {:week (:week w)
              :impressions (float (* (/ (- (:impressions w) impressions) impressions) 100))

From f4aa460e4fc722dd448a28a155f1a0e82c13066e Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Thu, 4 Dec 2025 10:54:44 +0200
Subject: [PATCH 180/391] added error messages if date format is incorrect or
 if date range is too small

---
 deps.edn                                      |  3 +-
 .../routes/analytics/creator/deltas.clj       | 31 +++++++++++++------
 2 files changed, 23 insertions(+), 11 deletions(-)

diff --git a/deps.edn b/deps.edn
index 50288a0e..2b1bc78f 100644
--- a/deps.edn
+++ b/deps.edn
@@ -32,8 +32,9 @@
         hickory/hickory {:mvn/version "0.7.1"}
         com.draines/postal {:mvn/version "2.0.5"}
         hiccup/hiccup {:mvn/version "2.0.0"}
+        clojure.java-time/clojure.java-time {:mvn/version "1.4.3"}
         metosin/jsonista {:mvn/version "0.3.13"}
-        metosin/reitit {:mvn/version "0.9.1"} 
+        metosin/reitit {:mvn/version "0.9.1"}
         metosin/reitit-middleware {:mvn/version "0.9.1"}
         metosin/reitit-swagger {:mvn/version "0.9.1"}
         metosin/reitit-swagger-ui {:mvn/version "0.9.1"}}}
diff --git a/src/source/routes/analytics/creator/deltas.clj b/src/source/routes/analytics/creator/deltas.clj
index 3877bbb0..89687164 100644
--- a/src/source/routes/analytics/creator/deltas.clj
+++ b/src/source/routes/analytics/creator/deltas.clj
@@ -1,6 +1,7 @@
 (ns source.routes.analytics.creator.deltas
   (:require [clojure.walk :as w]
             [ring.util.response :as res]
+            [java-time.api :as jt]
             [source.services.analytics.interface :as analytics]))
 
 (defn get
@@ -14,15 +15,25 @@
                             [:week :string]
                             [:impressions :float]
                             [:clicks :float]
-                            [:views :float]]]}}}
+                            [:views :float]]]}
+               400 {:body [:map [:message :string]]}}}
 
   [{:keys [ds user query-params] :as _request}]
-  (let [{:keys [mindate maxdate feed]} (w/keywordize-keys query-params)]
-    (let [results (analytics/weekly-growth-averages ds mindate maxdate {:creator-id (:id user)
-                                                                        :feed-id feed})
-          indexed-results (mapv (fn [{:keys [impressions clicks views]} i]
-                                  {:week (str "week " i)
-                                   :impressions impressions
-                                   :clicks clicks
-                                   :views views}) results (range 1 (inc (count results))))]
-      (res/response indexed-results))))
+  (let [{:keys [mindate maxdate feed]} (w/keywordize-keys query-params)
+        {:keys [parsed-mindate parsed-maxdate]} (try
+                                                  {:parsed-mindate (jt/local-date mindate)
+                                                   :parsed-maxdate (jt/local-date maxdate)}
+                                                  (catch Exception _ (-> (res/response {:message "Invalid date format. Date must be in the format YYYY-MM-DD."})
+                                                                         (res/status 400))))
+        days-between (.until parsed-mindate parsed-maxdate java.time.temporal.ChronoUnit/DAYS)
+        results (analytics/weekly-growth-averages ds mindate maxdate {:creator-id (:id user)
+                                                                      :feed-id feed})
+        indexed-results (mapv (fn [{:keys [impressions clicks views]} i]
+                                {:week (str "week " i)
+                                 :impressions impressions
+                                 :clicks clicks
+                                 :views views}) results (range 1 (inc (count results))))]
+    (if (>= days-between 14)
+      (res/response indexed-results)
+      (-> (res/response {:message "Date range too small. Date range must include at least 2 weeks."})
+          (res/status 400)))))

From 403f9ae7b5696038ea658d5e6bbc8a5c52803368 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Thu, 4 Dec 2025 14:21:39 +0200
Subject: [PATCH 181/391] changed weekly growth endpoint to aggregate days into
 weeks via clojure

---
 .../routes/analytics/creator/deltas.clj       | 14 ++++-------
 .../routes/analytics/creator/general.clj      |  3 ++-
 src/source/routes/analytics/creator/top.clj   |  3 ++-
 .../routes/analytics/creator/top_average.clj  |  3 ++-
 .../routes/analytics/distributor/general.clj  |  3 ++-
 .../routes/analytics/distributor/top.clj      |  3 ++-
 .../analytics/distributor/top_average.clj     |  3 ++-
 src/source/services/analytics/core.clj        | 25 ++++++++++++++++++-
 8 files changed, 41 insertions(+), 16 deletions(-)

diff --git a/src/source/routes/analytics/creator/deltas.clj b/src/source/routes/analytics/creator/deltas.clj
index 89687164..cc024129 100644
--- a/src/source/routes/analytics/creator/deltas.clj
+++ b/src/source/routes/analytics/creator/deltas.clj
@@ -5,7 +5,8 @@
             [source.services.analytics.interface :as analytics]))
 
 (defn get
-  {:summary "Returns the percentage of growth in impressions, clicks and views per week, over the given time period. Optionally filtered by feed."
+  {:summary "Returns the percentage of growth in impressions, clicks and views per week, over the given time period. Optionally filtered by feed. 
+   Date must be in the format yyyy-MM-dd"
    :parameters {:query [:map
                         [:mindate :string]
                         [:maxdate :string]
@@ -23,17 +24,12 @@
         {:keys [parsed-mindate parsed-maxdate]} (try
                                                   {:parsed-mindate (jt/local-date mindate)
                                                    :parsed-maxdate (jt/local-date maxdate)}
-                                                  (catch Exception _ (-> (res/response {:message "Invalid date format. Date must be in the format YYYY-MM-DD."})
+                                                  (catch Exception _ (-> (res/response {:message "Invalid date format. Date must be in the format yyyy-MM-dd."})
                                                                          (res/status 400))))
         days-between (.until parsed-mindate parsed-maxdate java.time.temporal.ChronoUnit/DAYS)
         results (analytics/weekly-growth-averages ds mindate maxdate {:creator-id (:id user)
-                                                                      :feed-id feed})
-        indexed-results (mapv (fn [{:keys [impressions clicks views]} i]
-                                {:week (str "week " i)
-                                 :impressions impressions
-                                 :clicks clicks
-                                 :views views}) results (range 1 (inc (count results))))]
+                                                                      :feed-id feed})]
     (if (>= days-between 14)
-      (res/response indexed-results)
+      (res/response results)
       (-> (res/response {:message "Date range too small. Date range must include at least 2 weeks."})
           (res/status 400)))))
diff --git a/src/source/routes/analytics/creator/general.clj b/src/source/routes/analytics/creator/general.clj
index bba8f402..984084f3 100644
--- a/src/source/routes/analytics/creator/general.clj
+++ b/src/source/routes/analytics/creator/general.clj
@@ -4,7 +4,8 @@
             [clojure.walk :as w]))
 
 (defn get
-  {:summary "Gets the number of impressions, clicks and views per day for a creator over the given time period. Optionally filtered by feed."
+  {:summary "Gets the number of impressions, clicks and views per day for a creator over the given time period. Optionally filtered by feed.
+   Date must be in the format yyyy-MM-dd"
    :parameters {:query [:map
                         [:mindate :string]
                         [:maxdate :string]
diff --git a/src/source/routes/analytics/creator/top.clj b/src/source/routes/analytics/creator/top.clj
index 9d0001d3..5040bece 100644
--- a/src/source/routes/analytics/creator/top.clj
+++ b/src/source/routes/analytics/creator/top.clj
@@ -17,7 +17,8 @@
           (services/bundles ds {:where [:in :id ids]}))))
 
 (defn get
-  {:summary "Get the top n records with the highest number of impressions, clicks and views, in terms of the given top field. Optionally filtered by content type."
+  {:summary "Get the top n records with the highest number of impressions, clicks and views, in terms of the given top field. Optionally filtered by content type.
+   Date must be in the format yyyy-MM-dd"
    :parameters {:query [:map
                         [:n :int]
                         [:mindate :string]
diff --git a/src/source/routes/analytics/creator/top_average.clj b/src/source/routes/analytics/creator/top_average.clj
index 971ee6ca..4f50a396 100644
--- a/src/source/routes/analytics/creator/top_average.clj
+++ b/src/source/routes/analytics/creator/top_average.clj
@@ -4,7 +4,8 @@
             [source.util :as utils]))
 
 (defn get
-  {:summary "Get the average engagement (clicks and views) for a creator, optionally filtered by content type."
+  {:summary "Get the average engagement (clicks and views) for a creator, optionally filtered by content type.
+   Date must be in the format yyyy-MM-dd"
    :parameters {:query [:map
                         [:mindate :string]
                         [:maxdate :string]
diff --git a/src/source/routes/analytics/distributor/general.clj b/src/source/routes/analytics/distributor/general.clj
index b0b94535..77216a9a 100644
--- a/src/source/routes/analytics/distributor/general.clj
+++ b/src/source/routes/analytics/distributor/general.clj
@@ -4,7 +4,8 @@
             [clojure.walk :as w]))
 
 (defn get
-  {:summary "Gets the number of impressions, clicks and views per day for a distributor over the given time period. Optionally filtered by bundle."
+  {:summary "Gets the number of impressions, clicks and views per day for a distributor over the given time period. Optionally filtered by bundle.
+   Date must be in the format yyyy-MM-dd"
    :parameters {:query [:map
                         [:mindate :string]
                         [:maxdate :string]
diff --git a/src/source/routes/analytics/distributor/top.clj b/src/source/routes/analytics/distributor/top.clj
index 2da1298f..4eed1c46 100644
--- a/src/source/routes/analytics/distributor/top.clj
+++ b/src/source/routes/analytics/distributor/top.clj
@@ -12,7 +12,8 @@
     (services/feeds ds {:where [:in :id ids]})))
 
 (defn get
-  {:summary "Get the top n records with the highest number of impressions, clicks and views, in terms of the given top field. Optionally filtered by content type."
+  {:summary "Get the top n records with the highest number of impressions, clicks and views, in terms of the given top field. Optionally filtered by content type.
+   Date must be in the format yyyy-MM-dd"
    :parameters {:query [:map
                         [:n :int]
                         [:mindate :string]
diff --git a/src/source/routes/analytics/distributor/top_average.clj b/src/source/routes/analytics/distributor/top_average.clj
index 805b55b5..e4f09f08 100644
--- a/src/source/routes/analytics/distributor/top_average.clj
+++ b/src/source/routes/analytics/distributor/top_average.clj
@@ -4,7 +4,8 @@
             [source.util :as utils]))
 
 (defn get
-  {:summary "Get the average engagement (clicks and views) for a distributor, optionally filtered by content type."
+  {:summary "Get the average engagement (clicks and views) for a distributor, optionally filtered by content type.
+   Date must be in the format yyyy-MM-dd"
    :parameters {:query [:map
                         [:mindate :string]
                         [:maxdate :string]
diff --git a/src/source/services/analytics/core.clj b/src/source/services/analytics/core.clj
index ac19f5cc..1dae5e5e 100644
--- a/src/source/services/analytics/core.clj
+++ b/src/source/services/analytics/core.clj
@@ -93,7 +93,14 @@
   Uses the first week as a basis for comparison.
   Can be filtered by any other arguments accepted by metric-query."
   [ds min-date max-date opts]
-  (let [weeks (interval-statistics-query ds :weekly min-date max-date opts)
+  (let [days (interval-statistics-query ds :daily min-date max-date opts)
+        parts (partition-all 7 days)
+        weeks (mapv (fn [week i]
+                      {:week (str "week " i)
+                       :impressions (apply + (mapv :impressions week))
+                       :clicks (apply + (mapv :clicks week))
+                       :views (apply + (mapv :views week))})
+                    parts (range 1 (inc (count parts))))
         {:keys [impressions clicks views]} (first weeks)
         {:keys [impressions clicks views]} (cond-> {:impressions impressions
                                                     :clicks clicks
@@ -387,5 +394,21 @@
 
   (time (top-statistics-query ds "2025-11-17" "2025-11-24" 10 :post-id {}))
 
+  (def data [{:a 1 :n 64}
+             {:a 2 :n 65}
+             {:a 3 :n 66}
+             {:a 4 :n 67}
+             {:a 5 :n 68}
+             {:a 6 :n 69}
+             {:a 7 :n 70}
+             {:a 8 :n 71}
+             {:a 9 :n 72}
+             {:a 10 :n 73}])
+
+  (mapv (fn [week i]
+          {:week i
+           :n (apply + (mapv :n week))})
+        (partition-all 7 data) (range 1 (inc (count (partition-all 7 data)))))
+
   ())
 

From dd7c61933d7baab589137f4e0249cf884d8bdf07 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Thu, 4 Dec 2025 15:09:03 +0200
Subject: [PATCH 182/391] prevented duplicate rss feed urls being used

---
 src/source/routes/feeds.clj | 109 +++++++++++++++++++-----------------
 1 file changed, 58 insertions(+), 51 deletions(-)

diff --git a/src/source/routes/feeds.clj b/src/source/routes/feeds.clj
index c7b35bc2..f64e61c1 100644
--- a/src/source/routes/feeds.clj
+++ b/src/source/routes/feeds.clj
@@ -3,7 +3,8 @@
             [source.util :as utils]
             [congest.jobs :as congest]
             [source.jobs.core :as jobs]
-            [ring.util.response :as res]))
+            [ring.util.response :as res]
+            [source.jobs.handlers :as handlers]))
 
 (defn get
   {:summary "get all feeds"
@@ -58,59 +59,65 @@
   [{:keys [js ds store user body] :as _request}]
   (let [{:keys [provider-id rss-url content-type-id]} body
         datetime (utils/get-utc-timestamp-string)
-        selection-schemas (->> [:= :provider-id provider-id]
-                               (assoc {} :where)
-                               (services/selection-schemas ds))
-        latest-ss (->> selection-schemas
-                       (reduce (fn [acc {:keys [id]}]
-                                 (conj acc id)) [])
-                       (apply max -1))
-        extracted (when-not (= latest-ss -1)
-                    (services/extract-data store {:schema-id latest-ss
-                                                  :url rss-url}))
-        extracted-posts (get-in extracted [:feed :posts])
-        new-feed (services/insert-feed!
-                  ds
-                  {:data (merge body {:title (get-in extracted [:feed :title])
-                                      :display-picture (get-in extracted [:feed :display-picture])
-                                      :user-id (:id user)
-                                      :created-at datetime
-                                      :state "pending"})})
-        extended-posts (mapv (fn [post]
-                               (merge post
-                                      {:feed-id (:id new-feed)
-                                       :creator-id (:id user)
-                                       :content-type-id content-type-id
-                                       :thumbnail (or (:thumbnail post) (:display-picture new-feed))}))
-                             extracted-posts)
-        {:keys [email]} (services/user ds {:id (:id user)})]
+        used-feed (services/feeds {:where [:= :rss-url rss-url]
+                                   :ret :1})]
+    (if (some? used-feed)
+      (-> (res/response {:message "There is already a feed with the given RSS feed"})
+          (res/status 400))
+
+      (let [selection-schemas (->> [:= :provider-id provider-id]
+                                   (assoc {} :where)
+                                   (services/selection-schemas ds))
+            latest-ss (->> selection-schemas
+                           (reduce (fn [acc {:keys [id]}]
+                                     (conj acc id)) [])
+                           (apply max -1))
+            extracted (when-not (= latest-ss -1)
+                        (services/extract-data store {:schema-id latest-ss
+                                                      :url rss-url}))
+            extracted-posts (get-in extracted [:feed :posts])
+            new-feed (services/insert-feed!
+                      ds
+                      {:data (merge body {:title (get-in extracted [:feed :title])
+                                          :display-picture (get-in extracted [:feed :display-picture])
+                                          :user-id (:id user)
+                                          :created-at datetime
+                                          :state "pending"})})
+            extended-posts (mapv (fn [post]
+                                   (merge post
+                                          {:feed-id (:id new-feed)
+                                           :creator-id (:id user)
+                                           :content-type-id content-type-id
+                                           :thumbnail (or (:thumbnail post) (:display-picture new-feed))}))
+                                 extracted-posts)
+            {:keys [email]} (services/user ds {:id (:id user)})]
 
-    (if (some? extracted-posts)
-      (do
-        (services/insert-incoming-post! ds {:data extended-posts})
+        (if (some? extracted-posts)
+          (do
+            (services/insert-incoming-post! ds {:data extended-posts})
 
-        (->> (jobs/prepare-congest-metadata
-              ds
-              store
-              {:id (str email "-" (:id new-feed))
-               :initial-delay (* 1000 60 60 24)
-               :auto-start true
-               :stop-after-fail false,
-               :interval (* 1000 60 60 24)
-               :recurring? true
-               :args {:feed-id (:id new-feed)
-                      :creator-id (:id user)
-                      :content-type-id content-type-id
-                      :provider-id provider-id
-                      :url rss-url}
-               :handler :update-feed-posts
-               :created-at (utils/get-utc-timestamp-string)
-               :sleep false})
-             (congest/register! js))
-        (res/response new-feed))
+            (->> (jobs/prepare-congest-metadata
+                  ds
+                  store
+                  {:id (handlers/update-feed-posts-job-id email (:id new-feed))
+                   :initial-delay (* 1000 60 60 24)
+                   :auto-start true
+                   :stop-after-fail false,
+                   :interval (* 1000 60 60 24)
+                   :recurring? true
+                   :args {:feed-id (:id new-feed)
+                          :creator-id (:id user)
+                          :content-type-id content-type-id
+                          :provider-id provider-id
+                          :url rss-url}
+                   :handler :update-feed-posts
+                   :created-at (utils/get-utc-timestamp-string)
+                   :sleep false})
+                 (congest/register! js))
+            (res/response new-feed))
 
-      (-> (res/response {:message "failed to extract data"})
-          (res/status 500)))))
+          (-> (res/response {:message "failed to extract data"})
+              (res/status 500)))))))
 
 (comment
   (require '[source.db.util :as db.util]

From bb77847b9d62f85123f4f431bd7cf909325acca7 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Thu, 4 Dec 2025 16:22:17 +0200
Subject: [PATCH 183/391] addressed requested changes

---
 src/source/routes/feeds.clj | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/src/source/routes/feeds.clj b/src/source/routes/feeds.clj
index f64e61c1..c9d1cf36 100644
--- a/src/source/routes/feeds.clj
+++ b/src/source/routes/feeds.clj
@@ -4,7 +4,7 @@
             [congest.jobs :as congest]
             [source.jobs.core :as jobs]
             [ring.util.response :as res]
-            [source.jobs.handlers :as handlers]))
+            [source.db.honey :as hon]))
 
 (defn get
   {:summary "get all feeds"
@@ -59,9 +59,10 @@
   [{:keys [js ds store user body] :as _request}]
   (let [{:keys [provider-id rss-url content-type-id]} body
         datetime (utils/get-utc-timestamp-string)
-        used-feed (services/feeds {:where [:= :rss-url rss-url]
-                                   :ret :1})]
-    (if (some? used-feed)
+        exists (hon/exists? ds {:tname :feeds
+                                :where [:= :rss-url rss-url]
+                                :ret :1})]
+    (if exists
       (-> (res/response {:message "There is already a feed with the given RSS feed"})
           (res/status 400))
 
@@ -99,7 +100,7 @@
             (->> (jobs/prepare-congest-metadata
                   ds
                   store
-                  {:id (handlers/update-feed-posts-job-id email (:id new-feed))
+                  {:id (str email "-" (:id new-feed))
                    :initial-delay (* 1000 60 60 24)
                    :auto-start true
                    :stop-after-fail false,

From f70ca161cd309e9105b5beb80452047253538850 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Thu, 4 Dec 2025 16:28:25 +0200
Subject: [PATCH 184/391] addressed requested changes

---
 src/source/services/analytics/core.clj | 13 ++++---------
 1 file changed, 4 insertions(+), 9 deletions(-)

diff --git a/src/source/services/analytics/core.clj b/src/source/services/analytics/core.clj
index 1dae5e5e..0fedde8d 100644
--- a/src/source/services/analytics/core.clj
+++ b/src/source/services/analytics/core.clj
@@ -102,17 +102,12 @@
                        :views (apply + (mapv :views week))})
                     parts (range 1 (inc (count parts))))
         {:keys [impressions clicks views]} (first weeks)
-        {:keys [impressions clicks views]} (cond-> {:impressions impressions
-                                                    :clicks clicks
-                                                    :views views}
-                                             (= impressions 0) (assoc :impressions 1)
-                                             (= clicks 0) (assoc :clicks 1)
-                                             (= views 0) (assoc :views 1))]
+        denom #(if (= % 0) 1 %)]
     (mapv (fn [w]
             {:week (:week w)
-             :impressions (float (* (/ (- (:impressions w) impressions) impressions) 100))
-             :clicks (float (* (/ (- (:clicks w) clicks) clicks) 100))
-             :views (float (* (/ (- (:views w) views) views) 100))})
+             :impressions (float (* (/ (- (:impressions w) impressions) (denom impressions)) 100))
+             :clicks (float (* (/ (- (:clicks w) clicks) (denom clicks)) 100))
+             :views (float (* (/ (- (:views w) views) (denom views)) 100))})
           weeks)))
 
 (defn average-engagement

From 38360dda0d623eaf0b9bcbdcb148718d56aa8f25 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Fri, 5 Dec 2025 09:48:57 +0200
Subject: [PATCH 185/391] fixed content type id being ignored

---
 src/source/services/analytics/core.clj | 21 +++++++++++++++++----
 1 file changed, 17 insertions(+), 4 deletions(-)

diff --git a/src/source/services/analytics/core.clj b/src/source/services/analytics/core.clj
index 0fedde8d..698196d5 100644
--- a/src/source/services/analytics/core.clj
+++ b/src/source/services/analytics/core.clj
@@ -9,6 +9,7 @@
   "Generic select query function for returning analytics data from the events table"
   [ds {:keys [select order-by group-by limit metric feed-id post-id content-type-id bundle-id creator-id distributor-id min-date max-date category-ids where ret]}]
   (let [clauses (cond-> {}
+                  (some? where) (merge where)
                   (some? metric) (hsql/where [:= :event metric])
                   (some? feed-id) (hsql/where [:= :feed-id feed-id])
                   (some? post-id) (hsql/where [:= :post-id post-id])
@@ -16,7 +17,6 @@
                   (some? bundle-id) (hsql/where [:= :bundle-id bundle-id])
                   (some? creator-id) (hsql/where [:= :creator-id creator-id])
                   (some? distributor-id) (hsql/where [:= :distributor-id distributor-id])
-                  (some? where) (merge where)
                   (and (some? min-date) (nil? max-date)) (hsql/where [:>= :timestamp min-date])
                   (and (some? max-date) (nil? min-date)) (hsql/where [:<= :timestamp max-date])
                   (and (some? min-date) (some? max-date)) (hsql/where [:between :timestamp min-date max-date])
@@ -315,7 +315,7 @@
       (seed-event! maximums)))
 
   (time (metric-query ds {:min-date "2025-11-25T15:00:00Z"
-                          :feed-id "1"}))
+                          :feed-id 2}))
 
   (time (statistics-query ds {:ret :*}))
 
@@ -355,7 +355,7 @@
 
   (time (weekly-growth-averages ds "2025-11-01" "2025-11-30" {:feed-id 4}))
 
-  (time (average-engagement ds "2025-11-24T00:00:00Z" "2025-11-24T23:59:59Z" {:feed-id 4}))
+  (time (average-engagement ds "2025-11-24" "2025-11-30" {:feed-id 4}))
 
   (time (click-through-rate ds {:min-date "2025-11-24T00:00:00Z"
                                 :max-date "2025-11-24T23:59:59Z"
@@ -387,7 +387,7 @@
                                   :bundle-id 1
                                   :distributor-id 1}}))
 
-  (time (top-statistics-query ds "2025-11-17" "2025-11-24" 10 :post-id {}))
+  (time (top-statistics-query ds "2025-11-17" "2025-11-24" 10 :post-id {:content-type-id 3}))
 
   (def data [{:a 1 :n 64}
              {:a 2 :n 65}
@@ -405,5 +405,18 @@
            :n (apply + (mapv :n week))})
         (partition-all 7 data) (range 1 (inc (count (partition-all 7 data)))))
 
+  (metric-query ds (merge {:select (hsql/select :post-id
+                                                [(hsql/filter :%count.* (hsql/where := :event "impression")) :impressions]
+                                                [(hsql/filter :%count.* (hsql/where := :event "click")) :clicks]
+                                                [(hsql/filter :%count.* (hsql/where := :event "view")) :views])
+                           :min-date "2025-11-17"
+                           :max-date "2025-11-24"
+                           :where (hsql/where [:!= :post-id nil])
+                           :group-by (hsql/group-by :post-id)
+                           :order-by (hsql/order-by [:impressions :desc] [:clicks :desc] [:views :desc])
+                           :limit 10
+                           :ret :*}
+                          {:content-type-id 3}))
+
   ())
 

From 804a9198cc04f538300e1959a207ef05479096d6 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Fri, 5 Dec 2025 11:35:07 +0200
Subject: [PATCH 186/391] fixed mindate being exclusive

---
 src/source/services/analytics/core.clj | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/source/services/analytics/core.clj b/src/source/services/analytics/core.clj
index 698196d5..65b7689b 100644
--- a/src/source/services/analytics/core.clj
+++ b/src/source/services/analytics/core.clj
@@ -3,7 +3,8 @@
             [source.db.honey :as hon]
             [source.services.bundles :as bundles]
             [source.util :as util]
-            [source.services.feed-categories :as feed-categories]))
+            [source.services.feed-categories :as feed-categories]
+            [honey.sql :as sql]))
 
 (defn metric-query
   "Generic select query function for returning analytics data from the events table"
@@ -66,7 +67,7 @@
                                                   [(hsql/filter :%count.* (hsql/where := :event "impression")) :impressions]
                                                   [(hsql/filter :%count.* (hsql/where := :event "click")) :clicks]
                                                   [(hsql/filter :%count.* (hsql/where := :event "view")) :views])
-                             :min-date (str min-date "T00:00:00Z")
+                             :min-date (str min-date)
                              :max-date (str max-date "T23:59:59Z")
                              :group-by (hsql/group-by column)
                              :order-by (hsql/order-by column)}
@@ -270,8 +271,7 @@
     (insert-post-event-categories! ds event' post)))
 
 (comment
-  (require '[source.db.util :as db.util]
-           '[honey.sql :as sql])
+  (require '[source.db.util :as db.util])
 
   (defonce ds (db.util/conn))
 
@@ -337,7 +337,7 @@
                              (hsql/group-by :day)
                              (hsql/order-by :day)) {:ret :*}))
 
-  (time (interval-statistics-query ds :daily "2025-11-17" "2025-11-24" {:feed-id 4}))
+  (time (interval-statistics-query ds :daily "2025-11-17" "2025-11-24" {}))
 
   (time (hon/execute! ds (-> (hsql/select [[:strftime "%W" :timestamp] :week]
                                           [(hsql/filter :%count.* (hsql/where := :event "impression")) :impressions]

From e65fb0a944495e941e71fd9aae7437bd6ca4c2b7 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 8 Dec 2025 09:59:51 +0200
Subject: [PATCH 187/391] updated uuid query param to be keyword instead of
 string

---
 src/source/middleware/auth/core.clj | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/source/middleware/auth/core.clj b/src/source/middleware/auth/core.clj
index 9a679696..ab039670 100644
--- a/src/source/middleware/auth/core.clj
+++ b/src/source/middleware/auth/core.clj
@@ -49,7 +49,7 @@
   [handler]
   (fn [request]
     (let [ds (db.util/conn :master)
-          bundle-uuid (get-in request [:query-params "uuid"])
+          bundle-uuid (get-in request [:query-params :uuid])
           {:keys [id]} (db/find-one ds {:tname :bundles
                                         :where [:= :uuid bundle-uuid]})]
       (if (some? id)
@@ -104,10 +104,10 @@
     (println "Test passed"))
 
   (require '[source.util :as utils])
-  (let [garbage-request {:query-params {"uuid" "garbage"}}
+  (let [garbage-request {:query-params {:uuid "garbage"}}
         ds (db.util/conn)
         uuid (utils/uuid)
-        bundle-request {:query-params {"uuid" uuid}}
+        bundle-request {:query-params {:uuid uuid}}
         test-handler (-> (fn [request]
                            request)
                          (wrap-bundle-id))]

From 09613aa5ba7ade26701565949399e246a0ceff8a Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 8 Dec 2025 14:17:35 +0200
Subject: [PATCH 188/391] added services to allow deletion on multiple
 different tables related to feeds

---
 src/source/services/analytics/core.clj      | 9 +++++++++
 src/source/services/analytics/interface.clj | 3 +++
 src/source/services/feeds.clj               | 9 +++++++++
 src/source/services/incoming_posts.clj      | 9 +++++++++
 src/source/services/interface.clj           | 6 ++++++
 5 files changed, 36 insertions(+)

diff --git a/src/source/services/analytics/core.clj b/src/source/services/analytics/core.clj
index 7aeadbd7..89794d20 100644
--- a/src/source/services/analytics/core.clj
+++ b/src/source/services/analytics/core.clj
@@ -131,6 +131,15 @@
        (merge opts)
        (hon/insert! ds)))
 
+(defn delete-event! [ds {:keys [id where] :as opts}]
+  (->> {:tname :events
+        :where (if (some? id)
+                 [:= :id id]
+                 where)
+        :ret :1}
+       (merge opts)
+       (hon/delete! ds)))
+
 (defn insert-feed-event-categories!
   "Given a list of events and a list of feeds (or a single event/feed), 
   inserts an event category record for each event and each category 
diff --git a/src/source/services/analytics/interface.clj b/src/source/services/analytics/interface.clj
index 6de4478a..ff581175 100644
--- a/src/source/services/analytics/interface.clj
+++ b/src/source/services/analytics/interface.clj
@@ -48,6 +48,9 @@
 (defn insert-event-categories! [ds {:keys [_data _ret] :as opts}]
   (core/insert-event-categories! ds opts))
 
+(defn delete-event! [ds {:keys [_id _where] :as opts}]
+  (core/delete-event! ds opts))
+
 (defn insert-feed-event-categories!
   "Given a list of events and a list of feeds (or a single event/feed), 
   inserts an event category record for each event and each category 
diff --git a/src/source/services/feeds.clj b/src/source/services/feeds.clj
index 91251770..5916f1ce 100644
--- a/src/source/services/feeds.clj
+++ b/src/source/services/feeds.clj
@@ -33,3 +33,12 @@
         :ret :1}
        (merge opts)
        (db/find ds)))
+
+(defn delete-feed! [ds {:keys [id where] :as opts}]
+  (->> {:tname :feeds
+        :where (if (some? id)
+                 [:= :id id]
+                 where)
+        :ret :1}
+       (merge opts)
+       (db/delete! ds)))
diff --git a/src/source/services/incoming_posts.clj b/src/source/services/incoming_posts.clj
index 0578d65c..9c4892d6 100644
--- a/src/source/services/incoming_posts.clj
+++ b/src/source/services/incoming_posts.clj
@@ -55,3 +55,12 @@
                   :right-join [:categories [:= :categories.id :feed-categories.category-id]]}
                  opts)
                 {:ret :*}))
+
+(defn delete-incoming-post! [ds {:keys [id where] :as opts}]
+  (->> {:tname :incoming-posts
+        :where (if (some? id)
+                 [:= :id id]
+                 where)
+        :ret :1}
+       (merge opts)
+       (db/delete! ds)))
diff --git a/src/source/services/interface.clj b/src/source/services/interface.clj
index 612b7cc2..1f59a2a1 100644
--- a/src/source/services/interface.clj
+++ b/src/source/services/interface.clj
@@ -197,6 +197,9 @@
 (defn feed [ds {:keys [_id] :as opts}]
   (feeds/feed ds opts))
 
+(defn delete-feed! [ds {:keys [_id _where] :as opts}]
+  (feeds/delete-feed! ds opts))
+
 (defn insert-incoming-post! [ds {:keys [_data _ret] :as opts}]
   (incoming-posts/insert-incoming-post! ds opts))
 
@@ -216,6 +219,9 @@
 (defn incoming-post [ds opts]
   (incoming-posts/incoming-post ds opts))
 
+(defn delete-incoming-post! [ds {:keys [_id _where] :as opts}]
+  (incoming-posts/delete-incoming-post! ds opts))
+
 (defn insert-cadence! [ds {:keys [_values _ret] :as opts}]
   (cadences/insert-cadence! ds opts))
 

From a4d3cdc283405ee37984ea00cacc2b7b9ec5d6e2 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 8 Dec 2025 14:18:06 +0200
Subject: [PATCH 189/391] added functions to resolve job names

---
 src/source/jobs/handlers.clj | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/src/source/jobs/handlers.clj b/src/source/jobs/handlers.clj
index bc734883..46e503ce 100644
--- a/src/source/jobs/handlers.clj
+++ b/src/source/jobs/handlers.clj
@@ -18,6 +18,11 @@
   (fn [{:keys [args]}]
     (println "hello" (get args :name) args)))
 
+(defn update-feed-posts-job-id 
+  "returns the job id of an update-feed-posts job with the given email and feed-id"
+  [email feed-id]
+  (str email "-" feed-id))
+
 (defmethod handler :update-feed-posts [_]
   (fn [{:keys [args ds store]}]
     (try
@@ -63,6 +68,11 @@
          extended-posts))
       (catch Exception _ :fail))))
 
+(defn update-bundle-job-id 
+  "returns the job id of an update-bundle job with the given bundle id"
+  [bundle-id]
+  (str "bundle_" bundle-id))
+
 ; run long heuristics and pull the highest scoring incoming posts into the bundle's outgoing posts
 (defmethod handler :update-bundle [_]
   (fn [{:keys [args ds]}]

From 73d58674d38189311314b18b2cab1ab00df43e85 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 8 Dec 2025 14:18:54 +0200
Subject: [PATCH 190/391] added endpoint to delete feed

---
 src/source/routes/feed.clj   | 30 +++++++++++++++++++++++++++++-
 src/source/routes/reitit.clj |  3 ++-
 2 files changed, 31 insertions(+), 2 deletions(-)

diff --git a/src/source/routes/feed.clj b/src/source/routes/feed.clj
index 077e28b4..d5f340d6 100644
--- a/src/source/routes/feed.clj
+++ b/src/source/routes/feed.clj
@@ -1,6 +1,9 @@
 (ns source.routes.feed
   (:require [source.services.interface :as services]
-            [ring.util.response :as res]))
+            [ring.util.response :as res]
+            [congest.jobs :as congest]
+            [source.jobs.handlers :as handlers]
+            [source.services.analytics.interface :as analytics]))
 
 (defn get
   {:summary "get feed by id"
@@ -45,3 +48,28 @@
   (services/update-feed! ds {:id (:id path-params)
                              :data body})
   (res/response {:message "successfully updated feed"}))
+
+(defn hard-delete-feed! [ds js creator-email feed-id]
+  (let [job-id (handlers/update-feed-posts-job-id creator-email feed-id)
+        post-ids (mapv :id (services/incoming-posts ds {:where [:= :feed-id feed-id]}))]
+    (services/delete-filtered-feed! ds {:where [:= :feed-id feed-id]})
+    (services/delete-filtered-post! ds {:where [:in :post-id post-ids]})
+    (services/delete-incoming-post! ds {:where [:= :feed-id feed-id]})
+    (services/delete-feed-category! ds {:where [:= :feed-id feed-id]})
+    (analytics/delete-event! ds {:where [:= :feed-id feed-id]})
+    (services/delete-feed! ds {:where [:= :id feed-id]})
+    (congest/deregister! js job-id)))
+
+(defn delete
+  {:summary "delete feed by id"
+   :parameters {:path [:map [:id {:title "id"
+                                  :description "feed id"} :int]]}
+   :responses {200 {:body [:map [:message :string]]}
+               401 {:body [:map [:message :string]]}
+               403 {:body [:map [:message :string]]}}}
+
+  [{:keys [ds js user path-params] :as _request}]
+  (let [id (:id path-params)
+        {:keys [email]} (services/user ds {:id (:id user)})]
+    (hard-delete-feed! ds js email id)
+    (res/response {:message "successfully deleted feed"})))
diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index 38e92abe..e0363dbe 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -213,7 +213,8 @@
                                 :post feeds/post})]
       ["/:id"
        [""              (route {:get feed/get
-                                :post feed/post})]
+                                :post feed/post
+                                :delete feed/delete})]
        ["/posts"
         [""             (route {:get posts/get})]
         ["/:post-id"

From 0ad672991a0470298f28ac0a17f6640fe6bb74b8 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 8 Dec 2025 14:19:31 +0200
Subject: [PATCH 191/391] updated bundle posts endpoint to ensure posts from
 deleted feeds aren't fetched

---
 src/source/routes/bundle_posts.clj | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/source/routes/bundle_posts.clj b/src/source/routes/bundle_posts.clj
index 3b75381c..79c23ebd 100644
--- a/src/source/routes/bundle_posts.clj
+++ b/src/source/routes/bundle_posts.clj
@@ -43,12 +43,15 @@
           start (when start (try (Integer/parseInt start) (catch Exception _)))
           limit (when limit (try (Integer/parseInt limit) (catch Exception _)))
 
+          all-feed-ids (mapv :id (services/feeds ds))
           blocked-feed-ids (mapv :feed-id (services/filtered-feeds ds {:where [:= :bundle-id bundle-id]}))
+          available-feed-ids (vec (remove (set blocked-feed-ids) all-feed-ids))
+
           blocked-post-ids (mapv :post-id (services/filtered-posts ds {:where [:= :bundle-id bundle-id]}))
 
           filtered-posts (services/outgoing-posts bundle-ds (-> (hsql/where content-type-comp
                                                                             [:not [:in :id blocked-post-ids]]
-                                                                            [:not [:in :feed-id blocked-feed-ids]])
+                                                                            [:in :feed-id available-feed-ids])
                                                                 (hsql/order-by (when (= latest "true") [[:posted-at :desc]]))))
 
           categorised-posts (vec

From ea1a55e2883b681b1b3ba82c628a90e2867de43b Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 8 Dec 2025 14:20:03 +0200
Subject: [PATCH 192/391] updated integration endpoints to use job name
 resolution functions

---
 src/source/routes/integration.clj  | 3 ++-
 src/source/routes/integrations.clj | 5 +++--
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/source/routes/integration.clj b/src/source/routes/integration.clj
index 43f46182..2149e92b 100644
--- a/src/source/routes/integration.clj
+++ b/src/source/routes/integration.clj
@@ -4,7 +4,8 @@
             [source.db.util :as db.util]
             [congest.jobs :as congest]
             [source.util :as utils]
-            [source.jobs.core :as jobs]))
+            [source.jobs.core :as jobs]
+            [source.jobs.handlers :as handlers]))
 
 (defn get
   {:summary "get integration by id"
diff --git a/src/source/routes/integrations.clj b/src/source/routes/integrations.clj
index 34a34f38..615ae95e 100644
--- a/src/source/routes/integrations.clj
+++ b/src/source/routes/integrations.clj
@@ -5,7 +5,8 @@
             [source.migrate :as migrate]
             [congest.jobs :as congest]
             [source.jobs.core :as jobs]
-            [source.db.util :as db.util]))
+            [source.db.util :as db.util]
+            [source.jobs.handlers :as handlers]))
 
 (defn get
   {:summary "get all integrations"
@@ -82,7 +83,7 @@
     (->> (jobs/prepare-congest-metadata
           ds
           store
-          {:id (str "bundle_" (:id new-bundle))
+          {:id (handlers/update-bundle-job-id (:id new-bundle))
            :initial-delay 0
            :auto-start true
            :stop-after-fail false,

From 8f468330fb0eb214118e12c7ebe215ff487be241 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 10 Dec 2025 15:40:07 +0200
Subject: [PATCH 193/391] updated fly toml

---
 fly.dev.toml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/fly.dev.toml b/fly.dev.toml
index 84d2f69f..ea05eff3 100644
--- a/fly.dev.toml
+++ b/fly.dev.toml
@@ -25,9 +25,9 @@ primary_region = 'jnb'
   processes = ['app']
 
 [[vm]]
-  memory = '1gb'
+  memory = '2gb'
   cpu_kind = 'shared'
-  cpus = 1
+  cpus = 2
 
 [mounts]
   source = 'source_storage_staging'

From a49c7857a51d5cd33f3dd0d0f0a54c74bc3a8929 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Thu, 11 Dec 2025 16:09:15 +0200
Subject: [PATCH 194/391] db schema changes and migrations

---
 src/source/db/master.clj                      |  9 +++++
 src/source/migrations/007_business_fields.clj | 35 +++++++++++++++++++
 2 files changed, 44 insertions(+)
 create mode 100644 src/source/migrations/007_business_fields.clj

diff --git a/src/source/db/master.clj b/src/source/db/master.clj
index 8a2b5a42..251933f5 100644
--- a/src/source/db/master.clj
+++ b/src/source/db/master.clj
@@ -248,6 +248,12 @@
    (tables/foreign-key :event-id :events :id)
    (tables/foreign-key :category-id :categories :id)))
 
+(def business-types
+  (tables/create-table-sql
+    :business-types 
+    (tables/table-id)
+    [:name :text :not nil]))
+
 (comment
   (require '[honey.sql :as sql])
 
@@ -269,6 +275,9 @@
   (sql/format incoming-posts)
   (sql/format jobs)
   (sql/format job-metadata)
+  (sql/format filtered-feeds)
+  (sql/format filtered-posts)
+  (sql/format business-types)
 
   (sql/format events)
   (sql/format event-categories)
diff --git a/src/source/migrations/007_business_fields.clj b/src/source/migrations/007_business_fields.clj
new file mode 100644
index 00000000..6a82a496
--- /dev/null
+++ b/src/source/migrations/007_business_fields.clj
@@ -0,0 +1,35 @@
+(ns source.migrations.007-business-fields
+  (:require [source.db.master]
+            [source.db.honey :as hon]
+            [source.db.tables :as tables]
+            [honey.sql.helpers :as hsql]))
+
+(def business-types
+  [{:name "Independent"}
+   {:name "Commercial"}
+   {:name "Non-profit"}
+   {:name "State-owned"}])
+
+(defn run-up! [context]
+  (let [ds-master (:db-master context)]
+    (tables/create-table!
+     ds-master
+     :source.db.master
+     :business-types)
+
+    (hon/execute!
+     ds-master
+     (-> (hsql/alter-table :businesses)
+         (hsql/add-column :business-type-id :integer)
+         (hsql/add-column :registration :text)))))
+
+(defn run-down! [context]
+  (let [ds-master (:db-master context)]
+    (hon/execute!
+     ds-master
+     (-> (hsql/alter-table :businesses)
+         (hsql/drop-column :business-type-id :registration)))
+
+    (tables/drop-table!
+     ds-master
+     :business-types)))

From 228fadbc41b56c57377561a44dae6a10c29787a2 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Thu, 11 Dec 2025 16:09:35 +0200
Subject: [PATCH 195/391] updated business aond sector related services

---
 src/source/services/businesses.clj   | 8 ++++++++
 src/source/services/interface.clj    | 6 +++++-
 src/source/services/user_sectors.clj | 6 +++---
 3 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/src/source/services/businesses.clj b/src/source/services/businesses.clj
index 7b8dffc6..4c361324 100644
--- a/src/source/services/businesses.clj
+++ b/src/source/services/businesses.clj
@@ -1,6 +1,14 @@
 (ns source.services.businesses
   (:require [source.db.interface :as db]))
 
+(defn business
+  [ds {:keys [id where] :as opts}]
+  (->> {:tname :businesses
+        :where (if id [:= :id id] where)
+        :ret :1}
+       (merge opts)
+       (db/find ds)))
+
 (defn businesses
   ([ds] (businesses ds {}))
   ([ds opts]
diff --git a/src/source/services/interface.clj b/src/source/services/interface.clj
index 1f59a2a1..d3db99e6 100644
--- a/src/source/services/interface.clj
+++ b/src/source/services/interface.clj
@@ -35,6 +35,10 @@
 (defn update-user! [ds {:keys [_id _values _where] :as opts}]
   (users/update-user! ds opts))
 
+(defn business
+  [ds {:keys [_id _where] :as opts}]
+  (businesses/business ds opts))
+
 (defn businesses
   ([ds] (businesses ds {}))
   ([ds opts]
@@ -285,7 +289,7 @@
 (defn delete-user-sector! [ds {:keys [_id _where] :as opts}]
   (user-sectors/delete-user-sector! ds opts))
 
-(defn sectors-by-user [ds {:keys [_sector-id _where] :as opts}]
+(defn sectors-by-user [ds {:keys [_user-id _where] :as opts}]
   (user-sectors/sectors-by-user ds opts))
 
 (defn sector-id [ds {:keys [_user-id _where] :as opts}]
diff --git a/src/source/services/user_sectors.clj b/src/source/services/user_sectors.clj
index 3ac971b9..a7d4cfad 100644
--- a/src/source/services/user_sectors.clj
+++ b/src/source/services/user_sectors.clj
@@ -25,13 +25,13 @@
        (merge opts)
        (db/delete! ds)))
 
-(defn sectors-by-user [ds {:keys [sector-id where] :as _opts}]
+(defn sectors-by-user [ds {:keys [user-id where] :as _opts}]
   (hon/execute! ds
                 {:select [[:user-sectors.sector-id :id] :name]
                  :from :sectors
                  :join [:user-sectors [:= :user-sectors.sector-id :sectors.id]]
-                 :where (if (some? sector-id)
-                          [:= :sector-id sector-id]
+                 :where (if (some? user-id)
+                          [:= :sector-id user-id]
                           where)}
                 {:ret :*}))
 

From a6edeb4d1da31ceea9cb771ff59b803a0536e2c2 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Thu, 11 Dec 2025 16:09:47 +0200
Subject: [PATCH 196/391] added and updated endpoints

---
 src/source/routes/business_types.clj | 13 ++++++++++
 src/source/routes/me_business.clj    | 38 +++++++++++++++++++++++-----
 src/source/routes/me_sectors.clj     | 13 +++++++++-
 src/source/routes/reitit.clj         | 10 +++++---
 4 files changed, 64 insertions(+), 10 deletions(-)
 create mode 100644 src/source/routes/business_types.clj

diff --git a/src/source/routes/business_types.clj b/src/source/routes/business_types.clj
new file mode 100644
index 00000000..31a5b2cb
--- /dev/null
+++ b/src/source/routes/business_types.clj
@@ -0,0 +1,13 @@
+(ns source.routes.business-types
+  (:require [source.db.honey :as db]
+            [ring.util.response :as res]))
+
+(defn get
+  {:summary "Get all business types"
+   :responses {200 {:body [:vector
+                           [:map
+                            [:id :int]
+                            [:name :string]]]}}}
+  [ds]
+  (res/response (db/find ds {:tname :business-types
+                             :ret :*})))
diff --git a/src/source/routes/me_business.clj b/src/source/routes/me_business.clj
index 347b0e8f..60f6d92c 100644
--- a/src/source/routes/me_business.clj
+++ b/src/source/routes/me_business.clj
@@ -3,14 +3,34 @@
             [ring.util.response :as res]
             [source.services.interface :as services]))
 
+(defn get
+  {:summary "get business for logged-in user"
+   :responses {200 {:body [:map
+                           [:id :int]
+                           [:name [:maybe :string]]
+                           [:address [:maybe :string]]
+                           [:url [:maybe :string]]
+                           [:linkedin [:maybe :string]]
+                           [:twitter [:maybe :string]]
+                           [:registration [:maybe :string]]
+                           [:business-type-id {:optional true} :int]]}
+               401 {:body [:map [:message :string]]}
+               403 {:body [:map [:message :string]]}}}
+
+  [{:keys [ds user] :as _request}]
+  (let [{:keys [business-id]} (services/user ds {:id (:id user)})]
+    (res/response (services/business ds {:id business-id}))))
+
 (defn post
-  {:summary "add business for logged-in user"
+  {:summary "add or update business for logged-in user"
    :parameters {:body [:map
                        [:name {:optional true} :string]
                        [:address {:optional true} :string]
                        [:url {:optional true} :string]
                        [:linkedin {:optional true} :string]
-                       [:twitter {:optional true} :string]]}
+                       [:twitter {:optional true} :string]
+                       [:registration {:optional true} :string]
+                       [:business-type-id {:optional true} :int]]}
    :responses {200 {:body [:map [:message :string]]}
                400 {:body [:map [:message :string]]}}}
 
@@ -20,8 +40,14 @@
       (-> (res/response {:message error})
           (res/status 400))
 
-      (let [business (services/insert-business! ds {:data data
-                                                    :ret :1})]
-        (services/update-user! ds {:id (:id user)
-                                   :data {:business-id (:id business)}})
+      (let [{:keys [business-id]} (services/user ds {:id (:id user)})
+            business (when (nil? business-id)
+                       (services/insert-business! ds {:data data
+                                                      :ret :1}))]
+        (if (nil? business-id)
+          (services/update-user! ds {:id (:id user)
+                                     :data {:business-id (:id business)}})
+          (services/update-business! ds {:id business-id
+                                         :data data}))
+
         (res/response {:message "successfully added business"})))))
diff --git a/src/source/routes/me_sectors.clj b/src/source/routes/me_sectors.clj
index baaa1627..e534e51f 100644
--- a/src/source/routes/me_sectors.clj
+++ b/src/source/routes/me_sectors.clj
@@ -1,7 +1,18 @@
-(ns source.routes.me-sectors 
+(ns source.routes.me-sectors
   (:require [source.services.interface :as services]
             [ring.util.response :as res]))
 
+(defn get
+  {:summary "get sectors for the logged-in user"
+   :responses {200 {:body [:vector
+                           [:map
+                            [:id :int]
+                            [:name :string]]]}
+               401 {:body [:map [:message :string]]}
+               403 {:body [:map [:message :string]]}}}
+  [{:keys [ds user] :as _request}]
+  (res/response (services/sectors-by-user ds {:user-id (:id user)})))
+
 (defn post
   {:summary "update sectors for the logged-in user"
    :parameters {:body [:vector
diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index 5012f564..c729173e 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -23,6 +23,7 @@
             [source.routes.authorized :as authorized]
             [source.routes.business :as business]
             [source.routes.businesses :as businesses]
+            [source.routes.business-types :as business-types]
             [source.routes.sectors :as sectors]
             [source.routes.selection-schemas :as selection-schemas]
             [source.routes.selection-schema :as selection-schema]
@@ -132,8 +133,10 @@
                          :openapi {:security [{:bearerAuth []}]}}
       [""               (route {:get me/get
                                 :post me/post})]
-      ["/business"      (route {:post me-business/post})]
-      ["/sectors"       (route {:post me-sectors/post})]]
+      ["/business"      (route {:get me-business/get
+                                :post me-business/post})]
+      ["/sectors"       (route {:get me-sectors/get
+                                :post me-sectors/post})]]
 
      ["/mail"             {:middleware [[mw/apply-auth]]
                            :tags #{"mail"}
@@ -147,7 +150,8 @@
                          :openapi {:security [{:bearerAuth []}]}}
       [""               (route {:get businesses/get
                                 :post business/post})]
-      ["/:id"           (route {:patch business/patch})]]
+      ["/:id"           (route {:patch business/patch})]
+      ["/types"         (route {:get business-types/get})]]
 
      ["/sectors"        {:tags #{"sectors"}}
       [""               (route {:get sectors/get})]]

From 06afcc05940c57710b6608b5ff4a461f06e2fe4e Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Fri, 12 Dec 2025 11:28:12 +0200
Subject: [PATCH 197/391] fixed services

---
 src/source/services/user_sectors.clj | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/source/services/user_sectors.clj b/src/source/services/user_sectors.clj
index a7d4cfad..32a5078a 100644
--- a/src/source/services/user_sectors.clj
+++ b/src/source/services/user_sectors.clj
@@ -31,7 +31,7 @@
                  :from :sectors
                  :join [:user-sectors [:= :user-sectors.sector-id :sectors.id]]
                  :where (if (some? user-id)
-                          [:= :sector-id user-id]
+                          [:= :user-id user-id]
                           where)}
                 {:ret :*}))
 

From 27ede70131fb97bef0ee120462bcc1bb4cf16b4d Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Fri, 12 Dec 2025 11:28:40 +0200
Subject: [PATCH 198/391] fixed migration alter table

---
 src/source/migrations/007_business_fields.clj | 22 +++++++++++++------
 1 file changed, 15 insertions(+), 7 deletions(-)

diff --git a/src/source/migrations/007_business_fields.clj b/src/source/migrations/007_business_fields.clj
index 6a82a496..8d0456cf 100644
--- a/src/source/migrations/007_business_fields.clj
+++ b/src/source/migrations/007_business_fields.clj
@@ -2,13 +2,15 @@
   (:require [source.db.master]
             [source.db.honey :as hon]
             [source.db.tables :as tables]
-            [honey.sql.helpers :as hsql]))
+            [honey.sql.helpers :as hsql]
+            [source.db.util :as db.util]))
 
-(def business-types
-  [{:name "Independent"}
-   {:name "Commercial"}
-   {:name "Non-profit"}
-   {:name "State-owned"}])
+(def business-types-seed
+  {:tname :business-types
+   :data [{:name "Independent"}
+          {:name "Commercial"}
+          {:name "Non-profit"}
+          {:name "State-owned"}]})
 
 (defn run-up! [context]
   (let [ds-master (:db-master context)]
@@ -17,10 +19,16 @@
      :source.db.master
      :business-types)
 
+    (hon/insert! ds-master business-types-seed)
+
+    (hon/execute!
+     ds-master
+     (-> (hsql/alter-table :businesses)
+         (hsql/add-column :business-type-id :integer)))
+
     (hon/execute!
      ds-master
      (-> (hsql/alter-table :businesses)
-         (hsql/add-column :business-type-id :integer)
          (hsql/add-column :registration :text)))))
 
 (defn run-down! [context]

From efb75ed3303422a6877365a322ee9f236b3740f5 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Fri, 12 Dec 2025 11:28:52 +0200
Subject: [PATCH 199/391] fixed bugs in routes

---
 src/source/routes/business_types.clj | 2 +-
 src/source/routes/me_business.clj    | 9 ++++++---
 src/source/routes/me_sectors.clj     | 3 ++-
 src/source/routes/reitit.clj         | 6 ++++--
 4 files changed, 13 insertions(+), 7 deletions(-)

diff --git a/src/source/routes/business_types.clj b/src/source/routes/business_types.clj
index 31a5b2cb..700eaca0 100644
--- a/src/source/routes/business_types.clj
+++ b/src/source/routes/business_types.clj
@@ -8,6 +8,6 @@
                            [:map
                             [:id :int]
                             [:name :string]]]}}}
-  [ds]
+  [{:keys [ds] :as _request}]
   (res/response (db/find ds {:tname :business-types
                              :ret :*})))
diff --git a/src/source/routes/me_business.clj b/src/source/routes/me_business.clj
index 60f6d92c..343ecaa0 100644
--- a/src/source/routes/me_business.clj
+++ b/src/source/routes/me_business.clj
@@ -18,8 +18,11 @@
                403 {:body [:map [:message :string]]}}}
 
   [{:keys [ds user] :as _request}]
-  (let [{:keys [business-id]} (services/user ds {:id (:id user)})]
-    (res/response (services/business ds {:id business-id}))))
+  (let [{:keys [business-id]} (services/user ds {:id (:id user)})
+        business (if business-id
+                   (services/business ds {:id business-id})
+                   {})]
+    (res/response business)))
 
 (defn post
   {:summary "add or update business for logged-in user"
@@ -50,4 +53,4 @@
           (services/update-business! ds {:id business-id
                                          :data data}))
 
-        (res/response {:message "successfully added business"})))))
+        (res/response {:message "successfully added or updated business"})))))
diff --git a/src/source/routes/me_sectors.clj b/src/source/routes/me_sectors.clj
index e534e51f..11780cb9 100644
--- a/src/source/routes/me_sectors.clj
+++ b/src/source/routes/me_sectors.clj
@@ -1,6 +1,7 @@
 (ns source.routes.me-sectors
   (:require [source.services.interface :as services]
-            [ring.util.response :as res]))
+            [ring.util.response :as res]
+            [source.db.util :as db.util]))
 
 (defn get
   {:summary "get sectors for the logged-in user"
diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index c729173e..15013d0d 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -149,8 +149,10 @@
                          :swagger {:security [{"auth" []}]}
                          :openapi {:security [{:bearerAuth []}]}}
       [""               (route {:get businesses/get
-                                :post business/post})]
-      ["/:id"           (route {:patch business/patch})]
+                                :post business/post})] 
+      ["/:id"           (route {:patch business/patch})]]
+
+     ["/business"
       ["/types"         (route {:get business-types/get})]]
 
      ["/sectors"        {:tags #{"sectors"}}

From ce9e0eafaa9dc74b2ffaba57efb62bf9c569d3eb Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Fri, 12 Dec 2025 11:29:53 +0200
Subject: [PATCH 200/391] removed unnecessary imports

---
 src/source/routes/me_sectors.clj | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/source/routes/me_sectors.clj b/src/source/routes/me_sectors.clj
index 11780cb9..e534e51f 100644
--- a/src/source/routes/me_sectors.clj
+++ b/src/source/routes/me_sectors.clj
@@ -1,7 +1,6 @@
 (ns source.routes.me-sectors
   (:require [source.services.interface :as services]
-            [ring.util.response :as res]
-            [source.db.util :as db.util]))
+            [ring.util.response :as res]))
 
 (defn get
   {:summary "get sectors for the logged-in user"

From 708961c11c3e9607963f1629f166e084090bc5f4 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Fri, 12 Dec 2025 12:42:26 +0200
Subject: [PATCH 201/391] added deletion services

---
 src/source/services/bundles.clj   | 9 +++++++++
 src/source/services/interface.clj | 3 +++
 2 files changed, 12 insertions(+)

diff --git a/src/source/services/bundles.clj b/src/source/services/bundles.clj
index 52df888f..4fee17f3 100644
--- a/src/source/services/bundles.clj
+++ b/src/source/services/bundles.clj
@@ -31,3 +31,12 @@
         :ret :1}
        (merge opts)
        (db/find ds)))
+
+(defn delete-bundle! [ds {:keys [id where] :as opts}]
+  (->> {:tname :bundles
+        :where (if (some? id)
+                 [:= :id id]
+                 where)
+        :ret :1}
+       (merge opts)
+       (db/delete! ds)))
diff --git a/src/source/services/interface.clj b/src/source/services/interface.clj
index 1f59a2a1..257c7137 100644
--- a/src/source/services/interface.clj
+++ b/src/source/services/interface.clj
@@ -72,6 +72,9 @@
 (defn bundle [ds {:keys [_id _where] :as opts}]
   (bundles/bundle ds opts))
 
+(defn delete-bundle [ds {:keys [_id _where] :as opts}]
+  (bundles/delete-bundle! ds opts))
+
 (defn bundle-categories
   ([ds] (bundle-categories ds {}))
   ([ds {:keys [_where] :as opts}]

From efe2e7f076c8c2fae0c740ff4838cec1f720b0e0 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Fri, 12 Dec 2025 12:43:22 +0200
Subject: [PATCH 202/391] added delete integration endpoint

---
 src/source/routes/integration.clj | 24 +++++++++++++++++++++++-
 src/source/routes/reitit.clj      |  3 ++-
 2 files changed, 25 insertions(+), 2 deletions(-)

diff --git a/src/source/routes/integration.clj b/src/source/routes/integration.clj
index 2149e92b..0a9d0681 100644
--- a/src/source/routes/integration.clj
+++ b/src/source/routes/integration.clj
@@ -5,7 +5,9 @@
             [congest.jobs :as congest]
             [source.util :as utils]
             [source.jobs.core :as jobs]
-            [source.jobs.handlers :as handlers]))
+            [source.jobs.handlers :as handlers]
+            [source.services.analytics.interface :as analytics]
+            [source.db.tables :as tables]))
 
 (defn get
   {:summary "get integration by id"
@@ -97,3 +99,23 @@
          (congest/register! js))
 
     (res/response {:message "successfully updated integration"})))
+
+(defn hard-delete-bundle! [ds js bundle-id]
+  (let [job-id (handlers/update-bundle-job-id bundle-id)]
+    (services/delete-filtered-feed! ds {:where [:= :bundle-id bundle-id]})
+    (services/delete-filtered-post! ds {:where [:= :bundle-id bundle-id]})
+    (services/delete-bundle-content-types! ds {:where [:= :bundle-id bundle-id]})
+    (analytics/delete-event! ds {:where [:= :bundle-id bundle-id]})
+    (tables/drop-all-tables! (db.util/conn :bundle bundle-id))
+    (services/delete-bundle ds {:id bundle-id})
+    (congest/deregister! js job-id)))
+
+(defn delete
+  {:summary "delete the integration with the given id"
+   :parameters {:path [:map [:id {:title "id"
+                                  :description "integration id"} :int]]}
+   :responses {200 {:body [:map [:message :string]]}}}
+
+  [{:keys [ds js path-params] :as _request}]
+    (hard-delete-bundle! ds js (:id path-params))
+    (res/response {:message "successfully deleted integration"}))
diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index 5012f564..5ee9db62 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -195,7 +195,8 @@
                                 :post integrations/post})]
       ["/:id"
        [""              (route {:get integration/get
-                                :post integration/post})]
+                                :post integration/post
+                                :delete integration/delete})]
        ["/key"          (route {:post integration-key/post})]
        ["/categories"   (route {:get integration-categories/get
                                 :post integration-categories/post})]

From 59a601bd606ed84ca9e552ee49a8caa835d84ba3 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Fri, 12 Dec 2025 13:05:25 +0200
Subject: [PATCH 203/391] fixed permissions error

---
 src/source/routes/integration.clj | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/src/source/routes/integration.clj b/src/source/routes/integration.clj
index 0a9d0681..d340250f 100644
--- a/src/source/routes/integration.clj
+++ b/src/source/routes/integration.clj
@@ -114,8 +114,16 @@
   {:summary "delete the integration with the given id"
    :parameters {:path [:map [:id {:title "id"
                                   :description "integration id"} :int]]}
-   :responses {200 {:body [:map [:message :string]]}}}
+   :responses {200 {:body [:map [:message :string]]}
+               403 {:body [:map [:message :string]]}}}
 
-  [{:keys [ds js path-params] :as _request}]
-    (hard-delete-bundle! ds js (:id path-params))
-    (res/response {:message "successfully deleted integration"}))
+  [{:keys [ds js user path-params] :as _request}]
+  (let [bundle (services/bundle ds {:where [:and
+                                            [:= :id (:id path-params)]
+                                            [:= :user-id (:id user)]]})]
+    (if (some? bundle)
+      (do
+        (hard-delete-bundle! ds js (:id path-params))
+        (res/response {:message "successfully deleted integration"}))
+      (-> (res/response {:message "unauthorized"})
+          (res/status 403)))))

From d120fcf259c48ba8f3d4ae2d877295c9fd9b52f0 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Fri, 12 Dec 2025 13:09:53 +0200
Subject: [PATCH 204/391] fixed permissions issue for feed deletion

---
 src/source/routes/feed.clj | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/src/source/routes/feed.clj b/src/source/routes/feed.clj
index d5f340d6..cd7acd31 100644
--- a/src/source/routes/feed.clj
+++ b/src/source/routes/feed.clj
@@ -70,6 +70,13 @@
 
   [{:keys [ds js user path-params] :as _request}]
   (let [id (:id path-params)
+        feed (services/feed ds {:where [:and
+                                        [:= :user-id (:id user)
+                                         := :id id]]})
         {:keys [email]} (services/user ds {:id (:id user)})]
-    (hard-delete-feed! ds js email id)
-    (res/response {:message "successfully deleted feed"})))
+    (if (some? feed)
+      (do
+        (hard-delete-feed! ds js email id)
+        (res/response {:message "successfully deleted feed"}))
+      (-> (res/response {:message "unauthorized"})
+          (res/status 403)))))

From e533ec36aae0e88870d645345652e2e1fd7f8c83 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Fri, 12 Dec 2025 16:37:10 +0200
Subject: [PATCH 205/391] fixed schemas

---
 src/source/routes/me_business.clj | 2 +-
 src/source/routes/reitit.clj      | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/source/routes/me_business.clj b/src/source/routes/me_business.clj
index 343ecaa0..8db2bcdf 100644
--- a/src/source/routes/me_business.clj
+++ b/src/source/routes/me_business.clj
@@ -13,7 +13,7 @@
                            [:linkedin [:maybe :string]]
                            [:twitter [:maybe :string]]
                            [:registration [:maybe :string]]
-                           [:business-type-id {:optional true} :int]]}
+                           [:business-type-id [:maybe :int]]]}
                401 {:body [:map [:message :string]]}
                403 {:body [:map [:message :string]]}}}
 
diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index 15013d0d..a93b496a 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -152,7 +152,7 @@
                                 :post business/post})] 
       ["/:id"           (route {:patch business/patch})]]
 
-     ["/business"
+     ["/business"       {:tags #{"businesses"}}
       ["/types"         (route {:get business-types/get})]]
 
      ["/sectors"        {:tags #{"sectors"}}

From 5b362f01dd94a186b5f7121edaf9bac1cd086e1b Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 15 Dec 2025 11:55:03 +0200
Subject: [PATCH 206/391] added admin endpoints for managing business types and
 removed seed data

---
 src/source/migrations/007_business_fields.clj | 12 +------
 src/source/routes/business_types.clj          | 36 +++++++++++++++++++
 src/source/routes/reitit.clj                  |  6 +++-
 3 files changed, 42 insertions(+), 12 deletions(-)

diff --git a/src/source/migrations/007_business_fields.clj b/src/source/migrations/007_business_fields.clj
index 8d0456cf..b07dba8e 100644
--- a/src/source/migrations/007_business_fields.clj
+++ b/src/source/migrations/007_business_fields.clj
@@ -2,15 +2,7 @@
   (:require [source.db.master]
             [source.db.honey :as hon]
             [source.db.tables :as tables]
-            [honey.sql.helpers :as hsql]
-            [source.db.util :as db.util]))
-
-(def business-types-seed
-  {:tname :business-types
-   :data [{:name "Independent"}
-          {:name "Commercial"}
-          {:name "Non-profit"}
-          {:name "State-owned"}]})
+            [honey.sql.helpers :as hsql]))
 
 (defn run-up! [context]
   (let [ds-master (:db-master context)]
@@ -19,8 +11,6 @@
      :source.db.master
      :business-types)
 
-    (hon/insert! ds-master business-types-seed)
-
     (hon/execute!
      ds-master
      (-> (hsql/alter-table :businesses)
diff --git a/src/source/routes/business_types.clj b/src/source/routes/business_types.clj
index 700eaca0..0c55d645 100644
--- a/src/source/routes/business_types.clj
+++ b/src/source/routes/business_types.clj
@@ -11,3 +11,39 @@
   [{:keys [ds] :as _request}]
   (res/response (db/find ds {:tname :business-types
                              :ret :*})))
+
+(defn post
+  {:summary "add one or more new business types"
+   :parameters {:body [:vector
+                       [:map
+                        [:name :string]]]}
+   :responses {201 {:body [:map [:message :string]]}}}
+
+  [{:keys [ds body] :as _request}]
+  (db/insert! ds {:tname :business-types
+                  :data body})
+  (res/response {:message "successfully added business type(s)"}))
+
+(defn patch
+  {:summary "update business type with the given id"
+   :parameters {:path [:id {:title "id"
+                            :description "business type id"} :int]
+                :body [:map [:name :string]]}
+   :responses {200 {:body [:map [:message :string]]}}}
+
+  [{:keys [ds path-params body] :as _request}]
+  (db/update! ds {:tname :business-types
+                  :data body
+                  :where [:= :id (:id path-params)]})
+  (res/response {:message "successfully updated business type"}))
+
+(defn delete
+  {:summary "delete business type with the given id"
+   :parameters {:path [:id {:title "id"
+                            :description "business type id"} :int]}
+   :responses {200 {:body [:map [:message :string]]}}}
+
+  [{:keys [ds path-params] :as _request}]
+  (db/delete! ds {:tname :business-types
+                  :where [:= :id (:id path-params)]})
+  (res/response {:message "successfully deleted business type"}))
diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index a93b496a..869c5c33 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -149,7 +149,7 @@
                          :swagger {:security [{"auth" []}]}
                          :openapi {:security [{:bearerAuth []}]}}
       [""               (route {:get businesses/get
-                                :post business/post})] 
+                                :post business/post})]
       ["/:id"           (route {:patch business/patch})]]
 
      ["/business"       {:tags #{"businesses"}}
@@ -293,6 +293,10 @@
                                  :tags #{"admin"}
                                  :swagger {:security [{"auth" []}]}
                                  :openapi {:security [{:bearerAuth []}]}}
+      ["/business"
+       ["/types"                (route {:post business-types/post
+                                        :patch business-types/patch
+                                        :delete business-types/delete})]]
       ["/feeds"
        [""                      (route {:get admin-feeds/get})]
        ["/:id"

From 5a8f8399e607b85eaea64c4488cb33ec0a741779 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 15 Dec 2025 13:35:30 +0200
Subject: [PATCH 207/391] fixed schemas

---
 src/source/routes/business_types.clj | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/source/routes/business_types.clj b/src/source/routes/business_types.clj
index 0c55d645..3a5fca27 100644
--- a/src/source/routes/business_types.clj
+++ b/src/source/routes/business_types.clj
@@ -26,8 +26,8 @@
 
 (defn patch
   {:summary "update business type with the given id"
-   :parameters {:path [:id {:title "id"
-                            :description "business type id"} :int]
+   :parameters {:path [:map [:id {:title "id"
+                                  :description "business type id"} :int]]
                 :body [:map [:name :string]]}
    :responses {200 {:body [:map [:message :string]]}}}
 
@@ -39,8 +39,8 @@
 
 (defn delete
   {:summary "delete business type with the given id"
-   :parameters {:path [:id {:title "id"
-                            :description "business type id"} :int]}
+   :parameters {:path [:map [:id {:title "id"
+                                  :description "business type id"} :int]]}
    :responses {200 {:body [:map [:message :string]]}}}
 
   [{:keys [ds path-params] :as _request}]

From 2a45b7f419b75d4415e3f1b51625e81fff16ad92 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 15 Dec 2025 14:00:46 +0200
Subject: [PATCH 208/391] fixed malli schemas for onboarding endpoints

---
 src/source/routes/me.clj          |  4 ++--
 src/source/routes/me_business.clj | 14 +++++++-------
 2 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/src/source/routes/me.clj b/src/source/routes/me.clj
index 0f654a0a..154ed1d4 100644
--- a/src/source/routes/me.clj
+++ b/src/source/routes/me.clj
@@ -28,13 +28,13 @@
 (defn post
   {:summary "update logged-in user by access token"
    :parameters {:body [:map
-                       [:address {:optional true} :string]
+                       [:address {:optional true} [:maybe :string]]
                        [:profile-image {:optional true} [:maybe :string]]
                        [:firstname {:optional true} :string]
                        [:lastname {:optional true} :string]
                        [:email-verified {:optional true} :int]
                        [:onboarded {:optional true} :int]
-                       [:mobile {:optional true} :string]]}
+                       [:mobile {:optional true} [:maybe :string]]]}
    :responses {200 {:body [:map [:message :string]]}
                400 {:body [:map [:message :string]]}}}
 
diff --git a/src/source/routes/me_business.clj b/src/source/routes/me_business.clj
index 8db2bcdf..6f3b1ad5 100644
--- a/src/source/routes/me_business.clj
+++ b/src/source/routes/me_business.clj
@@ -27,13 +27,13 @@
 (defn post
   {:summary "add or update business for logged-in user"
    :parameters {:body [:map
-                       [:name {:optional true} :string]
-                       [:address {:optional true} :string]
-                       [:url {:optional true} :string]
-                       [:linkedin {:optional true} :string]
-                       [:twitter {:optional true} :string]
-                       [:registration {:optional true} :string]
-                       [:business-type-id {:optional true} :int]]}
+                       [:name {:optional true} [:maybe :string]]
+                       [:address {:optional true} [:maybe :string]]
+                       [:url {:optional true} [:maybe :string]]
+                       [:linkedin {:optional true} [:maybe :string]]
+                       [:twitter {:optional true} [:maybe :string]]
+                       [:registration {:optional true} [:maybe :string]]
+                       [:business-type-id {:optional true} [:maybe :int]]]}
    :responses {200 {:body [:map [:message :string]]}
                400 {:body [:map [:message :string]]}}}
 

From 0bc7c7c9822065890943ea3001484f9c221617ba Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 17 Dec 2025 12:02:50 +0200
Subject: [PATCH 209/391] added migrations for db update

---
 .../008_provider_rss_instructions.clj         | 23 +++++++++++++++++++
 1 file changed, 23 insertions(+)
 create mode 100644 src/source/migrations/008_provider_rss_instructions.clj

diff --git a/src/source/migrations/008_provider_rss_instructions.clj b/src/source/migrations/008_provider_rss_instructions.clj
new file mode 100644
index 00000000..6174ce3c
--- /dev/null
+++ b/src/source/migrations/008_provider_rss_instructions.clj
@@ -0,0 +1,23 @@
+(ns source.migrations.008-provider-rss-instructions
+  (:require [source.db.master]
+            [source.db.honey :as hon]
+            [honey.sql.helpers :as hsql]))
+
+(defn run-up! [context]
+  (let [ds-master (:db-master context)]
+    (hon/execute!
+     ds-master
+     (-> (hsql/alter-table :providers)
+         (hsql/add-column :instructions :string)))
+
+    (hon/execute!
+     ds-master
+     (-> (hsql/alter-table :businesses)
+         (hsql/add-column :placeholder-url :string)))))
+
+(defn run-down! [context]
+  (let [ds-master (:db-master context)]
+    (hon/execute!
+     ds-master
+     (-> (hsql/alter-table :businesses)
+         (hsql/drop-column :instructions :placeholder-url)))))

From a6e96e5be546cfb82ce98a2b1489020d87b2cc56 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 17 Dec 2025 12:03:06 +0200
Subject: [PATCH 210/391] updated services

---
 src/source/services/interface.clj | 3 +++
 src/source/services/providers.clj | 7 +++++++
 2 files changed, 10 insertions(+)

diff --git a/src/source/services/interface.clj b/src/source/services/interface.clj
index 55a68e5d..e6dfda8f 100644
--- a/src/source/services/interface.clj
+++ b/src/source/services/interface.clj
@@ -177,6 +177,9 @@
 (defn delete-provider! [ds provider-id]
   (providers/delete-provider! ds provider-id))
 
+(defn update-provider! [ds {:keys [_id _data _where] :as opts}]
+  (providers/update-provider! ds opts))
+
 (defn insert-provider! [ds {:keys [_values _ret] :as opts}]
   (providers/insert-provider! ds opts))
 
diff --git a/src/source/services/providers.clj b/src/source/services/providers.clj
index 61fe4af4..e0d2bf77 100644
--- a/src/source/services/providers.clj
+++ b/src/source/services/providers.clj
@@ -25,6 +25,13 @@
        (merge opts)
        (db/find ds)))
 
+(defn update-provider! [ds {:keys [id data where] :as opts}]
+  (->> {:tname :providers
+        :data data
+        :where (if (some? id) [:= :id id] where)}
+       (merge opts)
+       (db/update! ds)))
+
 (defn delete-provider! [ds {:keys [id where] :as opts}]
   (->> {:tname :providers
         :where (if (some? id)

From 6e69f20b59703d9132702ec4a6e5932ed60874ab Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 17 Dec 2025 12:03:21 +0200
Subject: [PATCH 211/391] added and updated provider routes

---
 src/source/routes/provider.clj  | 27 +++++++++++++++++++++++----
 src/source/routes/providers.clj | 20 +++++++++++---------
 src/source/routes/reitit.clj    |  3 ++-
 3 files changed, 36 insertions(+), 14 deletions(-)

diff --git a/src/source/routes/provider.clj b/src/source/routes/provider.clj
index a6f4f4c0..a020cac5 100644
--- a/src/source/routes/provider.clj
+++ b/src/source/routes/provider.clj
@@ -9,20 +9,39 @@
    :responses {200 {:body [:map
                            [:id :int]
                            [:name :string]
-                           [:domain :string]
-                           [:content-type-id :int]]}}}
+                           [:domain [:maybe :string]]
+                           [:content-type-id :int]
+                           [:instructions [:maybe :string]]
+                           [:placeholder-url [:maybe :string]]]}}}
 
   [{:keys [ds path-params] :as _request}]
   (->> path-params
        (services/provider ds)
        (res/response)))
 
-(defn delete 
+(defn post
+  {:summary "update provider by id"
+   :parameters {:path [:map [:id {:title "id"
+                                  :description "provider id"} :int]]
+                :body [:map
+                       [:name :string]
+                       [:domain {:optional true} [:maybe :string]]
+                       [:content-type-id :int]
+                       [:instructions {:optional true} [:maybe :string]]
+                       [:placeholder-url {:optional true} [:maybe :string]]]}
+   :responses {200 {:body [:map [:message :string]]}}}
+
+  [{:keys [ds path-params body] :as _request}]
+  (services/update-provider! ds {:id (:id path-params)
+                                 :data body})
+  (res/response {:message "successfully updated provider"}))
+
+(defn delete
   {:summary "given a provider id, delete provider and associated selection schemas"
    :parameters {:path [:map [:id {:title "id"
                                   :description "provider id"} :int]]}
    :responses {200 {:body [:map [:message :string]]}}}
-  
+
   [{:keys [store ds path-params] :as _request}]
   (->> path-params
        (services/delete-provider! ds))
diff --git a/src/source/routes/providers.clj b/src/source/routes/providers.clj
index 4ce4548e..533c5185 100644
--- a/src/source/routes/providers.clj
+++ b/src/source/routes/providers.clj
@@ -8,8 +8,10 @@
                            [:map
                             [:id :int]
                             [:name :string]
-                            [:domain :string]
-                            [:content-type-id :int]]]}}}
+                            [:domain [:maybe :string]]
+                            [:content-type-id :int]
+                            [:instructions [:maybe :string]]
+                            [:placeholder-url [:maybe :string]]]]}}}
 
   [{:keys [ds] :as _request}]
   (-> (services/providers ds)
@@ -20,14 +22,14 @@
    :parameters {:body [:map
                        [:provider
                         [:map
-                         [:id :int]
                          [:name :string]
-                         [:domain :string]
-                         [:content-type-id :int]]]]}
+                         [:domain {:optional true} [:maybe :string]]
+                         [:content-type-id :int]
+                         [:instructions {:optional true} [:maybe :string]]
+                         [:placeholder-url {:optional true} [:maybe :string]]]]]}
    :responses {200 {:body [:map [:message :string]]}}}
 
   [{:keys [ds body] :as _request}]
-  (let [{:keys [provider]} body]
-    (services/insert-provider! ds {:data provider
-                                   :ret :1})
-    (res/response {:message "successfully added provider"})))
+  (services/insert-provider! ds {:data body
+                                 :ret :1})
+  (res/response {:message "successfully added provider"}))
diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index bf5626da..922a9670 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -326,7 +326,8 @@
        ["/:id"                  {:get output-schema/get}]]
       ["/providers"
        [""                      (route {:post providers/post})]
-       ["/:id"                  (route {:delete provider/delete})]]
+       ["/:id"                  (route {:post provider/post
+                                        :delete provider/delete})]]
       ["/ast"                   {:post xml/post}]
       ["/extract-data"          {:post data/post}]]]
 

From 709f7f6216e05c8a8713526ba3f59089d0b28ad7 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 17 Dec 2025 13:39:39 +0200
Subject: [PATCH 212/391] fixed migrations

---
 src/source/migrations/008_provider_rss_instructions.clj | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/source/migrations/008_provider_rss_instructions.clj b/src/source/migrations/008_provider_rss_instructions.clj
index 6174ce3c..b39b3ff6 100644
--- a/src/source/migrations/008_provider_rss_instructions.clj
+++ b/src/source/migrations/008_provider_rss_instructions.clj
@@ -12,12 +12,12 @@
 
     (hon/execute!
      ds-master
-     (-> (hsql/alter-table :businesses)
+     (-> (hsql/alter-table :providers)
          (hsql/add-column :placeholder-url :string)))))
 
 (defn run-down! [context]
   (let [ds-master (:db-master context)]
     (hon/execute!
      ds-master
-     (-> (hsql/alter-table :businesses)
+     (-> (hsql/alter-table :providers)
          (hsql/drop-column :instructions :placeholder-url)))))

From 6015f8126cbc3015702842b34741299ca19e6482 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 17 Dec 2025 14:35:13 +0200
Subject: [PATCH 213/391] fixed schema in add provider

---
 src/source/routes/providers.clj | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/src/source/routes/providers.clj b/src/source/routes/providers.clj
index 533c5185..9ef7cfa7 100644
--- a/src/source/routes/providers.clj
+++ b/src/source/routes/providers.clj
@@ -20,13 +20,11 @@
 (defn post
   {:summary "add a provider"
    :parameters {:body [:map
-                       [:provider
-                        [:map
-                         [:name :string]
-                         [:domain {:optional true} [:maybe :string]]
-                         [:content-type-id :int]
-                         [:instructions {:optional true} [:maybe :string]]
-                         [:placeholder-url {:optional true} [:maybe :string]]]]]}
+                       [:name :string]
+                       [:domain {:optional true} [:maybe :string]]
+                       [:content-type-id :int]
+                       [:instructions {:optional true} [:maybe :string]]
+                       [:placeholder-url {:optional true} [:maybe :string]]]}
    :responses {200 {:body [:map [:message :string]]}}}
 
   [{:keys [ds body] :as _request}]

From 3f01faf3366acb0cdb5c76e3ab4763690724fbfb Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Tue, 20 Jan 2026 16:24:54 +0200
Subject: [PATCH 214/391] refactored reitit router to use router utilities and
 automatic malli validation

---
 src/source/middleware/core.clj                |  24 +-
 src/source/middleware/interface.clj           |   3 +
 src/source/routes/admin.clj                   |  12 +-
 src/source/routes/analytics/creator/top.clj   |  40 ++-
 .../routes/analytics/creator/top_average.clj  |  16 +-
 .../routes/analytics/distributor/top.clj      |  47 ++-
 .../analytics/distributor/top_average.clj     |  16 +-
 src/source/routes/bundle_feeds.clj            |  10 +-
 src/source/routes/business.clj                |  26 +-
 src/source/routes/login.clj                   |  17 +-
 src/source/routes/me.clj                      |  11 +-
 src/source/routes/me_business.clj             |  28 +-
 src/source/routes/openapi.clj                 | 167 +++++++++++
 src/source/routes/register.clj                |  12 +-
 src/source/routes/reitit.clj                  | 277 +++++++-----------
 src/source/routes/user.clj                    |  18 +-
 src/source/routes/util.clj                    | 105 +++++++
 src/source/util.clj                           |  81 +++--
 18 files changed, 568 insertions(+), 342 deletions(-)
 create mode 100644 src/source/routes/openapi.clj
 create mode 100644 src/source/routes/util.clj

diff --git a/src/source/middleware/core.clj b/src/source/middleware/core.clj
index af0553b5..cdc5bf24 100644
--- a/src/source/middleware/core.clj
+++ b/src/source/middleware/core.clj
@@ -10,7 +10,9 @@
             [ring.middleware.defaults :refer [wrap-defaults site-defaults]]
             [ring.middleware.json :as ring]
             [ring.middleware.cookies :as cookies]
-            [clojure.walk :as walk]))
+            [clojure.walk :as walk]
+            [source.util :as util]
+            [clojure.string :as string]))
 
 (defn wrap-ds [handler ds]
   (fn [request]
@@ -55,6 +57,26 @@
         (-> (res/response {:message "Internal Server Error"})
             (res/status 500))))))
 
+(defn- validate-param [request [param-type schema]]
+  (let [{:keys [error] :as validated} (-> (cond
+                                            (= param-type :body) (:body request)
+                                            (= param-type :path) (:path-params request)
+                                            (= param-type :query) (:query-params request))
+                                          (util/validate schema))]
+    (->> (when error
+           (str "In " (name param-type) ":\n" error))
+         (assoc validated :error))))
+
+(defn wrap-input-validation [handler openapi-meta]
+  (fn [request]
+    (let [errors (->> (mapv (partial validate-param request) (:parameters openapi-meta))
+                      (filter #(:error %))
+                      (mapv :error))]
+      (if (seq errors)
+        (-> (res/response {:message (string/join "\n" errors)})
+            (res/status 400))
+        (handler request)))))
+
 (defn process-body [{:keys [body] :as req} t-fn]
   (assoc req
          :body
diff --git a/src/source/middleware/interface.clj b/src/source/middleware/interface.clj
index 065e6d9f..636aaa27 100644
--- a/src/source/middleware/interface.clj
+++ b/src/source/middleware/interface.clj
@@ -14,3 +14,6 @@
 
 (defn apply-api-key [app]
   (mw/apply-api-key app))
+
+(defn apply-validation [app openapi-meta]
+  (mw/wrap-input-validation app openapi-meta))
diff --git a/src/source/routes/admin.clj b/src/source/routes/admin.clj
index 8486907f..5f788fee 100644
--- a/src/source/routes/admin.clj
+++ b/src/source/routes/admin.clj
@@ -1,7 +1,6 @@
 (ns source.routes.admin
   (:require [source.services.users :as users]
             [source.password :as pw]
-            [source.util :as util]
             [ring.util.response :as res]))
 
 (defn post
@@ -16,14 +15,9 @@
 
   [{:keys [ds body] :as _request}]
 
-  (let [{:keys [data error success]} (util/validate post body)
-        user (users/user ds {:where [:= :email (:email data)]})
-        {:keys [password confirm-password]} data]
+  (let [user (users/user ds {:where [:= :email (:email body)]})
+        {:keys [password confirm-password]} body]
     (cond
-
-      (not success) (-> (res/response error)
-                        (res/status 400))
-
       (not (= password confirm-password))
       (-> (res/response {:message "passwords do not match!"})
           (res/status 400))
@@ -34,7 +28,7 @@
 
       :else
       (let [pw (pw/hash-password password)
-            new-user (-> (assoc data
+            new-user (-> (assoc body
                                 :password pw
                                 :type "admin")
                          (dissoc :confirm-password))]
diff --git a/src/source/routes/analytics/creator/top.clj b/src/source/routes/analytics/creator/top.clj
index 5040bece..dd6a68b3 100644
--- a/src/source/routes/analytics/creator/top.clj
+++ b/src/source/routes/analytics/creator/top.clj
@@ -2,8 +2,7 @@
   (:require [source.services.analytics.interface :as analytics]
             [ring.util.response :as res]
             [clojure.set :as set]
-            [source.services.interface :as services]
-            [source.util :as utils]))
+            [source.services.interface :as services]))
 
 (defn record-names [ds top-field ids]
   (if (= top-field :post-id)
@@ -33,25 +32,20 @@
                             [:views :int]]]}}}
 
   [{:keys [ds user query-params] :as _request}]
-  (let [{:keys [data success error]} (utils/validate get query-params :query)
-        {:keys [n mindate maxdate top contenttype]} data
-        top-field (if (= top "post") :post-id :bundle-id)]
-    (if success
-      (let [results (->> {:creator-id (:id user)
-                          :content-type-id contenttype}
-                         (analytics/top-statistics-query ds mindate maxdate n top-field)
-                         (mapv (fn [result]
-                                 (set/rename-keys result {top-field :top}))))
-            ids (mapv :top results)
-            names (record-names ds top-field ids)
-            juxted (->> names
-                        (mapv (juxt :id :name))
-                        (into {}))
-            named-results (mapv (fn [{:keys [top] :as r}]
-                                  (assoc r :top (clojure.core/get juxted top (str top))))
-                                results)]
+  (let [{:keys [n mindate maxdate top contenttype]} query-params
+        top-field (if (= top "post") :post-id :bundle-id)
+        results (->> {:creator-id (:id user)
+                      :content-type-id contenttype}
+                     (analytics/top-statistics-query ds mindate maxdate n top-field)
+                     (mapv (fn [result]
+                             (set/rename-keys result {top-field :top}))))
+        ids (mapv :top results)
+        names (record-names ds top-field ids)
+        juxted (->> names
+                    (mapv (juxt :id :name))
+                    (into {}))
+        named-results (mapv (fn [{:keys [top] :as r}]
+                              (assoc r :top (clojure.core/get juxted top (str top))))
+                            results)]
 
-        (res/response named-results))
-
-      (-> (res/response error)
-          (res/status 400)))))
+    (res/response named-results)))
diff --git a/src/source/routes/analytics/creator/top_average.clj b/src/source/routes/analytics/creator/top_average.clj
index 4f50a396..8a63451a 100644
--- a/src/source/routes/analytics/creator/top_average.clj
+++ b/src/source/routes/analytics/creator/top_average.clj
@@ -1,7 +1,6 @@
 (ns source.routes.analytics.creator.top-average
   (:require [source.services.analytics.interface :as analytics]
-            [ring.util.response :as res]
-            [source.util :as utils]))
+            [ring.util.response :as res]))
 
 (defn get
   {:summary "Get the average engagement (clicks and views) for a creator, optionally filtered by content type.
@@ -13,12 +12,7 @@
    :responses {200 {:body [:map [:average :float]]}}}
 
   [{:keys [ds user query-params] :as _request}]
-  (let [{:keys [data success error]} (utils/validate get query-params :query)
-        {:keys [mindate maxdate contenttype]} data]
-    (if success
-      (let [result (analytics/average-engagement ds mindate maxdate {:creator-id (:id user)
-                                                                     :content-type-id contenttype})]
-        (res/response {:average result}))
-
-      (-> (res/response error)
-          (res/status 400)))))
+  (let [{:keys [mindate maxdate contenttype]} query-params
+        result (analytics/average-engagement ds mindate maxdate {:creator-id (:id user)
+                                                                 :content-type-id contenttype})]
+    (res/response {:average result})))
diff --git a/src/source/routes/analytics/distributor/top.clj b/src/source/routes/analytics/distributor/top.clj
index 4eed1c46..8e2ab29d 100644
--- a/src/source/routes/analytics/distributor/top.clj
+++ b/src/source/routes/analytics/distributor/top.clj
@@ -2,9 +2,7 @@
   (:require [source.services.analytics.interface :as analytics]
             [ring.util.response :as res]
             [clojure.set :as set]
-            [source.services.interface :as services]
-            [source.util :as utils]
-            [clojure.walk :as walk]))
+            [source.services.interface :as services]))
 
 (defn record-names [ds top-field ids]
   (if (= top-field :post-id)
@@ -28,27 +26,22 @@
                             [:views :int]]]}}}
 
   [{:keys [ds user query-params] :as _request}]
-  (let [{:keys [data success error]} (utils/validate get query-params :query)
-        {:keys [n mindate maxdate top contenttype]} data
-        top-field (if (= top "post") :post-id :feed-id)]
-    (if success
-      (let [results (->> {:distributor-id (:id user)
-                          :content-type-id contenttype}
-                         (analytics/top-statistics-query ds mindate maxdate n top-field)
-                         (mapv (fn [result]
-                                 (set/rename-keys result {top-field :top}))))
-            ids (mapv :top results)
-            names (record-names ds top-field ids)
-            juxted (->> names
-                        (mapv (fn [{:keys [id title]}]
-                                {:id id
-                                 :name title}))
-                        (mapv (juxt :id :name))
-                        (into {}))
-            named-results (mapv (fn [{:keys [top] :as r}]
-                                  (assoc r :top (clojure.core/get juxted top (str top))))
-                                results)]
-        (res/response named-results))
-
-      (-> (res/response error)
-          (res/status 400)))))
+  (let [{:keys [n mindate maxdate top contenttype]} query-params
+        top-field (if (= top "post") :post-id :feed-id)
+        results (->> {:distributor-id (:id user)
+                      :content-type-id contenttype}
+                     (analytics/top-statistics-query ds mindate maxdate n top-field)
+                     (mapv (fn [result]
+                             (set/rename-keys result {top-field :top}))))
+        ids (mapv :top results)
+        names (record-names ds top-field ids)
+        juxted (->> names
+                    (mapv (fn [{:keys [id title]}]
+                            {:id id
+                             :name title}))
+                    (mapv (juxt :id :name))
+                    (into {}))
+        named-results (mapv (fn [{:keys [top] :as r}]
+                              (assoc r :top (clojure.core/get juxted top (str top))))
+                            results)]
+    (res/response named-results)))
diff --git a/src/source/routes/analytics/distributor/top_average.clj b/src/source/routes/analytics/distributor/top_average.clj
index e4f09f08..b0778dbd 100644
--- a/src/source/routes/analytics/distributor/top_average.clj
+++ b/src/source/routes/analytics/distributor/top_average.clj
@@ -1,7 +1,6 @@
 (ns source.routes.analytics.distributor.top-average
   (:require [source.services.analytics.interface :as analytics]
-            [ring.util.response :as res]
-            [source.util :as utils]))
+            [ring.util.response :as res]))
 
 (defn get
   {:summary "Get the average engagement (clicks and views) for a distributor, optionally filtered by content type.
@@ -13,12 +12,7 @@
    :responses {200 {:body [:map [:average :float]]}}}
 
   [{:keys [ds user query-params] :as _request}]
-  (let [{:keys [data success error]} (utils/validate get query-params :query)
-        {:keys [mindate maxdate contenttype]} data]
-    (if success
-      (let [result (analytics/average-engagement ds mindate maxdate {:distributor-id (:id user)
-                                                                     :content-type-id contenttype})]
-        (res/response {:average result}))
-
-      (-> (res/response error)
-          (res/status 400)))))
+  (let [{:keys [mindate maxdate contenttype]} query-params
+        result (analytics/average-engagement ds mindate maxdate {:distributor-id (:id user)
+                                                                 :content-type-id contenttype})]
+    (res/response {:average result})))
diff --git a/src/source/routes/bundle_feeds.clj b/src/source/routes/bundle_feeds.clj
index 722227f6..d1944d41 100644
--- a/src/source/routes/bundle_feeds.clj
+++ b/src/source/routes/bundle_feeds.clj
@@ -4,7 +4,8 @@
             [source.db.util :as db.util]
             [clojure.walk :as walk]
             [honey.sql.helpers :as hsql]
-            [source.services.analytics.interface :as analytics]))
+            [source.services.analytics.interface :as analytics]
+            [source.db.honey :as hon]))
 
 (defn post
   {:summary "get all feeds present in the bundle authorised by uuid"
@@ -55,3 +56,10 @@
 
       (analytics/insert-feed-impressions! ds type-filtered bundle-id)
       (res/response type-filtered))))
+
+(comment
+  (def ds (db.util/conn :master))
+
+  
+
+  ())
diff --git a/src/source/routes/business.clj b/src/source/routes/business.clj
index 8df54a66..4a50cf38 100644
--- a/src/source/routes/business.clj
+++ b/src/source/routes/business.clj
@@ -1,7 +1,6 @@
 (ns source.routes.business
   (:require [source.services.businesses :as businesses]
-            [ring.util.response :as res]
-            [source.util :as utils]))
+            [ring.util.response :as res]))
 
 (defn post
   {:summary "insert a business"
@@ -14,13 +13,8 @@
    :responses {201 {:body [:map [:message :string]]}}}
 
   [{:keys [ds body] :as _request}]
-
-  (let [{:keys [data error success]} (utils/validate post body)]
-    (if (not success) (-> (res/response error)
-                          (res/status 400))
-
-        (do (businesses/insert-business! ds {:values data})
-            (res/response {:message "successfully added business"})))))
+  (businesses/insert-business! ds {:values body})
+  (res/response {:message "successfully added business"}))
 
 (defn patch
   {:summary "update a business by id"
@@ -35,16 +29,9 @@
    :responses {200 {:body [:map [:message :string]]}}}
 
   [{:keys [ds body path-params] :as _request}]
-
-  (let [{:keys [data error success]} (utils/validate patch body)]
-    (if (not success)
-
-      (-> (res/response error)
-          (res/status 400))
-
-      (do (businesses/update-business! ds {:id (:id path-params)
-                                           :values data})
-          (res/response {:message "successfully updated business"})))))
+  (businesses/update-business! ds {:id (:id path-params)
+                                   :values body})
+  (res/response {:message "successfully updated business"}))
 
 (comment
   (require '[source.db.util :as db.util])
@@ -52,4 +39,3 @@
          :body {:name "modulr"
                 :url "https://modulr.com"}})
   ())
-
diff --git a/src/source/routes/login.clj b/src/source/routes/login.clj
index e45452f1..fb07a102 100644
--- a/src/source/routes/login.clj
+++ b/src/source/routes/login.clj
@@ -2,8 +2,7 @@
   (:require [source.services.auth :as auth]
             [ring.util.response :as res]
             [source.services.users :as users]
-            [source.password :as pw]
-            [source.util :as util]))
+            [source.password :as pw]))
 
 (defn post
   {:summary "get user data and access token provided valid credentials"
@@ -29,19 +28,15 @@
 
   [{:keys [ds body] :as _request}]
 
-  (let [{:keys [data error success]} (util/validate post body)
-        {:keys [email password]} data
+  (let [{:keys [email password]} body
         user (users/user ds {:where [:= :email email]})]
 
-    (cond
-      (not success) (-> (res/response error)
-                        (res/status 400))
-
-      (or (not (some? user))
-          (not (pw/verify-password password (:password user))))
+    (if
+     (or (not (some? user))
+         (not (pw/verify-password password (:password user))))
       {:status 401 :body {:message "Invalid username or password!"}}
 
-      :else (res/response (auth/login ds {:user user})))))
+      (res/response (auth/login ds {:user user})))))
 
 (comment
   (require '[source.db.interface :as db])
diff --git a/src/source/routes/me.clj b/src/source/routes/me.clj
index 154ed1d4..e53ce3e6 100644
--- a/src/source/routes/me.clj
+++ b/src/source/routes/me.clj
@@ -39,11 +39,6 @@
                400 {:body [:map [:message :string]]}}}
 
   [{:keys [ds user body] :as _request}]
-  (let [{:keys [data error success]} (util/validate post body)]
-    (if (not success)
-      (-> (res/response {:message error})
-          (res/status 400))
-      (do (services/update-user! ds {:id (:id user)
-                                     :data data})
-          (res/response {:message "successfully updated user"})))))
-
+  (services/update-user! ds {:id (:id user)
+                             :data body})
+  (res/response {:message "successfully updated user"}))
diff --git a/src/source/routes/me_business.clj b/src/source/routes/me_business.clj
index 6f3b1ad5..ffd21d25 100644
--- a/src/source/routes/me_business.clj
+++ b/src/source/routes/me_business.clj
@@ -1,6 +1,5 @@
 (ns source.routes.me-business
-  (:require [source.util :as util]
-            [ring.util.response :as res]
+  (:require [ring.util.response :as res]
             [source.services.interface :as services]))
 
 (defn get
@@ -38,19 +37,14 @@
                400 {:body [:map [:message :string]]}}}
 
   [{:keys [ds user body] :as _request}]
-  (let [{:keys [data error success]} (util/validate post body)]
-    (if (not success)
-      (-> (res/response {:message error})
-          (res/status 400))
-
-      (let [{:keys [business-id]} (services/user ds {:id (:id user)})
-            business (when (nil? business-id)
-                       (services/insert-business! ds {:data data
-                                                      :ret :1}))]
-        (if (nil? business-id)
-          (services/update-user! ds {:id (:id user)
-                                     :data {:business-id (:id business)}})
-          (services/update-business! ds {:id business-id
-                                         :data data}))
+  (let [{:keys [business-id]} (services/user ds {:id (:id user)})
+        business (when (nil? business-id)
+                   (services/insert-business! ds {:data body
+                                                  :ret :1}))]
+    (if (nil? business-id)
+      (services/update-user! ds {:id (:id user)
+                                 :data {:business-id (:id business)}})
+      (services/update-business! ds {:id business-id
+                                     :data body}))
 
-        (res/response {:message "successfully added or updated business"})))))
+    (res/response {:message "successfully added or updated business"})))
diff --git a/src/source/routes/openapi.clj b/src/source/routes/openapi.clj
new file mode 100644
index 00000000..065e8a0c
--- /dev/null
+++ b/src/source/routes/openapi.clj
@@ -0,0 +1,167 @@
+(ns source.routes.openapi
+  (:require [malli.util :as mu]))
+
+(defn response-schema
+  ([] (response-schema nil))
+  ([data-schema]
+   (vec
+    (cond-> [:map [:message :string]]
+      (some? data-schema)
+      (concat [[:data data-schema]])))))
+
+(defn error
+  "A function that wraps an optional error data schema in a standard error message schema"
+  ([] (error nil))
+  ([data-schema]
+   (response-schema data-schema)))
+
+(defn- assoc-response [acc [key schema]]
+  (assoc acc key {:body schema}))
+
+(defn- assoc-param [acc [key schema]]
+  (assoc acc key schema))
+
+;; what i want: (api/response {} 200 (api/success)  404 (api/error)...) => {200 {:body openapi/VocabSearchResult}}
+(defn response
+  [& opts]
+  (let [map-first? (map? (first opts))
+        responses (if map-first? (first opts) {})
+        opts (if map-first? (rest opts) opts)]
+    (merge responses (reduce assoc-response {} (partition 2 opts)))))
+
+(defn success
+  "Returns a map that can be used as the responses field 
+   for an openapi handler meta data. Optionally takes an existing
+   openapi responses map as the first param and updates its 200 field
+   and then returns it."
+  ([schema] (success {} schema))
+  ([responses data-schema]
+   (response (if (map? responses) responses {})
+             200
+             data-schema)))
+
+(defn not-found
+  "Retuns a map that can be used as the responses field for an
+   openapi handler meta data"
+  ([] (not-found {} nil))
+  ([schema-or-responses]
+   (let [responses? (map? schema-or-responses)
+         schema? (not responses?)
+         responses (if responses? schema-or-responses nil)
+         schema (if schema? schema-or-responses nil)]
+     (not-found responses schema)))
+  ([responses data-schema]
+   (response (if (map? responses) responses {})
+             404 (error data-schema))))
+
+(defn bad-request
+  "Retuns a map that can be used as the responses field for an
+   openapi handler meta data"
+  ([] (bad-request {} nil))
+  ([schema-or-responses]
+   (let [responses? (map? schema-or-responses)
+         schema? (not responses?)
+         responses (if responses? schema-or-responses nil)
+         schema (if schema? schema-or-responses nil)]
+     (bad-request responses schema)))
+  ([responses data-schema]
+   (response (if (map? responses) responses {})
+             400 (error data-schema))))
+
+(defn unauthenticated
+  ([schema] (unauthenticated {} schema))
+  ([responses schema]
+   (response (if (map? responses) responses {})
+             401 (error schema))))
+
+(defn unauthorized
+  ([schema] (unauthorized {} schema))
+  ([responses schema]
+   (response (if (map? responses) responses {})
+             401 (error schema))))
+
+(defn params
+  "Returns a map of the openapi parameter schemas path, body, query.
+   Optionally accepts an openapi parameters map as the first argument
+   updating it and returning the result."
+  [& opts]
+  (let [map-first? (map? (first opts))
+        parameters (if map-first? (first opts) {})
+        opts (if map-first? (rest opts) opts)]
+    (merge parameters (reduce assoc-param {} (partition 2 opts)))))
+
+(defn- sometimes-entry [[k _ s]] [k {:optional true} [:maybe s]])
+(defn- maybe-keys [schema]
+  (mu/transform-entries
+   schema
+   #(mapv sometimes-entry %)))
+
+;; MALLI SCHEMAS
+
+(defn optional [key type]
+  [key {:optional true} type])
+
+(defn maybe [type]
+  [:maybe type])
+
+(defn sometimes [key type]
+  (optional key (maybe type)))
+
+(def RegisterParams
+  [:map
+   [:email :string]
+   [:password :string]
+   [:confirm-password :string]
+   [:type [:enum "creator" "distributor"]]])
+
+(def LoginParams
+  [:map
+   [:email :string]
+   [:password :string]])
+
+(def InsertVocab
+  [:map
+   [:xhosa :string]
+   [:english :string]
+   (sometimes :illustration :string)
+   (sometimes :noun-class :string)
+   [:type :string]])
+
+(def InsertVocabParams
+  [:vector InsertVocab])
+
+(def CreateUnitParam
+  [:map
+   [:name :string]
+   [:description :string]
+   [:level :int]])
+
+(def CreateUnitsParam
+  [:vector CreateUnitParam])
+
+(def AnswerParam
+  [:or [:vector :string]
+   :string])
+
+(def AnswerParams
+  [:vector AnswerParam])
+
+(def ExerciseParam
+  [:map
+   [:question-type [:enum ["translation" "multiple-choice"]]]
+   [:question :string]
+   [:options [:vector :string]]
+   (sometimes :answers AnswerParams)])
+
+(def ExerciseParams
+  [:vector ExerciseParam])
+
+(def GetExerciseResult
+  [:map
+   [:id :int]
+   [:unit-id :int]
+   [:question-type [:enum ["translation" "multiple-choice"]]]
+   [:question :string]
+   [:options [:vector :string]]
+   [:answers AnswerParams]])
+
diff --git a/src/source/routes/register.clj b/src/source/routes/register.clj
index 505cc5ab..69d24328 100644
--- a/src/source/routes/register.clj
+++ b/src/source/routes/register.clj
@@ -1,7 +1,6 @@
 (ns source.routes.register
   (:require [source.services.interface :as services]
-            [ring.util.response :as res]
-            [source.util :as util]))
+            [ring.util.response :as res]))
 
 (defn post
   {:summary "register a new user"
@@ -28,14 +27,9 @@
 
   [{:keys [ds body] :as _request}]
 
-  (let [{:keys [data error success]} (util/validate post body)
-        {:keys [email password confirm-password]} data
+  (let [{:keys [email password confirm-password]} body
         existing-user (services/user ds {:where [:= :email email]})]
     (cond
-
-      (not success) (-> (res/response error)
-                        (res/status 400))
-
       (not (= password confirm-password))
       (-> (res/response {:error "Passwords do not match!"}))
 
@@ -43,7 +37,7 @@
       (-> (res/response {:error "An account for this email already exists!"}))
 
       :else
-      (-> (services/register ds data)
+      (-> (services/register ds body)
           (res/response)))))
 
 (comment
diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index 922a9670..11f61ec5 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -1,14 +1,10 @@
 (ns source.routes.reitit
   (:require [reitit.ring :as ring]
-            [reitit.swagger :as swagger]
-            [reitit.swagger-ui :as swagger-ui]
-            [reitit.openapi :as openapi]
             [reitit.coercion.malli]
             [reitit.ring.malli]
-            [malli.util :as mu]
             [source.middleware.interface :as mw]
             [clojure.data.json :as json]
-            [source.util :as util]
+            [source.routes.util :refer [get patch post delete] :as rutil]
             [source.routes.user :as user]
             [source.routes.users :as users]
             [source.routes.me :as me]
@@ -81,270 +77,223 @@
             [source.routes.approve-feed :as approve-feed]
             [source.routes.reject-feed :as reject-feed]))
 
-(defn route [handlers]
-  (reduce (fn [acc [k v]]
-            (let [{:keys [middleware summary parameters responses]} (util/metadata v)]
-              (merge acc {k {:middleware middleware
-                             :summary summary
-                             :parameters parameters
-                             :responses responses
-                             :handler v}})))
-          {} handlers))
-
 (defn create-app [{:keys [ds store js]}]
   (ring/ring-handler
    (ring/router
-    [["/swagger.json"   {:get {:no-doc true
-                               :swagger {:info {:title "source-api"
-                                                :description "swagger docs for source api with malli and reitit-ring"
-                                                :version "0.0.1"}
-                                         :securityDefinitions {"auth" {:type :apiKey
-                                                                       :in :header
-                                                                       :name "Authorization"}
-                                                               "apiKey" {:type :apiKey
-                                                                         :in :header
-                                                                         :name "Authorization"}}}
-                               :handler (swagger/create-swagger-handler)}}]
-
-     ["/openapi.json"   {:get {:no-doc true
-                               :openapi {:info {:title "source-api"
-                                                :description "openapi3 docs for source api with malli and reitit-ring"
-                                                :version "0.0.1"}
-                                         :components {:securitySchemes {"bearerAuth" {:type :http
-                                                                                      :scheme :bearer
-                                                                                      :bearerFormat "JWT"
-                                                                                      :description "JWT Authorization using the Bearer scheme"}
-                                                                        "apiKey" {:type :http
-                                                                                  :scheme :bearer
-                                                                                  :description "API Key authorization using the Bearer scheme"}}}}
-                               :handler (openapi/create-openapi-handler)}}]
+    [(rutil/swagger-route)
+     (rutil/openapi-route)
 
      ["/users"          {:middleware [[mw/apply-auth {:required-type :admin}]]
                          :tags #{"users"}
                          :swagger {:security [{"auth" []}]}
                          :openapi {:security [{:bearerAuth []}]}}
-      [""               (route {:get users/get})]
-      ["/:id"           (route {:get user/get
-                                :patch user/patch})]]
+      [""               (-> (get users/get))]
+      ["/:id"           (-> (get user/get)
+                            (patch user/patch))]]
 
      ["/me"             {:middleware [[mw/apply-auth]]
                          :tags #{"me"}
                          :swagger {:security [{"auth" []}]}
                          :openapi {:security [{:bearerAuth []}]}}
-      [""               (route {:get me/get
-                                :post me/post})]
-      ["/business"      (route {:get me-business/get
-                                :post me-business/post})]
-      ["/sectors"       (route {:get me-sectors/get
-                                :post me-sectors/post})]]
+      [""               (-> (get me/get)
+                            (post me/post))]
+      ["/business"      (-> (get me-business/get)
+                            (post me-business/post))]
+      ["/sectors"       (-> (get me-sectors/get)
+                            (post me-sectors/post))]]
 
      ["/mail"             {:middleware [[mw/apply-auth]]
                            :tags #{"mail"}
                            :swagger {:security [{"auth" []}]}
                            :openapi {:security [{:bearerAuth []}]}}
-      ["/report"          (route {:post report/post})]]
+      ["/report"          (-> (post report/post))]]
 
      ["/businesses"     {:middleware [[mw/apply-auth {:required-type :admin}]]
                          :tags #{"businesses"}
                          :swagger {:security [{"auth" []}]}
                          :openapi {:security [{:bearerAuth []}]}}
-      [""               (route {:get businesses/get
-                                :post business/post})]
-      ["/:id"           (route {:patch business/patch})]]
+      [""               (-> (get businesses/get)
+                            (post business/post))]
+      ["/:id"           (-> (patch business/patch))]]
 
      ["/business"       {:tags #{"businesses"}}
-      ["/types"         (route {:get business-types/get})]]
+      ["/types"         (-> (get business-types/get))]]
 
      ["/sectors"        {:tags #{"sectors"}}
-      [""               (route {:get sectors/get})]]
+      [""               (-> (get sectors/get))]]
 
      ["/login"          {:tags #{"auth"}}
-      [""               (route {:post login/post})]]
+      [""               (-> (post login/post))]]
 
      ["/register"       {:tags #{"auth"}}
-      [""               (route {:post register/post})]]
+      [""               (-> (post register/post))]]
 
      ["/oauth2"
       ["/google"        {:tags #{"google"}}
-       [""              (route {:get google-launch/get})]
-       ["/callback"     {:no-doc true
-                         :get google-redirect/get}]
-       ["/user"         (route {:get google-user/get})]]]
+       [""              (-> (get google-launch/get))]
+       ["/callback"     {:no-doc true}
+        [""             (-> (get google-redirect/get))]]
+       ["/user"         (-> (get google-user/get))]]]
 
      ["/protected"      {:middleware [[mw/apply-auth]]
                          :tags #{"protected"}
                          :swagger {:security [{"auth" []}]}
                          :openapi {:security [{:bearerAuth []}]}}
-      ["/authorized"    (route {:get authorized/get})]]
+      ["/authorized"    (-> (get authorized/get))]]
 
      ["/providers"      {:tags #{"providers"}}
-      [""               (route {:get providers/get})]
-      ["/:id"           (route {:get provider/get})]]
+      [""               (-> (get providers/get))]
+      ["/:id"           (-> (get provider/get))]]
 
      ["/cadences"       {:tags #{"cadences"}}
-      [""               (route {:get cadences/get})]]
+      [""               (-> (get cadences/get))]]
 
      ["/categories"     {:tags #{"categories"}}
-      [""               (route {:get categories/get})]
-      ["/:id"           (route {:get category/get})]]
+      [""               (-> (get categories/get))]
+      ["/:id"           (-> (get category/get))]]
 
      ["/baselines"      {:tags #{"baselines"}}
-      [""               (route {:get baselines/get})]]
+      [""               (-> (get baselines/get))]]
 
      ["/contentTypes"  {:tags #{"content types"}}
-      [""               (route {:get content-types/get})]
-      ["/:id"           (route {:get content-type/get})]]
+      [""               (-> (get content-types/get))]
+      ["/:id"           (-> (get content-type/get))]]
 
      ["/integrations"   {:middleware [[mw/apply-auth]]
                          :tags #{"integrations"}}
-      [""               (route {:get integrations/get
-                                :post integrations/post})]
+      [""               (->  (get integrations/get)
+                             (post integrations/post))]
       ["/:id"
-       [""              (route {:get integration/get
-                                :post integration/post
-                                :delete integration/delete})]
-       ["/key"          (route {:post integration-key/post})]
-       ["/categories"   (route {:get integration-categories/get
-                                :post integration-categories/post})]
+       [""              (-> (get integration/get)
+                            (post integration/post)
+                            (delete integration/delete))]
+       ["/key"          (-> (post integration-key/post))]
+       ["/categories"   (->  (get integration-categories/get)
+                             (post integration-categories/post))]
        ["/filter"
         ["/feeds"
-         [""            (route {:get integration-filter-feeds/get})]
-         ["/:feed-id"   (route {:get integration-filter-feed/get
-                                :post integration-filter-feed/post})]]
+         [""            (-> (get integration-filter-feeds/get))]
+         ["/:feed-id"   (->  (get integration-filter-feed/get)
+                             (post integration-filter-feed/post))]]
         ["/posts"
-         [""            (route {:get integration-filter-posts/get})]
-         ["/:post-id"   (route {:get integration-filter-post/get
-                                :post integration-filter-post/post})]]]]]
+         [""            (-> (get integration-filter-posts/get))]
+         ["/:post-id"   (-> (get integration-filter-post/get)
+                            (post integration-filter-post/post))]]]]]
 
      ["/feeds"          {:middleware [[mw/apply-auth]]
                          :tags #{"feeds"}}
-      [""               (route {:get feeds/get
-                                :post feeds/post})]
+      [""               (-> (get feeds/get)
+                            (post feeds/post))]
       ["/:id"
-       [""              (route {:get feed/get
-                                :post feed/post
-                                :delete feed/delete})]
+       [""              (-> (get feed/get)
+                            (post feed/post)
+                            (delete feed/delete))]
        ["/posts"
-        [""             (route {:get posts/get})]
+        [""             (-> (get posts/get))]
         ["/:post-id"
-         [""            (route {:get post/get})]
-         ["/prune"      (route {:post post-prune/post})]]]
-       ["/categories"   (route {:get feed-categories/get
-                                :post feed-categories/post})]]]
+         [""            (-> (get post/get))]
+         ["/prune"      (-> (post post-prune/post))]]]
+       ["/categories"   (-> (get feed-categories/get)
+                            (post feed-categories/post))]]]
 
      ["/analytics"      {:tags #{"analytics"}}
       ["/creator"       {:middleware [[mw/apply-auth {:required-type :creator}]]}
-       ["/general"      (route {:get analytics-creator-general/get})]
-       ["/deltas"       (route {:get analytics-creator-deltas/get})]
+       ["/general"      (-> (get analytics-creator-general/get))]
+       ["/deltas"       (-> (get analytics-creator-deltas/get))]
        ["/top"
-        [""             (route {:get analytics-creator-top/get})]
-        ["/average"     (route {:get analytics-creator-top-average/get})]]]
+        [""             (-> (get analytics-creator-top/get))]
+        ["/average"     (-> (get analytics-creator-top-average/get))]]]
       ["/distributor"   {:middleware [[mw/apply-auth {:required-type :distributor}]]}
-       ["/general"      (route {:get analytics-distributor-general/get})]
+       ["/general"      (-> (get analytics-distributor-general/get))]
        ["/top"
-        [""             (route {:get analytics-distributor-top/get})]
-        ["/average"     (route {:get analytics-distributor-top-average/get})]]]
+        [""             (-> (get analytics-distributor-top/get))]
+        ["/average"     (-> (get analytics-distributor-top-average/get))]]]
       ["/bundle"        {:middleware [[mw/apply-bundle]]}
        ["/posts"
         ["/:id"
-         ["/views"      (route {:post analytics-bundle-posts-id-views/post})]]]]
+         ["/views"      (-> (post analytics-bundle-posts-id-views/post))]]]]
       ["admin"          {:middleware [[mw/apply-auth {:required-type :admin}]]}
        ["/general"]
        ["/top"]]]
 
      ["/bundle"        {:middleware [[mw/apply-bundle]]
                         :tags #{"bundles"}}
-      [""               (route {:get bundle/get})]
-      ["/categories"
-       [""              (route {:get bundle-categories/get})]]
-      ["/feeds"
-       [""              (route {:post bundle-feeds/post})]
-       ["/:id"
-        [""             (route {:get bundle-feed/get})]
-        ["/posts"
-         [""            (route {:get bundle-feed-posts/get})]
-         ["/:post-id"   (route {:get bundle-feed-post/get})]]]]
-      ["/posts"
-       [""              (route {:post bundle-posts/post})]
-       ["/:id"          (route {:get bundle-post/get})]]]
+
+      ["" (get bundle/get)]
+      ["/categories" (get bundle-categories/get)]
+      ["/feeds" (post bundle-feeds/post)]
+      ["/feeds/:id" (get bundle-feed/get)]
+      ["/feeds/:id/posts" (get bundle-feed-posts/get)]
+      ["/feeds/:id/posts/:post-id" (get bundle-feed-post/get)]
+      ["/posts" (post bundle-posts/post)]
+      ["/posts/:id" (get bundle-post/get)]]
 
      ["/api"             {:middleware [[mw/apply-api-key]]
                           :tags #{"api"}
                           :swagger {:security [{"apiKey" []}]}
                           :openapi {:security [{:apiKey []}]}}
       ["/bundle"         {:middleware [[mw/apply-bundle]]}
-       [""               (route {:get bundle/get})]
+       [""               (-> (get bundle/get))]
        ["/categories"
-        [""              (route {:get bundle-categories/get})]]
+        [""              (-> (get bundle-categories/get))]]
        ["/feeds"
-        [""              (route {:post bundle-feeds/post})]
+        [""              (-> (post bundle-feeds/post))]
         ["/:id"
-         [""             (route {:get bundle-feed/get})]
+         [""             (-> (get bundle-feed/get))]
          ["/posts"
-          [""            (route {:get bundle-feed-posts/get})]
-          ["/:post-id"   (route {:get bundle-feed-post/get})]]]]
+          [""            (-> (get bundle-feed-posts/get))]
+          ["/:post-id"   (-> (get bundle-feed-post/get))]]]]
        ["/posts"
-        [""              (route {:post bundle-posts/post})]
-        ["/:id"          (route {:get bundle-post/get})]]]
+        [""              (-> (post bundle-posts/post))]
+        ["/:id"          (-> (get bundle-post/get))]]]
       ["/categories"
-       [""               (route {:get categories/get})]
-       ["/:id"           (route {:get category/get})]]]
+       [""               (-> (get categories/get))]
+       ["/:id"           (-> (get category/get))]]]
 
      ["/admin"                  {:middleware [[mw/apply-auth {:required-type :admin}]]
                                  :tags #{"admin"}
                                  :swagger {:security [{"auth" []}]}
                                  :openapi {:security [{:bearerAuth []}]}}
       ["/business"
-       ["/types"                (route {:post business-types/post
-                                        :patch business-types/patch
-                                        :delete business-types/delete})]]
+       ["/types"                (-> (post business-types/post)
+                                    (patch business-types/patch)
+                                    (delete business-types/delete))]]
       ["/feeds"
-       [""                      (route {:get admin-feeds/get})]
+       [""                      (-> (get admin-feeds/get))]
        ["/:id"
-        ["/approve"             (route {:post approve-feed/post})]
-        ["/reject"              (route {:post reject-feed/post})]]]
+        ["/approve"             (-> (post approve-feed/post))]
+        ["/reject"              (-> (post reject-feed/post))]]]
       ["/jobs"
-       [""                      {:get jobs/get}]
+       [""                      (-> (get jobs/get))]
        ["/manage"
-        ["/view"                {:get jobs-view/get}]
-        ["/register"            {:post jobs/post}]]
+        ["/view"                (-> (get jobs-view/get))]
+        ["/register"            (-> (post jobs/post))]]
        ["/:id"
-        [""                     {:get job/get}]
+        [""                     (-> (get job/get))]
         ["/manage"
-         ["/deregister"         {:get job-deregister/get}]
-         ["/start"              {:get job-start/get}]
-         ["/stop"               {:get job-stop/get}]]]]
-      ["/add-admin"             (route {:post admin/post})]
+         ["/deregister"         (-> (get job-deregister/get))]
+         ["/start"              (-> (get job-start/get))]
+         ["/stop"               (-> (get job-stop/get))]]]]
+      ["/add-admin"             (-> (post admin/post))]
       ["/selection-schemas"
-       [""                      {:get selection-schemas/get
-                                 :post selection-schemas/post}]
-       ["/:id"                  {:get selection-schema/get}]
-       ["/providers/:id"        {:get provider-selection-schemas/get}]]
+       [""                      (-> (get selection-schemas/get)
+                                    (post selection-schemas/post))]
+       ["/:id"                  (-> (get selection-schema/get))]
+       ["/providers/:id"        (-> (get provider-selection-schemas/get))]]
       ["/output-schemas"
-       [""                      {:get output-schemas/get
-                                 :post output-schemas/post}]
-       ["/:id"                  {:get output-schema/get}]]
+       [""                      (-> (get output-schemas/get)
+                                    (post output-schemas/post))]
+       ["/:id"                  (-> (get output-schema/get))]]
       ["/providers"
-       [""                      (route {:post providers/post})]
-       ["/:id"                  (route {:post provider/post
-                                        :delete provider/delete})]]
-      ["/ast"                   {:post xml/post}]
-      ["/extract-data"          {:post data/post}]]]
-
-    {:data {:coercion (reitit.coercion.malli/create
-                       {:error-keys #{#_:type :coercion :in :schema :value :errors :humanized #_:transformed}
-                        :compile mu/closed-schema
-                        :strip-extra-keys true
-                        :default-values true
-                        :options nil})
-            :middleware [[mw/apply-generic :ds ds :store store :js js]]}})
+       [""                      (-> (post providers/post))]
+       ["/:id"                  (-> (post provider/post)
+                                    (delete provider/delete))]]
+      ["/ast"                   (-> (post xml/post))]
+      ["/extract-data"          (-> (post data/post))]]]
+
+    (rutil/data-map ds store js))
    (ring/routes
-    (swagger-ui/create-swagger-ui-handler {:path "/"
-                                           :config {:validatorUrl nil
-                                                    :urls [{:name "swagger", :url "swagger.json"}
-                                                           {:name "openapi", :url "openapi.json"}]
-                                                    :urls.primaryName "swagger"
-                                                    :operationsSorter "alpha"}})
+    (rutil/swagger-ui-handler)
     (ring/create-default-handler))))
 
 (comment
diff --git a/src/source/routes/user.clj b/src/source/routes/user.clj
index 413ddd48..26b3e09f 100644
--- a/src/source/routes/user.clj
+++ b/src/source/routes/user.clj
@@ -1,7 +1,6 @@
 (ns source.routes.user
   (:require [source.services.interface :as services]
-            [ring.util.response :as res]
-            [source.util :as util]))
+            [ring.util.response :as res]))
 
 (defn get
   {:summary "get user by id"
@@ -61,17 +60,10 @@
                403 {:body [:map [:message :string]]}}}
 
   [{:keys [ds body path-params] :as _request}]
-
-  (let [{:keys [data error success]} (util/validate patch body)]
-    (if (not success)
-
-      (-> (res/response error)
-          (res/status 400))
-
-      (do (services/update-user! ds
-                                 {:id (:id path-params)
-                                  :values data})
-          (res/response {:message "successfully updated user"})))))
+  (services/update-user! ds
+                         {:id (:id path-params)
+                          :values body})
+  (res/response {:message "successfully updated user"}))
 
 (comment
   (require '[source.db.interface :as db])
diff --git a/src/source/routes/util.clj b/src/source/routes/util.clj
new file mode 100644
index 00000000..9f2ab345
--- /dev/null
+++ b/src/source/routes/util.clj
@@ -0,0 +1,105 @@
+(ns source.routes.util
+  (:require [source.util :as util]
+            [reitit.swagger :as swagger]
+            [reitit.swagger-ui :as swagger-ui]
+            [reitit.openapi :as openapi]
+            [reitit.coercion.malli :as coercion]
+            [source.middleware.interface :as mw]
+            [malli.util :as mu]
+            [source.routes.openapi :as api]))
+
+(defn- extract-openapi-meta [handler]
+  (-> (util/metadata handler)
+      (dissoc :arglists :line :column :file :name :ns)))
+
+(defn- attach-handler [handler validation-mw openapi-meta]
+  (cond-> openapi-meta
+    (some? (:parameters openapi-meta))
+    (assoc :middleware [[validation-mw openapi-meta]]
+           :responses (merge (:responses openapi-meta)
+                             (api/bad-request)))
+    true (assoc :handler handler)))
+
+(defn- merge-route-map [validation-mw acc [method handler]]
+  (->> (extract-openapi-meta handler)
+       (attach-handler handler validation-mw)
+       (assoc {} method)
+       (merge acc)))
+
+(defn- parse-route-opts [& opts]
+  (let [map-first? (map? (first opts))
+        route-map (if map-first? (first opts) {})
+        opts (if map-first? (rest opts) opts)]
+    [route-map (vec opts)]))
+
+(defn route
+  [& opts]
+  (let [[route-map opts] (apply parse-route-opts opts)]
+    (->> (partition 2 opts)
+         (reduce (partial merge-route-map mw/apply-validation) {})
+         (merge route-map))))
+
+(defn- resolve-route-map [method]
+  (fn
+    ([handler] (route {} method handler))
+    ([route-map handler]
+     (when (not (map? route-map))
+       (throw (ex-info "Invalid argument for resolve-route-map: route-map must be a map"
+                       {:panic? "not really"})))
+     (route route-map method handler))))
+
+(defn get [& opts]
+  (apply (resolve-route-map :get) (vec opts)))
+
+(defn post [& opts]
+  (apply (resolve-route-map :post) (vec opts)))
+
+(defn patch [& opts]
+  (apply (resolve-route-map :patch) (vec opts)))
+
+(defn delete [& opts]
+  (apply (resolve-route-map :delete) (vec opts)))
+
+(defn swagger-ui-handler []
+  (swagger-ui/create-swagger-ui-handler {:path "/"
+                                         :config {:validatorUrl nil
+                                                  :urls [{:name "swagger", :url "swagger.json"}
+                                                         {:name "openapi", :url "openapi.json"}]
+                                                  :urls.primaryName "swagger"
+                                                  :operationsSorter "alpha"}}))
+
+(defn swagger-route []
+  ["/swagger.json" {:get {:no-doc true
+                          :swagger {:info {:title "source-api"
+                                           :description "swagger docs for source api with malli and reitit-ring"
+                                           :version "0.0.1"}
+                                    :securityDefinitions {"auth" {:type :apiKey
+                                                                  :in :header
+                                                                  :name "Authorization"}
+                                                          "apiKey" {:type :apiKey
+                                                                    :in :header
+                                                                    :name "Authorization"}}}
+                          :handler (swagger/create-swagger-handler)}}])
+
+(defn openapi-route []
+  ["/openapi.json" {:get {:no-doc true
+                          :openapi {:info {:title "source-api"
+                                           :description "openapi3 docs for source api with malli and reitit-ring"
+                                           :version "0.0.1"}
+                                    :components {:securitySchemes {"bearerAuth" {:type :http
+                                                                                 :scheme :bearer
+                                                                                 :bearerFormat "JWT"
+                                                                                 :description "JWT Authorization using the Bearer scheme"}
+                                                                   "apiKey" {:type :http
+                                                                             :scheme :bearer
+                                                                             :description "API Key authorization using the Bearer scheme"}}}}
+                          :handler (openapi/create-openapi-handler)}}])
+
+(defn data-map [ds store js]
+  {:data {:coercion (reitit.coercion.malli/create
+                     {:error-keys #{#_:type :coercion :in :schema :value :errors :humanized #_:transformed}
+                      :compile mu/closed-schema
+                      :strip-extra-keys true
+                      :default-values true
+                      :options nil})
+          :middleware [[mw/apply-generic :ds ds :store store :js js]]}})
diff --git a/src/source/util.clj b/src/source/util.clj
index 2240eb1d..49d8062f 100644
--- a/src/source/util.clj
+++ b/src/source/util.clj
@@ -4,7 +4,8 @@
             [clojure.main :refer [demunge]]
             [malli.core :as m]
             [malli.error :as me]
-            [malli.transform :as mt])
+            [malli.transform :as mt]
+            [clojure.string :as string])
   (:import (java.math BigInteger)
            (java.security MessageDigest)))
 
@@ -54,18 +55,54 @@
         hash-bytes (.digest digest bytes)]
     (format "%064x" (BigInteger. 1 hash-bytes))))
 
+(defn parse-type-from-schema [schema]
+  (cond
+    (keyword? schema)
+    (name schema)
+
+    (and (seq schema) (= (first schema) :vector))
+    (try
+      (str "Array[" (string/join " " (mapv name (rest schema))) "]")
+      (catch Exception _ "Array[]"))
+
+    (and (seq schema) (= (first schema) :map))
+    (try
+      (str "Object { " (reduce (fn [acc pair]
+                                 (str acc (string/join ": " (mapv name pair)) "; "))
+                               "" (rest schema)) "}")
+      (catch Exception _ "Object {}"))))
+
+(defn append-humanised-error [acc error-map]
+  (let [path (string/join "." (mapv (fn [p]
+                                      (if (keyword? p)
+                                        (name p)
+                                        (str "[" p "]"))) (:path error-map)))
+        k (if (or (nil? path) (= path "")) "" (str path ": "))
+        value (:value error-map)
+        value (if (nil? value) "null" (:value error-map))
+        schema (m/form (:schema error-map))
+        error-type (cond
+                     (= (:type error-map) :malli.core/missing-key) :missing
+                     (= (:type error-map) :malli.core/invalid-type) :invalid
+                     :else :type-error)
+        expected (parse-type-from-schema schema)]
+    (cond
+      (= error-type :missing) (str acc "Missing required key: '" path "'\n")
+      (= error-type :invalid) (str acc k "Expected '" expected "', found '" value "'\n")
+      :else (str acc k "Expected '" expected "', found '" value "'\n"))))
+
+(defn humanise [{:keys [errors] :as _error}]
+  (reduce append-humanised-error "" errors))
+
 (defn validate
-  ([handler data]
-   (validate handler data :body))
-  ([handler data schema-type]
-   (let [schema (get-in (metadata handler) [:parameters schema-type])
-         transformed (m/decode schema data mt/string-transformer)
-         success (m/validate schema transformed)]
-     {:data (when success transformed)
-      :success success
-      :error (when-not success (->> transformed
-                                    (m/explain schema)
-                                    (me/humanize)))})))
+  [data schema]
+  (let [transformed (m/decode schema data mt/string-transformer)
+        success (m/validate schema transformed)]
+    {:data (when success transformed)
+     :success success
+     :error (when-not success (->> transformed
+                                   (m/explain schema)
+                                   (humanise)))}))
 
 (defn format-rss-date
   "Takes a date as a string in RFC 1123 format and returns it in a format that meets ISO 8601 standards for SQLite.
@@ -80,10 +117,20 @@
     (catch Exception _ s)))
 
 (comment
-  (require '[source.routes.business :as business])
-  (require '[source.routes.analytics.creator.top :as ctop])
-  (validate business/post {:cheese "modulr"})
-  (validate ctop/get {:mindate "2025-12-02" :maxdate "2025-12-02" :n "10" :contenttype 1 :top "post"} :query)
   (sha256 "1")
-  ())
 
+  (validate {:message "yeet"
+             :b 1
+             :array {}
+             :test []
+             :units [{:name "cheese"}]}
+            [:map
+             [:message :string]
+             [:a :int]
+             [:b :string]
+             [:array [:vector :int]]
+             [:test [:map [:a :int]]]
+             [:units [:vector [:map
+                               [:name [:maybe :string]]
+                               [:description [:maybe :string]]]]]])
+  ())

From 4aa5170243f3b079e2de04795f6322ca474da7a5 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 21 Jan 2026 14:08:55 +0200
Subject: [PATCH 215/391] addressed requested changes and cleaned up reitit
 router

---
 src/source/routes/admin.clj                   |  13 +-
 src/source/routes/analytics/creator/top.clj   |  40 +-
 .../routes/analytics/creator/top_average.clj  |  16 +-
 .../routes/analytics/distributor/top.clj      |  46 ++-
 .../analytics/distributor/top_average.clj     |  16 +-
 src/source/routes/business.clj                |  25 +-
 src/source/routes/login.clj                   |  17 +-
 src/source/routes/me.clj                      |  10 +-
 src/source/routes/me_business.clj             |  28 +-
 src/source/routes/register.clj                |  13 +-
 src/source/routes/reitit.clj                  | 370 +++++++++---------
 src/source/routes/user.clj                    |  18 +-
 src/source/routes/util.clj                    |   2 +-
 src/source/util.clj                           |  29 +-
 14 files changed, 356 insertions(+), 287 deletions(-)

diff --git a/src/source/routes/admin.clj b/src/source/routes/admin.clj
index 5f788fee..98bd7e6b 100644
--- a/src/source/routes/admin.clj
+++ b/src/source/routes/admin.clj
@@ -1,6 +1,7 @@
 (ns source.routes.admin
   (:require [source.services.users :as users]
             [source.password :as pw]
+            [source.util :as util]
             [ring.util.response :as res]))
 
 (defn post
@@ -15,9 +16,14 @@
 
   [{:keys [ds body] :as _request}]
 
-  (let [user (users/user ds {:where [:= :email (:email body)]})
-        {:keys [password confirm-password]} body]
+  (let [{:keys [data error success]} (util/validate post body)
+        user (users/user ds {:where [:= :email (:email data)]})
+        {:keys [password confirm-password]} data]
     (cond
+
+      (not success) (-> (res/response error)
+                        (res/status 400))
+
       (not (= password confirm-password))
       (-> (res/response {:message "passwords do not match!"})
           (res/status 400))
@@ -28,9 +34,10 @@
 
       :else
       (let [pw (pw/hash-password password)
-            new-user (-> (assoc body
+            new-user (-> (assoc data
                                 :password pw
                                 :type "admin")
                          (dissoc :confirm-password))]
         (users/insert-user! ds {:data new-user})
         (res/response {:message "successfully created user"})))))
+
diff --git a/src/source/routes/analytics/creator/top.clj b/src/source/routes/analytics/creator/top.clj
index dd6a68b3..5040bece 100644
--- a/src/source/routes/analytics/creator/top.clj
+++ b/src/source/routes/analytics/creator/top.clj
@@ -2,7 +2,8 @@
   (:require [source.services.analytics.interface :as analytics]
             [ring.util.response :as res]
             [clojure.set :as set]
-            [source.services.interface :as services]))
+            [source.services.interface :as services]
+            [source.util :as utils]))
 
 (defn record-names [ds top-field ids]
   (if (= top-field :post-id)
@@ -32,20 +33,25 @@
                             [:views :int]]]}}}
 
   [{:keys [ds user query-params] :as _request}]
-  (let [{:keys [n mindate maxdate top contenttype]} query-params
-        top-field (if (= top "post") :post-id :bundle-id)
-        results (->> {:creator-id (:id user)
-                      :content-type-id contenttype}
-                     (analytics/top-statistics-query ds mindate maxdate n top-field)
-                     (mapv (fn [result]
-                             (set/rename-keys result {top-field :top}))))
-        ids (mapv :top results)
-        names (record-names ds top-field ids)
-        juxted (->> names
-                    (mapv (juxt :id :name))
-                    (into {}))
-        named-results (mapv (fn [{:keys [top] :as r}]
-                              (assoc r :top (clojure.core/get juxted top (str top))))
-                            results)]
+  (let [{:keys [data success error]} (utils/validate get query-params :query)
+        {:keys [n mindate maxdate top contenttype]} data
+        top-field (if (= top "post") :post-id :bundle-id)]
+    (if success
+      (let [results (->> {:creator-id (:id user)
+                          :content-type-id contenttype}
+                         (analytics/top-statistics-query ds mindate maxdate n top-field)
+                         (mapv (fn [result]
+                                 (set/rename-keys result {top-field :top}))))
+            ids (mapv :top results)
+            names (record-names ds top-field ids)
+            juxted (->> names
+                        (mapv (juxt :id :name))
+                        (into {}))
+            named-results (mapv (fn [{:keys [top] :as r}]
+                                  (assoc r :top (clojure.core/get juxted top (str top))))
+                                results)]
 
-    (res/response named-results)))
+        (res/response named-results))
+
+      (-> (res/response error)
+          (res/status 400)))))
diff --git a/src/source/routes/analytics/creator/top_average.clj b/src/source/routes/analytics/creator/top_average.clj
index 8a63451a..4f50a396 100644
--- a/src/source/routes/analytics/creator/top_average.clj
+++ b/src/source/routes/analytics/creator/top_average.clj
@@ -1,6 +1,7 @@
 (ns source.routes.analytics.creator.top-average
   (:require [source.services.analytics.interface :as analytics]
-            [ring.util.response :as res]))
+            [ring.util.response :as res]
+            [source.util :as utils]))
 
 (defn get
   {:summary "Get the average engagement (clicks and views) for a creator, optionally filtered by content type.
@@ -12,7 +13,12 @@
    :responses {200 {:body [:map [:average :float]]}}}
 
   [{:keys [ds user query-params] :as _request}]
-  (let [{:keys [mindate maxdate contenttype]} query-params
-        result (analytics/average-engagement ds mindate maxdate {:creator-id (:id user)
-                                                                 :content-type-id contenttype})]
-    (res/response {:average result})))
+  (let [{:keys [data success error]} (utils/validate get query-params :query)
+        {:keys [mindate maxdate contenttype]} data]
+    (if success
+      (let [result (analytics/average-engagement ds mindate maxdate {:creator-id (:id user)
+                                                                     :content-type-id contenttype})]
+        (res/response {:average result}))
+
+      (-> (res/response error)
+          (res/status 400)))))
diff --git a/src/source/routes/analytics/distributor/top.clj b/src/source/routes/analytics/distributor/top.clj
index 8e2ab29d..a95bfd85 100644
--- a/src/source/routes/analytics/distributor/top.clj
+++ b/src/source/routes/analytics/distributor/top.clj
@@ -2,7 +2,8 @@
   (:require [source.services.analytics.interface :as analytics]
             [ring.util.response :as res]
             [clojure.set :as set]
-            [source.services.interface :as services]))
+            [source.services.interface :as services]
+            [source.util :as utils]))
 
 (defn record-names [ds top-field ids]
   (if (= top-field :post-id)
@@ -26,22 +27,27 @@
                             [:views :int]]]}}}
 
   [{:keys [ds user query-params] :as _request}]
-  (let [{:keys [n mindate maxdate top contenttype]} query-params
-        top-field (if (= top "post") :post-id :feed-id)
-        results (->> {:distributor-id (:id user)
-                      :content-type-id contenttype}
-                     (analytics/top-statistics-query ds mindate maxdate n top-field)
-                     (mapv (fn [result]
-                             (set/rename-keys result {top-field :top}))))
-        ids (mapv :top results)
-        names (record-names ds top-field ids)
-        juxted (->> names
-                    (mapv (fn [{:keys [id title]}]
-                            {:id id
-                             :name title}))
-                    (mapv (juxt :id :name))
-                    (into {}))
-        named-results (mapv (fn [{:keys [top] :as r}]
-                              (assoc r :top (clojure.core/get juxted top (str top))))
-                            results)]
-    (res/response named-results)))
+  (let [{:keys [data success error]} (utils/validate get query-params :query)
+        {:keys [n mindate maxdate top contenttype]} data
+        top-field (if (= top "post") :post-id :feed-id)]
+    (if success
+      (let [results (->> {:distributor-id (:id user)
+                          :content-type-id contenttype}
+                         (analytics/top-statistics-query ds mindate maxdate n top-field)
+                         (mapv (fn [result]
+                                 (set/rename-keys result {top-field :top}))))
+            ids (mapv :top results)
+            names (record-names ds top-field ids)
+            juxted (->> names
+                        (mapv (fn [{:keys [id title]}]
+                                {:id id
+                                 :name title}))
+                        (mapv (juxt :id :name))
+                        (into {}))
+            named-results (mapv (fn [{:keys [top] :as r}]
+                                  (assoc r :top (clojure.core/get juxted top (str top))))
+                                results)]
+        (res/response named-results))
+
+      (-> (res/response error)
+          (res/status 400)))))
diff --git a/src/source/routes/analytics/distributor/top_average.clj b/src/source/routes/analytics/distributor/top_average.clj
index b0778dbd..e4f09f08 100644
--- a/src/source/routes/analytics/distributor/top_average.clj
+++ b/src/source/routes/analytics/distributor/top_average.clj
@@ -1,6 +1,7 @@
 (ns source.routes.analytics.distributor.top-average
   (:require [source.services.analytics.interface :as analytics]
-            [ring.util.response :as res]))
+            [ring.util.response :as res]
+            [source.util :as utils]))
 
 (defn get
   {:summary "Get the average engagement (clicks and views) for a distributor, optionally filtered by content type.
@@ -12,7 +13,12 @@
    :responses {200 {:body [:map [:average :float]]}}}
 
   [{:keys [ds user query-params] :as _request}]
-  (let [{:keys [mindate maxdate contenttype]} query-params
-        result (analytics/average-engagement ds mindate maxdate {:distributor-id (:id user)
-                                                                 :content-type-id contenttype})]
-    (res/response {:average result})))
+  (let [{:keys [data success error]} (utils/validate get query-params :query)
+        {:keys [mindate maxdate contenttype]} data]
+    (if success
+      (let [result (analytics/average-engagement ds mindate maxdate {:distributor-id (:id user)
+                                                                     :content-type-id contenttype})]
+        (res/response {:average result}))
+
+      (-> (res/response error)
+          (res/status 400)))))
diff --git a/src/source/routes/business.clj b/src/source/routes/business.clj
index 4a50cf38..1d34aadb 100644
--- a/src/source/routes/business.clj
+++ b/src/source/routes/business.clj
@@ -1,6 +1,7 @@
 (ns source.routes.business
   (:require [source.services.businesses :as businesses]
-            [ring.util.response :as res]))
+            [ring.util.response :as res]
+            [source.util :as utils]))
 
 (defn post
   {:summary "insert a business"
@@ -13,8 +14,13 @@
    :responses {201 {:body [:map [:message :string]]}}}
 
   [{:keys [ds body] :as _request}]
-  (businesses/insert-business! ds {:values body})
-  (res/response {:message "successfully added business"}))
+
+  (let [{:keys [data error success]} (utils/validate post body)]
+    (if (not success) (-> (res/response error)
+                          (res/status 400))
+
+        (do (businesses/insert-business! ds {:values data})
+            (res/response {:message "successfully added business"})))))
 
 (defn patch
   {:summary "update a business by id"
@@ -29,9 +35,16 @@
    :responses {200 {:body [:map [:message :string]]}}}
 
   [{:keys [ds body path-params] :as _request}]
-  (businesses/update-business! ds {:id (:id path-params)
-                                   :values body})
-  (res/response {:message "successfully updated business"}))
+
+  (let [{:keys [data error success]} (utils/validate patch body)]
+    (if (not success)
+
+      (-> (res/response error)
+          (res/status 400))
+
+      (do (businesses/update-business! ds {:id (:id path-params)
+                                           :values data})
+          (res/response {:message "successfully updated business"})))))
 
 (comment
   (require '[source.db.util :as db.util])
diff --git a/src/source/routes/login.clj b/src/source/routes/login.clj
index fb07a102..e45452f1 100644
--- a/src/source/routes/login.clj
+++ b/src/source/routes/login.clj
@@ -2,7 +2,8 @@
   (:require [source.services.auth :as auth]
             [ring.util.response :as res]
             [source.services.users :as users]
-            [source.password :as pw]))
+            [source.password :as pw]
+            [source.util :as util]))
 
 (defn post
   {:summary "get user data and access token provided valid credentials"
@@ -28,15 +29,19 @@
 
   [{:keys [ds body] :as _request}]
 
-  (let [{:keys [email password]} body
+  (let [{:keys [data error success]} (util/validate post body)
+        {:keys [email password]} data
         user (users/user ds {:where [:= :email email]})]
 
-    (if
-     (or (not (some? user))
-         (not (pw/verify-password password (:password user))))
+    (cond
+      (not success) (-> (res/response error)
+                        (res/status 400))
+
+      (or (not (some? user))
+          (not (pw/verify-password password (:password user))))
       {:status 401 :body {:message "Invalid username or password!"}}
 
-      (res/response (auth/login ds {:user user})))))
+      :else (res/response (auth/login ds {:user user})))))
 
 (comment
   (require '[source.db.interface :as db])
diff --git a/src/source/routes/me.clj b/src/source/routes/me.clj
index e53ce3e6..01f003dd 100644
--- a/src/source/routes/me.clj
+++ b/src/source/routes/me.clj
@@ -39,6 +39,10 @@
                400 {:body [:map [:message :string]]}}}
 
   [{:keys [ds user body] :as _request}]
-  (services/update-user! ds {:id (:id user)
-                             :data body})
-  (res/response {:message "successfully updated user"}))
+  (let [{:keys [data error success]} (util/validate post body)]
+    (if (not success)
+      (-> (res/response {:message error})
+          (res/status 400))
+      (do (services/update-user! ds {:id (:id user)
+                                     :data data})
+          (res/response {:message "successfully updated user"})))))
diff --git a/src/source/routes/me_business.clj b/src/source/routes/me_business.clj
index ffd21d25..6f3b1ad5 100644
--- a/src/source/routes/me_business.clj
+++ b/src/source/routes/me_business.clj
@@ -1,5 +1,6 @@
 (ns source.routes.me-business
-  (:require [ring.util.response :as res]
+  (:require [source.util :as util]
+            [ring.util.response :as res]
             [source.services.interface :as services]))
 
 (defn get
@@ -37,14 +38,19 @@
                400 {:body [:map [:message :string]]}}}
 
   [{:keys [ds user body] :as _request}]
-  (let [{:keys [business-id]} (services/user ds {:id (:id user)})
-        business (when (nil? business-id)
-                   (services/insert-business! ds {:data body
-                                                  :ret :1}))]
-    (if (nil? business-id)
-      (services/update-user! ds {:id (:id user)
-                                 :data {:business-id (:id business)}})
-      (services/update-business! ds {:id business-id
-                                     :data body}))
+  (let [{:keys [data error success]} (util/validate post body)]
+    (if (not success)
+      (-> (res/response {:message error})
+          (res/status 400))
+
+      (let [{:keys [business-id]} (services/user ds {:id (:id user)})
+            business (when (nil? business-id)
+                       (services/insert-business! ds {:data data
+                                                      :ret :1}))]
+        (if (nil? business-id)
+          (services/update-user! ds {:id (:id user)
+                                     :data {:business-id (:id business)}})
+          (services/update-business! ds {:id business-id
+                                         :data data}))
 
-    (res/response {:message "successfully added or updated business"})))
+        (res/response {:message "successfully added or updated business"})))))
diff --git a/src/source/routes/register.clj b/src/source/routes/register.clj
index 69d24328..4f0737d5 100644
--- a/src/source/routes/register.clj
+++ b/src/source/routes/register.clj
@@ -1,6 +1,7 @@
 (ns source.routes.register
   (:require [source.services.interface :as services]
-            [ring.util.response :as res]))
+            [ring.util.response :as res]
+            [source.util :as util]))
 
 (defn post
   {:summary "register a new user"
@@ -27,9 +28,14 @@
 
   [{:keys [ds body] :as _request}]
 
-  (let [{:keys [email password confirm-password]} body
+  (let [{:keys [data error success]} (util/validate post body)
+        {:keys [email password confirm-password]} data
         existing-user (services/user ds {:where [:= :email email]})]
     (cond
+
+      (not success) (-> (res/response error)
+                        (res/status 400))
+
       (not (= password confirm-password))
       (-> (res/response {:error "Passwords do not match!"}))
 
@@ -37,7 +43,7 @@
       (-> (res/response {:error "An account for this email already exists!"}))
 
       :else
-      (-> (services/register ds body)
+      (-> (services/register ds data)
           (res/response)))))
 
 (comment
@@ -47,3 +53,4 @@
                                     :type "distributor"
                                     :confirm-password "test"}})
   ())
+
diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index 11f61ec5..7d29ceac 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -83,142 +83,138 @@
     [(rutil/swagger-route)
      (rutil/openapi-route)
 
-     ["/users"          {:middleware [[mw/apply-auth {:required-type :admin}]]
-                         :tags #{"users"}
-                         :swagger {:security [{"auth" []}]}
-                         :openapi {:security [{:bearerAuth []}]}}
-      [""               (-> (get users/get))]
-      ["/:id"           (-> (get user/get)
-                            (patch user/patch))]]
-
-     ["/me"             {:middleware [[mw/apply-auth]]
-                         :tags #{"me"}
-                         :swagger {:security [{"auth" []}]}
-                         :openapi {:security [{:bearerAuth []}]}}
-      [""               (-> (get me/get)
-                            (post me/post))]
-      ["/business"      (-> (get me-business/get)
-                            (post me-business/post))]
-      ["/sectors"       (-> (get me-sectors/get)
-                            (post me-sectors/post))]]
-
-     ["/mail"             {:middleware [[mw/apply-auth]]
-                           :tags #{"mail"}
-                           :swagger {:security [{"auth" []}]}
-                           :openapi {:security [{:bearerAuth []}]}}
-      ["/report"          (-> (post report/post))]]
-
-     ["/businesses"     {:middleware [[mw/apply-auth {:required-type :admin}]]
-                         :tags #{"businesses"}
-                         :swagger {:security [{"auth" []}]}
-                         :openapi {:security [{:bearerAuth []}]}}
-      [""               (-> (get businesses/get)
-                            (post business/post))]
-      ["/:id"           (-> (patch business/patch))]]
-
-     ["/business"       {:tags #{"businesses"}}
-      ["/types"         (-> (get business-types/get))]]
-
-     ["/sectors"        {:tags #{"sectors"}}
-      [""               (-> (get sectors/get))]]
-
-     ["/login"          {:tags #{"auth"}}
-      [""               (-> (post login/post))]]
-
-     ["/register"       {:tags #{"auth"}}
-      [""               (-> (post register/post))]]
+     ["/users" {:middleware [[mw/apply-auth {:required-type :admin}]]
+                :tags #{"users"}
+                :swagger {:security [{"auth" []}]}
+                :openapi {:security [{:bearerAuth []}]}}
+
+      ["" (get users/get)]
+      ["/:id" (-> (get user/get)
+                  (patch user/patch))]]
+
+     ["/me" {:middleware [[mw/apply-auth]]
+             :tags #{"me"}
+             :swagger {:security [{"auth" []}]}
+             :openapi {:security [{:bearerAuth []}]}}
+
+      ["" (-> (get me/get)
+              (post me/post))]
+      ["/business" (-> (get me-business/get)
+                       (post me-business/post))]
+      ["/sectors" (-> (get me-sectors/get)
+                      (post me-sectors/post))]]
+
+     ["/mail" {:middleware [[mw/apply-auth]]
+               :tags #{"mail"}
+               :swagger {:security [{"auth" []}]}
+               :openapi {:security [{:bearerAuth []}]}}
+
+      ["/report" (post report/post)]]
+
+     ["/businesses" {:middleware [[mw/apply-auth {:required-type :admin}]]
+                     :tags #{"businesses"}
+                     :swagger {:security [{"auth" []}]}
+                     :openapi {:security [{:bearerAuth []}]}}
+
+      ["" (-> (get businesses/get)
+              (post business/post))]
+      ["/:id" (patch business/patch)]]
+
+     ["/business" {:tags #{"businesses"}}
+      ["/types" (get business-types/get)]]
+
+     ["/sectors" {:tags #{"sectors"}}
+      ["" (get sectors/get)]]
+
+     ["/login" {:tags #{"auth"}}
+      ["" (post login/post)]]
+
+     ["/register" {:tags #{"auth"}}
+      ["" (post register/post)]]
 
      ["/oauth2"
-      ["/google"        {:tags #{"google"}}
-       [""              (-> (get google-launch/get))]
-       ["/callback"     {:no-doc true}
-        [""             (-> (get google-redirect/get))]]
-       ["/user"         (-> (get google-user/get))]]]
-
-     ["/protected"      {:middleware [[mw/apply-auth]]
-                         :tags #{"protected"}
-                         :swagger {:security [{"auth" []}]}
-                         :openapi {:security [{:bearerAuth []}]}}
-      ["/authorized"    (-> (get authorized/get))]]
-
-     ["/providers"      {:tags #{"providers"}}
-      [""               (-> (get providers/get))]
-      ["/:id"           (-> (get provider/get))]]
-
-     ["/cadences"       {:tags #{"cadences"}}
-      [""               (-> (get cadences/get))]]
-
-     ["/categories"     {:tags #{"categories"}}
-      [""               (-> (get categories/get))]
-      ["/:id"           (-> (get category/get))]]
-
-     ["/baselines"      {:tags #{"baselines"}}
-      [""               (-> (get baselines/get))]]
-
-     ["/contentTypes"  {:tags #{"content types"}}
-      [""               (-> (get content-types/get))]
-      ["/:id"           (-> (get content-type/get))]]
-
-     ["/integrations"   {:middleware [[mw/apply-auth]]
-                         :tags #{"integrations"}}
-      [""               (->  (get integrations/get)
-                             (post integrations/post))]
-      ["/:id"
-       [""              (-> (get integration/get)
-                            (post integration/post)
-                            (delete integration/delete))]
-       ["/key"          (-> (post integration-key/post))]
-       ["/categories"   (->  (get integration-categories/get)
-                             (post integration-categories/post))]
-       ["/filter"
-        ["/feeds"
-         [""            (-> (get integration-filter-feeds/get))]
-         ["/:feed-id"   (->  (get integration-filter-feed/get)
-                             (post integration-filter-feed/post))]]
-        ["/posts"
-         [""            (-> (get integration-filter-posts/get))]
-         ["/:post-id"   (-> (get integration-filter-post/get)
-                            (post integration-filter-post/post))]]]]]
-
-     ["/feeds"          {:middleware [[mw/apply-auth]]
-                         :tags #{"feeds"}}
-      [""               (-> (get feeds/get)
-                            (post feeds/post))]
-      ["/:id"
-       [""              (-> (get feed/get)
-                            (post feed/post)
-                            (delete feed/delete))]
-       ["/posts"
-        [""             (-> (get posts/get))]
-        ["/:post-id"
-         [""            (-> (get post/get))]
-         ["/prune"      (-> (post post-prune/post))]]]
-       ["/categories"   (-> (get feed-categories/get)
-                            (post feed-categories/post))]]]
-
-     ["/analytics"      {:tags #{"analytics"}}
-      ["/creator"       {:middleware [[mw/apply-auth {:required-type :creator}]]}
-       ["/general"      (-> (get analytics-creator-general/get))]
-       ["/deltas"       (-> (get analytics-creator-deltas/get))]
-       ["/top"
-        [""             (-> (get analytics-creator-top/get))]
-        ["/average"     (-> (get analytics-creator-top-average/get))]]]
-      ["/distributor"   {:middleware [[mw/apply-auth {:required-type :distributor}]]}
-       ["/general"      (-> (get analytics-distributor-general/get))]
-       ["/top"
-        [""             (-> (get analytics-distributor-top/get))]
-        ["/average"     (-> (get analytics-distributor-top-average/get))]]]
-      ["/bundle"        {:middleware [[mw/apply-bundle]]}
-       ["/posts"
-        ["/:id"
-         ["/views"      (-> (post analytics-bundle-posts-id-views/post))]]]]
-      ["admin"          {:middleware [[mw/apply-auth {:required-type :admin}]]}
+      ["/google" {:tags #{"google"}}
+
+       ["" (get google-launch/get)]
+       ["/callback" {:no-doc true}
+        ["" (get google-redirect/get)]]
+       ["/user" (get google-user/get)]]]
+
+     ["/protected" {:middleware [[mw/apply-auth]]
+                    :tags #{"protected"}
+                    :swagger {:security [{"auth" []}]}
+                    :openapi {:security [{:bearerAuth []}]}}
+
+      ["/authorized" (get authorized/get)]]
+
+     ["/providers" {:tags #{"providers"}}
+      ["" (get providers/get)]
+      ["/:id" (get provider/get)]]
+
+     ["/cadences" {:tags #{"cadences"}}
+      ["" (get cadences/get)]]
+
+     ["/categories" {:tags #{"categories"}}
+      ["" (get categories/get)]
+      ["/:id" (get category/get)]]
+
+     ["/baselines" {:tags #{"baselines"}}
+      ["" (get baselines/get)]]
+
+     ["/contentTypes" {:tags #{"content types"}}
+      ["" (get content-types/get)]
+      ["/:id" (get content-type/get)]]
+
+     ["/integrations" {:middleware [[mw/apply-auth]]
+                       :tags #{"integrations"}}
+
+      ["" (->  (get integrations/get)
+               (post integrations/post))]
+      ["/:id" (-> (get integration/get)
+                  (post integration/post)
+                  (delete integration/delete))]
+      ["/:id/key" (post integration-key/post)]
+      ["/:id/categories" (->  (get integration-categories/get)
+                              (post integration-categories/post))]
+      ["/:id/filter/feeds" (get integration-filter-feeds/get)]
+      ["/:id/filter/feeds/:feed-id" (->  (get integration-filter-feed/get)
+                                         (post integration-filter-feed/post))]
+      ["/:id/filter/posts" (get integration-filter-posts/get)]
+      ["/:id/filter/posts/:post-id" (-> (get integration-filter-post/get)
+                                        (post integration-filter-post/post))]]
+
+     ["/feeds" {:middleware [[mw/apply-auth]]
+                :tags #{"feeds"}}
+
+      ["" (-> (get feeds/get)
+              (post feeds/post))]
+      ["/:id" (-> (get feed/get)
+                  (post feed/post)
+                  (delete feed/delete))]
+      ["/:id/posts" (get posts/get)]
+      ["/:id/posts/:post-id" (get post/get)]
+      ["/:id/posts/:post-id/prune" (post post-prune/post)]
+      ["/:id/categories" (-> (get feed-categories/get)
+                             (post feed-categories/post))]]
+
+     ["/analytics" {:tags #{"analytics"}}
+
+      ["/creator" {:middleware [[mw/apply-auth {:required-type :creator}]]}
+       ["/general" (get analytics-creator-general/get)]
+       ["/deltas" (get analytics-creator-deltas/get)]
+       ["/top" (get analytics-creator-top/get)]
+       ["/top/average" (get analytics-creator-top-average/get)]]
+      ["/distributor" {:middleware [[mw/apply-auth {:required-type :distributor}]]}
+       ["/general" (get analytics-distributor-general/get)]
+       ["/top" (get analytics-distributor-top/get)]
+       ["/top/average" (get analytics-distributor-top-average/get)]]
+      ["/bundle" {:middleware [[mw/apply-bundle]]}
+       ["/posts/:id/views" (post analytics-bundle-posts-id-views/post)]]
+      ["admin" {:middleware [[mw/apply-auth {:required-type :admin}]]}
        ["/general"]
        ["/top"]]]
-
-     ["/bundle"        {:middleware [[mw/apply-bundle]]
-                        :tags #{"bundles"}}
-
+     ["/bundle" {:middleware [[mw/apply-bundle]]
+                 :tags #{"bundles"}}
       ["" (get bundle/get)]
       ["/categories" (get bundle-categories/get)]
       ["/feeds" (post bundle-feeds/post)]
@@ -228,68 +224,54 @@
       ["/posts" (post bundle-posts/post)]
       ["/posts/:id" (get bundle-post/get)]]
 
-     ["/api"             {:middleware [[mw/apply-api-key]]
-                          :tags #{"api"}
-                          :swagger {:security [{"apiKey" []}]}
-                          :openapi {:security [{:apiKey []}]}}
-      ["/bundle"         {:middleware [[mw/apply-bundle]]}
-       [""               (-> (get bundle/get))]
-       ["/categories"
-        [""              (-> (get bundle-categories/get))]]
-       ["/feeds"
-        [""              (-> (post bundle-feeds/post))]
-        ["/:id"
-         [""             (-> (get bundle-feed/get))]
-         ["/posts"
-          [""            (-> (get bundle-feed-posts/get))]
-          ["/:post-id"   (-> (get bundle-feed-post/get))]]]]
-       ["/posts"
-        [""              (-> (post bundle-posts/post))]
-        ["/:id"          (-> (get bundle-post/get))]]]
-      ["/categories"
-       [""               (-> (get categories/get))]
-       ["/:id"           (-> (get category/get))]]]
-
-     ["/admin"                  {:middleware [[mw/apply-auth {:required-type :admin}]]
-                                 :tags #{"admin"}
-                                 :swagger {:security [{"auth" []}]}
-                                 :openapi {:security [{:bearerAuth []}]}}
-      ["/business"
-       ["/types"                (-> (post business-types/post)
-                                    (patch business-types/patch)
-                                    (delete business-types/delete))]]
-      ["/feeds"
-       [""                      (-> (get admin-feeds/get))]
-       ["/:id"
-        ["/approve"             (-> (post approve-feed/post))]
-        ["/reject"              (-> (post reject-feed/post))]]]
-      ["/jobs"
-       [""                      (-> (get jobs/get))]
-       ["/manage"
-        ["/view"                (-> (get jobs-view/get))]
-        ["/register"            (-> (post jobs/post))]]
-       ["/:id"
-        [""                     (-> (get job/get))]
-        ["/manage"
-         ["/deregister"         (-> (get job-deregister/get))]
-         ["/start"              (-> (get job-start/get))]
-         ["/stop"               (-> (get job-stop/get))]]]]
-      ["/add-admin"             (-> (post admin/post))]
-      ["/selection-schemas"
-       [""                      (-> (get selection-schemas/get)
-                                    (post selection-schemas/post))]
-       ["/:id"                  (-> (get selection-schema/get))]
-       ["/providers/:id"        (-> (get provider-selection-schemas/get))]]
-      ["/output-schemas"
-       [""                      (-> (get output-schemas/get)
-                                    (post output-schemas/post))]
-       ["/:id"                  (-> (get output-schema/get))]]
-      ["/providers"
-       [""                      (-> (post providers/post))]
-       ["/:id"                  (-> (post provider/post)
-                                    (delete provider/delete))]]
-      ["/ast"                   (-> (post xml/post))]
-      ["/extract-data"          (-> (post data/post))]]]
+     ["/api" {:middleware [[mw/apply-api-key]]
+              :tags #{"api"}
+              :swagger {:security [{"apiKey" []}]}
+              :openapi {:security [{:apiKey []}]}}
+
+      ["/bundle" {:middleware [[mw/apply-bundle]]}
+       ["" (get bundle/get)]
+       ["/categories" (get bundle-categories/get)]
+       ["/feeds" (post bundle-feeds/post)]
+       ["/feeds/:id" (get bundle-feed/get)]
+       ["/feeds/:id/posts" (get bundle-feed-posts/get)]
+       ["/feeds/:id/posts/:post-id" (get bundle-feed-post/get)]
+       ["/posts" (post bundle-posts/post)]
+       ["/posts/:id" (get bundle-post/get)]]
+      ["/categories" (get categories/get)]
+      ["/categories/:id" (get category/get)]]
+
+     ["/admin" {:middleware [[mw/apply-auth {:required-type :admin}]]
+                :tags #{"admin"}
+                :swagger {:security [{"auth" []}]}
+                :openapi {:security [{:bearerAuth []}]}}
+
+      ["/business/types" (-> (post business-types/post)
+                             (patch business-types/patch)
+                             (delete business-types/delete))]
+      ["/feeds" (get admin-feeds/get)]
+      ["/feeds/:id/approve" (post approve-feed/post)]
+      ["/feeds/:id/reject" (post reject-feed/post)]
+      ["/jobs" (get jobs/get)]
+      ["/jobs/manage/view" (get jobs-view/get)]
+      ["/jobs/manage/register" (post jobs/post)]
+      ["/jobs/:id" (get job/get)]
+      ["/jobs/manage/deregister" (get job-deregister/get)]
+      ["/jobs/manage/start" (get job-start/get)]
+      ["/jobs/manage/stop" (get job-stop/get)]
+      ["/add-admin" (post admin/post)]
+      ["/selection-schemas" (-> (get selection-schemas/get)
+                                (post selection-schemas/post))]
+      ["/selection-schemas/:id" (get selection-schema/get)]
+      ["/selection-schemas/providers/:id" (get provider-selection-schemas/get)]
+      ["/output-schemas" (-> (get output-schemas/get)
+                             (post output-schemas/post))]
+      ["/output-schemas/:id" (get output-schema/get)]
+      ["/providers" (post providers/post)]
+      ["/providers/:id" (-> (post provider/post)
+                            (delete provider/delete))]
+      ["/ast" (post xml/post)]
+      ["/extract-data" (post data/post)]]]
 
     (rutil/data-map ds store js))
    (ring/routes
diff --git a/src/source/routes/user.clj b/src/source/routes/user.clj
index 26b3e09f..413ddd48 100644
--- a/src/source/routes/user.clj
+++ b/src/source/routes/user.clj
@@ -1,6 +1,7 @@
 (ns source.routes.user
   (:require [source.services.interface :as services]
-            [ring.util.response :as res]))
+            [ring.util.response :as res]
+            [source.util :as util]))
 
 (defn get
   {:summary "get user by id"
@@ -60,10 +61,17 @@
                403 {:body [:map [:message :string]]}}}
 
   [{:keys [ds body path-params] :as _request}]
-  (services/update-user! ds
-                         {:id (:id path-params)
-                          :values body})
-  (res/response {:message "successfully updated user"}))
+
+  (let [{:keys [data error success]} (util/validate patch body)]
+    (if (not success)
+
+      (-> (res/response error)
+          (res/status 400))
+
+      (do (services/update-user! ds
+                                 {:id (:id path-params)
+                                  :values data})
+          (res/response {:message "successfully updated user"})))))
 
 (comment
   (require '[source.db.interface :as db])
diff --git a/src/source/routes/util.clj b/src/source/routes/util.clj
index 9f2ab345..dfe275de 100644
--- a/src/source/routes/util.clj
+++ b/src/source/routes/util.clj
@@ -22,7 +22,7 @@
 
 (defn- merge-route-map [validation-mw acc [method handler]]
   (->> (extract-openapi-meta handler)
-       (attach-handler handler validation-mw)
+       ;(attach-handler handler validation-mw)
        (assoc {} method)
        (merge acc)))
 
diff --git a/src/source/util.clj b/src/source/util.clj
index 49d8062f..e515bf44 100644
--- a/src/source/util.clj
+++ b/src/source/util.clj
@@ -94,15 +94,28 @@
 (defn humanise [{:keys [errors] :as _error}]
   (reduce append-humanised-error "" errors))
 
+; (defn validate
+;   [data schema]
+;   (let [transformed (m/decode schema data mt/string-transformer)
+;         success (m/validate schema transformed)]
+;     {:data (when success transformed)
+;      :success success
+;      :error (when-not success (->> transformed
+;                                    (m/explain schema)
+;                                    (humanise)))}))
+
 (defn validate
-  [data schema]
-  (let [transformed (m/decode schema data mt/string-transformer)
-        success (m/validate schema transformed)]
-    {:data (when success transformed)
-     :success success
-     :error (when-not success (->> transformed
-                                   (m/explain schema)
-                                   (humanise)))}))
+  ([handler data]
+   (validate handler data :body))
+  ([handler data schema-type]
+   (let [schema (get-in (metadata handler) [:parameters schema-type])
+         transformed (m/decode schema data mt/string-transformer)
+         success (m/validate schema transformed)]
+     {:data (when success transformed)
+      :success success
+      :error (when-not success (->> transformed
+                                    (m/explain schema)
+                                    (me/humanize)))})))
 
 (defn format-rss-date
   "Takes a date as a string in RFC 1123 format and returns it in a format that meets ISO 8601 standards for SQLite.

From 9921555566ea3f0fe769d316d01556aef6e0e7c8 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 21 Jan 2026 14:12:38 +0200
Subject: [PATCH 216/391] moved middleware to data-map function to make it
 visible in the reitit router

---
 src/source/routes/reitit.clj | 2 +-
 src/source/routes/util.clj   | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index 7d29ceac..17973e26 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -273,7 +273,7 @@
       ["/ast" (post xml/post)]
       ["/extract-data" (post data/post)]]]
 
-    (rutil/data-map ds store js))
+    (rutil/data-map [[mw/apply-generic :ds ds :store store :js js]]))
    (ring/routes
     (rutil/swagger-ui-handler)
     (ring/create-default-handler))))
diff --git a/src/source/routes/util.clj b/src/source/routes/util.clj
index dfe275de..455075a2 100644
--- a/src/source/routes/util.clj
+++ b/src/source/routes/util.clj
@@ -95,11 +95,11 @@
                                                                              :description "API Key authorization using the Bearer scheme"}}}}
                           :handler (openapi/create-openapi-handler)}}])
 
-(defn data-map [ds store js]
+(defn data-map [middleware]
   {:data {:coercion (reitit.coercion.malli/create
                      {:error-keys #{#_:type :coercion :in :schema :value :errors :humanized #_:transformed}
                       :compile mu/closed-schema
                       :strip-extra-keys true
                       :default-values true
                       :options nil})
-          :middleware [[mw/apply-generic :ds ds :store store :js js]]}})
+          :middleware middleware}})

From bbd7a0531cce7d508672ab1992cd2c2091931f70 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 21 Jan 2026 14:24:45 +0200
Subject: [PATCH 217/391] fixed bug in route utils

---
 src/source/routes/util.clj | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/source/routes/util.clj b/src/source/routes/util.clj
index 455075a2..8cb18850 100644
--- a/src/source/routes/util.clj
+++ b/src/source/routes/util.clj
@@ -14,15 +14,15 @@
 
 (defn- attach-handler [handler validation-mw openapi-meta]
   (cond-> openapi-meta
-    (some? (:parameters openapi-meta))
-    (assoc :middleware [[validation-mw openapi-meta]]
-           :responses (merge (:responses openapi-meta)
-                             (api/bad-request)))
+    #_(some? (:parameters openapi-meta))
+    #_(assoc :middleware [[validation-mw openapi-meta]]
+             :responses (merge (:responses openapi-meta)
+                               (api/bad-request)))
     true (assoc :handler handler)))
 
 (defn- merge-route-map [validation-mw acc [method handler]]
   (->> (extract-openapi-meta handler)
-       ;(attach-handler handler validation-mw)
+       (attach-handler handler validation-mw)
        (assoc {} method)
        (merge acc)))
 

From 725b77c8cb6425810ab18924a5cbae777671eec3 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 26 Jan 2026 13:28:14 +0200
Subject: [PATCH 218/391] added schemas for new services

---
 src/source/services_new/auth.clj           |   3 +
 src/source/services_new/categories.clj     |   4 +
 src/source/services_new/content_types.clj  |   4 +
 src/source/services_new/incoming_posts.clj |   4 +
 src/source/services_new/schemas.clj        | 207 +++++++++++++++++++++
 src/source/services_new/sectors.clj        |  12 ++
 src/source/services_new/users.clj          |   8 +
 7 files changed, 242 insertions(+)
 create mode 100644 src/source/services_new/auth.clj
 create mode 100644 src/source/services_new/categories.clj
 create mode 100644 src/source/services_new/content_types.clj
 create mode 100644 src/source/services_new/incoming_posts.clj
 create mode 100644 src/source/services_new/schemas.clj
 create mode 100644 src/source/services_new/sectors.clj
 create mode 100644 src/source/services_new/users.clj

diff --git a/src/source/services_new/auth.clj b/src/source/services_new/auth.clj
new file mode 100644
index 00000000..1cb4b336
--- /dev/null
+++ b/src/source/services_new/auth.clj
@@ -0,0 +1,3 @@
+(ns source.services-new.auth
+  (:require [source.services-new.schemas :as schemas]))
+
diff --git a/src/source/services_new/categories.clj b/src/source/services_new/categories.clj
new file mode 100644
index 00000000..01631b16
--- /dev/null
+++ b/src/source/services_new/categories.clj
@@ -0,0 +1,4 @@
+(ns source.services-new.categories
+  (:require [source.services-new.schemas :as schemas]))
+
+;; Service schemas
diff --git a/src/source/services_new/content_types.clj b/src/source/services_new/content_types.clj
new file mode 100644
index 00000000..9e4bf762
--- /dev/null
+++ b/src/source/services_new/content_types.clj
@@ -0,0 +1,4 @@
+(ns source.services-new.content-types
+  (:require [source.services-new.schemas :as schemas]))
+
+;; Service schemas
diff --git a/src/source/services_new/incoming_posts.clj b/src/source/services_new/incoming_posts.clj
new file mode 100644
index 00000000..cee88247
--- /dev/null
+++ b/src/source/services_new/incoming_posts.clj
@@ -0,0 +1,4 @@
+(ns source.services-new.incoming-posts
+  (:require [source.services-new.schemas :as schemas]))
+
+
diff --git a/src/source/services_new/schemas.clj b/src/source/services_new/schemas.clj
new file mode 100644
index 00000000..5ad93d85
--- /dev/null
+++ b/src/source/services_new/schemas.clj
@@ -0,0 +1,207 @@
+(ns source.services-new.schemas
+  (:require [malli.util :as mu]))
+
+(def Business
+  [:map
+   [:id :int]
+   [:name :string]
+   [:address :string]
+   [:url :string]
+   [:linkedin :string]
+   [:twitter :string]])
+
+(def User
+  [:map
+   [:id :int]
+   [:email :string]
+   [:firstname :string]
+   [:lastname :string]
+   [:type [:enum ["creator" "distributor" "admin"]]]
+   [:email-verified :int]
+   [:onboarded :int]
+   [:address :string]
+   [:mobile :string]
+   [:profile-image :string]])
+
+(def UserWithBusiness
+  (-> User
+      (mu/assoc :business Business)))
+
+(def AuthSchema
+  [:map
+   [:access-token :string]
+   [:refresh-token :string]])
+
+(def Login
+  (-> AuthSchema
+      (mu/assoc :user User)))
+
+(def Register
+  (-> AuthSchema
+      (mu/assoc :user User)))
+
+(def ConstantSchema
+  [:map
+   [:id :int]
+   [:name :string]])
+
+(def Sector
+  ConstantSchema)
+
+(def Category
+  (-> ConstantSchema
+      (mu/assoc :display-picture :string)))
+
+(def Categories
+  [:vector Category])
+
+(def ContentType
+  ConstantSchema)
+
+(def ContentTypes
+  [:vector ContentType])
+
+(def Provider
+  (-> ConstantSchema
+      (mu/assoc :domain :string)))
+
+(def ProviderWithContentType
+  (-> Provider
+      (mu/assoc :content-type ContentType)))
+
+(def BusinessType
+  ConstantSchema)
+
+(def Cadence
+  [:map
+   [:id :int]
+   [:label :string]
+   [:days :int]])
+
+(def Baseline
+  [:map
+   [:id :int]
+   [:label :string]
+   [:min :int]
+   [:max :int]])
+
+(def Feed
+  [:map
+   [:id :int]
+   [:title :string]
+   [:display-picture :string]
+   [:url :string]
+   [:rss-url :string]
+   [:provider-id :int]
+   [:created-at :string]
+   [:updated-at :string]
+   [:ts-and-cs :int]
+   [:state [:enum ["live" "not live" "pending"]]]])
+
+(def FeedWithChildren
+  (-> Feed
+      (mu/assoc :user User)
+      (mu/assoc :content-type ContentType)
+      (mu/assoc :cadence Cadence)
+      (mu/assoc :baseline Baseline)))
+
+(def IncomingPost
+  [:map
+   [:id :int]
+   [:post-id :string]
+   [:title :string]
+   [:thumbnail :string]
+   [:info :string]
+   [:url :string]
+   [:stream-url :string]
+   [:season :int]
+   [:episode :int]
+   [:redacted :int]
+   [:posted-at :string]])
+
+(def IncomingPostWithChildren
+  (-> IncomingPost
+      (mu/assoc :feed FeedWithChildren)))
+
+;; This is exactly the same as IncomingPost except without redacted
+;; We could probably just make a Post schema with (sometimes :redacted :int)
+(def OutgoingPost
+  [:map
+   [:id :int]
+   [:post-id :string]
+   [:title :string]
+   [:thumbnail :string]
+   [:info :string]
+   [:url :string]
+   [:stream-url :string]
+   [:season :int]
+   [:episode :int]
+   [:posted-at :string]])
+
+(def Job
+  [:map
+   [:id :int]
+   [:job-id :string]
+   [:status [:enum ["running" "stopped"]]]
+   [:args :string]
+   [:handler :string]
+   [:last-heartbeat :string]])
+
+(def JobMetadata
+  [:map
+   [:initial-delay :int]
+   [:auto-start :int]
+   [:stop-after-fail :int]
+   [:kill-after :int]
+   [:num-calls :int]
+   [:interval :int]
+   [:recurring :int]
+   [:created-at :string]
+   [:sleep :int]])
+
+(def JobWithMetadata
+  (-> Job
+      (mu/assoc :job-metadata JobMetadata)))
+
+(def Bundle
+  [:map
+   [:id :int]
+   [:name :string]
+   [:uuid :string]
+   [:video :int]
+   [:podcast :int]
+   [:blog :int]
+   [:hash :string]
+   [:ts-and-cs :int]])
+
+(def BundleWithUser
+  (-> Bundle
+      (mu/assoc :user User)))
+
+(def GeneralAnalytics
+  [:vector
+   [:map
+    [:day :string]
+    [:impressions :int]
+    [:clicks :int]
+    [:views :int]]])
+
+(def DeltasAnalytics
+  [:vector
+   [:map
+    [:week :string]
+    [:impressions :float]
+    [:clicks :float]
+    [:views :float]]])
+
+(def TopAnalytics
+  [:vector
+   [:map
+    [:top :string]
+    [:impressions :int]
+    [:clicks :int]
+    [:views :int]]])
+
+(def AverageEngagementAnalytics
+  [:map
+   [:average :float]])
diff --git a/src/source/services_new/sectors.clj b/src/source/services_new/sectors.clj
new file mode 100644
index 00000000..8addaa4f
--- /dev/null
+++ b/src/source/services_new/sectors.clj
@@ -0,0 +1,12 @@
+(ns source.services-new.sectors
+  (:require [source.services-new.schemas :as schemas]))
+
+;; Service schemas
+(def GetSectors
+  [:vector schemas/Sector])
+
+(def GetSector 
+  schemas/Sector)
+
+(def CreateSectors 
+  [:vector schemas/Sector])
diff --git a/src/source/services_new/users.clj b/src/source/services_new/users.clj
new file mode 100644
index 00000000..45a22e90
--- /dev/null
+++ b/src/source/services_new/users.clj
@@ -0,0 +1,8 @@
+(ns source.services-new.users
+  (:require [source.services-new.schemas :as schemas]))
+
+(def GetUser
+  schemas/User)
+
+(def UpdateUser
+  schemas/User)

From 3a20a6824d18acc511753e4d7ee04cd26c2acac5 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 26 Jan 2026 13:38:07 +0200
Subject: [PATCH 219/391] addressed change request

---
 src/source/services_new/schemas.clj | 76 +++++++++++++++++------------
 1 file changed, 44 insertions(+), 32 deletions(-)

diff --git a/src/source/services_new/schemas.clj b/src/source/services_new/schemas.clj
index 5ad93d85..de4901f9 100644
--- a/src/source/services_new/schemas.clj
+++ b/src/source/services_new/schemas.clj
@@ -10,13 +10,15 @@
    [:linkedin :string]
    [:twitter :string]])
 
+(def UserType [:enum ["creator" "distributor" "admin"]])
+
 (def User
   [:map
    [:id :int]
    [:email :string]
    [:firstname :string]
    [:lastname :string]
-   [:type [:enum ["creator" "distributor" "admin"]]]
+   [:type UserType]
    [:email-verified :int]
    [:onboarded :int]
    [:address :string]
@@ -27,17 +29,17 @@
   (-> User
       (mu/assoc :business Business)))
 
-(def AuthSchema
+(def SessionCredentials
   [:map
    [:access-token :string]
    [:refresh-token :string]])
 
 (def Login
-  (-> AuthSchema
+  (-> SessionCredentials
       (mu/assoc :user User)))
 
 (def Register
-  (-> AuthSchema
+  (-> SessionCredentials
       (mu/assoc :user User)))
 
 (def ConstantSchema
@@ -85,27 +87,29 @@
    [:min :int]
    [:max :int]])
 
-(def Feed
+(def FeedStatus [:enum ["live" "not live" "pending"]])
+
+(def FeedRecord
   [:map
    [:id :int]
    [:title :string]
    [:display-picture :string]
    [:url :string]
    [:rss-url :string]
-   [:provider-id :int]
    [:created-at :string]
    [:updated-at :string]
    [:ts-and-cs :int]
-   [:state [:enum ["live" "not live" "pending"]]]])
+   [:state FeedStatus]])
 
-(def FeedWithChildren
-  (-> Feed
+(def Feed
+  (-> FeedRecord
       (mu/assoc :user User)
       (mu/assoc :content-type ContentType)
       (mu/assoc :cadence Cadence)
-      (mu/assoc :baseline Baseline)))
+      (mu/assoc :baseline Baseline)
+      (mu/assoc :provider Provider)))
 
-(def IncomingPost
+(def IncomingPostRecord
   [:map
    [:id :int]
    [:post-id :string]
@@ -119,9 +123,9 @@
    [:redacted :int]
    [:posted-at :string]])
 
-(def IncomingPostWithChildren
+(def IncomingPost
   (-> IncomingPost
-      (mu/assoc :feed FeedWithChildren)))
+      (mu/assoc :feed FeedRecord)))
 
 ;; This is exactly the same as IncomingPost except without redacted
 ;; We could probably just make a Post schema with (sometimes :redacted :int)
@@ -138,11 +142,13 @@
    [:episode :int]
    [:posted-at :string]])
 
+(def JobStatus [:enum ["running" "stopped"]])
+
 (def Job
   [:map
    [:id :int]
    [:job-id :string]
-   [:status [:enum ["running" "stopped"]]]
+   [:status JobStatus]
    [:args :string]
    [:handler :string]
    [:last-heartbeat :string]])
@@ -178,29 +184,35 @@
   (-> Bundle
       (mu/assoc :user User)))
 
+(def GeneralStatistic
+  [:map
+   [:day :string]
+   [:impressions :int]
+   [:clicks :int]
+   [:views :int]])
+
 (def GeneralAnalytics
-  [:vector
-   [:map
-    [:day :string]
-    [:impressions :int]
-    [:clicks :int]
-    [:views :int]]])
+  [:vector GeneralStatistic])
+
+(def DeltasStatistic
+  [:map
+   [:week :string]
+   [:impressions :float]
+   [:clicks :float]
+   [:views :float]])
 
 (def DeltasAnalytics
-  [:vector
-   [:map
-    [:week :string]
-    [:impressions :float]
-    [:clicks :float]
-    [:views :float]]])
+  [:vector DeltasStatistic])
+
+(def TopStatistic
+  [:map
+   [:top :string]
+   [:impressions :int]
+   [:clicks :int]
+   [:views :int]])
 
 (def TopAnalytics
-  [:vector
-   [:map
-    [:top :string]
-    [:impressions :int]
-    [:clicks :int]
-    [:views :int]]])
+  [:vector TopStatistic])
 
 (def AverageEngagementAnalytics
   [:map

From ca63263096aabf5cb40657b61732df9fa7527991 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 26 Jan 2026 15:10:55 +0200
Subject: [PATCH 220/391] fixed schema name

---
 src/source/services_new/schemas.clj | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/source/services_new/schemas.clj b/src/source/services_new/schemas.clj
index de4901f9..e8948b66 100644
--- a/src/source/services_new/schemas.clj
+++ b/src/source/services_new/schemas.clj
@@ -124,7 +124,7 @@
    [:posted-at :string]])
 
 (def IncomingPost
-  (-> IncomingPost
+  (-> IncomingPostRecord
       (mu/assoc :feed FeedRecord)))
 
 ;; This is exactly the same as IncomingPost except without redacted

From 3968faf3984f35822e6730de6cade8e94b822de8 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 26 Jan 2026 15:52:51 +0200
Subject: [PATCH 221/391] added new services

---
 src/source/services/bundle_categories.clj    | 13 ++++++++++++
 src/source/services/bundle_content_types.clj | 21 ++++++++++++++++----
 src/source/services/bundles.clj              | 11 +++++++++-
 src/source/services/user_sectors.clj         |  8 ++++++++
 4 files changed, 48 insertions(+), 5 deletions(-)

diff --git a/src/source/services/bundle_categories.clj b/src/source/services/bundle_categories.clj
index c63b4aa0..9275c36d 100644
--- a/src/source/services/bundle_categories.clj
+++ b/src/source/services/bundle_categories.clj
@@ -32,3 +32,16 @@
         :ret :*}
        (merge opts)
        (db/find ds)))
+
+(defn insert-bundle-categories! [ds {:keys [bundle-id categories]}]
+  (let [bundle-categories (mapv (fn [{:keys [id]}]
+                                  {:bundle-id bundle-id
+                                   :category-id id}) categories)]
+    (insert-bundle-category! ds {:data bundle-categories})))
+
+(defn update-bundle-categories! [ds {:keys [bundle-id categories]}]
+  (let [bundle-categories (mapv (fn [{:keys [id]}]
+                                  {:bundle-id bundle-id
+                                   :category-id id}) categories)]
+    (delete-bundle-category! ds {:where [:= :bundle-id bundle-id]})
+    (insert-bundle-category! ds {:data bundle-categories})))
diff --git a/src/source/services/bundle_content_types.clj b/src/source/services/bundle_content_types.clj
index a0b4562f..6bd3bc21 100644
--- a/src/source/services/bundle_content_types.clj
+++ b/src/source/services/bundle_content_types.clj
@@ -11,10 +11,15 @@
         (merge opts)
         (db/find ds))))
 
-(defn insert-bundle-content-types! [ds {:keys [_data _ret] :as opts}]
-  (->> {:tname :bundle-content-types}
-       (merge opts)
-       (db/insert! ds)))
+;;NEW
+(defn insert-bundle-content-types! [ds {:keys [bundle-id content-types]}]
+  (let [content-types (mapv (fn [{:keys [id]}]
+                              {:bundle-id bundle-id
+                               :content-type-id id}) content-types)]
+    (->> {:tname :bundle-content-types
+          :data content-types
+          :ret :*}
+         (db/insert! ds))))
 
 (defn delete-bundle-content-types! [ds {:keys [id where] :as opts}]
   (->> {:tname :bundle-content-types
@@ -43,3 +48,11 @@
         :ret :*}
        (merge opts)
        (db/find ds)))
+
+;;NEW
+(defn update-bundle-content-types! [ds {:keys [bundle-id content-types]}]
+  (let [content-types (mapv (fn [{:keys [id]}]
+                              {:bundle-id bundle-id
+                               :content-type-id id}) content-types)]
+    (delete-bundle-content-types! ds {:where [:= :bundle-id bundle-id]})
+    (insert-bundle-content-types! ds {:data content-types})))
diff --git a/src/source/services/bundles.clj b/src/source/services/bundles.clj
index 4fee17f3..d395af93 100644
--- a/src/source/services/bundles.clj
+++ b/src/source/services/bundles.clj
@@ -1,11 +1,20 @@
 (ns source.services.bundles
-  (:require [source.db.interface :as db]))
+  (:require [source.db.interface :as db]
+            [source.util :as utils]))
 
 (defn insert-bundle! [ds {:keys [_values _ret] :as opts}]
   (->> {:tname :bundles}
        (merge opts)
        (db/insert! ds)))
 
+;;NEW
+(defn create-bundle! [ds {:keys [user-id bundle-metadata]}]
+  (insert-bundle! ds {:data (merge {:user-id user-id
+                                    :content-type-id 1 ; temporarily assign garbo id
+                                    :uuid (utils/uuid)}
+                                   bundle-metadata)
+                      :ret :1}))
+
 (defn update-bundle! [ds {:keys [id data where] :as opts}]
   (->> {:tname :bundles
         :values data
diff --git a/src/source/services/user_sectors.clj b/src/source/services/user_sectors.clj
index 32a5078a..c7e93609 100644
--- a/src/source/services/user_sectors.clj
+++ b/src/source/services/user_sectors.clj
@@ -43,3 +43,11 @@
         :ret :1}
        (merge opts)
        (db/find ds)))
+
+;;NEW
+(defn update-user-sectors! [ds {:keys [user-id sectors]}]
+  (let [update-data (reduce (fn [acc {:keys [id]}]
+                              (conj acc {:user-id user-id
+                                         :sector-id id})) [] sectors)]
+    (delete-user-sector! ds {:where [:= :user-id user-id]})
+    (insert-user-sector! ds {:data update-data})))

From 18058af5338b2f61e676a606ba76b8393172fc09 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 26 Jan 2026 15:55:11 +0200
Subject: [PATCH 222/391] updated endpoints with new small services

---
 src/source/routes/integration.clj  | 74 ++++++++++++++--------------
 src/source/routes/integrations.clj | 77 ++++++++++++++----------------
 src/source/routes/me_sectors.clj   | 15 +++---
 3 files changed, 78 insertions(+), 88 deletions(-)

diff --git a/src/source/routes/integration.clj b/src/source/routes/integration.clj
index d340250f..7cf71e84 100644
--- a/src/source/routes/integration.clj
+++ b/src/source/routes/integration.clj
@@ -7,7 +7,10 @@
             [source.jobs.core :as jobs]
             [source.jobs.handlers :as handlers]
             [source.services.analytics.interface :as analytics]
-            [source.db.tables :as tables]))
+            [source.db.tables :as tables]
+            [source.services.bundle-categories :as bundle-categories]
+            [source.services.bundle-content-types :as bundle-content-types]
+            [source.services.categories :as categories]))
 
 (defn get
   {:summary "get integration by id"
@@ -55,51 +58,46 @@
                403 {:body [:map [:message :string]]}}}
 
   [{:keys [js ds store path-params body] :as _request}]
-  (let [_ (services/update-bundle! ds {:id (:id path-params)
-                                       :data (dissoc body :categories :content-types)})
+  (with-open [bundle-ds (db.util/conn :bundle (:id path-params))]
+    (let [_ (services/update-bundle! ds {:id (:id path-params)
+                                         :data (dissoc body :categories :content-types)})
 
-        bundle-categories (mapv (fn [{:keys [id]}]
-                                  {:bundle-id (:id path-params)
-                                   :category-id id}) (:categories body))
-        bundle-content-types (mapv (fn [{:keys [id]}]
-                                     {:bundle-id (:id path-params)
-                                      :content-type-id id}) (:content-types body))
-
-        job-id (str "bundle_" (:id path-params))
+          job-id (str "bundle_" (:id path-params))
 
         ; update bundle categories
-        bundle-ds (db.util/conn :bundle (:id path-params))
-        _ (services/delete-bundle-category! bundle-ds {:where [:= :bundle-id (:id path-params)]})
-        _ (services/insert-bundle-category! bundle-ds {:data bundle-categories})
-
+          _ (bundle-categories/update-bundle-categories! bundle-ds {:bundle-id (:id path-params)
+                                                                    :categories (:categories body)})
         ; update bundle content types
-        _ (services/delete-bundle-content-types! ds {:where [:= :bundle-id (:id path-params)]})
-        _ (services/insert-bundle-content-types! ds {:data bundle-content-types})
+          _ (bundle-content-types/update-bundle-content-types! ds {:bundle-id (:id path-params)
+                                                                   :content-types (:content-types body)})
 
-        category-ids (services/category-id-by-bundle bundle-ds {:bundle-id (:id path-params)})
-        id-vec (mapv (fn [{:keys [category-id]}] category-id) category-ids)
-        categories-by-bundle (services/categories ds {:where [:in :id id-vec]})]
+          ; service needed
+          category-ids (bundle-categories/category-id bundle-ds {:bundle-id (:id path-params)})
+          id-vec (mapv (fn [{:keys [category-id]}] category-id) category-ids)
+          categories-by-bundle (categories/categories ds {:where [:in :id id-vec]})]
 
-    (congest/deregister! js job-id)
-    (->> (jobs/prepare-congest-metadata
-          ds
-          store
-          {:id job-id
-           :initial-delay (* 1000 60 60 24)
-           :auto-start true
-           :stop-after-fail false,
-           :interval (* 1000 60 60 24)
-           :recurring? true
-           :ds ds
-           :args {:bundle-id (:id path-params)
-                  :categories categories-by-bundle}
-           :handler :update-bundle
-           :created-at (utils/get-utc-timestamp-string)
-           :sleep false})
-         (congest/register! js))
+      ; service needed
+      (congest/deregister! js job-id)
+      (->> (jobs/prepare-congest-metadata
+            ds
+            store
+            {:id job-id
+             :initial-delay (* 1000 60 60 24)
+             :auto-start true
+             :stop-after-fail false,
+             :interval (* 1000 60 60 24)
+             :recurring? true
+             :ds ds
+             :args {:bundle-id (:id path-params)
+                    :categories categories-by-bundle}
+             :handler :update-bundle
+             :created-at (utils/get-utc-timestamp-string)
+             :sleep false})
+           (congest/register! js))
 
-    (res/response {:message "successfully updated integration"})))
+      (res/response {:message "successfully updated integration"}))))
 
+; cleanup needed
 (defn hard-delete-bundle! [ds js bundle-id]
   (let [job-id (handlers/update-bundle-job-id bundle-id)]
     (services/delete-filtered-feed! ds {:where [:= :bundle-id bundle-id]})
diff --git a/src/source/routes/integrations.clj b/src/source/routes/integrations.clj
index 615ae95e..847b46ae 100644
--- a/src/source/routes/integrations.clj
+++ b/src/source/routes/integrations.clj
@@ -1,12 +1,15 @@
 (ns source.routes.integrations
-  (:require [source.services.interface :as services]
-            [ring.util.response :as res]
+  (:require [ring.util.response :as res]
             [source.util :as utils]
             [source.migrate :as migrate]
             [congest.jobs :as congest]
             [source.jobs.core :as jobs]
             [source.db.util :as db.util]
-            [source.jobs.handlers :as handlers]))
+            [source.jobs.handlers :as handlers]
+            [source.services.bundle-categories :as bundle-categories]
+            [source.services.bundle-content-types :as bundle-content-types]
+            [source.services.bundles :as bundles]
+            [source.services.categories :as categories]))
 
 (defn get
   {:summary "get all integrations"
@@ -26,7 +29,7 @@
                403 {:body [:map [:message :string]]}}}
 
   [{:keys [ds user] :as _request}]
-  (res/response (services/bundles ds {:where [:= :user-id (:id user)]})))
+  (res/response (bundles/bundles ds {:where [:= :user-id (:id user)]})))
 
 (defn post
   {:summary "add an integration"
@@ -56,45 +59,37 @@
                403 {:body [:map [:message :string]]}}}
 
   [{:keys [js ds store user body] :as _request}]
-  (let [new-bundle (services/insert-bundle! ds {:data (merge
-                                                       (dissoc body :categories :content-types)
-                                                       {:user-id (:id user)
-                                                        :content-type-id 1 ; temporarily assign garbo id
-                                                        :uuid (utils/uuid)})
-                                                :ret :1})
-        bundle-categories (mapv (fn [{:keys [id]}]
-                                  {:bundle-id (:id new-bundle)
-                                   :category-id id}) (:categories body))
-        bundle-content-types (mapv (fn [{:keys [id]}]
-                                     {:bundle-id (:id new-bundle)
-                                      :content-type-id id}) (:content-types body))
+  (let [new-bundle (bundles/create-bundle! ds {:user-id (:id user)
+                                               :bundle-metadata (dissoc body :categories :content-types)})
+        _ (migrate/migrate-bundle (:id new-bundle) ["up"])]
 
-        _ (migrate/migrate-bundle (:id new-bundle) ["up"])
-        ; insert bundle categories
-        bundle-ds (db.util/conn :bundle (:id new-bundle))
+    (with-open [bundle-ds (db.util/conn :bundle (:id new-bundle))]
+      (let [_ (bundle-categories/insert-bundle-categories! bundle-ds {:bundle-id (:id new-bundle)
+                                                                      :categories (:categories body)})
+            _ (bundle-content-types/insert-bundle-content-types! ds {:bundle-id (:id new-bundle)
+                                                                     :content-types (:content-types body)})
 
-        _ (services/insert-bundle-category! bundle-ds {:data bundle-categories})
-        _ (services/insert-bundle-content-types! ds {:data bundle-content-types})
+            ; service needed
+            category-ids (bundle-categories/category-id bundle-ds {:bundle-id (:id new-bundle)})
+            id-vec (mapv (fn [{:keys [category-id]}] category-id) category-ids)
+            categories-by-bundle (categories/categories ds {:where [:in :id id-vec]})]
 
-        category-ids (services/category-id-by-bundle bundle-ds {:bundle-id (:id new-bundle)})
-        id-vec (mapv (fn [{:keys [category-id]}] category-id) category-ids)
-        categories-by-bundle (services/categories ds {:where [:in :id id-vec]})]
-
-    (->> (jobs/prepare-congest-metadata
-          ds
-          store
-          {:id (handlers/update-bundle-job-id (:id new-bundle))
-           :initial-delay 0
-           :auto-start true
-           :stop-after-fail false,
-           :interval (* 1000 60 60 24)
-           :recurring? true
-           :ds ds
-           :args {:bundle-id (:id new-bundle)
-                  :categories categories-by-bundle}
-           :handler :update-bundle
-           :created-at (utils/get-utc-timestamp-string)
-           :sleep false})
-         (congest/register! js))
+        ; service needed
+        (->> (jobs/prepare-congest-metadata
+              ds
+              store
+              {:id (handlers/update-bundle-job-id (:id new-bundle))
+               :initial-delay 0
+               :auto-start true
+               :stop-after-fail false,
+               :interval (* 1000 60 60 24)
+               :recurring? true
+               :ds ds
+               :args {:bundle-id (:id new-bundle)
+                      :categories categories-by-bundle}
+               :handler :update-bundle
+               :created-at (utils/get-utc-timestamp-string)
+               :sleep false})
+             (congest/register! js))))
 
     (res/response new-bundle)))
diff --git a/src/source/routes/me_sectors.clj b/src/source/routes/me_sectors.clj
index e534e51f..f1132ead 100644
--- a/src/source/routes/me_sectors.clj
+++ b/src/source/routes/me_sectors.clj
@@ -1,6 +1,6 @@
 (ns source.routes.me-sectors
-  (:require [source.services.interface :as services]
-            [ring.util.response :as res]))
+  (:require [ring.util.response :as res]
+            [source.services.user-sectors :as user-sectors]))
 
 (defn get
   {:summary "get sectors for the logged-in user"
@@ -11,7 +11,7 @@
                401 {:body [:map [:message :string]]}
                403 {:body [:map [:message :string]]}}}
   [{:keys [ds user] :as _request}]
-  (res/response (services/sectors-by-user ds {:user-id (:id user)})))
+  (res/response (user-sectors/sectors-by-user ds {:user-id (:id user)})))
 
 (defn post
   {:summary "update sectors for the logged-in user"
@@ -21,9 +21,6 @@
                         [:name :string]]]}
    :responses {200 {:body [:map [:message :string]]}}}
   [{:keys [ds user body] :as _request}]
-  (let [update-data (reduce (fn [acc {:keys [id]}]
-                              (conj acc {:user-id (:id user)
-                                         :sector-id id})) [] body)]
-    (services/delete-user-sector! ds {:where [:= :user-id (:id user)]})
-    (services/insert-user-sector! ds {:data update-data})
-    (res/response {:message "successfully updated user sectors"})))
+  (user-sectors/update-user-sectors! ds {:user-id (:id user)
+                                         :sectors body})
+  (res/response {:message "successfully updated user sectors"}))

From a26744f5d789e734ea53bb0d597782307e486029 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Tue, 27 Jan 2026 11:23:37 +0200
Subject: [PATCH 223/391] changed services_new to workers and made service for
 integration creation

---
 src/source/routes/integrations.clj            | 50 +++----------------
 src/source/services/bundles.clj               |  9 +++-
 src/source/services_new/auth.clj              |  3 --
 src/source/services_new/categories.clj        |  4 --
 src/source/services_new/content_types.clj     |  4 --
 src/source/services_new/incoming_posts.clj    |  4 --
 src/source/services_new/sectors.clj           | 12 -----
 src/source/services_new/users.clj             |  8 ---
 src/source/workers/auth.clj                   |  3 ++
 src/source/workers/categories.clj             |  4 ++
 src/source/workers/content_types.clj          |  4 ++
 src/source/workers/incoming_posts.clj         |  2 +
 src/source/workers/integrations.clj           | 42 ++++++++++++++++
 .../{services_new => workers}/schemas.clj     |  2 +-
 src/source/workers/sectors.clj                |  2 +
 src/source/workers/users.clj                  |  2 +
 16 files changed, 75 insertions(+), 80 deletions(-)
 delete mode 100644 src/source/services_new/auth.clj
 delete mode 100644 src/source/services_new/categories.clj
 delete mode 100644 src/source/services_new/content_types.clj
 delete mode 100644 src/source/services_new/incoming_posts.clj
 delete mode 100644 src/source/services_new/sectors.clj
 delete mode 100644 src/source/services_new/users.clj
 create mode 100644 src/source/workers/auth.clj
 create mode 100644 src/source/workers/categories.clj
 create mode 100644 src/source/workers/content_types.clj
 create mode 100644 src/source/workers/incoming_posts.clj
 create mode 100644 src/source/workers/integrations.clj
 rename src/source/{services_new => workers}/schemas.clj (99%)
 create mode 100644 src/source/workers/sectors.clj
 create mode 100644 src/source/workers/users.clj

diff --git a/src/source/routes/integrations.clj b/src/source/routes/integrations.clj
index 847b46ae..dd5f5b95 100644
--- a/src/source/routes/integrations.clj
+++ b/src/source/routes/integrations.clj
@@ -1,15 +1,7 @@
 (ns source.routes.integrations
   (:require [ring.util.response :as res]
-            [source.util :as utils]
-            [source.migrate :as migrate]
-            [congest.jobs :as congest]
-            [source.jobs.core :as jobs]
-            [source.db.util :as db.util]
-            [source.jobs.handlers :as handlers]
-            [source.services.bundle-categories :as bundle-categories]
-            [source.services.bundle-content-types :as bundle-content-types]
             [source.services.bundles :as bundles]
-            [source.services.categories :as categories]))
+            [source.workers.integrations :as integrations]))
 
 (defn get
   {:summary "get all integrations"
@@ -59,37 +51,9 @@
                403 {:body [:map [:message :string]]}}}
 
   [{:keys [js ds store user body] :as _request}]
-  (let [new-bundle (bundles/create-bundle! ds {:user-id (:id user)
-                                               :bundle-metadata (dissoc body :categories :content-types)})
-        _ (migrate/migrate-bundle (:id new-bundle) ["up"])]
-
-    (with-open [bundle-ds (db.util/conn :bundle (:id new-bundle))]
-      (let [_ (bundle-categories/insert-bundle-categories! bundle-ds {:bundle-id (:id new-bundle)
-                                                                      :categories (:categories body)})
-            _ (bundle-content-types/insert-bundle-content-types! ds {:bundle-id (:id new-bundle)
-                                                                     :content-types (:content-types body)})
-
-            ; service needed
-            category-ids (bundle-categories/category-id bundle-ds {:bundle-id (:id new-bundle)})
-            id-vec (mapv (fn [{:keys [category-id]}] category-id) category-ids)
-            categories-by-bundle (categories/categories ds {:where [:in :id id-vec]})]
-
-        ; service needed
-        (->> (jobs/prepare-congest-metadata
-              ds
-              store
-              {:id (handlers/update-bundle-job-id (:id new-bundle))
-               :initial-delay 0
-               :auto-start true
-               :stop-after-fail false,
-               :interval (* 1000 60 60 24)
-               :recurring? true
-               :ds ds
-               :args {:bundle-id (:id new-bundle)
-                      :categories categories-by-bundle}
-               :handler :update-bundle
-               :created-at (utils/get-utc-timestamp-string)
-               :sleep false})
-             (congest/register! js))))
-
-    (res/response new-bundle)))
+  (->> {:user-id (:id user)
+        :bundle-metadata (dissoc body :categories :content-types)
+        :categories (:categories body)
+        :content-types (:content-types body)}
+       (integrations/create-integration! ds js store)
+       (res/response)))
diff --git a/src/source/services/bundles.clj b/src/source/services/bundles.clj
index d395af93..8928c9bd 100644
--- a/src/source/services/bundles.clj
+++ b/src/source/services/bundles.clj
@@ -1,6 +1,8 @@
 (ns source.services.bundles
   (:require [source.db.interface :as db]
-            [source.util :as utils]))
+            [source.util :as utils]
+            [source.services.categories :as categories]
+            [source.services.bundle-categories :as bundle-categories]))
 
 (defn insert-bundle! [ds {:keys [_values _ret] :as opts}]
   (->> {:tname :bundles}
@@ -49,3 +51,8 @@
         :ret :1}
        (merge opts)
        (db/delete! ds)))
+
+(defn categories-in-bundle [ds bundle-id]
+  (let [category-ids (bundle-categories/category-id ds {:bundle-id bundle-id})
+        id-vec (mapv (fn [{:keys [category-id]}] category-id) category-ids)]
+    (categories/categories ds {:where [:in :id id-vec]})))
diff --git a/src/source/services_new/auth.clj b/src/source/services_new/auth.clj
deleted file mode 100644
index 1cb4b336..00000000
--- a/src/source/services_new/auth.clj
+++ /dev/null
@@ -1,3 +0,0 @@
-(ns source.services-new.auth
-  (:require [source.services-new.schemas :as schemas]))
-
diff --git a/src/source/services_new/categories.clj b/src/source/services_new/categories.clj
deleted file mode 100644
index 01631b16..00000000
--- a/src/source/services_new/categories.clj
+++ /dev/null
@@ -1,4 +0,0 @@
-(ns source.services-new.categories
-  (:require [source.services-new.schemas :as schemas]))
-
-;; Service schemas
diff --git a/src/source/services_new/content_types.clj b/src/source/services_new/content_types.clj
deleted file mode 100644
index 9e4bf762..00000000
--- a/src/source/services_new/content_types.clj
+++ /dev/null
@@ -1,4 +0,0 @@
-(ns source.services-new.content-types
-  (:require [source.services-new.schemas :as schemas]))
-
-;; Service schemas
diff --git a/src/source/services_new/incoming_posts.clj b/src/source/services_new/incoming_posts.clj
deleted file mode 100644
index cee88247..00000000
--- a/src/source/services_new/incoming_posts.clj
+++ /dev/null
@@ -1,4 +0,0 @@
-(ns source.services-new.incoming-posts
-  (:require [source.services-new.schemas :as schemas]))
-
-
diff --git a/src/source/services_new/sectors.clj b/src/source/services_new/sectors.clj
deleted file mode 100644
index 8addaa4f..00000000
--- a/src/source/services_new/sectors.clj
+++ /dev/null
@@ -1,12 +0,0 @@
-(ns source.services-new.sectors
-  (:require [source.services-new.schemas :as schemas]))
-
-;; Service schemas
-(def GetSectors
-  [:vector schemas/Sector])
-
-(def GetSector 
-  schemas/Sector)
-
-(def CreateSectors 
-  [:vector schemas/Sector])
diff --git a/src/source/services_new/users.clj b/src/source/services_new/users.clj
deleted file mode 100644
index 45a22e90..00000000
--- a/src/source/services_new/users.clj
+++ /dev/null
@@ -1,8 +0,0 @@
-(ns source.services-new.users
-  (:require [source.services-new.schemas :as schemas]))
-
-(def GetUser
-  schemas/User)
-
-(def UpdateUser
-  schemas/User)
diff --git a/src/source/workers/auth.clj b/src/source/workers/auth.clj
new file mode 100644
index 00000000..3035af53
--- /dev/null
+++ b/src/source/workers/auth.clj
@@ -0,0 +1,3 @@
+(ns source.workers.auth
+  (:require [source.workers.schemas :as schemas]))
+
diff --git a/src/source/workers/categories.clj b/src/source/workers/categories.clj
new file mode 100644
index 00000000..e4cd7905
--- /dev/null
+++ b/src/source/workers/categories.clj
@@ -0,0 +1,4 @@
+(ns source.workers.categories
+  (:require [source.workers.schemas :as schemas]))
+
+;; Service schemas
diff --git a/src/source/workers/content_types.clj b/src/source/workers/content_types.clj
new file mode 100644
index 00000000..e2cdcb29
--- /dev/null
+++ b/src/source/workers/content_types.clj
@@ -0,0 +1,4 @@
+(ns source.workers.content-types
+  (:require [source.workers.schemas :as schemas]))
+
+;; Service schemas
diff --git a/src/source/workers/incoming_posts.clj b/src/source/workers/incoming_posts.clj
new file mode 100644
index 00000000..f2482f35
--- /dev/null
+++ b/src/source/workers/incoming_posts.clj
@@ -0,0 +1,2 @@
+(ns source.workers.incoming-posts
+  (:require [source.workers.schemas :as schemas]))
diff --git a/src/source/workers/integrations.clj b/src/source/workers/integrations.clj
new file mode 100644
index 00000000..567fe975
--- /dev/null
+++ b/src/source/workers/integrations.clj
@@ -0,0 +1,42 @@
+(ns source.workers.integrations
+  (:require [source.services.bundles :as bundles]
+            [source.migrate :as migrate]
+            [source.db.util :as db.util]
+            [source.services.bundle-categories :as bundle-categories]
+            [source.services.bundle-content-types :as bundle-content-types]
+            [source.jobs.core :as jobs]
+            [source.jobs.handlers :as handlers]
+            [source.util :as utils]
+            [congest.jobs :as congest]))
+
+(defn create-integration! [ds js store {:keys [user-id bundle-metadata categories content-types]}]
+  (let [new-bundle (bundles/create-bundle! ds {:user-id user-id
+                                               :bundle-metadata bundle-metadata})
+        _ (migrate/migrate-bundle (:id new-bundle) ["up"])]
+
+    (with-open [bundle-ds (db.util/conn :bundle (:id new-bundle))]
+      (let [_ (bundle-categories/insert-bundle-categories! bundle-ds {:bundle-id (:id new-bundle)
+                                                                      :categories categories})
+            _ (bundle-content-types/insert-bundle-content-types! ds {:bundle-id (:id new-bundle)
+                                                                     :content-types content-types})
+
+            categories-by-bundle (bundles/categories-in-bundle ds (:id new-bundle))]
+
+        ; service needed
+        (->> (jobs/prepare-congest-metadata
+              ds
+              store
+              {:id (handlers/update-bundle-job-id (:id new-bundle))
+               :initial-delay 0
+               :auto-start true
+               :stop-after-fail false,
+               :interval (* 1000 60 60 24)
+               :recurring? true
+               :ds ds
+               :args {:bundle-id (:id new-bundle)
+                      :categories categories-by-bundle}
+               :handler :update-bundle
+               :created-at (utils/get-utc-timestamp-string)
+               :sleep false})
+             (congest/register! js))))
+    new-bundle))
diff --git a/src/source/services_new/schemas.clj b/src/source/workers/schemas.clj
similarity index 99%
rename from src/source/services_new/schemas.clj
rename to src/source/workers/schemas.clj
index de4901f9..c12d7486 100644
--- a/src/source/services_new/schemas.clj
+++ b/src/source/workers/schemas.clj
@@ -1,4 +1,4 @@
-(ns source.services-new.schemas
+(ns source.workers.schemas
   (:require [malli.util :as mu]))
 
 (def Business
diff --git a/src/source/workers/sectors.clj b/src/source/workers/sectors.clj
new file mode 100644
index 00000000..9e0fce29
--- /dev/null
+++ b/src/source/workers/sectors.clj
@@ -0,0 +1,2 @@
+(ns source.workers.sectors
+  (:require [source.workers.schemas :as schemas]))
diff --git a/src/source/workers/users.clj b/src/source/workers/users.clj
new file mode 100644
index 00000000..4ca1c0e6
--- /dev/null
+++ b/src/source/workers/users.clj
@@ -0,0 +1,2 @@
+(ns source.workers.users
+  (:require [source.workers.schemas :as schemas]))

From cabb397ad9362c3007a0e16acd0b7fb2034cee4d Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Tue, 27 Jan 2026 11:40:45 +0200
Subject: [PATCH 224/391] extracted integration update, delete and key
 generation into services

---
 src/source/routes/integration.clj     | 74 ++++-----------------------
 src/source/routes/integration_key.clj | 13 ++---
 src/source/workers/integrations.clj   | 59 ++++++++++++++++++++-
 3 files changed, 74 insertions(+), 72 deletions(-)

diff --git a/src/source/routes/integration.clj b/src/source/routes/integration.clj
index 7cf71e84..57de0f85 100644
--- a/src/source/routes/integration.clj
+++ b/src/source/routes/integration.clj
@@ -1,16 +1,8 @@
 (ns source.routes.integration
   (:require [ring.util.response :as res]
             [source.services.interface :as services]
-            [source.db.util :as db.util]
-            [congest.jobs :as congest]
-            [source.util :as utils]
-            [source.jobs.core :as jobs]
-            [source.jobs.handlers :as handlers]
-            [source.services.analytics.interface :as analytics]
-            [source.db.tables :as tables]
-            [source.services.bundle-categories :as bundle-categories]
-            [source.services.bundle-content-types :as bundle-content-types]
-            [source.services.categories :as categories]))
+            [source.services.bundles :as bundles]
+            [source.workers.integrations :as integrations]))
 
 (defn get
   {:summary "get integration by id"
@@ -58,55 +50,11 @@
                403 {:body [:map [:message :string]]}}}
 
   [{:keys [js ds store path-params body] :as _request}]
-  (with-open [bundle-ds (db.util/conn :bundle (:id path-params))]
-    (let [_ (services/update-bundle! ds {:id (:id path-params)
-                                         :data (dissoc body :categories :content-types)})
-
-          job-id (str "bundle_" (:id path-params))
-
-        ; update bundle categories
-          _ (bundle-categories/update-bundle-categories! bundle-ds {:bundle-id (:id path-params)
-                                                                    :categories (:categories body)})
-        ; update bundle content types
-          _ (bundle-content-types/update-bundle-content-types! ds {:bundle-id (:id path-params)
-                                                                   :content-types (:content-types body)})
-
-          ; service needed
-          category-ids (bundle-categories/category-id bundle-ds {:bundle-id (:id path-params)})
-          id-vec (mapv (fn [{:keys [category-id]}] category-id) category-ids)
-          categories-by-bundle (categories/categories ds {:where [:in :id id-vec]})]
-
-      ; service needed
-      (congest/deregister! js job-id)
-      (->> (jobs/prepare-congest-metadata
-            ds
-            store
-            {:id job-id
-             :initial-delay (* 1000 60 60 24)
-             :auto-start true
-             :stop-after-fail false,
-             :interval (* 1000 60 60 24)
-             :recurring? true
-             :ds ds
-             :args {:bundle-id (:id path-params)
-                    :categories categories-by-bundle}
-             :handler :update-bundle
-             :created-at (utils/get-utc-timestamp-string)
-             :sleep false})
-           (congest/register! js))
-
-      (res/response {:message "successfully updated integration"}))))
-
-; cleanup needed
-(defn hard-delete-bundle! [ds js bundle-id]
-  (let [job-id (handlers/update-bundle-job-id bundle-id)]
-    (services/delete-filtered-feed! ds {:where [:= :bundle-id bundle-id]})
-    (services/delete-filtered-post! ds {:where [:= :bundle-id bundle-id]})
-    (services/delete-bundle-content-types! ds {:where [:= :bundle-id bundle-id]})
-    (analytics/delete-event! ds {:where [:= :bundle-id bundle-id]})
-    (tables/drop-all-tables! (db.util/conn :bundle bundle-id))
-    (services/delete-bundle ds {:id bundle-id})
-    (congest/deregister! js job-id)))
+  (integrations/update-integration! ds js store {:bundle-id (:id path-params)
+                                                 :bundle-metadata (dissoc body :categories :content-types)
+                                                 :categories (:categories body)
+                                                 :content-types (:content-types body)})
+  (res/response {:message "successfully updated integration"}))
 
 (defn delete
   {:summary "delete the integration with the given id"
@@ -116,12 +64,12 @@
                403 {:body [:map [:message :string]]}}}
 
   [{:keys [ds js user path-params] :as _request}]
-  (let [bundle (services/bundle ds {:where [:and
-                                            [:= :id (:id path-params)]
-                                            [:= :user-id (:id user)]]})]
+  (let [bundle (bundles/bundle ds {:where [:and
+                                           [:= :id (:id path-params)]
+                                           [:= :user-id (:id user)]]})]
     (if (some? bundle)
       (do
-        (hard-delete-bundle! ds js (:id path-params))
+        (integrations/hard-delete-bundle! ds js (:id path-params))
         (res/response {:message "successfully deleted integration"}))
       (-> (res/response {:message "unauthorized"})
           (res/status 403)))))
diff --git a/src/source/routes/integration_key.clj b/src/source/routes/integration_key.clj
index 6a282688..6521eaaf 100644
--- a/src/source/routes/integration_key.clj
+++ b/src/source/routes/integration_key.clj
@@ -1,7 +1,6 @@
 (ns source.routes.integration-key
-  (:require [source.util :as util]
-            [ring.util.response :as res]
-            [source.services.interface :as services]))
+  (:require [ring.util.response :as res]
+            [source.workers.integrations :as integrations]))
 
 (defn post
   {:summary "generate an API key for the integration with the given id"
@@ -12,8 +11,6 @@
                403 {:body [:map [:message :string]]}}}
 
   [{:keys [ds user path-params] :as _request}]
-  (let [uuid (util/uuid)
-        api-key (util/sha256 (str (:id user) (:id path-params) uuid))]
-    (services/update-bundle! ds {:id (:id path-params)
-                                 :data {:hash api-key}})
-    (res/response {:key api-key})))
+  (->> (integrations/generate-api-key! ds (:id user) (:id path-params))
+       (assoc {} :key)
+       (res/response)))
diff --git a/src/source/workers/integrations.clj b/src/source/workers/integrations.clj
index 567fe975..d92017a4 100644
--- a/src/source/workers/integrations.clj
+++ b/src/source/workers/integrations.clj
@@ -7,7 +7,12 @@
             [source.jobs.core :as jobs]
             [source.jobs.handlers :as handlers]
             [source.util :as utils]
-            [congest.jobs :as congest]))
+            [congest.jobs :as congest]
+            [source.services.filtered-feeds :as filtered-feeds]
+            [source.services.filtered-posts :as filtered-posts]
+            [source.services.analytics.interface :as analytics]
+            [source.db.tables :as tables]
+            [source.util :as util]))
 
 (defn create-integration! [ds js store {:keys [user-id bundle-metadata categories content-types]}]
   (let [new-bundle (bundles/create-bundle! ds {:user-id user-id
@@ -40,3 +45,55 @@
                :sleep false})
              (congest/register! js))))
     new-bundle))
+
+(defn update-integration! [ds js store {:keys [bundle-id bundle-metadata categories content-types]}]
+  (with-open [bundle-ds (db.util/conn :bundle bundle-id)]
+    (let [_ (bundles/update-bundle! ds {:id bundle-id
+                                        :data bundle-metadata})
+
+          job-id (str "bundle_" bundle-id)
+
+        ; update bundle categories
+          _ (bundle-categories/update-bundle-categories! bundle-ds {:bundle-id bundle-id
+                                                                    :categories categories})
+        ; update bundle content types
+          _ (bundle-content-types/update-bundle-content-types! ds {:bundle-id bundle-id
+                                                                   :content-types content-types})
+
+          categories-by-bundle (bundles/categories-in-bundle ds bundle-id)]
+
+      ; service needed
+      (congest/deregister! js job-id)
+      (->> (jobs/prepare-congest-metadata
+            ds
+            store
+            {:id job-id
+             :initial-delay (* 1000 60 60 24)
+             :auto-start true
+             :stop-after-fail false,
+             :interval (* 1000 60 60 24)
+             :recurring? true
+             :ds ds
+             :args {:bundle-id bundle-id
+                    :categories categories-by-bundle}
+             :handler :update-bundle
+             :created-at (utils/get-utc-timestamp-string)
+             :sleep false})
+           (congest/register! js)))))
+
+(defn hard-delete-bundle! [ds js bundle-id]
+  (let [job-id (handlers/update-bundle-job-id bundle-id)]
+    (filtered-feeds/delete-filtered-feed! ds {:where [:= :bundle-id bundle-id]})
+    (filtered-posts/delete-filtered-post! ds {:where [:= :bundle-id bundle-id]})
+    (bundle-content-types/delete-bundle-content-types! ds {:where [:= :bundle-id bundle-id]})
+    (analytics/delete-event! ds {:where [:= :bundle-id bundle-id]})
+    (tables/drop-all-tables! (db.util/conn :bundle bundle-id))
+    (bundles/delete-bundle! ds {:id bundle-id})
+    (congest/deregister! js job-id)))
+
+(defn generate-api-key! [ds user-id bundle-id]
+  (let [uuid (util/uuid)
+        api-key (util/sha256 (str user-id bundle-id uuid))]
+    (bundles/update-bundle! ds {:id bundle-id
+                                :data {:hash api-key}})
+    api-key))

From b66b9ba3abdca87cdcbdf279deb208f606e9cc71 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Tue, 27 Jan 2026 11:50:22 +0200
Subject: [PATCH 225/391] extracted bundle category updating into service

---
 src/source/routes/integration_categories.clj | 19 +++++++------------
 src/source/services/bundles.clj              | 10 ++++++----
 2 files changed, 13 insertions(+), 16 deletions(-)

diff --git a/src/source/routes/integration_categories.clj b/src/source/routes/integration_categories.clj
index 2b600a15..b35d511d 100644
--- a/src/source/routes/integration_categories.clj
+++ b/src/source/routes/integration_categories.clj
@@ -1,7 +1,9 @@
 (ns source.routes.integration-categories
   (:require [source.services.interface :as services]
             [ring.util.response :as res]
-            [source.db.util :as db.util]))
+            [source.db.util :as db.util]
+            [source.services.bundles :as bundles]
+            [source.services.bundle-categories :as bundle-categories]))
 
 (defn get
   {:summary "get all categories belonging to the integration with the given id"
@@ -13,11 +15,7 @@
                             [:name :string]]]}}}
 
   [{:keys [ds path-params] :as _request}]
-  (with-open [bundle-ds (db.util/conn :bundle (:id path-params))]
-    (let [category-ids (services/category-id-by-bundle bundle-ds {:bundle-id (:id path-params)})
-          id-vec (mapv (fn [{:keys [category-id]}] category-id) category-ids)
-          categories (services/categories ds {:where [:in :id id-vec]})]
-      (res/response categories))))
+  (res/response (bundles/categories-in-bundle ds (:id path-params))))
 
 (defn post
   {:summary "update categories belonging to the integration with the given id"
@@ -31,9 +29,6 @@
 
   [{:keys [path-params body] :as _request}]
   (with-open [bundle-ds (db.util/conn :bundle (:id path-params))]
-    (let [update-data (reduce (fn [acc {:keys [id]}]
-                                (conj acc {:bundle-id (:id path-params)
-                                           :category-id id})) [] body)]
-      (services/delete-bundle-category! bundle-ds {:where [:= :bundle-id (:id path-params)]})
-      (services/insert-bundle-category! bundle-ds {:data update-data})
-      (res/response {:message "successfully updated integration categories"}))))
+    (bundle-categories/update-bundle-categories! bundle-ds {:bundle-id (:id path-params)
+                                                            :categories body})
+    (res/response {:message "successfully updated integration categories"})))
diff --git a/src/source/services/bundles.clj b/src/source/services/bundles.clj
index 8928c9bd..9d4b21dc 100644
--- a/src/source/services/bundles.clj
+++ b/src/source/services/bundles.clj
@@ -2,7 +2,8 @@
   (:require [source.db.interface :as db]
             [source.util :as utils]
             [source.services.categories :as categories]
-            [source.services.bundle-categories :as bundle-categories]))
+            [source.services.bundle-categories :as bundle-categories]
+            [source.db.util :as db.util]))
 
 (defn insert-bundle! [ds {:keys [_values _ret] :as opts}]
   (->> {:tname :bundles}
@@ -53,6 +54,7 @@
        (db/delete! ds)))
 
 (defn categories-in-bundle [ds bundle-id]
-  (let [category-ids (bundle-categories/category-id ds {:bundle-id bundle-id})
-        id-vec (mapv (fn [{:keys [category-id]}] category-id) category-ids)]
-    (categories/categories ds {:where [:in :id id-vec]})))
+  (with-open [bundle-ds (db.util/conn :bundle bundle-id)]
+    (let [category-ids (bundle-categories/category-id bundle-ds {:bundle-id bundle-id})
+          id-vec (mapv (fn [{:keys [category-id]}] category-id) category-ids)]
+      (categories/categories ds {:where [:in :id id-vec]}))))

From b6235c4ecd9fbc5f41098fc8d29c976602d296e4 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Tue, 27 Jan 2026 11:57:33 +0200
Subject: [PATCH 226/391] extracted filtered feed updating into service

---
 src/source/routes/integration_filter_feed.clj | 21 +++++++------------
 src/source/services/bundles.clj               |  1 +
 src/source/workers/integrations.clj           |  8 +++++++
 3 files changed, 16 insertions(+), 14 deletions(-)

diff --git a/src/source/routes/integration_filter_feed.clj b/src/source/routes/integration_filter_feed.clj
index 1af65f99..302cb664 100644
--- a/src/source/routes/integration_filter_feed.clj
+++ b/src/source/routes/integration_filter_feed.clj
@@ -1,6 +1,7 @@
 (ns source.routes.integration-filter-feed
   (:require [source.services.interface :as services]
-            [ring.util.response :as res]))
+            [ring.util.response :as res]
+            [source.workers.integrations :as integrations]))
 
 (defn get
   {:summary "Returns true if the feed with the given id is filtered out by the integration with the given id"
@@ -35,16 +36,8 @@
                       403 [:map [:message :string]]}}}
 
   [{:keys [ds path-params body] :as _request}]
-
-  (let [{:keys [filtered]} body
-        {:keys [id feed-id]} path-params]
-
-    (if filtered
-      (services/insert-filtered-feeds! ds {:data {:feed-id feed-id
-                                                  :bundle-id id}})
-
-      (services/delete-filtered-feed! ds {:where [:and
-                                                  [:= :feed-id feed-id]
-                                                  [:= :bundle-id id]]}))
-
-    (res/response {:message "Successfully updated feed filtering."})))
+  (->> {:filtered (:filtered body)
+        :bundle-id (:id path-params)
+        :feed-id (:feed-id path-params)}
+       (integrations/update-filtered-feeds! ds))
+  (res/response {:message "Successfully updated feed filtering."}))
diff --git a/src/source/services/bundles.clj b/src/source/services/bundles.clj
index 9d4b21dc..c1c8e294 100644
--- a/src/source/services/bundles.clj
+++ b/src/source/services/bundles.clj
@@ -53,6 +53,7 @@
        (merge opts)
        (db/delete! ds)))
 
+;;NEW
 (defn categories-in-bundle [ds bundle-id]
   (with-open [bundle-ds (db.util/conn :bundle bundle-id)]
     (let [category-ids (bundle-categories/category-id bundle-ds {:bundle-id bundle-id})
diff --git a/src/source/workers/integrations.clj b/src/source/workers/integrations.clj
index d92017a4..dfa98bb9 100644
--- a/src/source/workers/integrations.clj
+++ b/src/source/workers/integrations.clj
@@ -97,3 +97,11 @@
     (bundles/update-bundle! ds {:id bundle-id
                                 :data {:hash api-key}})
     api-key))
+
+(defn update-filtered-feeds! [ds {:keys [filtered bundle-id feed-id]}]
+  (if filtered
+    (filtered-feeds/insert-filtered-feeds! ds {:data {:feed-id feed-id
+                                                      :bundle-id bundle-id}})
+    (filtered-feeds/delete-filtered-feed! ds {:where [:and
+                                                      [:= :feed-id feed-id]
+                                                      [:= :bundle-id bundle-id]]})))

From 3f02ff0f87c1fcae6593bc26ae14d44abab90069 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Tue, 27 Jan 2026 12:02:12 +0200
Subject: [PATCH 227/391] extracted update filtered posts into service

---
 src/source/routes/integration_filter_post.clj | 18 +++++++-----------
 src/source/workers/integrations.clj           | 15 +++++++++++----
 2 files changed, 18 insertions(+), 15 deletions(-)

diff --git a/src/source/routes/integration_filter_post.clj b/src/source/routes/integration_filter_post.clj
index 35be03e6..d93e82c4 100644
--- a/src/source/routes/integration_filter_post.clj
+++ b/src/source/routes/integration_filter_post.clj
@@ -1,6 +1,7 @@
 (ns source.routes.integration-filter-post
   (:require [source.services.interface :as services]
-            [ring.util.response :as res]))
+            [ring.util.response :as res]
+            [source.workers.integrations :as integrations]))
 
 (defn get
   {:summary "Returns true if the post with the given id is filtered out by the integration with the given id"
@@ -35,13 +36,8 @@
                       403 [:map [:message :string]]}}}
 
   [{:keys [ds path-params body] :as _request}]
-
-  (let [{:keys [filtered]} body
-        {:keys [id post-id]} path-params]
-    (if filtered
-      (services/insert-filtered-posts! ds {:data {:post-id post-id
-                                                  :bundle-id id}})
-      (services/delete-filtered-post! ds {:where [:and
-                                                  [:= :post-id post-id]
-                                                  [:= :bundle-id id]]}))
-    (res/response {:message "successfully updated post filtering"})))
+  (->> {:filtered (:filtered body)
+        :bundle-id (:id path-params)
+        :post-id (:post-id path-params)}
+       (integrations/update-filtered-posts! ds))
+  (res/response {:message "successfully updated post filtering"}))
diff --git a/src/source/workers/integrations.clj b/src/source/workers/integrations.clj
index dfa98bb9..61d6504b 100644
--- a/src/source/workers/integrations.clj
+++ b/src/source/workers/integrations.clj
@@ -11,8 +11,7 @@
             [source.services.filtered-feeds :as filtered-feeds]
             [source.services.filtered-posts :as filtered-posts]
             [source.services.analytics.interface :as analytics]
-            [source.db.tables :as tables]
-            [source.util :as util]))
+            [source.db.tables :as tables]))
 
 (defn create-integration! [ds js store {:keys [user-id bundle-metadata categories content-types]}]
   (let [new-bundle (bundles/create-bundle! ds {:user-id user-id
@@ -92,8 +91,8 @@
     (congest/deregister! js job-id)))
 
 (defn generate-api-key! [ds user-id bundle-id]
-  (let [uuid (util/uuid)
-        api-key (util/sha256 (str user-id bundle-id uuid))]
+  (let [uuid (utils/uuid)
+        api-key (utils/sha256 (str user-id bundle-id uuid))]
     (bundles/update-bundle! ds {:id bundle-id
                                 :data {:hash api-key}})
     api-key))
@@ -105,3 +104,11 @@
     (filtered-feeds/delete-filtered-feed! ds {:where [:and
                                                       [:= :feed-id feed-id]
                                                       [:= :bundle-id bundle-id]]})))
+
+(defn update-filtered-posts! [ds {:keys [filtered bundle-id post-id]}]
+  (if filtered
+    (filtered-posts/insert-filtered-posts! ds {:data {:post-id post-id
+                                                      :bundle-id bundle-id}})
+    (filtered-posts/delete-filtered-post! ds {:where [:and
+                                                      [:= :post-id post-id]
+                                                      [:= :bundle-id bundle-id]]})))

From 18dab723265807f1731695085a76baebf4b32674 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Tue, 27 Jan 2026 13:39:48 +0200
Subject: [PATCH 228/391] extracted services for feeds

---
 src/source/routes/feed.clj   | 38 ++++++---------
 src/source/routes/feeds.clj  | 66 ++++----------------------
 src/source/workers/feeds.clj | 91 ++++++++++++++++++++++++++++++++++++
 3 files changed, 114 insertions(+), 81 deletions(-)
 create mode 100644 src/source/workers/feeds.clj

diff --git a/src/source/routes/feed.clj b/src/source/routes/feed.clj
index cd7acd31..b4ead652 100644
--- a/src/source/routes/feed.clj
+++ b/src/source/routes/feed.clj
@@ -1,9 +1,7 @@
 (ns source.routes.feed
-  (:require [source.services.interface :as services]
-            [ring.util.response :as res]
-            [congest.jobs :as congest]
-            [source.jobs.handlers :as handlers]
-            [source.services.analytics.interface :as analytics]))
+  (:require [ring.util.response :as res]
+            [source.db.honey :as hon]
+            [source.workers.feeds :as feeds]))
 
 (defn get
   {:summary "get feed by id"
@@ -26,7 +24,8 @@
                             [:state [:enum "live" "not live" "pending"]]]}}}
 
   [{:keys [ds path-params] :as _request}]
-  (-> (services/feed ds path-params)
+  (-> (hon/find-one ds {:tname :feeds
+                        :where [:= :id (:id path-params)]})
       (res/response)))
 
 (defn post
@@ -45,21 +44,10 @@
                 403 {:body [:map [:message :string]]}}}
 
   [{:keys [ds path-params body] :as _request}]
-  (services/update-feed! ds {:id (:id path-params)
-                             :data body})
+  (feeds/update-feed! ds {:feed-id (:id path-params)
+                          :feed-metadata body})
   (res/response {:message "successfully updated feed"}))
 
-(defn hard-delete-feed! [ds js creator-email feed-id]
-  (let [job-id (handlers/update-feed-posts-job-id creator-email feed-id)
-        post-ids (mapv :id (services/incoming-posts ds {:where [:= :feed-id feed-id]}))]
-    (services/delete-filtered-feed! ds {:where [:= :feed-id feed-id]})
-    (services/delete-filtered-post! ds {:where [:in :post-id post-ids]})
-    (services/delete-incoming-post! ds {:where [:= :feed-id feed-id]})
-    (services/delete-feed-category! ds {:where [:= :feed-id feed-id]})
-    (analytics/delete-event! ds {:where [:= :feed-id feed-id]})
-    (services/delete-feed! ds {:where [:= :id feed-id]})
-    (congest/deregister! js job-id)))
-
 (defn delete
   {:summary "delete feed by id"
    :parameters {:path [:map [:id {:title "id"
@@ -70,13 +58,15 @@
 
   [{:keys [ds js user path-params] :as _request}]
   (let [id (:id path-params)
-        feed (services/feed ds {:where [:and
-                                        [:= :user-id (:id user)
-                                         := :id id]]})
-        {:keys [email]} (services/user ds {:id (:id user)})]
+        feed (hon/find-one ds {:tname :feeds
+                               :where [:and
+                                       [:= :user-id (:id user)
+                                        := :id id]]})
+        {:keys [email]} (hon/find-one ds {:tname :users
+                                          :where [:= :id (:id user)]})]
     (if (some? feed)
       (do
-        (hard-delete-feed! ds js email id)
+        (feeds/hard-delete-feed! ds js email id)
         (res/response {:message "successfully deleted feed"}))
       (-> (res/response {:message "unauthorized"})
           (res/status 403)))))
diff --git a/src/source/routes/feeds.clj b/src/source/routes/feeds.clj
index c9d1cf36..34cb2f9f 100644
--- a/src/source/routes/feeds.clj
+++ b/src/source/routes/feeds.clj
@@ -1,6 +1,7 @@
 (ns source.routes.feeds
   (:require [source.services.interface :as services]
             [source.util :as utils]
+            [source.workers.feeds :as feeds]
             [congest.jobs :as congest]
             [source.jobs.core :as jobs]
             [ring.util.response :as res]
@@ -57,68 +58,19 @@
                            [:state [:enum "live" "not live" "pending"]]]}}}
 
   [{:keys [js ds store user body] :as _request}]
-  (let [{:keys [provider-id rss-url content-type-id]} body
-        datetime (utils/get-utc-timestamp-string)
-        exists (hon/exists? ds {:tname :feeds
-                                :where [:= :rss-url rss-url]
+  (let [exists (hon/exists? ds {:tname :feeds
+                                :where [:= :rss-url (:rss-url body)]
                                 :ret :1})]
     (if exists
       (-> (res/response {:message "There is already a feed with the given RSS feed"})
           (res/status 400))
 
-      (let [selection-schemas (->> [:= :provider-id provider-id]
-                                   (assoc {} :where)
-                                   (services/selection-schemas ds))
-            latest-ss (->> selection-schemas
-                           (reduce (fn [acc {:keys [id]}]
-                                     (conj acc id)) [])
-                           (apply max -1))
-            extracted (when-not (= latest-ss -1)
-                        (services/extract-data store {:schema-id latest-ss
-                                                      :url rss-url}))
-            extracted-posts (get-in extracted [:feed :posts])
-            new-feed (services/insert-feed!
-                      ds
-                      {:data (merge body {:title (get-in extracted [:feed :title])
-                                          :display-picture (get-in extracted [:feed :display-picture])
-                                          :user-id (:id user)
-                                          :created-at datetime
-                                          :state "pending"})})
-            extended-posts (mapv (fn [post]
-                                   (merge post
-                                          {:feed-id (:id new-feed)
-                                           :creator-id (:id user)
-                                           :content-type-id content-type-id
-                                           :thumbnail (or (:thumbnail post) (:display-picture new-feed))}))
-                                 extracted-posts)
-            {:keys [email]} (services/user ds {:id (:id user)})]
-
-        (if (some? extracted-posts)
-          (do
-            (services/insert-incoming-post! ds {:data extended-posts})
-
-            (->> (jobs/prepare-congest-metadata
-                  ds
-                  store
-                  {:id (str email "-" (:id new-feed))
-                   :initial-delay (* 1000 60 60 24)
-                   :auto-start true
-                   :stop-after-fail false,
-                   :interval (* 1000 60 60 24)
-                   :recurring? true
-                   :args {:feed-id (:id new-feed)
-                          :creator-id (:id user)
-                          :content-type-id content-type-id
-                          :provider-id provider-id
-                          :url rss-url}
-                   :handler :update-feed-posts
-                   :created-at (utils/get-utc-timestamp-string)
-                   :sleep false})
-                 (congest/register! js))
-            (res/response new-feed))
-
-          (-> (res/response {:message "failed to extract data"})
-              (res/status 500)))))))
+      (let [new-feed (feeds/create-feed! ds js store {:user-id (:id user)
+                                                      :feed-metadata body})]
+        (if new-feed
+          (res/response new-feed)
+          (-> (res/response {:message "Failed to parse RSS feed"})
+              (res/status 422)))))))
 
 (comment
   (require '[source.db.util :as db.util]
diff --git a/src/source/workers/feeds.clj b/src/source/workers/feeds.clj
new file mode 100644
index 00000000..c8e537b3
--- /dev/null
+++ b/src/source/workers/feeds.clj
@@ -0,0 +1,91 @@
+(ns source.workers.feeds
+  (:require [source.util :as utils]
+            [source.services.xml-schemas :as xml]
+            [source.jobs.core :as jobs]
+            [congest.jobs :as congest]
+            [source.db.honey :as hon]
+            [source.jobs.handlers :as handlers]))
+
+(defn create-feed!
+  "Creates feed with incoming posts pulled from RSS feed and starts associated job"
+  [ds js store {:keys [user-id feed-metadata]}]
+  (let [{:keys [provider-id rss-url content-type-id]} feed-metadata
+        datetime (utils/get-utc-timestamp-string)
+        selection-schemas (->> [:= :provider-id provider-id]
+                               (assoc {} :where)
+                               (xml/selection-schemas ds))
+        latest-ss (->> selection-schemas
+                       (reduce (fn [acc {:keys [id]}]
+                                 (conj acc id)) [])
+                       (apply max -1))
+        extracted (when-not (= latest-ss -1)
+                    (xml/extract-data store latest-ss rss-url))
+        extracted-posts (get-in extracted [:feed :posts])
+        new-feed (hon/insert!
+                  ds
+                  {:tname :feeds
+                   :data (merge feed-metadata {:title (get-in extracted [:feed :title])
+                                               :display-picture (get-in extracted [:feed :display-picture])
+                                               :user-id user-id
+                                               :created-at datetime
+                                               :state "pending"})})
+        extended-posts (mapv (fn [post]
+                               (merge post
+                                      {:feed-id (:id new-feed)
+                                       :creator-id user-id
+                                       :content-type-id content-type-id
+                                       :thumbnail (or (:thumbnail post) (:display-picture new-feed))}))
+                             extracted-posts)
+        {:keys [email]} (hon/find-one ds {:tname :users
+                                          :where [:= :id user-id]})]
+
+    (if (some? extracted-posts)
+      (do
+        (hon/insert! ds {:tname :incoming-posts
+                         :data extended-posts})
+
+        (->> (jobs/prepare-congest-metadata
+              ds
+              store
+              {:id (str email "-" (:id new-feed))
+               :initial-delay (* 1000 60 60 24)
+               :auto-start true
+               :stop-after-fail false,
+               :interval (* 1000 60 60 24)
+               :recurring? true
+               :args {:feed-id (:id new-feed)
+                      :creator-id user-id
+                      :content-type-id content-type-id
+                      :provider-id provider-id
+                      :url rss-url}
+               :handler :update-feed-posts
+               :created-at (utils/get-utc-timestamp-string)
+               :sleep false})
+             (congest/register! js))
+        new-feed)
+      false)))
+
+(defn update-feed! [ds {:keys [feed-id feed-metadata]}]
+  (hon/update! ds {:tname :feeds
+                   :where [:= :id feed-id]
+                   :data feed-metadata
+                   :ret :1}))
+
+(defn hard-delete-feed! [ds js creator-email feed-id]
+  (let [job-id (handlers/update-feed-posts-job-id creator-email feed-id)
+        post-ids (mapv :id (hon/find ds {:tname :incoming-posts
+                                         :where [:= :feed-id feed-id]
+                                         :ret :*}))]
+    (hon/delete! ds {:tname :filtered-feeds
+                     :where [:= :feed-id feed-id]})
+    (hon/delete! ds {:tname :filtered-posts
+                     :where [:in :post-id post-ids]})
+    (hon/delete! ds {:tname :incoming-posts
+                     :where [:= :feed-id feed-id]})
+    (hon/delete! ds {:tname :feed-categories
+                     :where [:= :feed-id feed-id]})
+    (hon/delete! ds {:tname :events
+                     :where [:= :feed-id feed-id]})
+    (hon/delete! ds {:tname :feeds
+                     :where [:= :id feed-id]})
+    (congest/deregister! js job-id)))

From 168c0d019cda462cdd01b9ea508d873db2aa7853 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Tue, 27 Jan 2026 13:49:38 +0200
Subject: [PATCH 229/391] extracted services for feed categories

---
 src/source/routes/feed_categories.clj | 13 +++++--------
 src/source/routes/post.clj            | 11 ++++++-----
 src/source/routes/post_prune.clj      | 13 +++++++------
 src/source/routes/posts.clj           |  8 +++++---
 src/source/workers/feeds.clj          | 11 +++++++++++
 5 files changed, 34 insertions(+), 22 deletions(-)

diff --git a/src/source/routes/feed_categories.clj b/src/source/routes/feed_categories.clj
index b3d994b1..6605d525 100644
--- a/src/source/routes/feed_categories.clj
+++ b/src/source/routes/feed_categories.clj
@@ -1,6 +1,7 @@
 (ns source.routes.feed-categories
   (:require [source.services.interface :as services]
-            [ring.util.response :as res]))
+            [ring.util.response :as res]
+            [source.workers.feeds :as feeds]))
 
 (defn get
   {:summary "get all categories belonging to the feed with the given id"
@@ -26,10 +27,6 @@
                         [:name :string]]]}
    :responses {200 {:body [:map [:message :string]]}}}
   [{:keys [ds path-params body] :as _request}]
-  (let [update-data (reduce (fn [acc {:keys [id]}]
-                              (conj acc {:feed-id (:id path-params)
-                                         :category-id id})) [] body)]
-    (when (seq update-data)
-      (services/delete-feed-category! ds {:where [:= :feed-id (:id path-params)]})
-      (services/insert-feed-category! ds {:data update-data}))
-    (res/response {:message "successfully updated feed categories"})))
+  (feeds/update-feed-categories! ds {:feed-id (:id path-params)
+                                     :categories body})
+  (res/response {:message "successfully updated feed categories"}))
diff --git a/src/source/routes/post.clj b/src/source/routes/post.clj
index 832529a9..5fd9f6f4 100644
--- a/src/source/routes/post.clj
+++ b/src/source/routes/post.clj
@@ -1,6 +1,6 @@
 (ns source.routes.post
-  (:require [source.services.interface :as services]
-            [ring.util.response :as res]))
+  (:require [ring.util.response :as res]
+            [source.db.honey :as hon]))
 
 (defn get
   {:summary "get post by id"
@@ -23,7 +23,8 @@
                             [:posted-at [:maybe :string]]]}}}
 
   [{:keys [ds user path-params] :as _request}]
-  (-> (services/incoming-post ds {:where [:and
-                                          [:= :id (:post-id path-params)]
-                                          [:= :creator-id (:id user)]]})
+  (-> (hon/find-one ds {:tname :incoming-posts
+                        :where [:and
+                                [:= :id (:post-id path-params)]
+                                [:= :creator-id (:id user)]]})
       (res/response)))
diff --git a/src/source/routes/post_prune.clj b/src/source/routes/post_prune.clj
index 4b252ea2..79f198c6 100644
--- a/src/source/routes/post_prune.clj
+++ b/src/source/routes/post_prune.clj
@@ -1,6 +1,6 @@
 (ns source.routes.post-prune
-  (:require [source.services.interface :as services]
-            [ring.util.response :as res]))
+  (:require [ring.util.response :as res]
+            [source.db.honey :as hon]))
 
 (defn post
   {:summary "Update redacted status of post with the given id"
@@ -13,8 +13,9 @@
                 403 {:body [:map [:message :string]]}}}
 
   [{:keys [ds path-params user body] :as _request}]
-  (services/update-incoming-post! ds {:where [:and
-                                              [:= :id (:post-id path-params)]
-                                              [:= :creator-id (:id user)]]
-                                      :data {:redacted (if (:redacted body) 1 0)}})
+  (hon/update! ds {:tname :incoming-posts
+                   :where [:and
+                           [:= :id (:post-id path-params)]
+                           [:= :creator-id (:id user)]]
+                   :data {:redacted (if (:redacted body) 1 0)}})
   (res/response {:message "successfully updated post"}))
diff --git a/src/source/routes/posts.clj b/src/source/routes/posts.clj
index e3d93cfe..8de11a8f 100644
--- a/src/source/routes/posts.clj
+++ b/src/source/routes/posts.clj
@@ -1,6 +1,6 @@
 (ns source.routes.posts
-  (:require [source.services.interface :as services]
-            [ring.util.response :as res]))
+  (:require [ring.util.response :as res]
+            [source.db.honey :as hon]))
 
 (defn get
   {:summary "get all posts by feed id"
@@ -26,5 +26,7 @@
                403 {:body [:map [:message :string]]}}}
 
   [{:keys [ds path-params] :as _request}]
-  (-> (services/incoming-posts ds {:where [:= :feed-id (:id path-params)]})
+  (-> (hon/find ds {:tname :incoming-posts
+                    :where [:= :feed-id (:id path-params)]
+                    :ret :*})
       (res/response)))
diff --git a/src/source/workers/feeds.clj b/src/source/workers/feeds.clj
index c8e537b3..53ce86b8 100644
--- a/src/source/workers/feeds.clj
+++ b/src/source/workers/feeds.clj
@@ -89,3 +89,14 @@
     (hon/delete! ds {:tname :feeds
                      :where [:= :id feed-id]})
     (congest/deregister! js job-id)))
+
+(defn update-feed-categories! [ds {:keys [feed-id categories]}]
+  (let [update-data (mapv (fn [{:keys [id]}]
+                            {:feed-id feed-id
+                             :category-id id}) categories)]
+    (when (seq update-data)
+      (hon/delete! ds {:tname :feed-categories
+                       :where [:= :feed-id feed-id]})
+      (hon/insert! ds {:tname :feed-categories
+                       :data update-data
+                       :ret :*}))))

From 3cb91014c19d7e6e0b126f9f2622927ecc9a0274 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Tue, 27 Jan 2026 13:58:43 +0200
Subject: [PATCH 230/391] extracted services for bundle categories

---
 src/source/routes/bundle.clj            |  5 +++--
 src/source/routes/bundle_categories.clj | 10 ++--------
 src/source/routes/reitit.clj            |  1 +
 src/source/workers/bundles.clj          | 16 ++++++++++++++++
 4 files changed, 22 insertions(+), 10 deletions(-)
 create mode 100644 src/source/workers/bundles.clj

diff --git a/src/source/routes/bundle.clj b/src/source/routes/bundle.clj
index ac922c95..0addbadb 100644
--- a/src/source/routes/bundle.clj
+++ b/src/source/routes/bundle.clj
@@ -1,6 +1,6 @@
 (ns source.routes.bundle
   (:require [ring.util.response :as res]
-            [source.services.interface :as services]))
+            [source.db.honey :as hon]))
 
 (defn get
   {:summary "get bundle metadata by authorized uuid"
@@ -19,4 +19,5 @@
                404 {:body [:map [:message :string]]}}}
 
   [{:keys [ds bundle-id] :as _request}]
-  (res/response (services/bundle ds {:id bundle-id})))
+  (res/response (hon/find-one ds {:tname :bundles
+                                  :where [:= :id bundle-id]})))
diff --git a/src/source/routes/bundle_categories.clj b/src/source/routes/bundle_categories.clj
index cbf9ef60..6c362bad 100644
--- a/src/source/routes/bundle_categories.clj
+++ b/src/source/routes/bundle_categories.clj
@@ -1,6 +1,5 @@
 (ns source.routes.bundle-categories
-  (:require [source.services.interface :as services]
-            [source.db.util :as db.util]
+  (:require [source.workers.bundles :as bundles]
             [ring.util.response :as res]))
 
 (defn get
@@ -14,9 +13,4 @@
                404 {:body [:map [:message :string]]}}}
 
   [{:keys [bundle-id ds] :as _request}]
-  (with-open [bundle-ds (db.util/conn :bundle bundle-id)]
-    (let [feed-ids (->> (services/outgoing-posts bundle-ds)
-                        (mapv :feed-id))
-          category-ids (->> (services/feed-categories ds {:where [:in :feed-id feed-ids]})
-                            (mapv :category-id))]
-      (res/response (services/categories ds {:where [:in :id category-ids]})))))
+  (res/response (bundles/get-bundle-categories ds bundle-id)))
diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index 17973e26..b857abd6 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -213,6 +213,7 @@
       ["admin" {:middleware [[mw/apply-auth {:required-type :admin}]]}
        ["/general"]
        ["/top"]]]
+
      ["/bundle" {:middleware [[mw/apply-bundle]]
                  :tags #{"bundles"}}
       ["" (get bundle/get)]
diff --git a/src/source/workers/bundles.clj b/src/source/workers/bundles.clj
new file mode 100644
index 00000000..153867bb
--- /dev/null
+++ b/src/source/workers/bundles.clj
@@ -0,0 +1,16 @@
+(ns source.workers.bundles
+  (:require [source.db.util :as db.util]
+            [source.db.honey :as hon]))
+
+(defn get-bundle-categories [ds bundle-id]
+  (with-open [bundle-ds (db.util/conn :bundle bundle-id)]
+    (let [feed-ids (->> (hon/find bundle-ds {:tname :outgoing-posts
+                                             :ret :*})
+                        (mapv :feed-id))
+          category-ids (->> (hon/find ds {:tname :feed-categories
+                                          :where [:in :feed-id feed-ids]
+                                          :ret :*})
+                            (mapv :category-id))]
+      (hon/find ds {:tname :categories
+                    :where [:in :id category-ids]
+                    :ret :*}))))

From ad4bfb2f8671584bad793fbe34f87c19ef8f578e Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Tue, 27 Jan 2026 14:13:25 +0200
Subject: [PATCH 231/391] extracted service for getting filtered feeds in
 bundle

---
 src/source/routes/bundle_feeds.clj | 35 +++++++++---------------------
 src/source/workers/bundles.clj     | 35 +++++++++++++++++++++++++++++-
 2 files changed, 44 insertions(+), 26 deletions(-)

diff --git a/src/source/routes/bundle_feeds.clj b/src/source/routes/bundle_feeds.clj
index d1944d41..e03f63be 100644
--- a/src/source/routes/bundle_feeds.clj
+++ b/src/source/routes/bundle_feeds.clj
@@ -5,7 +5,8 @@
             [clojure.walk :as walk]
             [honey.sql.helpers :as hsql]
             [source.services.analytics.interface :as analytics]
-            [source.db.honey :as hon]))
+            [source.db.honey :as hon]
+            [source.workers.bundles :as bundles]))
 
 (defn post
   {:summary "get all feeds present in the bundle authorised by uuid"
@@ -34,32 +35,16 @@
                404 {:body [:map [:message :string]]}}}
 
   [{:keys [ds bundle-id query-params body] :as _request}]
-  (with-open [bundle-ds (db.util/conn :bundle bundle-id)]
-    (let [{:keys [category-ids]} body
-          {:keys [type latest nonfiltered]} (walk/keywordize-keys query-params)
-          feed-ids (mapv :feed-id (services/outgoing-posts bundle-ds))
-          category-filtered-feed-ids (if (empty? category-ids)
-                                       feed-ids
-                                       (->> (hsql/where
-                                             [:in :feed-id feed-ids]
-                                             [:in :category-id category-ids])
-                                            (services/feed-categories ds)
-                                            (mapv :feed-id)))
-          blocked-feed-ids (if (some? nonfiltered)
-                             []
-                             (mapv :feed-id (services/filtered-feeds ds {:where [:= :bundle-id bundle-id]})))
-          query (-> (when type [:= :content-type-id type])
-                    (hsql/where [:in :id category-filtered-feed-ids]
-                                [:not [:in :id blocked-feed-ids]])
-                    (hsql/order-by (when latest [:created-at :desc])))
-          type-filtered (services/feeds ds query)]
-
-      (analytics/insert-feed-impressions! ds type-filtered bundle-id)
-      (res/response type-filtered))))
+  (let [{:keys [type latest nonfiltered]} (walk/keywordize-keys query-params)]
+    (->> {:bundle-id bundle-id
+          :type type
+          :latest latest
+          :category-ids (:category-ids body)
+          :nonfiltered nonfiltered}
+         (bundles/get-feeds-in-bundle! ds)
+         (res/response))))
 
 (comment
   (def ds (db.util/conn :master))
 
-  
-
   ())
diff --git a/src/source/workers/bundles.clj b/src/source/workers/bundles.clj
index 153867bb..ea7650e9 100644
--- a/src/source/workers/bundles.clj
+++ b/src/source/workers/bundles.clj
@@ -1,6 +1,8 @@
 (ns source.workers.bundles
   (:require [source.db.util :as db.util]
-            [source.db.honey :as hon]))
+            [source.db.honey :as hon]
+            [honey.sql.helpers :as hsql]
+            [source.services.analytics.interface :as analytics]))
 
 (defn get-bundle-categories [ds bundle-id]
   (with-open [bundle-ds (db.util/conn :bundle bundle-id)]
@@ -14,3 +16,34 @@
       (hon/find ds {:tname :categories
                     :where [:in :id category-ids]
                     :ret :*}))))
+
+(defn get-feeds-in-bundle!
+  "Gets a filtered list of feeds from the bundle. Updates analytics impressions."
+  [ds {:keys [bundle-id type latest category-ids nonfiltered]}]
+  (with-open [bundle-ds (db.util/conn :bundle bundle-id)]
+    (let [feed-ids (mapv :feed-id (hon/find bundle-ds {:tname :outgoing-posts
+                                                       :ret :*}))
+          category-filtered-feed-ids (if (empty? category-ids)
+                                       feed-ids
+                                       (->> (hsql/where
+                                             [:in :feed-id feed-ids]
+                                             [:in :category-id category-ids])
+                                            (merge {:tname :feed-categories
+                                                    :ret :*})
+                                            (hon/find ds)
+                                            (mapv :feed-id)))
+          blocked-feed-ids (if (some? nonfiltered)
+                             []
+                             (mapv :feed-id (hon/find ds {:tname :filtered-feeds
+                                                          :where [:= :bundle-id bundle-id]
+                                                          :ret :*})))
+          query (-> (when type [:= :content-type-id type])
+                    (hsql/where [:in :id category-filtered-feed-ids]
+                                [:not [:in :id blocked-feed-ids]])
+                    (hsql/order-by (when latest [:created-at :desc]))
+                    (merge {:tname :feeds
+                            :ret :*}))
+          type-filtered (hon/find ds query)]
+
+      (analytics/insert-feed-impressions! ds type-filtered bundle-id)
+      type-filtered)))

From 32db1970d1d14ab8e9c3b07f82b6ccaece80c244 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Tue, 27 Jan 2026 14:38:59 +0200
Subject: [PATCH 232/391] extracted bundle handler work into services

---
 src/source/routes/bundle_feed.clj       | 12 ++--
 src/source/routes/bundle_feed_post.clj  | 12 ++--
 src/source/routes/bundle_feed_posts.clj | 12 ++--
 src/source/routes/bundle_feeds.clj      |  4 --
 src/source/routes/bundle_post.clj       | 13 ++--
 src/source/routes/bundle_posts.clj      | 57 ++++-------------
 src/source/workers/bundles.clj          | 82 ++++++++++++++++++++++++-
 7 files changed, 116 insertions(+), 76 deletions(-)

diff --git a/src/source/routes/bundle_feed.clj b/src/source/routes/bundle_feed.clj
index 714c6e6c..f75e88a4 100644
--- a/src/source/routes/bundle_feed.clj
+++ b/src/source/routes/bundle_feed.clj
@@ -1,7 +1,6 @@
 (ns source.routes.bundle-feed
-  (:require [source.services.interface :as services]
-            [ring.util.response :as res]
-            [source.services.analytics.interface :as analytics]))
+  (:require [ring.util.response :as res]
+            [source.workers.bundles :as bundles]))
 
 (defn get
   {:summary "get feed by id"
@@ -26,6 +25,7 @@
                 404 {:body [:map [:message :string]]}}}
 
   [{:keys [ds bundle-id path-params] :as _request}]
-  (let [feed (services/feed ds path-params)]
-    (analytics/insert-feed-click! ds feed bundle-id)
-    (res/response feed)))
+  (->> {:bundle-id bundle-id
+        :feed-id (:id path-params)}
+       (bundles/get-feed-in-bundle! ds)
+       (res/response)))
diff --git a/src/source/routes/bundle_feed_post.clj b/src/source/routes/bundle_feed_post.clj
index 87d01f5b..db590562 100644
--- a/src/source/routes/bundle_feed_post.clj
+++ b/src/source/routes/bundle_feed_post.clj
@@ -1,7 +1,6 @@
 (ns source.routes.bundle-feed-post
-  (:require [source.services.interface :as services]
-            [ring.util.response :as res]
-            [source.services.analytics.interface :as analytics]))
+  (:require [ring.util.response :as res]
+            [source.workers.bundles :as bundles]))
 
 (defn get
   {:summary "get post by post id"
@@ -27,6 +26,7 @@
                404 {:body [:map [:message :string]]}}}
 
   [{:keys [ds bundle-id path-params] :as _request}]
-  (let [post (services/incoming-post ds {:id (:post-id path-params)})]
-    (analytics/insert-post-click! ds post bundle-id)
-    (res/response post)))
+  (->> {:bundle-id bundle-id
+        :post-id (:id path-params)}
+       (bundles/get-post-by-feed-in-bundle! ds)
+       (res/response)))
diff --git a/src/source/routes/bundle_feed_posts.clj b/src/source/routes/bundle_feed_posts.clj
index f36bf94b..18c6ad41 100644
--- a/src/source/routes/bundle_feed_posts.clj
+++ b/src/source/routes/bundle_feed_posts.clj
@@ -1,7 +1,6 @@
 (ns source.routes.bundle-feed-posts
-  (:require [source.services.interface :as services]
-            [ring.util.response :as res]
-            [source.services.analytics.interface :as analytics]))
+  (:require [ring.util.response :as res]
+            [source.workers.bundles :as bundles]))
 
 (defn get
   {:summary "get all posts by feed id"
@@ -28,6 +27,7 @@
                403 {:body [:map [:message :string]]}}}
 
   [{:keys [ds bundle-id path-params] :as _request}]
-  (let [posts (services/incoming-posts ds {:where [:= :feed-id (:id path-params)]})]
-    (analytics/insert-post-impressions! ds posts bundle-id)
-    (res/response posts)))
+  (->> {:bundle-id bundle-id
+        :feed-id (:id path-params)}
+       (bundles/get-posts-by-feed-in-bundle! ds)
+       (res/response)))
diff --git a/src/source/routes/bundle_feeds.clj b/src/source/routes/bundle_feeds.clj
index e03f63be..4c85bcb9 100644
--- a/src/source/routes/bundle_feeds.clj
+++ b/src/source/routes/bundle_feeds.clj
@@ -1,11 +1,7 @@
 (ns source.routes.bundle-feeds
   (:require [ring.util.response :as res]
-            [source.services.interface :as services]
             [source.db.util :as db.util]
             [clojure.walk :as walk]
-            [honey.sql.helpers :as hsql]
-            [source.services.analytics.interface :as analytics]
-            [source.db.honey :as hon]
             [source.workers.bundles :as bundles]))
 
 (defn post
diff --git a/src/source/routes/bundle_post.clj b/src/source/routes/bundle_post.clj
index 259607d7..943c6a56 100644
--- a/src/source/routes/bundle_post.clj
+++ b/src/source/routes/bundle_post.clj
@@ -2,7 +2,8 @@
   (:require [source.services.interface :as services]
             [source.db.util :as db.util]
             [ring.util.response :as res]
-            [source.services.analytics.interface :as analytics]))
+            [source.services.analytics.interface :as analytics]
+            [source.workers.bundles :as bundles]))
 
 (defn get
   {:summary "get a single outgoing post in the uuid-authorized bundle by post id"
@@ -27,9 +28,7 @@
                404 {:body [:map [:message :string]]}}}
 
   [{:keys [ds bundle-id path-params] :as _request}]
-  (with-open [bundle-ds (db.util/conn :bundle bundle-id)]
-    (let [id (:id path-params)
-          post (services/outgoing-post bundle-ds {:id id})]
-      (analytics/insert-post-click! ds post bundle-id)
-      (res/response post))))
-
+  (->> {:bundle-id bundle-id
+        :post-id (:id path-params)}
+       (bundles/get-outgoing-post! ds)
+       (res/response)))
diff --git a/src/source/routes/bundle_posts.clj b/src/source/routes/bundle_posts.clj
index 79c23ebd..1023b310 100644
--- a/src/source/routes/bundle_posts.clj
+++ b/src/source/routes/bundle_posts.clj
@@ -5,7 +5,8 @@
             [ring.util.response :as res]
             [clojure.set :as set]
             [honey.sql.helpers :as hsql]
-            [source.services.analytics.interface :as analytics]))
+            [source.services.analytics.interface :as analytics]
+            [source.workers.bundles :as bundles]))
 
 (defn post
   {:summary "get all outgoing posts in the uuid-authorized bundle"
@@ -35,48 +36,12 @@
                404 {:boy [:map [:message :string]]}}}
 
   [{:keys [ds bundle-id query-params body] :as _request}]
-  (with-open [bundle-ds (db.util/conn :bundle bundle-id)]
-    (let [{:keys [limit start type latest]} (walk/keywordize-keys query-params)
-          {:keys [category-ids]} body
-
-          content-type-comp (when type [:= :content-type-id type])
-          start (when start (try (Integer/parseInt start) (catch Exception _)))
-          limit (when limit (try (Integer/parseInt limit) (catch Exception _)))
-
-          all-feed-ids (mapv :id (services/feeds ds))
-          blocked-feed-ids (mapv :feed-id (services/filtered-feeds ds {:where [:= :bundle-id bundle-id]}))
-          available-feed-ids (vec (remove (set blocked-feed-ids) all-feed-ids))
-
-          blocked-post-ids (mapv :post-id (services/filtered-posts ds {:where [:= :bundle-id bundle-id]}))
-
-          filtered-posts (services/outgoing-posts bundle-ds (-> (hsql/where content-type-comp
-                                                                            [:not [:in :id blocked-post-ids]]
-                                                                            [:in :feed-id available-feed-ids])
-                                                                (hsql/order-by (when (= latest "true") [[:posted-at :desc]]))))
-
-          categorised-posts (vec
-                             (if (seq category-ids)
-                               (->> filtered-posts
-                                    (mapv
-                                     (fn [post]
-                                       (when (seq (set/intersection
-                                                   (set category-ids)
-                                                   (->> {:feed-id (:feed-id post)}
-                                                        (services/categories-by-feed ds)
-                                                        (mapv :id)
-                                                        (set))))
-                                         post)))
-                                    (remove nil?))
-                               filtered-posts))
-
-          valid-start? (and (some? start) (>= start 0) (< start (count categorised-posts)))
-          started-posts (if valid-start?
-                          (subvec categorised-posts start)
-                          categorised-posts)
-
-          limited-posts (if (and (some? limit) (> (count started-posts) limit))
-                          (subvec started-posts 0 limit)
-                          started-posts)]
-
-      (analytics/insert-post-impressions! ds limited-posts bundle-id)
-      (res/response limited-posts))))
+  (let [{:keys [limit start type latest]} (walk/keywordize-keys query-params)]
+    (->> {:bundle-id bundle-id
+          :limit limit
+          :start start
+          :type type
+          :latest latest
+          :category-ids (:category-ids body)}
+         (bundles/get-outgoing-posts! ds)
+         (res/response))))
diff --git a/src/source/workers/bundles.clj b/src/source/workers/bundles.clj
index ea7650e9..d0da0d62 100644
--- a/src/source/workers/bundles.clj
+++ b/src/source/workers/bundles.clj
@@ -2,7 +2,9 @@
   (:require [source.db.util :as db.util]
             [source.db.honey :as hon]
             [honey.sql.helpers :as hsql]
-            [source.services.analytics.interface :as analytics]))
+            [source.services.analytics.interface :as analytics]
+            [clojure.set :as set]
+            [source.services.feed-categories :as feed-categories]))
 
 (defn get-bundle-categories [ds bundle-id]
   (with-open [bundle-ds (db.util/conn :bundle bundle-id)]
@@ -47,3 +49,81 @@
 
       (analytics/insert-feed-impressions! ds type-filtered bundle-id)
       type-filtered)))
+
+(defn get-feed-in-bundle! [ds {:keys [bundle-id feed-id]}]
+  (let [feed (hon/find-one ds {:tname :feeds
+                               :where [:= :id feed-id]})]
+    (analytics/insert-feed-click! ds feed bundle-id)
+    feed))
+
+(defn get-posts-by-feed-in-bundle! [ds {:keys [bundle-id feed-id]}]
+  (let [posts (hon/find ds {:tname :incoming-posts
+                            :where [:= :feed-id feed-id]
+                            :ret :*})]
+    (analytics/insert-post-impressions! ds posts bundle-id)
+    posts))
+
+(defn get-post-by-feed-in-bundle! [ds {:keys [bundle-id post-id]}]
+  (let [post (hon/find-one ds {:tname :incoming-posts
+                               :where [:= :id post-id]})]
+    (analytics/insert-post-click! ds post bundle-id)
+    post))
+
+(defn get-outgoing-posts! [ds {:keys [bundle-id limit start type latest category-ids]}]
+  (with-open [bundle-ds (db.util/conn :bundle bundle-id)]
+    (let [content-type-comp (when type [:= :content-type-id type])
+          start (when start (try (Integer/parseInt start) (catch Exception _)))
+          limit (when limit (try (Integer/parseInt limit) (catch Exception _)))
+
+          all-feed-ids (mapv :id (hon/find ds {:tname :feeds
+                                               :ret :*}))
+          blocked-feed-ids (mapv :feed-id (hon/find ds {:tname :filtered-feeds
+                                                        :where [:= :bundle-id bundle-id]
+                                                        :ret :*}))
+          available-feed-ids (vec (remove (set blocked-feed-ids) all-feed-ids))
+
+          blocked-post-ids (mapv :post-id (hon/find ds {:tname :filtered-posts
+                                                        :where [:= :bundle-id bundle-id]
+                                                        :ret :*}))
+
+          filtered-posts (hon/find bundle-ds (-> (hsql/where content-type-comp
+                                                             [:not [:in :id blocked-post-ids]]
+                                                             [:in :feed-id available-feed-ids])
+                                                 (hsql/order-by (when (= latest "true") [[:posted-at :desc]]))
+                                                 (merge {:tname :outgoing-posts
+                                                         :ret :*})))
+
+          categorised-posts (vec
+                             (if (seq category-ids)
+                               (->> filtered-posts
+                                    (mapv
+                                     (fn [post]
+                                       (when (seq (set/intersection
+                                                   (set category-ids)
+                                                   (->> {:feed-id (:feed-id post)}
+                                                        (feed-categories/categories-by-feed ds)
+                                                        (mapv :id)
+                                                        (set))))
+                                         post)))
+                                    (remove nil?))
+                               filtered-posts))
+
+          valid-start? (and (some? start) (>= start 0) (< start (count categorised-posts)))
+          started-posts (if valid-start?
+                          (subvec categorised-posts start)
+                          categorised-posts)
+
+          limited-posts (if (and (some? limit) (> (count started-posts) limit))
+                          (subvec started-posts 0 limit)
+                          started-posts)]
+
+      (analytics/insert-post-impressions! ds limited-posts bundle-id)
+      limited-posts)))
+
+(defn get-outgoing-post! [ds {:keys [bundle-id post-id]}]
+  (with-open [bundle-ds (db.util/conn :bundle bundle-id)]
+    (let [post (hon/find-one bundle-ds {:tname :outgoing-posts
+                                        :where [:= :id post-id]
+                                        :ret :1})]
+      (analytics/insert-post-click! ds post bundle-id)
+      post)))

From ae0cabd55a31680b723b9d35606fa8a8b641b509 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Tue, 27 Jan 2026 14:47:25 +0200
Subject: [PATCH 233/391] cleaned up some admin endpoints

---
 src/source/db/honey.clj            |  2 +-
 src/source/routes/admin_feeds.clj  |  7 ++++---
 src/source/routes/approve_feed.clj | 14 +++++++++-----
 src/source/routes/reject_feed.clj  | 14 +++++++++-----
 4 files changed, 23 insertions(+), 14 deletions(-)

diff --git a/src/source/db/honey.clj b/src/source/db/honey.clj
index 51d94181..4691c200 100644
--- a/src/source/db/honey.clj
+++ b/src/source/db/honey.clj
@@ -39,7 +39,7 @@
                  (or (cske/transform-keys
                       csk/->snake_case_keyword where)
                      [])))
-            :ret ret))
+            :ret (or ret :*)))
 
 (defn find-one [ds opts]
   (->> {:ret :1}
diff --git a/src/source/routes/admin_feeds.clj b/src/source/routes/admin_feeds.clj
index c67ac05c..a4f7a0e9 100644
--- a/src/source/routes/admin_feeds.clj
+++ b/src/source/routes/admin_feeds.clj
@@ -1,6 +1,6 @@
 (ns source.routes.admin-feeds
-  (:require [source.services.interface :as services]
-            [ring.util.response :as res]))
+  (:require [ring.util.response :as res]
+            [source.db.honey :as hon]))
 
 (defn get
   {:summary "get all feeds"
@@ -22,5 +22,6 @@
                              [:state [:enum "live" "not live" "pending"]]]]}}}
 
   [{:keys [ds] :as _request}]
-  (-> (services/feeds ds)
+  (-> (hon/find ds {:tname :feeds
+                    :ret :*})
       (res/response)))
diff --git a/src/source/routes/approve_feed.clj b/src/source/routes/approve_feed.clj
index bcad0eb5..dc1746ec 100644
--- a/src/source/routes/approve_feed.clj
+++ b/src/source/routes/approve_feed.clj
@@ -2,7 +2,8 @@
   (:require [source.services.interface :as services]
             [source.email.gmail :as gmail]
             [source.email.templates :as templates]
-            [ring.util.response :as res]))
+            [ring.util.response :as res]
+            [source.db.honey :as hon]))
 
 (defn post
   {:summary "approve the feed with the given feed-id and allow it to go live"
@@ -13,10 +14,13 @@
                403 {:body [:map [:message :string]]}}}
 
   [{:keys [ds path-params] :as _request}]
-  (let [{:keys [id user-id title]} (services/feed ds path-params)
-        {:keys [email firstname]} (services/user ds {:id user-id})]
-    (services/update-feed! ds {:id (:id path-params)
-                               :data {:state "live"}})
+  (let [{:keys [id user-id title]} (hon/find-one ds {:tname :feeds
+                                                     :where [:= :id (:id path-params)]})
+        {:keys [email firstname]} (services/user ds {:tname :users
+                                                     :where [:= :id user-id]})]
+    (hon/update! ds {:tname :feeds
+                     :where [:= :id (:id path-params)]
+                     :data {:state "live"}})
     (gmail/send-email {:to email
                        :subject "Feed Approval"
                        :body (templates/feed-approval {:creator-name firstname
diff --git a/src/source/routes/reject_feed.clj b/src/source/routes/reject_feed.clj
index d483a5bb..b908d1dd 100644
--- a/src/source/routes/reject_feed.clj
+++ b/src/source/routes/reject_feed.clj
@@ -2,7 +2,8 @@
   (:require [source.services.interface :as services]
             [source.email.gmail :as gmail]
             [source.email.templates :as templates]
-            [ring.util.response :as res]))
+            [ring.util.response :as res]
+            [source.db.honey :as hon]))
 
 (defn post
   {:summary "reject the feed with the given feed-id and prevent it from going live"
@@ -14,10 +15,13 @@
                403 {:body [:map [:message :string]]}}}
 
   [{:keys [ds path-params body] :as _request}]
-  (let [{:keys [user-id title]} (services/feed ds path-params)
-        {:keys [email firstname]} (services/user ds {:id user-id})]
-    (services/update-feed! ds {:id (:id path-params)
-                               :data {:state "not live"}})
+  (let [{:keys [user-id title]} (hon/find-one ds {:tname :feeds
+                                                  :where [:= :id (:id path-params)]})
+        {:keys [email firstname]} (hon/find-one ds {:tname :users
+                                                    :where [:= :id user-id]})]
+    (hon/update! ds {:tname :feeds
+                     :where [:= :id (:id path-params)]
+                     :data {:state "not live"}})
     (gmail/send-email {:to email
                        :subject "Feed Rejection"
                        :body (templates/feed-rejection {:creator-name firstname

From 297e4df2b84ed28e9e4ded14b64b7dbc272ee9d1 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Tue, 27 Jan 2026 14:54:29 +0200
Subject: [PATCH 234/391] added docstrings

---
 src/source/routes/users.clj    |  5 +++--
 src/source/workers/bundles.clj | 24 ++++++++++++++++++------
 2 files changed, 21 insertions(+), 8 deletions(-)

diff --git a/src/source/routes/users.clj b/src/source/routes/users.clj
index 2a05f65b..e778a93b 100644
--- a/src/source/routes/users.clj
+++ b/src/source/routes/users.clj
@@ -1,6 +1,6 @@
 (ns source.routes.users
   (:require [ring.util.response :as res]
-            [source.services.users :as users]))
+            [source.db.honey :as hon]))
 
 (defn get
   {:summary "get all users"
@@ -22,7 +22,8 @@
                 403 {:body [:map [:message :string]]}}}
 
   [{:keys [ds] :as _request}]
-  (res/response {:users (users/users ds)}))
+  (res/response {:users (hon/find ds {:tname :users
+                                      :ret :*})}))
 
 (comment
   (require '[source.db.interface :as db])
diff --git a/src/source/workers/bundles.clj b/src/source/workers/bundles.clj
index d0da0d62..f13b1c96 100644
--- a/src/source/workers/bundles.clj
+++ b/src/source/workers/bundles.clj
@@ -6,7 +6,9 @@
             [clojure.set :as set]
             [source.services.feed-categories :as feed-categories]))
 
-(defn get-bundle-categories [ds bundle-id]
+(defn get-bundle-categories 
+  "Get all categories for feeds/posts in bundle"
+  [ds bundle-id]
   (with-open [bundle-ds (db.util/conn :bundle bundle-id)]
     (let [feed-ids (->> (hon/find bundle-ds {:tname :outgoing-posts
                                              :ret :*})
@@ -50,26 +52,34 @@
       (analytics/insert-feed-impressions! ds type-filtered bundle-id)
       type-filtered)))
 
-(defn get-feed-in-bundle! [ds {:keys [bundle-id feed-id]}]
+(defn get-feed-in-bundle! 
+  "Get feed in bundle and update analytics clicks"
+  [ds {:keys [bundle-id feed-id]}]
   (let [feed (hon/find-one ds {:tname :feeds
                                :where [:= :id feed-id]})]
     (analytics/insert-feed-click! ds feed bundle-id)
     feed))
 
-(defn get-posts-by-feed-in-bundle! [ds {:keys [bundle-id feed-id]}]
+(defn get-posts-by-feed-in-bundle! 
+  "Get all posts in a feed and update analytics impressions"
+  [ds {:keys [bundle-id feed-id]}]
   (let [posts (hon/find ds {:tname :incoming-posts
                             :where [:= :feed-id feed-id]
                             :ret :*})]
     (analytics/insert-post-impressions! ds posts bundle-id)
     posts))
 
-(defn get-post-by-feed-in-bundle! [ds {:keys [bundle-id post-id]}]
+(defn get-post-by-feed-in-bundle! 
+  "Get post in a feed and update analytics clicks"
+  [ds {:keys [bundle-id post-id]}]
   (let [post (hon/find-one ds {:tname :incoming-posts
                                :where [:= :id post-id]})]
     (analytics/insert-post-click! ds post bundle-id)
     post))
 
-(defn get-outgoing-posts! [ds {:keys [bundle-id limit start type latest category-ids]}]
+(defn get-outgoing-posts! 
+  "Get outgoing posts based on short heuristics and update analytics impressions"
+  [ds {:keys [bundle-id limit start type latest category-ids]}]
   (with-open [bundle-ds (db.util/conn :bundle bundle-id)]
     (let [content-type-comp (when type [:= :content-type-id type])
           start (when start (try (Integer/parseInt start) (catch Exception _)))
@@ -120,7 +130,9 @@
       (analytics/insert-post-impressions! ds limited-posts bundle-id)
       limited-posts)))
 
-(defn get-outgoing-post! [ds {:keys [bundle-id post-id]}]
+(defn get-outgoing-post! 
+  "Get single outgoing post and update analytics click"
+  [ds {:keys [bundle-id post-id]}]
   (with-open [bundle-ds (db.util/conn :bundle bundle-id)]
     (let [post (hon/find-one bundle-ds {:tname :outgoing-posts
                                         :where [:= :id post-id]

From e2c8149dcaa3746ceaa209f884f2df319acd1d41 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 28 Jan 2026 12:41:45 +0200
Subject: [PATCH 235/391] fixed requested changes in bundle services

---
 src/source/routes/bundle.clj            |  2 +-
 src/source/routes/bundle_categories.clj |  2 +-
 src/source/routes/bundle_feed.clj       | 13 +++----
 src/source/routes/bundle_feed_post.clj  | 13 +++----
 src/source/routes/bundle_feed_posts.clj | 14 ++++----
 src/source/routes/bundle_feeds.clj      | 22 ++++++------
 src/source/routes/bundle_post.clj       | 17 ++++-----
 src/source/routes/bundle_posts.clj      | 27 +++++++-------
 src/source/workers/bundles.clj          | 48 +++----------------------
 9 files changed, 61 insertions(+), 97 deletions(-)

diff --git a/src/source/routes/bundle.clj b/src/source/routes/bundle.clj
index 0addbadb..68b93e3d 100644
--- a/src/source/routes/bundle.clj
+++ b/src/source/routes/bundle.clj
@@ -3,7 +3,7 @@
             [source.db.honey :as hon]))
 
 (defn get
-  {:summary "get bundle metadata by authorized uuid"
+  {:summary "get metadata for the associated uuid-authorized bundle"
    :parameters {:query [:map [:uuid :string]]}
    :responses {200 {:body [:map
                            [:id :int]
diff --git a/src/source/routes/bundle_categories.clj b/src/source/routes/bundle_categories.clj
index 6c362bad..ada7ba21 100644
--- a/src/source/routes/bundle_categories.clj
+++ b/src/source/routes/bundle_categories.clj
@@ -3,7 +3,7 @@
             [ring.util.response :as res]))
 
 (defn get
-  {:summary "get categories in the uuid-authorized bundle"
+  {:summary "get all categories for content present in the uuid-authorized bundle"
    :parameters {:query [:map [:uuid :string]]}
    :responses {200 {:body [:vector
                            [:map
diff --git a/src/source/routes/bundle_feed.clj b/src/source/routes/bundle_feed.clj
index f75e88a4..23233eff 100644
--- a/src/source/routes/bundle_feed.clj
+++ b/src/source/routes/bundle_feed.clj
@@ -1,9 +1,10 @@
 (ns source.routes.bundle-feed
   (:require [ring.util.response :as res]
-            [source.workers.bundles :as bundles]))
+            [source.db.honey :as hon]
+            [source.services.analytics.interface :as analytics]))
 
 (defn get
-  {:summary "get feed by id"
+  {:summary "get feed associated with the uuid-authorized bundle by id, updates click analytics for the given feed"
    :parameters {:query [:map [:uuid :string]]
                 :path [:map [:id {:title "id"
                                   :description "feed id"} :int]]}
@@ -25,7 +26,7 @@
                 404 {:body [:map [:message :string]]}}}
 
   [{:keys [ds bundle-id path-params] :as _request}]
-  (->> {:bundle-id bundle-id
-        :feed-id (:id path-params)}
-       (bundles/get-feed-in-bundle! ds)
-       (res/response)))
+  (let [feed (hon/find-one ds {:tname :feeds
+                               :where [:= :id (:id path-params)]})]
+    (analytics/insert-feed-click! ds feed bundle-id)
+    (res/response feed)))
diff --git a/src/source/routes/bundle_feed_post.clj b/src/source/routes/bundle_feed_post.clj
index db590562..ee3d6f9e 100644
--- a/src/source/routes/bundle_feed_post.clj
+++ b/src/source/routes/bundle_feed_post.clj
@@ -1,9 +1,10 @@
 (ns source.routes.bundle-feed-post
   (:require [ring.util.response :as res]
-            [source.workers.bundles :as bundles]))
+            [source.db.honey :as hon]
+            [source.services.analytics.interface :as analytics]))
 
 (defn get
-  {:summary "get post by post id"
+  {:summary "get post in outgoing feed for the associated uuid-authorized bundle by post id, updates click analytics"
    :parameters {:query [:map [:uuid :string]]
                 :path [:map [:post-id {:title "post-id"
                                        :description "post id"} :int]]}
@@ -26,7 +27,7 @@
                404 {:body [:map [:message :string]]}}}
 
   [{:keys [ds bundle-id path-params] :as _request}]
-  (->> {:bundle-id bundle-id
-        :post-id (:id path-params)}
-       (bundles/get-post-by-feed-in-bundle! ds)
-       (res/response)))
+  (let [post (hon/find-one ds {:tname :incoming-posts
+                               :where [:= :id (:id path-params)]})]
+    (analytics/insert-post-click! ds post bundle-id)
+    (res/response post)))
diff --git a/src/source/routes/bundle_feed_posts.clj b/src/source/routes/bundle_feed_posts.clj
index 18c6ad41..cae4d957 100644
--- a/src/source/routes/bundle_feed_posts.clj
+++ b/src/source/routes/bundle_feed_posts.clj
@@ -1,9 +1,10 @@
 (ns source.routes.bundle-feed-posts
   (:require [ring.util.response :as res]
-            [source.workers.bundles :as bundles]))
+            [source.db.honey :as hon]
+            [source.services.analytics.interface :as analytics]))
 
 (defn get
-  {:summary "get all posts by feed id"
+  {:summary "get all posts in the outgoing feed for the associated uuid-authorized bundle by feed id"
    :parameters {:query [:map [:uuid :string]]
                 :path [:map [:id {:title "id"
                                   :description "feed id"} :int]]}
@@ -27,7 +28,8 @@
                403 {:body [:map [:message :string]]}}}
 
   [{:keys [ds bundle-id path-params] :as _request}]
-  (->> {:bundle-id bundle-id
-        :feed-id (:id path-params)}
-       (bundles/get-posts-by-feed-in-bundle! ds)
-       (res/response)))
+  (let [posts (hon/find ds {:tname :incoming-posts
+                            :where [:= :feed-id (:id path-params)]
+                            :ret :*})]
+    (analytics/insert-post-impressions! ds posts bundle-id)
+    (res/response posts)))
diff --git a/src/source/routes/bundle_feeds.clj b/src/source/routes/bundle_feeds.clj
index 4c85bcb9..bc91ce41 100644
--- a/src/source/routes/bundle_feeds.clj
+++ b/src/source/routes/bundle_feeds.clj
@@ -2,10 +2,11 @@
   (:require [ring.util.response :as res]
             [source.db.util :as db.util]
             [clojure.walk :as walk]
-            [source.workers.bundles :as bundles]))
+            [source.workers.bundles :as bundles]
+            [source.services.analytics.interface :as analytics]))
 
 (defn post
-  {:summary "get all feeds present in the bundle authorised by uuid"
+  {:summary "get all feeds present in the bundle authorised by uuid, updating impression analytics."
    :parameters {:query [:map
                         [:uuid :string]
                         [:type {:optional true} :int]
@@ -31,14 +32,15 @@
                404 {:body [:map [:message :string]]}}}
 
   [{:keys [ds bundle-id query-params body] :as _request}]
-  (let [{:keys [type latest nonfiltered]} (walk/keywordize-keys query-params)]
-    (->> {:bundle-id bundle-id
-          :type type
-          :latest latest
-          :category-ids (:category-ids body)
-          :nonfiltered nonfiltered}
-         (bundles/get-feeds-in-bundle! ds)
-         (res/response))))
+  (let [{:keys [type latest nonfiltered]} (walk/keywordize-keys query-params)
+        feeds (->> {:bundle-id bundle-id
+                    :type type
+                    :latest latest
+                    :category-ids (:category-ids body)
+                    :nonfiltered nonfiltered}
+                   (bundles/get-outgoing-feeds ds))]
+    (analytics/insert-feed-impressions! ds feeds bundle-id)
+    (res/response feeds)))
 
 (comment
   (def ds (db.util/conn :master))
diff --git a/src/source/routes/bundle_post.clj b/src/source/routes/bundle_post.clj
index 943c6a56..71a6270f 100644
--- a/src/source/routes/bundle_post.clj
+++ b/src/source/routes/bundle_post.clj
@@ -1,12 +1,11 @@
 (ns source.routes.bundle-post
-  (:require [source.services.interface :as services]
-            [source.db.util :as db.util]
+  (:require [source.db.util :as db.util]
             [ring.util.response :as res]
             [source.services.analytics.interface :as analytics]
-            [source.workers.bundles :as bundles]))
+            [source.db.honey :as hon]))
 
 (defn get
-  {:summary "get a single outgoing post in the uuid-authorized bundle by post id"
+  {:summary "get a single outgoing post in the uuid-authorized bundle by post id, updates click analytics"
    :parameters {:query [:map [:uuid :string]]
                 :path [:map [:id {:title "id"
                                   :description "post id"} :int]]}
@@ -28,7 +27,9 @@
                404 {:body [:map [:message :string]]}}}
 
   [{:keys [ds bundle-id path-params] :as _request}]
-  (->> {:bundle-id bundle-id
-        :post-id (:id path-params)}
-       (bundles/get-outgoing-post! ds)
-       (res/response)))
+  (with-open [bundle-ds (db.util/conn :bundle bundle-id)]
+    (let [post (hon/find-one bundle-ds {:tname :outgoing-posts
+                                        :where [:= :id (:id path-params)]
+                                        :ret :1})]
+      (analytics/insert-post-click! ds post bundle-id)
+      (res/response post))))
diff --git a/src/source/routes/bundle_posts.clj b/src/source/routes/bundle_posts.clj
index 1023b310..b4b8c5c6 100644
--- a/src/source/routes/bundle_posts.clj
+++ b/src/source/routes/bundle_posts.clj
@@ -1,15 +1,11 @@
 (ns source.routes.bundle-posts
-  (:require [source.services.interface :as services]
-            [source.db.util :as db.util]
-            [clojure.walk :as walk]
+  (:require [clojure.walk :as walk]
             [ring.util.response :as res]
-            [clojure.set :as set]
-            [honey.sql.helpers :as hsql]
             [source.services.analytics.interface :as analytics]
             [source.workers.bundles :as bundles]))
 
 (defn post
-  {:summary "get all outgoing posts in the uuid-authorized bundle"
+  {:summary "get all (optionally filtered) outgoing posts in the uuid-authorized bundle, updates impression analytics"
    :parameters {:body [:map [:category-ids [:vector :int]]]
                 :query [:map
                         [:uuid :string]
@@ -36,12 +32,13 @@
                404 {:boy [:map [:message :string]]}}}
 
   [{:keys [ds bundle-id query-params body] :as _request}]
-  (let [{:keys [limit start type latest]} (walk/keywordize-keys query-params)]
-    (->> {:bundle-id bundle-id
-          :limit limit
-          :start start
-          :type type
-          :latest latest
-          :category-ids (:category-ids body)}
-         (bundles/get-outgoing-posts! ds)
-         (res/response))))
+  (let [{:keys [limit start type latest]} (walk/keywordize-keys query-params)
+        posts (->> {:bundle-id bundle-id
+                    :limit limit
+                    :start start
+                    :type type
+                    :latest latest
+                    :category-ids (:category-ids body)}
+                   (bundles/get-outgoing-posts ds))]
+    (analytics/insert-post-impressions! ds posts bundle-id)
+    (res/response posts)))
diff --git a/src/source/workers/bundles.clj b/src/source/workers/bundles.clj
index f13b1c96..239b1b9f 100644
--- a/src/source/workers/bundles.clj
+++ b/src/source/workers/bundles.clj
@@ -2,11 +2,10 @@
   (:require [source.db.util :as db.util]
             [source.db.honey :as hon]
             [honey.sql.helpers :as hsql]
-            [source.services.analytics.interface :as analytics]
             [clojure.set :as set]
             [source.services.feed-categories :as feed-categories]))
 
-(defn get-bundle-categories 
+(defn get-bundle-categories
   "Get all categories for feeds/posts in bundle"
   [ds bundle-id]
   (with-open [bundle-ds (db.util/conn :bundle bundle-id)]
@@ -21,8 +20,8 @@
                     :where [:in :id category-ids]
                     :ret :*}))))
 
-(defn get-feeds-in-bundle!
-  "Gets a filtered list of feeds from the bundle. Updates analytics impressions."
+(defn get-outgoing-feeds
+  "Gets a filtered list of outgoing feeds for the associated bundle."
   [ds {:keys [bundle-id type latest category-ids nonfiltered]}]
   (with-open [bundle-ds (db.util/conn :bundle bundle-id)]
     (let [feed-ids (mapv :feed-id (hon/find bundle-ds {:tname :outgoing-posts
@@ -48,36 +47,9 @@
                     (merge {:tname :feeds
                             :ret :*}))
           type-filtered (hon/find ds query)]
-
-      (analytics/insert-feed-impressions! ds type-filtered bundle-id)
       type-filtered)))
 
-(defn get-feed-in-bundle! 
-  "Get feed in bundle and update analytics clicks"
-  [ds {:keys [bundle-id feed-id]}]
-  (let [feed (hon/find-one ds {:tname :feeds
-                               :where [:= :id feed-id]})]
-    (analytics/insert-feed-click! ds feed bundle-id)
-    feed))
-
-(defn get-posts-by-feed-in-bundle! 
-  "Get all posts in a feed and update analytics impressions"
-  [ds {:keys [bundle-id feed-id]}]
-  (let [posts (hon/find ds {:tname :incoming-posts
-                            :where [:= :feed-id feed-id]
-                            :ret :*})]
-    (analytics/insert-post-impressions! ds posts bundle-id)
-    posts))
-
-(defn get-post-by-feed-in-bundle! 
-  "Get post in a feed and update analytics clicks"
-  [ds {:keys [bundle-id post-id]}]
-  (let [post (hon/find-one ds {:tname :incoming-posts
-                               :where [:= :id post-id]})]
-    (analytics/insert-post-click! ds post bundle-id)
-    post))
-
-(defn get-outgoing-posts! 
+(defn get-outgoing-posts
   "Get outgoing posts based on short heuristics and update analytics impressions"
   [ds {:keys [bundle-id limit start type latest category-ids]}]
   (with-open [bundle-ds (db.util/conn :bundle bundle-id)]
@@ -126,16 +98,4 @@
           limited-posts (if (and (some? limit) (> (count started-posts) limit))
                           (subvec started-posts 0 limit)
                           started-posts)]
-
-      (analytics/insert-post-impressions! ds limited-posts bundle-id)
       limited-posts)))
-
-(defn get-outgoing-post! 
-  "Get single outgoing post and update analytics click"
-  [ds {:keys [bundle-id post-id]}]
-  (with-open [bundle-ds (db.util/conn :bundle bundle-id)]
-    (let [post (hon/find-one bundle-ds {:tname :outgoing-posts
-                                        :where [:= :id post-id]
-                                        :ret :1})]
-      (analytics/insert-post-click! ds post bundle-id)
-      post)))

From 18670a26c6ed116a33eb867e7745adfc37b1bf5b Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 28 Jan 2026 13:01:50 +0200
Subject: [PATCH 236/391] removed more unnecessary service calls

---
 src/source/workers/bundles.clj      |  5 ++-
 src/source/workers/integrations.clj | 50 +++++++++++++++++------------
 2 files changed, 31 insertions(+), 24 deletions(-)

diff --git a/src/source/workers/bundles.clj b/src/source/workers/bundles.clj
index 239b1b9f..254f954e 100644
--- a/src/source/workers/bundles.clj
+++ b/src/source/workers/bundles.clj
@@ -53,8 +53,7 @@
   "Get outgoing posts based on short heuristics and update analytics impressions"
   [ds {:keys [bundle-id limit start type latest category-ids]}]
   (with-open [bundle-ds (db.util/conn :bundle bundle-id)]
-    (let [content-type-comp (when type [:= :content-type-id type])
-          start (when start (try (Integer/parseInt start) (catch Exception _)))
+    (let [start (when start (try (Integer/parseInt start) (catch Exception _)))
           limit (when limit (try (Integer/parseInt limit) (catch Exception _)))
 
           all-feed-ids (mapv :id (hon/find ds {:tname :feeds
@@ -68,7 +67,7 @@
                                                         :where [:= :bundle-id bundle-id]
                                                         :ret :*}))
 
-          filtered-posts (hon/find bundle-ds (-> (hsql/where content-type-comp
+          filtered-posts (hon/find bundle-ds (-> (hsql/where (when type [:= :content-type-id type])
                                                              [:not [:in :id blocked-post-ids]]
                                                              [:in :feed-id available-feed-ids])
                                                  (hsql/order-by (when (= latest "true") [[:posted-at :desc]]))
diff --git a/src/source/workers/integrations.clj b/src/source/workers/integrations.clj
index 61d6504b..3a8910d2 100644
--- a/src/source/workers/integrations.clj
+++ b/src/source/workers/integrations.clj
@@ -8,10 +8,8 @@
             [source.jobs.handlers :as handlers]
             [source.util :as utils]
             [congest.jobs :as congest]
-            [source.services.filtered-feeds :as filtered-feeds]
-            [source.services.filtered-posts :as filtered-posts]
-            [source.services.analytics.interface :as analytics]
-            [source.db.tables :as tables]))
+            [source.db.tables :as tables]
+            [source.db.honey :as hon]))
 
 (defn create-integration! [ds js store {:keys [user-id bundle-metadata categories content-types]}]
   (let [new-bundle (bundles/create-bundle! ds {:user-id user-id
@@ -82,33 +80,43 @@
 
 (defn hard-delete-bundle! [ds js bundle-id]
   (let [job-id (handlers/update-bundle-job-id bundle-id)]
-    (filtered-feeds/delete-filtered-feed! ds {:where [:= :bundle-id bundle-id]})
-    (filtered-posts/delete-filtered-post! ds {:where [:= :bundle-id bundle-id]})
-    (bundle-content-types/delete-bundle-content-types! ds {:where [:= :bundle-id bundle-id]})
-    (analytics/delete-event! ds {:where [:= :bundle-id bundle-id]})
+    (hon/delete! ds {:tname :filtered-feeds
+                     :where [:= :bundle-id bundle-id]})
+    (hon/delete! ds {:tname :filtered-posts
+                     :where [:= :bundle-id bundle-id]})
+    (hon/delete! ds {:tname :bundle-content-types
+                     :where [:= :bundle-id bundle-id]})
+    (hon/delete! ds {:tname :events
+                     :where [:= :bundle-id bundle-id]})
     (tables/drop-all-tables! (db.util/conn :bundle bundle-id))
-    (bundles/delete-bundle! ds {:id bundle-id})
+    (hon/delete! ds {:tname :bundles
+                     :where [:= :id bundle-id]})
     (congest/deregister! js job-id)))
 
 (defn generate-api-key! [ds user-id bundle-id]
   (let [uuid (utils/uuid)
         api-key (utils/sha256 (str user-id bundle-id uuid))]
-    (bundles/update-bundle! ds {:id bundle-id
-                                :data {:hash api-key}})
+    (hon/update! ds {:tname :bundles
+                     :where [:= :id bundle-id]
+                     :data {:hash api-key}})
     api-key))
 
 (defn update-filtered-feeds! [ds {:keys [filtered bundle-id feed-id]}]
   (if filtered
-    (filtered-feeds/insert-filtered-feeds! ds {:data {:feed-id feed-id
-                                                      :bundle-id bundle-id}})
-    (filtered-feeds/delete-filtered-feed! ds {:where [:and
-                                                      [:= :feed-id feed-id]
-                                                      [:= :bundle-id bundle-id]]})))
+    (hon/insert! ds {:tname :filtered-feeds
+                     :data {:feed-id feed-id
+                            :bundle-id bundle-id}})
+    (hon/delete! ds {:tname :filtered-feeds
+                     :where [:and
+                             [:= :feed-id feed-id]
+                             [:= :bundle-id bundle-id]]})))
 
 (defn update-filtered-posts! [ds {:keys [filtered bundle-id post-id]}]
   (if filtered
-    (filtered-posts/insert-filtered-posts! ds {:data {:post-id post-id
-                                                      :bundle-id bundle-id}})
-    (filtered-posts/delete-filtered-post! ds {:where [:and
-                                                      [:= :post-id post-id]
-                                                      [:= :bundle-id bundle-id]]})))
+    (hon/insert! ds {:tname :filtered-posts
+                     :data {:post-id post-id
+                            :bundle-id bundle-id}})
+    (hon/delete! ds {:tname :filtered-posts
+                     :where [:and
+                             [:= :post-id post-id]
+                             [:= :bundle-id bundle-id]]})))

From 6e7760dc46028c923fbbd324e0d0922ce9176023 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 28 Jan 2026 13:12:02 +0200
Subject: [PATCH 237/391] deleted unused services

---
 src/source/services/baselines.clj            |  16 --
 src/source/services/bundle_categories.clj    |   9 --
 src/source/services/bundle_content_types.clj |  18 ---
 src/source/services/bundles.clj              |   9 --
 src/source/services/cadences.clj             |  16 --
 src/source/services/categories.clj           |   8 -
 src/source/services/content_types.clj        |   7 -
 src/source/services/feed_categories.clj      |   9 --
 src/source/services/feeds.clj                |  16 --
 src/source/services/filtered_feeds.clj       |  24 ---
 src/source/services/filtered_posts.clj       |  24 ---
 src/source/services/incoming_posts.clj       |   9 --
 src/source/services/interface.clj            | 159 -------------------
 src/source/services/outgoing_posts.clj       |  13 +-
 src/source/services/post_heuristics.clj      |  32 ----
 src/source/services/user_sectors.clj         |  18 ---
 src/source/services/users.clj                |   6 -
 17 files changed, 1 insertion(+), 392 deletions(-)

diff --git a/src/source/services/baselines.clj b/src/source/services/baselines.clj
index 4bb168dd..b790bd58 100644
--- a/src/source/services/baselines.clj
+++ b/src/source/services/baselines.clj
@@ -1,13 +1,6 @@
 (ns source.services.baselines
   (:require [source.db.interface :as db]))
 
-(defn insert-baseline! [ds {:keys [data ret] :as opts}]
-  (->> {:tname :baselines
-        :data data
-        :ret ret}
-       (merge opts)
-       (db/insert! ds)))
-
 (defn baselines
   ([ds] (baselines ds {}))
   ([ds opts]
@@ -15,12 +8,3 @@
          :ret :*}
         (merge opts)
         (db/find ds))))
-
-(defn baseline [ds {:keys [id where] :as opts}]
-  (->> {:tname :baselines
-        :where (if (some? id)
-                 [:= :id id]
-                 where)
-        :ret :1}
-       (merge opts)
-       (db/find ds)))
diff --git a/src/source/services/bundle_categories.clj b/src/source/services/bundle_categories.clj
index 9275c36d..0c4c004c 100644
--- a/src/source/services/bundle_categories.clj
+++ b/src/source/services/bundle_categories.clj
@@ -1,15 +1,6 @@
 (ns source.services.bundle-categories
   (:require [source.db.interface :as db]))
 
-(defn bundle-categories
-  ([ds] (bundle-categories ds {}))
-  ([ds {:keys [where] :as opts}]
-   (->> {:tname :bundle-categories
-         :where where
-         :ret :*}
-        (merge opts)
-        (db/find ds))))
-
 (defn insert-bundle-category! [ds {:keys [_data _ret] :as opts}]
   (->> {:tname :bundle-categories}
        (merge opts)
diff --git a/src/source/services/bundle_content_types.clj b/src/source/services/bundle_content_types.clj
index 6bd3bc21..38e972e8 100644
--- a/src/source/services/bundle_content_types.clj
+++ b/src/source/services/bundle_content_types.clj
@@ -2,15 +2,6 @@
   (:require [source.db.interface :as db]
             [source.db.honey :as hon]))
 
-(defn bundle-content-types
-  ([ds] (bundle-content-types ds {}))
-  ([ds {:keys [where] :as opts}]
-   (->> {:tname :bundle-content-types
-         :where where
-         :ret :*}
-        (merge opts)
-        (db/find ds))))
-
 ;;NEW
 (defn insert-bundle-content-types! [ds {:keys [bundle-id content-types]}]
   (let [content-types (mapv (fn [{:keys [id]}]
@@ -40,15 +31,6 @@
                           where)}
                 {:ret :*}))
 
-(defn content-type-id [ds {:keys [bundle-id where] :as opts}]
-  (->> {:tname :bundle-content-types
-        :where (if (some? bundle-id)
-                 [:= :bundle-id bundle-id]
-                 where)
-        :ret :*}
-       (merge opts)
-       (db/find ds)))
-
 ;;NEW
 (defn update-bundle-content-types! [ds {:keys [bundle-id content-types]}]
   (let [content-types (mapv (fn [{:keys [id]}]
diff --git a/src/source/services/bundles.clj b/src/source/services/bundles.clj
index c1c8e294..7f4ec533 100644
--- a/src/source/services/bundles.clj
+++ b/src/source/services/bundles.clj
@@ -44,15 +44,6 @@
        (merge opts)
        (db/find ds)))
 
-(defn delete-bundle! [ds {:keys [id where] :as opts}]
-  (->> {:tname :bundles
-        :where (if (some? id)
-                 [:= :id id]
-                 where)
-        :ret :1}
-       (merge opts)
-       (db/delete! ds)))
-
 ;;NEW
 (defn categories-in-bundle [ds bundle-id]
   (with-open [bundle-ds (db.util/conn :bundle bundle-id)]
diff --git a/src/source/services/cadences.clj b/src/source/services/cadences.clj
index d806938b..a03e3a79 100644
--- a/src/source/services/cadences.clj
+++ b/src/source/services/cadences.clj
@@ -1,13 +1,6 @@
 (ns source.services.cadences
   (:require [source.db.interface :as db]))
 
-(defn insert-cadence! [ds {:keys [data ret] :as opts}]
-  (->> {:tname :cadences
-        :data data
-        :ret ret}
-       (merge opts)
-       (db/insert! ds)))
-
 (defn cadences
   ([ds] (cadences ds {}))
   ([ds opts]
@@ -15,12 +8,3 @@
          :ret :*}
         (merge opts)
         (db/find ds))))
-
-(defn cadence [ds {:keys [id where] :as opts}]
-  (->> {:tname :cadences
-        :where (if (some? id)
-                 [:= :id id]
-                 where)
-        :ret :1}
-       (merge opts)
-       (db/find ds)))
diff --git a/src/source/services/categories.clj b/src/source/services/categories.clj
index b8f6e53b..d75ff75d 100644
--- a/src/source/services/categories.clj
+++ b/src/source/services/categories.clj
@@ -8,14 +8,6 @@
        (merge opts)
        (db/insert! ds)))
 
-(defn update-category! [ds {:keys [id data where] :as opts}]
-  (->> {:tname :categories
-        :values data
-        :where (if (some? id) [:= :id id] where)
-        :ret :1}
-       (merge opts)
-       (db/update! ds)))
-
 (defn categories
   ([ds] (categories ds {}))
   ([ds {:keys [where] :as opts}]
diff --git a/src/source/services/content_types.clj b/src/source/services/content_types.clj
index f124dec4..117086b8 100644
--- a/src/source/services/content_types.clj
+++ b/src/source/services/content_types.clj
@@ -1,13 +1,6 @@
 (ns source.services.content-types
   (:require [source.db.interface :as db]))
 
-(defn insert-content-type! [ds {:keys [data ret] :as opts}]
-  (->> {:tname :content-types
-        :data data
-        :ret ret}
-       (merge opts)
-       (db/insert! ds)))
-
 (defn content-types
   ([ds] (content-types ds {}))
   ([ds opts]
diff --git a/src/source/services/feed_categories.clj b/src/source/services/feed_categories.clj
index b64391d1..7a16cdd9 100644
--- a/src/source/services/feed_categories.clj
+++ b/src/source/services/feed_categories.clj
@@ -25,15 +25,6 @@
        (assoc :on-conflict [:feed-id :category-id])
        (assoc :do-update-set {:category-id :excluded.category-id}))))
 
-(defn delete-feed-category! [ds {:keys [id where] :as opts}]
-  (->> {:tname :feed-categories
-        :where (if (some? id)
-                 [:= :id id]
-                 where)
-        :ret :1}
-       (merge opts)
-       (db/delete! ds)))
-
 (defn categories-by-feed [ds {:keys [feed-id where] :as _opts}]
   (hon/execute! ds
                 {:select [[:feed-categories.category-id :id] :name]
diff --git a/src/source/services/feeds.clj b/src/source/services/feeds.clj
index 5916f1ce..7f6e2047 100644
--- a/src/source/services/feeds.clj
+++ b/src/source/services/feeds.clj
@@ -1,13 +1,6 @@
 (ns source.services.feeds
   (:require [source.db.interface :as db]))
 
-(defn insert-feed! [ds {:keys [data] :as opts}]
-  (->> {:tname :feeds
-        :data data
-        :ret :1}
-       (merge opts)
-       (db/insert! ds)))
-
 (defn update-feed! [ds {:keys [id data where] :as opts}]
   (->> {:tname :feeds
         :values data
@@ -33,12 +26,3 @@
         :ret :1}
        (merge opts)
        (db/find ds)))
-
-(defn delete-feed! [ds {:keys [id where] :as opts}]
-  (->> {:tname :feeds
-        :where (if (some? id)
-                 [:= :id id]
-                 where)
-        :ret :1}
-       (merge opts)
-       (db/delete! ds)))
diff --git a/src/source/services/filtered_feeds.clj b/src/source/services/filtered_feeds.clj
index 87da7ea0..3f26a5fa 100644
--- a/src/source/services/filtered_feeds.clj
+++ b/src/source/services/filtered_feeds.clj
@@ -1,21 +1,6 @@
 (ns source.services.filtered-feeds
   (:require [source.db.interface :as db]))
 
-(defn insert-filtered-feeds! [ds {:keys [data ret] :as opts}]
-  (->> {:tname :filtered-feeds
-        :data data
-        :ret ret}
-       (merge opts)
-       (db/insert! ds)))
-
-(defn update-filtered-feeds! [ds {:keys [id data where] :as opts}]
-  (->> {:tname :filtered-feeds
-        :values data
-        :where (if (some? id) [:= :id id] where)
-        :ret :1}
-       (merge opts)
-       (db/update! ds)))
-
 (defn filtered-feeds
   ([ds] (filtered-feeds ds {}))
   ([ds {:keys [where] :as opts}]
@@ -24,12 +9,3 @@
          :ret :*}
         (merge opts)
         (db/find ds))))
-
-(defn delete-filtered-feed! [ds {:keys [id where] :as opts}]
-  (->> {:tname :filtered-feeds
-        :where (if (some? id)
-                 [:= :id id]
-                 where)
-        :ret :1}
-       (merge opts)
-       (db/delete! ds)))
diff --git a/src/source/services/filtered_posts.clj b/src/source/services/filtered_posts.clj
index a7f6cddc..571c74c9 100644
--- a/src/source/services/filtered_posts.clj
+++ b/src/source/services/filtered_posts.clj
@@ -1,21 +1,6 @@
 (ns source.services.filtered-posts
   (:require [source.db.interface :as db]))
 
-(defn insert-filtered-posts! [ds {:keys [data ret] :as opts}]
-  (->> {:tname :filtered-posts
-        :data data
-        :ret ret}
-       (merge opts)
-       (db/insert! ds)))
-
-(defn update-filtered-posts! [ds {:keys [id data where] :as opts}]
-  (->> {:tname :filtered-posts
-        :values data
-        :where (if (some? id) [:= :id id] where)
-        :ret :1}
-       (merge opts)
-       (db/update! ds)))
-
 (defn filtered-posts
   ([ds] (filtered-posts ds {}))
   ([ds {:keys [where] :as opts}]
@@ -24,12 +9,3 @@
          :ret :*}
         (merge opts)
         (db/find ds))))
-
-(defn delete-filtered-post! [ds {:keys [id where] :as opts}]
-  (->> {:tname :filtered-posts
-        :where (if (some? id)
-                 [:= :id id]
-                 where)
-        :ret :1}
-       (merge opts)
-       (db/delete! ds)))
diff --git a/src/source/services/incoming_posts.clj b/src/source/services/incoming_posts.clj
index 9c4892d6..0578d65c 100644
--- a/src/source/services/incoming_posts.clj
+++ b/src/source/services/incoming_posts.clj
@@ -55,12 +55,3 @@
                   :right-join [:categories [:= :categories.id :feed-categories.category-id]]}
                  opts)
                 {:ret :*}))
-
-(defn delete-incoming-post! [ds {:keys [id where] :as opts}]
-  (->> {:tname :incoming-posts
-        :where (if (some? id)
-                 [:= :id id]
-                 where)
-        :ret :1}
-       (merge opts)
-       (db/delete! ds)))
diff --git a/src/source/services/interface.clj b/src/source/services/interface.clj
index e6dfda8f..575d6731 100644
--- a/src/source/services/interface.clj
+++ b/src/source/services/interface.clj
@@ -1,10 +1,8 @@
 (ns source.services.interface
   (:require  [source.services.users :as users]
-             [source.db.interface :as db]
              [source.services.auth :as auth]
              [source.services.xml-schemas :as xml]
              [source.services.bundles :as bundles]
-             [source.services.bundle-categories :as bundle-categories]
              [source.services.bundle-content-types :as bundle-content-types]
              [source.services.post-heuristics :as post-heuristics]
              [source.services.providers :as providers]
@@ -18,20 +16,12 @@
              [source.services.feed-categories :as feed-categories]
              [source.services.jobs :as jobs]
              [source.services.businesses :as businesses]
-             [source.services.user-sectors :as user-sectors]
              [source.services.filtered-feeds :as filtered-feeds]
              [source.services.filtered-posts :as filtered-posts]))
 
-(defn users
-  [& args]
-  (apply users/users args))
-
 (defn user [ds {:keys [_id] :as opts}]
   (users/user ds opts))
 
-(defn insert-user! [ds {:keys [_values _ret] :as opts}]
-  (users/insert-user! ds opts))
-
 (defn update-user! [ds {:keys [_id _values _where] :as opts}]
   (users/update-user! ds opts))
 
@@ -39,20 +29,12 @@
   [ds {:keys [_id _where] :as opts}]
   (businesses/business ds opts))
 
-(defn businesses
-  ([ds] (businesses ds {}))
-  ([ds opts]
-   (businesses/businesses ds opts)))
-
 (defn insert-business! [ds {:keys [_values _ret] :as opts}]
   (businesses/insert-business! ds opts))
 
 (defn update-business! [ds {:keys [_id _values _where] :as opts}]
   (businesses/update-business! ds opts))
 
-(defn login [ds {:keys [_email] :as opts}]
-  (auth/login ds opts))
-
 (defn register [ds user]
   (auth/register ds user))
 
@@ -62,9 +44,6 @@
 (defn selection-schema [ds {:keys [_id] :as opts}]
   (xml/selection-schema ds opts))
 
-(defn insert-bundle! [ds {:keys [_values _ret] :as opts}]
-  (bundles/insert-bundle! ds opts))
-
 (defn update-bundle! [ds {:keys [_id _data _where] :as opts}]
   (bundles/update-bundle! ds opts))
 
@@ -76,68 +55,18 @@
 (defn bundle [ds {:keys [_id _where] :as opts}]
   (bundles/bundle ds opts))
 
-(defn delete-bundle [ds {:keys [_id _where] :as opts}]
-  (bundles/delete-bundle! ds opts))
-
-(defn bundle-categories
-  ([ds] (bundle-categories ds {}))
-  ([ds {:keys [_where] :as opts}]
-   (bundle-categories/bundle-categories ds opts)))
-
-(defn insert-bundle-category! [ds {:keys [_data _ret] :as opts}]
-  (bundle-categories/insert-bundle-category! ds opts))
-
-(defn delete-bundle-category! [ds {:keys [_id _where] :as opts}]
-  (bundle-categories/delete-bundle-category! ds opts))
-
-(defn category-id-by-bundle [ds {:keys [_bundle-id _where] :as opts}]
-  (bundle-categories/category-id ds opts))
-
-(defn bundle-content-types
-  ([ds] (bundle-content-types ds {}))
-  ([ds {:keys [_where] :as opts}]
-   (bundle-content-types/bundle-content-types ds opts)))
-
 (defn insert-bundle-content-types! [ds {:keys [_data _ret] :as opts}]
   (bundle-content-types/insert-bundle-content-types! ds opts))
 
-(defn delete-bundle-content-types! [ds {:keys [_id _where] :as opts}]
-  (bundle-content-types/delete-bundle-content-types! ds opts))
-
 (defn content-types-by-bundle [ds {:keys [_bundle-id _where] :as opts}]
   (bundle-content-types/content-types-by-bundle ds opts))
 
-(defn content-type-id [ds {:keys [_bundle-id _where] :as opts}]
-  (bundle-content-types/content-type-id ds opts))
-
-(defn insert-post-heuristics! [ds {:keys [_data] :as opts}]
-  (post-heuristics/insert-post-heuristics! ds opts))
-
-(defn update-post-heuristics! [ds {:keys [_id _data _where] :as opts}]
-  (post-heuristics/update-post-heuristics! ds opts))
-
-(defn post-heuristics
-  ([ds] (post-heuristics ds {}))
-  ([ds {:keys [_where] :as opts}]
-   (post-heuristics/post-heuristics ds opts)))
-
 (defn upsert-post-heuristics! [ds {:keys [_data] :as opts}]
   (post-heuristics/upsert-post-heuristics! ds opts))
 
-(defn post-heuristic [ds {:keys [_id _where] :as opts}]
-  (post-heuristics/post-heuristic ds opts))
-
 (defn top-posts-by-heuristic [ds {:keys [_select _limit _heuristic] :as opts}]
   (post-heuristics/top-posts-by-heuristic ds opts))
 
-(defn outgoing-posts
-  ([ds] (outgoing-posts ds {}))
-  ([ds {:keys [_where] :as opts}]
-   (outgoing-posts/outgoing-posts ds opts)))
-
-(defn outgoing-post [ds {:keys [_id _where] :as opts}]
-  (outgoing-posts/outgoing-post ds opts))
-
 (defn insert-outgoing-post! [ds {:keys [_values _ret] :as opts}]
   (outgoing-posts/insert-outgoing-post! ds opts))
 
@@ -189,12 +118,6 @@
 (defn content-type [ds id]
   (content-types/content-type ds id))
 
-(defn insert-content-type! [ds {:keys [_values _ret] :as opts}]
-  (content-types/insert-content-type! ds opts))
-
-(defn insert-feed! [ds {:keys [_data] :as opts}]
-  (feeds/insert-feed! ds opts))
-
 (defn update-feed! [ds {:keys [_id _data _where] :as opts}]
   (feeds/update-feed! ds opts))
 
@@ -207,9 +130,6 @@
 (defn feed [ds {:keys [_id] :as opts}]
   (feeds/feed ds opts))
 
-(defn delete-feed! [ds {:keys [_id _where] :as opts}]
-  (feeds/delete-feed! ds opts))
-
 (defn insert-incoming-post! [ds {:keys [_data _ret] :as opts}]
   (incoming-posts/insert-incoming-post! ds opts))
 
@@ -229,33 +149,12 @@
 (defn incoming-post [ds opts]
   (incoming-posts/incoming-post ds opts))
 
-(defn delete-incoming-post! [ds {:keys [_id _where] :as opts}]
-  (incoming-posts/delete-incoming-post! ds opts))
-
-(defn insert-cadence! [ds {:keys [_values _ret] :as opts}]
-  (cadences/insert-cadence! ds opts))
-
 (defn cadences [ds]
   (cadences/cadences ds))
 
-(defn cadence [ds id]
-  (cadences/cadence ds id))
-
-(defn insert-baseline! [ds {:keys [_values _ret] :as opts}]
-  (baselines/insert-baseline! ds opts))
-
 (defn baselines [ds]
   (baselines/baselines ds))
 
-(defn baseline [ds id]
-  (baselines/baseline ds id))
-
-(defn insert-category! [ds {:keys [_data _ret] :as opts}]
-  (categories/insert-category! ds opts))
-
-(defn update-category! [ds {:keys [_id _data _where] :as opts}]
-  (categories/update-category! ds opts))
-
 (defn categories
   ([ds] (categories ds {}))
   ([ds {:keys [_where] :as opts}]
@@ -264,43 +163,9 @@
 (defn category [ds {:keys [_id _where] :as opts}]
   (categories/category ds opts))
 
-(defn feed-categories
-  ([ds] (feed-categories ds {}))
-  ([ds {:keys [_where] :as opts}]
-   (feed-categories/feed-categories ds opts)))
-
-(defn insert-feed-category! [ds {:keys [_data _ret] :as opts}]
-  (feed-categories/insert-feed-category! ds opts))
-
-(defn upsert-feed-categories! [ds {:keys [_data] :as opts}]
-  (feed-categories/upsert-feed-categories! ds opts))
-
-(defn delete-feed-category! [ds {:keys [_id _where] :as opts}]
-  (feed-categories/delete-feed-category! ds opts))
-
 (defn categories-by-feed [ds {:keys [_feed-id _where] :as opts}]
   (feed-categories/categories-by-feed ds opts))
 
-(defn category-id-by-feed [ds {:keys [_feed-id _where] :as opts}]
-  (feed-categories/category-id ds opts))
-
-(defn user-sectors
-  ([ds] (user-sectors ds {}))
-  ([ds {:keys [_where] :as opts}]
-   (user-sectors/user-sectors ds opts)))
-
-(defn insert-user-sector! [ds {:keys [_data _ret] :as opts}]
-  (user-sectors/insert-user-sector! ds opts))
-
-(defn delete-user-sector! [ds {:keys [_id _where] :as opts}]
-  (user-sectors/delete-user-sector! ds opts))
-
-(defn sectors-by-user [ds {:keys [_user-id _where] :as opts}]
-  (user-sectors/sectors-by-user ds opts))
-
-(defn sector-id [ds {:keys [_user-id _where] :as opts}]
-  (user-sectors/sector-id ds opts))
-
 (defn insert-job! [ds {:keys [_data _ret] :as opts}]
   (jobs/insert-job! ds opts))
 
@@ -330,36 +195,12 @@
 (defn job-metadata [ds {:keys [_id _where] :as opts}]
   (jobs/job-metadata ds opts))
 
-(defn insert-filtered-feeds! [ds {:keys [_data _ret] :as opts}]
-  (filtered-feeds/insert-filtered-feeds! ds opts))
-
-(defn update-filtered-feeds! [ds {:keys [_id _data _where] :as opts}]
-  (filtered-feeds/update-filtered-feeds! ds opts))
-
 (defn filtered-feeds
   ([ds] (filtered-feeds ds {}))
   ([ds {:keys [_where] :as opts}]
    (filtered-feeds/filtered-feeds ds opts)))
 
-(defn delete-filtered-feed! [ds {:keys [_id _where] :as opts}]
-  (filtered-feeds/delete-filtered-feed! ds opts))
-
-(defn insert-filtered-posts! [ds {:keys [_data _ret] :as opts}]
-  (filtered-posts/insert-filtered-posts! ds opts))
-
-(defn update-filtered-posts! [ds {:keys [_id _data _where] :as opts}]
-  (filtered-posts/update-filtered-posts! ds opts))
-
 (defn filtered-posts
   ([ds] (filtered-posts ds {}))
   ([ds {:keys [_where] :as opts}]
    (filtered-posts/filtered-posts ds opts)))
-
-(defn delete-filtered-post! [ds {:keys [_id _where] :as opts}]
-  (filtered-posts/delete-filtered-post! ds opts))
-
-(comment
-  (users (db/ds :master))
-  (user (db/ds :master) {:id 2})
-  (login (db/ds :master) {:email "merveillevaneck@gmail.com"})
-  ())
diff --git a/src/source/services/outgoing_posts.clj b/src/source/services/outgoing_posts.clj
index fcd13789..f45b89d3 100644
--- a/src/source/services/outgoing_posts.clj
+++ b/src/source/services/outgoing_posts.clj
@@ -1,16 +1,5 @@
 (ns source.services.outgoing-posts
-  (:require [source.db.interface :as db]
-            [source.db.honey :as hon]
-            [honey.sql.helpers :as hsql]))
-
-(defn outgoing-posts
-  ([ds] (outgoing-posts ds {}))
-  ([ds {:keys [where ret] :as opts}]
-   (->> {:tname :outgoing-posts
-         :where where
-         :ret (or ret :*)}
-        (merge opts)
-        (db/find ds))))
+  (:require [source.db.interface :as db]))
 
 (defn outgoing-post [ds {:keys [id where] :as opts}]
   (->> {:tname :outgoing-posts
diff --git a/src/source/services/post_heuristics.clj b/src/source/services/post_heuristics.clj
index 30c93c57..b682efa7 100644
--- a/src/source/services/post_heuristics.clj
+++ b/src/source/services/post_heuristics.clj
@@ -3,20 +3,6 @@
             [source.db.honey :as hon]
             [honey.sql.helpers :as hsql]))
 
-(defn insert-post-heuristics! [ds {:keys [data] :as opts}]
-  (->> {:tname :post-heuristics
-        :data data
-        :ret :1}
-       (merge opts)
-       (db/insert! ds)))
-
-(defn update-post-heuristics! [ds {:keys [id _data where] :as opts}]
-  (->> {:tname :post-heuristics
-        :where (if (some? id) [:= :id id] where)
-        :ret :1}
-       (merge opts)
-       (db/update! ds)))
-
 (defn upsert-post-heuristics! [ds {:keys [data]}]
   (hon/execute!
    ds
@@ -26,15 +12,6 @@
        (assoc :do-update-set {:long-heuristic :excluded.long-heuristic
                               :short-heuristic :excluded.short-heuristic}))))
 
-(defn post-heuristics
-  ([ds] (post-heuristics ds {}))
-  ([ds {:keys [where] :as opts}]
-   (->> {:tname :post-heuristics
-         :where where
-         :ret :*}
-        (merge opts)
-        (db/find ds))))
-
 (defn top-posts-by-heuristic [ds {:keys [select limit heuristic] :as _opts}]
   (hon/execute! ds
                 (merge {:select (or select :*)
@@ -42,12 +19,3 @@
                         :order-by [[heuristic :desc]]
                         :limit limit})
                 {:ret :*}))
-
-(defn post-heuristic [ds {:keys [id where] :as opts}]
-  (->> {:tname :post-heuristics
-        :where (if (some? id)
-                 [:= :id id]
-                 where)
-        :ret :1}
-       (merge opts)
-       (db/find ds)))
diff --git a/src/source/services/user_sectors.clj b/src/source/services/user_sectors.clj
index c7e93609..99e387f5 100644
--- a/src/source/services/user_sectors.clj
+++ b/src/source/services/user_sectors.clj
@@ -2,15 +2,6 @@
   (:require [source.db.interface :as db]
             [source.db.honey :as hon]))
 
-(defn user-sectors
-  ([ds] (user-sectors ds {}))
-  ([ds {:keys [where] :as opts}]
-   (->> {:tname :user-sectors
-         :where where
-         :ret :*}
-        (merge opts)
-        (db/find ds))))
-
 (defn insert-user-sector! [ds {:keys [_data _ret] :as opts}]
   (->> {:tname :user-sectors}
        (merge opts)
@@ -35,15 +26,6 @@
                           where)}
                 {:ret :*}))
 
-(defn sector-id [ds {:keys [user-id where] :as opts}]
-  (->> {:tname :user-sectors
-        :where (if (some? user-id)
-                 [:= :user-id user-id]
-                 where)
-        :ret :1}
-       (merge opts)
-       (db/find ds)))
-
 ;;NEW
 (defn update-user-sectors! [ds {:keys [user-id sectors]}]
   (let [update-data (reduce (fn [acc {:keys [id]}]
diff --git a/src/source/services/users.clj b/src/source/services/users.clj
index aa4e7a0f..2c37cb1f 100644
--- a/src/source/services/users.clj
+++ b/src/source/services/users.clj
@@ -25,12 +25,6 @@
        (merge opts)
        (db/insert! ds)))
 
-(defn delete-user! [ds {:keys [id where] :as opts}]
-  (->> {:tname :users
-        :where (if (some? id) [:= :id id] where)}
-       (merge opts)
-       (db/delete! ds)))
-
 (defn update-user! [ds {:keys [id values where] :as opts}]
   (->> {:tname :users
         :values values

From bb2f7317315d0d01b8d4cfe9aaee2cf701e61817 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 28 Jan 2026 13:40:08 +0200
Subject: [PATCH 238/391] removed unnecessary service calls

---
 src/source/routes/baselines.clj     |  6 +++---
 src/source/routes/business.clj      | 14 ++++++++------
 src/source/routes/businesses.clj    |  7 ++++---
 src/source/routes/cadences.clj      |  7 ++++---
 src/source/routes/categories.clj    |  7 +++----
 src/source/routes/category.clj      |  5 +++--
 src/source/routes/content_type.clj  |  8 ++++----
 src/source/routes/content_types.clj |  6 +++---
 src/source/routes/google_user.clj   | 15 +++++++++------
 src/source/routes/login.clj         |  7 ++++---
 src/source/routes/me.clj            | 15 ++++++++-------
 src/source/routes/me_business.clj   | 26 ++++++++++++++++----------
 src/source/routes/provider.clj      | 16 +++++++++-------
 src/source/routes/providers.clj     | 12 +++++++-----
 src/source/routes/register.clj      |  6 ++++--
 src/source/routes/report.clj        |  7 ++++---
 src/source/routes/sectors.clj       |  7 ++++---
 src/source/routes/user.clj          | 16 ++++++++--------
 18 files changed, 105 insertions(+), 82 deletions(-)

diff --git a/src/source/routes/baselines.clj b/src/source/routes/baselines.clj
index b1e8e4d5..2f69a561 100644
--- a/src/source/routes/baselines.clj
+++ b/src/source/routes/baselines.clj
@@ -1,6 +1,6 @@
 (ns source.routes.baselines
-  (:require [source.services.interface :as services]
-            [ring.util.response :as res]))
+  (:require [ring.util.response :as res]
+            [source.db.honey :as hon]))
 
 (defn get
   {:summary "get all baselines"
@@ -11,5 +11,5 @@
                             [:min :int]
                             [:max :int]]]}}}
   [{:keys [ds] :as _request}]
-  (->> (services/baselines ds)
+  (->> (hon/find ds {:tname :baselines})
        (res/response)))
diff --git a/src/source/routes/business.clj b/src/source/routes/business.clj
index 1d34aadb..c89bfa0d 100644
--- a/src/source/routes/business.clj
+++ b/src/source/routes/business.clj
@@ -1,7 +1,7 @@
 (ns source.routes.business
-  (:require [source.services.businesses :as businesses]
-            [ring.util.response :as res]
-            [source.util :as utils]))
+  (:require [ring.util.response :as res]
+            [source.util :as utils]
+            [source.db.honey :as hon]))
 
 (defn post
   {:summary "insert a business"
@@ -19,7 +19,8 @@
     (if (not success) (-> (res/response error)
                           (res/status 400))
 
-        (do (businesses/insert-business! ds {:values data})
+        (do (hon/insert! ds {:tname :businesses
+                             :data data})
             (res/response {:message "successfully added business"})))))
 
 (defn patch
@@ -42,8 +43,9 @@
       (-> (res/response error)
           (res/status 400))
 
-      (do (businesses/update-business! ds {:id (:id path-params)
-                                           :values data})
+      (do (hon/update! ds {:tname :businesses
+                           :where  [:= :id (:id path-params)]
+                           :data data})
           (res/response {:message "successfully updated business"})))))
 
 (comment
diff --git a/src/source/routes/businesses.clj b/src/source/routes/businesses.clj
index cec00eb2..c4ece13b 100644
--- a/src/source/routes/businesses.clj
+++ b/src/source/routes/businesses.clj
@@ -1,6 +1,6 @@
 (ns source.routes.businesses
-  (:require [source.services.businesses :as businesses]
-            [ring.util.response :as res]))
+  (:require [ring.util.response :as res]
+            [source.db.honey :as hon]))
 
 (defn get
   {:summary "get all businesses"
@@ -15,7 +15,8 @@
                              [:twitter [:maybe :string]]]]]}}}
 
   [{:keys [ds] :as _request}]
-  (res/response {:businesses (businesses/businesses ds)}))
+  (res/response {:businesses (hon/find ds {:tname :businesses
+                                           :ret :*})}))
 
 (comment
   (require '[source.db.util :as db.util])
diff --git a/src/source/routes/cadences.clj b/src/source/routes/cadences.clj
index c8809adf..47fad7c0 100644
--- a/src/source/routes/cadences.clj
+++ b/src/source/routes/cadences.clj
@@ -1,6 +1,6 @@
 (ns source.routes.cadences
-  (:require [source.services.interface :as services]
-            [ring.util.response :as res]))
+  (:require [ring.util.response :as res]
+            [source.db.honey :as hon]))
 
 (defn get
   {:summary "get all cadences"
@@ -10,5 +10,6 @@
                             [:label :string]
                             [:days :int]]]}}}
   [{:keys [ds] :as _request}]
-  (->> (services/cadences ds)
+  (->> (hon/find ds {:tname :cadences
+                     :ret :*})
        (res/response)))
diff --git a/src/source/routes/categories.clj b/src/source/routes/categories.clj
index 57b6b286..d77c1ac9 100644
--- a/src/source/routes/categories.clj
+++ b/src/source/routes/categories.clj
@@ -1,7 +1,6 @@
 (ns source.routes.categories
-  (:require [source.services.interface :as services]
-            [ring.util.response :as res]
-            [source.db.util :as db.util]))
+  (:require [ring.util.response :as res]
+            [source.db.honey :as hon]))
 
 (defn get
   {:summary "get all categories"
@@ -11,5 +10,5 @@
                             [:name :string]
                             [:display-picture {:optional true} [:maybe :string]]]]}}}
   [{:keys [ds] :as _request}]
-  (->> (services/categories ds)
+  (->> (hon/find ds {:tname :categories})
        (res/response)))
diff --git a/src/source/routes/category.clj b/src/source/routes/category.clj
index 3ffe6d9b..dfaf6b08 100644
--- a/src/source/routes/category.clj
+++ b/src/source/routes/category.clj
@@ -1,6 +1,6 @@
 (ns source.routes.category
   (:require [ring.util.response :as res]
-            [source.services.interface :as services]))
+            [source.db.honey :as hon]))
 
 (defn get
   {:summary "get category by id"
@@ -11,4 +11,5 @@
                            [:name :string]]}}}
 
   [{:keys [ds path-params] :as _request}]
-  (res/response (services/category ds path-params)))
+  (res/response (hon/find-one ds {:tname :categories
+                                  :where [:= :id (:id path-params)]})))
diff --git a/src/source/routes/content_type.clj b/src/source/routes/content_type.clj
index 521c2fb8..b6b26cf4 100644
--- a/src/source/routes/content_type.clj
+++ b/src/source/routes/content_type.clj
@@ -1,6 +1,6 @@
 (ns source.routes.content-type
-  (:require [source.services.interface :as services]
-            [ring.util.response :as res]))
+  (:require [ring.util.response :as res]
+            [source.db.honey :as hon]))
 
 (defn get
   {:summary "get content type by id"
@@ -11,6 +11,6 @@
                            [:name :string]]}}}
 
   [{:keys [ds path-params] :as _request}]
-  (->> path-params
-       (services/content-type ds)
+  (->> (hon/find-one ds {:tname :content-types
+                         :where [:= :id (:id path-params)]})
        (res/response)))
diff --git a/src/source/routes/content_types.clj b/src/source/routes/content_types.clj
index c7c2808d..c025c98f 100644
--- a/src/source/routes/content_types.clj
+++ b/src/source/routes/content_types.clj
@@ -1,6 +1,6 @@
 (ns source.routes.content-types
-  (:require [source.services.interface :as services]
-            [ring.util.response :as res]))
+  (:require [ring.util.response :as res]
+            [source.db.honey :as hon]))
 
 (defn get
   {:summary "get all content types"
@@ -10,5 +10,5 @@
                             [:name :string]]]}}}
 
   [{:keys [ds] :as _request}]
-  (-> (services/content-types ds)
+  (-> (hon/find ds {:tname :content-types})
       (res/response)))
diff --git a/src/source/routes/google_user.clj b/src/source/routes/google_user.clj
index b5e19541..79e25506 100644
--- a/src/source/routes/google_user.clj
+++ b/src/source/routes/google_user.clj
@@ -1,8 +1,8 @@
 (ns source.routes.google-user
   (:require [source.oauth2.google.interface :as google]
             [source.middleware.auth.core :as auth]
-            [source.services.users :as users]
-            [ring.util.response :as res]))
+            [ring.util.response :as res]
+            [source.db.honey :as hon]))
 
 (defn get
   {:summary "completes the google oauth2 flow and returns the authenticated user"
@@ -28,7 +28,8 @@
 
   (let [{:keys [uuid _uri]} body
         email (google/google-session-user uuid (:params req))
-        user (users/user ds {:where [:= :email email]})
+        user (hon/find-one ds {:tname :users
+                               :where [:= :email email]})
         user-type (get-in req [:cookies "user_type" :value])]
 
     (if (some? user)
@@ -36,9 +37,11 @@
             session (auth/create-session payload)]
         (res/response (merge {:user payload} session)))
       (do
-        (users/insert-user! ds {:data {:email email
-                                       :type user-type}})
-        (let [new-user (users/user ds {:where [:= :email email]})
+        (hon/insert! ds {:tname :users
+                         :data {:email email
+                                :type user-type}})
+        (let [new-user (hon/find-one ds {:tname :users
+                                         :where [:= :email email]})
               payload (dissoc new-user :password)
               session (auth/create-session payload)]
           (res/response (merge {:user payload} session)))))))
diff --git a/src/source/routes/login.clj b/src/source/routes/login.clj
index e45452f1..beab23b8 100644
--- a/src/source/routes/login.clj
+++ b/src/source/routes/login.clj
@@ -1,9 +1,9 @@
 (ns source.routes.login
   (:require [source.services.auth :as auth]
             [ring.util.response :as res]
-            [source.services.users :as users]
             [source.password :as pw]
-            [source.util :as util]))
+            [source.util :as util]
+            [source.db.honey :as hon]))
 
 (defn post
   {:summary "get user data and access token provided valid credentials"
@@ -31,7 +31,8 @@
 
   (let [{:keys [data error success]} (util/validate post body)
         {:keys [email password]} data
-        user (users/user ds {:where [:= :email email]})]
+        user (hon/find-one ds {:tname :users
+                               :where [:= :email email]})]
 
     (cond
       (not success) (-> (res/response error)
diff --git a/src/source/routes/me.clj b/src/source/routes/me.clj
index 01f003dd..d544fa19 100644
--- a/src/source/routes/me.clj
+++ b/src/source/routes/me.clj
@@ -1,7 +1,7 @@
 (ns source.routes.me
-  (:require [source.services.interface :as services]
-            [ring.util.response :as res]
-            [source.util :as util]))
+  (:require [ring.util.response :as res]
+            [source.util :as util]
+            [source.db.honey :as hon]))
 
 (defn get
   {:summary "get logged in user by access token"
@@ -20,8 +20,8 @@
                403 {:body [:map [:message :string]]}}}
 
   [{:keys [ds user] :as _request}]
-  (let [user (->> user
-                  (services/user ds))]
+  (let [user (hon/find-one ds {:tname :users
+                               :where [:= :id (:id user)]})]
     (->> (dissoc user :password)
          (res/response))))
 
@@ -43,6 +43,7 @@
     (if (not success)
       (-> (res/response {:message error})
           (res/status 400))
-      (do (services/update-user! ds {:id (:id user)
-                                     :data data})
+      (do (hon/update! ds {:tname :users
+                           :where [:= :id (:id user)]
+                           :data data})
           (res/response {:message "successfully updated user"})))))
diff --git a/src/source/routes/me_business.clj b/src/source/routes/me_business.clj
index 6f3b1ad5..76567c98 100644
--- a/src/source/routes/me_business.clj
+++ b/src/source/routes/me_business.clj
@@ -1,7 +1,7 @@
 (ns source.routes.me-business
   (:require [source.util :as util]
             [ring.util.response :as res]
-            [source.services.interface :as services]))
+            [source.db.honey :as hon]))
 
 (defn get
   {:summary "get business for logged-in user"
@@ -18,9 +18,11 @@
                403 {:body [:map [:message :string]]}}}
 
   [{:keys [ds user] :as _request}]
-  (let [{:keys [business-id]} (services/user ds {:id (:id user)})
+  (let [{:keys [business-id]} (hon/find-one ds {:tname :users
+                                                :where [:= :id (:id user)]})
         business (if business-id
-                   (services/business ds {:id business-id})
+                   (hon/find-one ds {:tname :businesses
+                                     :where [:= :id business-id]})
                    {})]
     (res/response business)))
 
@@ -43,14 +45,18 @@
       (-> (res/response {:message error})
           (res/status 400))
 
-      (let [{:keys [business-id]} (services/user ds {:id (:id user)})
+      (let [{:keys [business-id]} (hon/find-one ds {:tname :users
+                                                    :where [:= :id (:id user)]})
             business (when (nil? business-id)
-                       (services/insert-business! ds {:data data
-                                                      :ret :1}))]
+                       (hon/insert! ds {:tname :businesses
+                                        :data data
+                                        :ret :1}))]
         (if (nil? business-id)
-          (services/update-user! ds {:id (:id user)
-                                     :data {:business-id (:id business)}})
-          (services/update-business! ds {:id business-id
-                                         :data data}))
+          (hon/update! ds {:tname :users
+                           :where [:= :id (:id user)]
+                           :data {:business-id (:id business)}})
+          (hon/update! ds {:tname :businesses
+                           :where  [:= :id business-id]
+                           :data data}))
 
         (res/response {:message "successfully added or updated business"})))))
diff --git a/src/source/routes/provider.clj b/src/source/routes/provider.clj
index a020cac5..6b3d1249 100644
--- a/src/source/routes/provider.clj
+++ b/src/source/routes/provider.clj
@@ -1,6 +1,7 @@
 (ns source.routes.provider
   (:require [source.services.interface :as services]
-            [ring.util.response :as res]))
+            [ring.util.response :as res]
+            [source.db.honey :as hon]))
 
 (defn get
   {:summary "get provider by id"
@@ -15,8 +16,8 @@
                            [:placeholder-url [:maybe :string]]]}}}
 
   [{:keys [ds path-params] :as _request}]
-  (->> path-params
-       (services/provider ds)
+  (->> (services/provider ds {:tname :providers
+                              :where [:= :id (:id path-params)]})
        (res/response)))
 
 (defn post
@@ -32,8 +33,9 @@
    :responses {200 {:body [:map [:message :string]]}}}
 
   [{:keys [ds path-params body] :as _request}]
-  (services/update-provider! ds {:id (:id path-params)
-                                 :data body})
+  (hon/update! ds {:tname :providers
+                   :where [:= :id (:id path-params)]
+                   :data body})
   (res/response {:message "successfully updated provider"}))
 
 (defn delete
@@ -43,7 +45,7 @@
    :responses {200 {:body [:map [:message :string]]}}}
 
   [{:keys [store ds path-params] :as _request}]
-  (->> path-params
-       (services/delete-provider! ds))
+  (hon/delete! ds {:tname :providers
+                   :where [:= :id (:id path-params)]})
   (services/delete-selection-schemas-by-provider! store ds (:id path-params))
   (res/response {:message "successfully deleted provider"}))
diff --git a/src/source/routes/providers.clj b/src/source/routes/providers.clj
index 9ef7cfa7..1b8d39f8 100644
--- a/src/source/routes/providers.clj
+++ b/src/source/routes/providers.clj
@@ -1,6 +1,6 @@
 (ns source.routes.providers
-  (:require [source.services.interface :as services]
-            [ring.util.response :as res]))
+  (:require [ring.util.response :as res]
+            [source.db.honey :as hon]))
 
 (defn get
   {:summary "get all providers"
@@ -14,7 +14,8 @@
                             [:placeholder-url [:maybe :string]]]]}}}
 
   [{:keys [ds] :as _request}]
-  (-> (services/providers ds)
+  (-> (hon/find ds {:tname :providers
+                    :ret :*})
       (res/response)))
 
 (defn post
@@ -28,6 +29,7 @@
    :responses {200 {:body [:map [:message :string]]}}}
 
   [{:keys [ds body] :as _request}]
-  (services/insert-provider! ds {:data body
-                                 :ret :1})
+  (hon/insert! ds {:tname :providers
+                   :data body
+                   :ret :1})
   (res/response {:message "successfully added provider"}))
diff --git a/src/source/routes/register.clj b/src/source/routes/register.clj
index 4f0737d5..86557566 100644
--- a/src/source/routes/register.clj
+++ b/src/source/routes/register.clj
@@ -1,7 +1,8 @@
 (ns source.routes.register
   (:require [source.services.interface :as services]
             [ring.util.response :as res]
-            [source.util :as util]))
+            [source.util :as util]
+            [source.db.honey :as hon]))
 
 (defn post
   {:summary "register a new user"
@@ -30,7 +31,8 @@
 
   (let [{:keys [data error success]} (util/validate post body)
         {:keys [email password confirm-password]} data
-        existing-user (services/user ds {:where [:= :email email]})]
+        existing-user (hon/find-one ds {:tname :users
+                                        :where [:= :email email]})]
     (cond
 
       (not success) (-> (res/response error)
diff --git a/src/source/routes/report.clj b/src/source/routes/report.clj
index afc222f1..a8d43ef1 100644
--- a/src/source/routes/report.clj
+++ b/src/source/routes/report.clj
@@ -2,8 +2,8 @@
   (:require [source.config :as conf]
             [source.email.gmail :as gmail]
             [source.email.templates :as templates]
-            [source.services.interface :as services]
-            [ring.util.response :as res]))
+            [ring.util.response :as res]
+            [source.db.honey :as hon]))
 
 (defn post
   {:summary "sends us a message to let us know of a problem"
@@ -12,7 +12,8 @@
                401 {:body [:map [:message :string]]}
                403 {:body [:map [:message :string]]}}}
   [{:keys [ds user body] :as _req}]
-  (let [{:keys [email]} (services/user ds {:id (:id user)})
+  (let [{:keys [email]} (hon/find-one ds {:tname :users
+                                          :where [:= :id (:id user)]})
         email-body (templates/admin-reported-problem {:user-id (:id user)
                                                       :user-type (:type user)
                                                       :user-email email
diff --git a/src/source/routes/sectors.clj b/src/source/routes/sectors.clj
index 0f8fa450..3e77f966 100644
--- a/src/source/routes/sectors.clj
+++ b/src/source/routes/sectors.clj
@@ -1,6 +1,6 @@
 (ns source.routes.sectors
-  (:require [source.services.sectors :as sectors]
-            [ring.util.response :as res]))
+  (:require [ring.util.response :as res]
+            [source.db.honey :as hon]))
 
 (defn get
   {:summary "get all sectors"
@@ -10,7 +10,8 @@
                             [:name :string]]]}}}
 
   [{:keys [ds] :as _request}]
-  (res/response (sectors/sectors ds)))
+  (res/response (hon/find ds {:tname :sectors
+                              :ret :*})))
 
 (comment
   (require '[source.db.util :as db.util])
diff --git a/src/source/routes/user.clj b/src/source/routes/user.clj
index 413ddd48..78fe1bdc 100644
--- a/src/source/routes/user.clj
+++ b/src/source/routes/user.clj
@@ -1,7 +1,7 @@
 (ns source.routes.user
-  (:require [source.services.interface :as services]
-            [ring.util.response :as res]
-            [source.util :as util]))
+  (:require [ring.util.response :as res]
+            [source.util :as util]
+            [source.db.honey :as hon]))
 
 (defn get
   {:summary "get user by id"
@@ -24,8 +24,8 @@
                403 {:body [:map [:message :string]]}}}
 
   [{:keys [ds path-params] :as _request}]
-  (let [user (->> path-params
-                  (services/user ds))]
+  (let [user (hon/find-one ds {:tname :users
+                               :where [:= :id (:id path-params)]})]
     (->> (dissoc user :password)
          (assoc {} :user)
          (res/response))))
@@ -68,9 +68,9 @@
       (-> (res/response error)
           (res/status 400))
 
-      (do (services/update-user! ds
-                                 {:id (:id path-params)
-                                  :values data})
+      (do (hon/update! ds {:tname :users
+                           :where [:= :id (:id path-params)]
+                           :data data})
           (res/response {:message "successfully updated user"})))))
 
 (comment

From 16b991d940a5faf351f3414074b7b64c9b54a863 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 28 Jan 2026 13:54:35 +0200
Subject: [PATCH 239/391] removed more unnecessary service calls and deleted
 unused services

---
 src/source/routes/integration_filter_feed.clj | 13 ++--
 .../routes/integration_filter_feeds.clj       |  7 +-
 src/source/routes/integration_filter_post.clj | 13 ++--
 .../routes/integration_filter_posts.clj       |  7 +-
 src/source/services/baselines.clj             | 10 ---
 src/source/services/businesses.clj            | 30 ---------
 src/source/services/cadences.clj              | 10 ---
 src/source/services/categories.clj            |  9 ---
 src/source/services/content_types.clj         | 19 ------
 src/source/services/filtered_feeds.clj        | 11 ----
 src/source/services/filtered_posts.clj        | 11 ----
 src/source/services/interface.clj             | 64 +------------------
 src/source/services/providers.clj             | 31 ---------
 src/source/services/sectors.clj               | 10 ---
 14 files changed, 23 insertions(+), 222 deletions(-)
 delete mode 100644 src/source/services/baselines.clj
 delete mode 100644 src/source/services/businesses.clj
 delete mode 100644 src/source/services/cadences.clj
 delete mode 100644 src/source/services/content_types.clj
 delete mode 100644 src/source/services/filtered_feeds.clj
 delete mode 100644 src/source/services/filtered_posts.clj
 delete mode 100644 src/source/services/sectors.clj

diff --git a/src/source/routes/integration_filter_feed.clj b/src/source/routes/integration_filter_feed.clj
index 302cb664..ecffde6c 100644
--- a/src/source/routes/integration_filter_feed.clj
+++ b/src/source/routes/integration_filter_feed.clj
@@ -1,7 +1,7 @@
 (ns source.routes.integration-filter-feed
-  (:require [source.services.interface :as services]
-            [ring.util.response :as res]
-            [source.workers.integrations :as integrations]))
+  (:require [ring.util.response :as res]
+            [source.workers.integrations :as integrations]
+            [source.db.honey :as hon]))
 
 (defn get
   {:summary "Returns true if the feed with the given id is filtered out by the integration with the given id"
@@ -16,9 +16,10 @@
 
   [{:keys [ds path-params] :as _request}]
 
-  (let [returned (services/filtered-feeds ds {:where [:and
-                                                      [:= :bundle-id (:id path-params)]
-                                                      [:= :feed-id (:feed-id path-params)]]})
+  (let [returned (hon/find ds {:tname :filtered-feeds
+                               :where [:and
+                                       [:= :bundle-id (:id path-params)]
+                                       [:= :feed-id (:feed-id path-params)]]})
         blocked (if (seq returned) true false)]
     (res/response {:filtered blocked})))
 
diff --git a/src/source/routes/integration_filter_feeds.clj b/src/source/routes/integration_filter_feeds.clj
index 1442680f..e84ddfe2 100644
--- a/src/source/routes/integration_filter_feeds.clj
+++ b/src/source/routes/integration_filter_feeds.clj
@@ -1,6 +1,6 @@
 (ns source.routes.integration-filter-feeds
-  (:require [source.services.interface :as services]
-            [ring.util.response :as res]))
+  (:require [ring.util.response :as res]
+            [source.db.honey :as hon]))
 
 (defn get
   {:summary "gets all filtered feed ids by integration id"
@@ -15,4 +15,5 @@
 
   [{:keys [ds path-params] :as _request}]
 
-  (res/response (services/filtered-feeds ds {:where [:= :bundle-id (:id path-params)]})))
+  (res/response (hon/find ds {:tname :filtered-feeds
+                              :where [:= :bundle-id (:id path-params)]})))
diff --git a/src/source/routes/integration_filter_post.clj b/src/source/routes/integration_filter_post.clj
index d93e82c4..f124c980 100644
--- a/src/source/routes/integration_filter_post.clj
+++ b/src/source/routes/integration_filter_post.clj
@@ -1,7 +1,7 @@
 (ns source.routes.integration-filter-post
-  (:require [source.services.interface :as services]
-            [ring.util.response :as res]
-            [source.workers.integrations :as integrations]))
+  (:require [ring.util.response :as res]
+            [source.workers.integrations :as integrations]
+            [source.db.honey :as hon]))
 
 (defn get
   {:summary "Returns true if the post with the given id is filtered out by the integration with the given id"
@@ -16,9 +16,10 @@
 
   [{:keys [ds path-params] :as _request}]
 
-  (let [returned (services/filtered-feeds ds {:where [:and
-                                                      [:= :bundle-id (:id path-params)]
-                                                      [:= :post-id (:post-id path-params)]]})
+  (let [returned (hon/find ds {:tname :filtered-feeds
+                               :where [:and
+                                       [:= :bundle-id (:id path-params)]
+                                       [:= :post-id (:post-id path-params)]]})
         blocked (if (seq returned) true false)]
     (res/response {:filtered blocked})))
 
diff --git a/src/source/routes/integration_filter_posts.clj b/src/source/routes/integration_filter_posts.clj
index 3844de41..65c4e3b7 100644
--- a/src/source/routes/integration_filter_posts.clj
+++ b/src/source/routes/integration_filter_posts.clj
@@ -1,6 +1,6 @@
 (ns source.routes.integration-filter-posts
-  (:require [source.services.interface :as services]
-            [ring.util.response :as res]))
+  (:require [ring.util.response :as res]
+            [source.db.honey :as hon]))
 
 (defn get
   {:summary "gets all filtered post ids by integration id"
@@ -15,4 +15,5 @@
 
   [{:keys [ds path-params] :as _request}]
 
-  (res/response (services/filtered-posts ds {:where [:= :bundle-id (:id path-params)]})))
+  (res/response (hon/find ds {:tname :filtered-posts
+                              :where [:= :bundle-id (:id path-params)]})))
diff --git a/src/source/services/baselines.clj b/src/source/services/baselines.clj
deleted file mode 100644
index b790bd58..00000000
--- a/src/source/services/baselines.clj
+++ /dev/null
@@ -1,10 +0,0 @@
-(ns source.services.baselines
-  (:require [source.db.interface :as db]))
-
-(defn baselines
-  ([ds] (baselines ds {}))
-  ([ds opts]
-   (->> {:tname :baselines
-         :ret :*}
-        (merge opts)
-        (db/find ds))))
diff --git a/src/source/services/businesses.clj b/src/source/services/businesses.clj
deleted file mode 100644
index 4c361324..00000000
--- a/src/source/services/businesses.clj
+++ /dev/null
@@ -1,30 +0,0 @@
-(ns source.services.businesses
-  (:require [source.db.interface :as db]))
-
-(defn business
-  [ds {:keys [id where] :as opts}]
-  (->> {:tname :businesses
-        :where (if id [:= :id id] where)
-        :ret :1}
-       (merge opts)
-       (db/find ds)))
-
-(defn businesses
-  ([ds] (businesses ds {}))
-  ([ds opts]
-   (->> {:tname :businesses
-         :ret :*}
-        (merge opts)
-        (db/find ds))))
-
-(defn insert-business! [ds {:keys [_values _ret] :as opts}]
-  (->> {:tname :businesses}
-       (merge opts)
-       (db/insert! ds)))
-
-(defn update-business! [ds {:keys [id values where] :as opts}]
-  (->> {:tname :businesses
-        :values values
-        :where (if (some? id) [:= :id id] where)}
-       (merge opts)
-       (db/update! ds)))
diff --git a/src/source/services/cadences.clj b/src/source/services/cadences.clj
deleted file mode 100644
index a03e3a79..00000000
--- a/src/source/services/cadences.clj
+++ /dev/null
@@ -1,10 +0,0 @@
-(ns source.services.cadences
-  (:require [source.db.interface :as db]))
-
-(defn cadences
-  ([ds] (cadences ds {}))
-  ([ds opts]
-   (->> {:tname :cadences
-         :ret :*}
-        (merge opts)
-        (db/find ds))))
diff --git a/src/source/services/categories.clj b/src/source/services/categories.clj
index d75ff75d..b720e6b6 100644
--- a/src/source/services/categories.clj
+++ b/src/source/services/categories.clj
@@ -17,15 +17,6 @@
         (merge opts)
         (db/find ds))))
 
-(defn category [ds {:keys [id where] :as opts}]
-  (->> {:tname :categories
-        :where (if (some? id)
-                 [:= :id id]
-                 where)
-        :ret :1}
-       (merge opts)
-       (db/find ds)))
-
 (comment 
   (require '[source.db.util :as db.util])
   (def ds (db.util/conn))
diff --git a/src/source/services/content_types.clj b/src/source/services/content_types.clj
deleted file mode 100644
index 117086b8..00000000
--- a/src/source/services/content_types.clj
+++ /dev/null
@@ -1,19 +0,0 @@
-(ns source.services.content-types
-  (:require [source.db.interface :as db]))
-
-(defn content-types
-  ([ds] (content-types ds {}))
-  ([ds opts]
-   (->> {:tname :content-types
-         :ret :*}
-        (merge opts)
-        (db/find ds))))
-
-(defn content-type [ds {:keys [id where] :as opts}]
-  (->> {:tname :content-types
-        :where (if (some? id)
-                 [:= :id id]
-                 where)
-        :ret :1}
-       (merge opts)
-       (db/find ds)))
diff --git a/src/source/services/filtered_feeds.clj b/src/source/services/filtered_feeds.clj
deleted file mode 100644
index 3f26a5fa..00000000
--- a/src/source/services/filtered_feeds.clj
+++ /dev/null
@@ -1,11 +0,0 @@
-(ns source.services.filtered-feeds
-  (:require [source.db.interface :as db]))
-
-(defn filtered-feeds
-  ([ds] (filtered-feeds ds {}))
-  ([ds {:keys [where] :as opts}]
-   (->> {:tname :filtered-feeds
-         :where where
-         :ret :*}
-        (merge opts)
-        (db/find ds))))
diff --git a/src/source/services/filtered_posts.clj b/src/source/services/filtered_posts.clj
deleted file mode 100644
index 571c74c9..00000000
--- a/src/source/services/filtered_posts.clj
+++ /dev/null
@@ -1,11 +0,0 @@
-(ns source.services.filtered-posts
-  (:require [source.db.interface :as db]))
-
-(defn filtered-posts
-  ([ds] (filtered-posts ds {}))
-  ([ds {:keys [where] :as opts}]
-   (->> {:tname :filtered-posts
-         :where where
-         :ret :*}
-        (merge opts)
-        (db/find ds))))
diff --git a/src/source/services/interface.clj b/src/source/services/interface.clj
index 575d6731..e5a59d4a 100644
--- a/src/source/services/interface.clj
+++ b/src/source/services/interface.clj
@@ -9,32 +9,12 @@
              [source.services.feeds :as feeds]
              [source.services.incoming-posts :as incoming-posts]
              [source.services.outgoing-posts :as outgoing-posts]
-             [source.services.cadences :as cadences]
-             [source.services.baselines :as baselines]
-             [source.services.content-types :as content-types]
-             [source.services.categories :as categories]
              [source.services.feed-categories :as feed-categories]
-             [source.services.jobs :as jobs]
-             [source.services.businesses :as businesses]
-             [source.services.filtered-feeds :as filtered-feeds]
-             [source.services.filtered-posts :as filtered-posts]))
+             [source.services.jobs :as jobs]))
 
 (defn user [ds {:keys [_id] :as opts}]
   (users/user ds opts))
 
-(defn update-user! [ds {:keys [_id _values _where] :as opts}]
-  (users/update-user! ds opts))
-
-(defn business
-  [ds {:keys [_id _where] :as opts}]
-  (businesses/business ds opts))
-
-(defn insert-business! [ds {:keys [_values _ret] :as opts}]
-  (businesses/insert-business! ds opts))
-
-(defn update-business! [ds {:keys [_id _values _where] :as opts}]
-  (businesses/update-business! ds opts))
-
 (defn register [ds user]
   (auth/register ds user))
 
@@ -97,27 +77,9 @@
 (defn insert-output-schema! [store schema]
   (xml/insert-output-schema! store schema))
 
-(defn providers [ds]
-  (providers/providers ds))
-
 (defn provider [ds provider-id]
   (providers/provider ds provider-id))
 
-(defn delete-provider! [ds provider-id]
-  (providers/delete-provider! ds provider-id))
-
-(defn update-provider! [ds {:keys [_id _data _where] :as opts}]
-  (providers/update-provider! ds opts))
-
-(defn insert-provider! [ds {:keys [_values _ret] :as opts}]
-  (providers/insert-provider! ds opts))
-
-(defn content-types [ds]
-  (content-types/content-types ds))
-
-(defn content-type [ds id]
-  (content-types/content-type ds id))
-
 (defn update-feed! [ds {:keys [_id _data _where] :as opts}]
   (feeds/update-feed! ds opts))
 
@@ -149,20 +111,6 @@
 (defn incoming-post [ds opts]
   (incoming-posts/incoming-post ds opts))
 
-(defn cadences [ds]
-  (cadences/cadences ds))
-
-(defn baselines [ds]
-  (baselines/baselines ds))
-
-(defn categories
-  ([ds] (categories ds {}))
-  ([ds {:keys [_where] :as opts}]
-   (categories/categories ds opts)))
-
-(defn category [ds {:keys [_id _where] :as opts}]
-  (categories/category ds opts))
-
 (defn categories-by-feed [ds {:keys [_feed-id _where] :as opts}]
   (feed-categories/categories-by-feed ds opts))
 
@@ -194,13 +142,3 @@
 
 (defn job-metadata [ds {:keys [_id _where] :as opts}]
   (jobs/job-metadata ds opts))
-
-(defn filtered-feeds
-  ([ds] (filtered-feeds ds {}))
-  ([ds {:keys [_where] :as opts}]
-   (filtered-feeds/filtered-feeds ds opts)))
-
-(defn filtered-posts
-  ([ds] (filtered-posts ds {}))
-  ([ds {:keys [_where] :as opts}]
-   (filtered-posts/filtered-posts ds opts)))
diff --git a/src/source/services/providers.clj b/src/source/services/providers.clj
index e0d2bf77..df731c63 100644
--- a/src/source/services/providers.clj
+++ b/src/source/services/providers.clj
@@ -1,21 +1,6 @@
 (ns source.services.providers
   (:require [source.db.interface :as db]))
 
-(defn insert-provider! [ds {:keys [data ret] :as opts}]
-  (->> {:tname :providers
-        :data data
-        :ret ret}
-       (merge opts)
-       (db/insert! ds)))
-
-(defn providers
-  ([ds] (providers ds {}))
-  ([ds opts]
-   (->> {:tname :providers
-         :ret :*}
-        (merge opts)
-        (db/find ds))))
-
 (defn provider [ds {:keys [id where] :as opts}]
   (->> {:tname :providers
         :where (if (some? id)
@@ -24,19 +9,3 @@
         :ret :1}
        (merge opts)
        (db/find ds)))
-
-(defn update-provider! [ds {:keys [id data where] :as opts}]
-  (->> {:tname :providers
-        :data data
-        :where (if (some? id) [:= :id id] where)}
-       (merge opts)
-       (db/update! ds)))
-
-(defn delete-provider! [ds {:keys [id where] :as opts}]
-  (->> {:tname :providers
-        :where (if (some? id)
-                 [:= :id id]
-                 where)
-        :ret :1}
-       (merge opts)
-       (db/delete! ds)))
diff --git a/src/source/services/sectors.clj b/src/source/services/sectors.clj
deleted file mode 100644
index 82979c7b..00000000
--- a/src/source/services/sectors.clj
+++ /dev/null
@@ -1,10 +0,0 @@
-(ns source.services.sectors
-  (:require [source.db.interface :as db]))
-
-(defn sectors
-  ([ds] (sectors ds {}))
-  ([ds opts]
-   (->> {:tname :sectors
-         :ret :*}
-        (merge opts)
-        (db/find ds))))

From a745fbda1921f9957683ae31fbeee5b6977a9266 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 28 Jan 2026 14:23:15 +0200
Subject: [PATCH 240/391] deleted more unnecessary services

---
 src/source/db/event.clj                       | 17 ++++---
 src/source/jobs/handlers.clj                  | 44 +++++++++-------
 src/source/middleware/auth/core.clj           |  6 +--
 src/source/routes/admin.clj                   | 13 ++---
 .../routes/analytics/distributor/top.clj      | 10 ++--
 src/source/routes/approve_feed.clj            |  7 ++-
 src/source/routes/feeds.clj                   |  8 ++-
 src/source/routes/provider.clj                |  4 +-
 src/source/routes/reject_feed.clj             |  3 +-
 src/source/services/auth.clj                  | 14 +++---
 src/source/services/bundles.clj               |  7 +--
 src/source/services/categories.clj            | 26 ----------
 src/source/services/event_categories.clj      |  8 ---
 src/source/services/feeds.clj                 | 28 -----------
 src/source/services/incoming_posts.clj        | 15 ------
 src/source/services/interface.clj             | 50 +++----------------
 src/source/services/outgoing_posts.clj        | 25 ----------
 src/source/services/providers.clj             | 11 ----
 src/source/services/users.clj                 | 47 -----------------
 19 files changed, 80 insertions(+), 263 deletions(-)
 delete mode 100644 src/source/services/categories.clj
 delete mode 100644 src/source/services/event_categories.clj
 delete mode 100644 src/source/services/feeds.clj
 delete mode 100644 src/source/services/outgoing_posts.clj
 delete mode 100644 src/source/services/providers.clj
 delete mode 100644 src/source/services/users.clj

diff --git a/src/source/db/event.clj b/src/source/db/event.clj
index eb95bd1f..33ce7e13 100644
--- a/src/source/db/event.clj
+++ b/src/source/db/event.clj
@@ -1,14 +1,13 @@
 (ns source.db.event
-  (:require [source.services.event-categories :as ec]
-            [source.services.analytics.interface :as analytics]
-            [source.services.outgoing-posts :as outgoing-posts]
+  (:require [source.services.analytics.interface :as analytics]
             [source.services.feed-categories :as feed-categories]
             [source.db.util :as db.util]
             [source.util :as util]
             [source.db.honey :as db]))
 
 (defn get-post-categories [bundle-ds ds post-id]
-  (let [feed-id (-> (outgoing-posts/outgoing-post bundle-ds {:id post-id})
+  (let [feed-id (-> (db/find-one bundle-ds {:tname :outgoing-posts
+                                            :where [:= :id post-id]})
                     (:feed-id))]
     (feed-categories/category-id ds {:feed-id feed-id})))
 
@@ -29,12 +28,14 @@
                                                                   :timestamp timestamp}
                                                            :ret :*})
                        (first))]
-      (ec/insert-event-category! bundle-ds {:data {:event-id event-id
-                                                   :category-id (:category-id categories)}}))
+      (db/insert! bundle-ds {:tname :event-categories
+                             :data {:event-id event-id
+                                    :category-id (:category-id categories)}}))
     (let [event-id (-> (analytics/insert-event! creator-ds {:data {:post_id post-id
                                                                    :event_type type
                                                                    :timestamp timestamp}
                                                             :ret :*})
                        (first))]
-      (ec/insert-event-category! creator-ds {:data {:event-id event-id
-                                                    :category-id (:category-id categories)}}))))
+      (db/insert! creator-ds {:tname :event-categories
+                              :data {:event-id event-id
+                                     :category-id (:category-id categories)}}))))
diff --git a/src/source/jobs/handlers.clj b/src/source/jobs/handlers.clj
index 46e503ce..af20f91a 100644
--- a/src/source/jobs/handlers.clj
+++ b/src/source/jobs/handlers.clj
@@ -4,7 +4,8 @@
             [source.services.incoming-posts :as incoming-posts]
             [source.db.util :as db.util]
             [clojure.set :as set]
-            [clojure.string :as string]))
+            [clojure.string :as string]
+            [source.db.honey :as hon]))
 
 (defmulti handler
   (fn [opts]
@@ -18,7 +19,7 @@
   (fn [{:keys [args]}]
     (println "hello" (get args :name) args)))
 
-(defn update-feed-posts-job-id 
+(defn update-feed-posts-job-id
   "returns the job id of an update-feed-posts job with the given email and feed-id"
   [email feed-id]
   (str email "-" feed-id))
@@ -50,25 +51,30 @@
                                                         thumbnail
                                                         extracted-display)}))
                                  extracted-posts)
-            existing-posts (services/incoming-posts ds {:where [:= :creator-id creator-id]})
-            existing-feed (services/feed ds {:id feed-id})]
-        (services/update-feed! ds {:id feed-id
-                                   :data {:title (get-in extracted [:feed :title])
-                                          :display-picture (if (and (:display-picture existing-feed)
-                                                                    (seq (:display-picture existing-feed)))
-                                                             (:display-picture existing-feed)
-                                                             extracted-display)
-                                          :updated-at (util/get-utc-timestamp-string)}})
+            existing-posts (hon/find ds {:tname :incoming-posts
+                                         :where [:= :creator-id creator-id]})
+            existing-feed (hon/find-one ds {:tname :feeds
+                                            :where [:= :id feed-id]})]
+        (hon/update! ds {:tname :feeds
+                         :where [:= :id feed-id]
+                         :data {:title (get-in extracted [:feed :title])
+                                :display-picture (if (and (:display-picture existing-feed)
+                                                          (seq (:display-picture existing-feed)))
+                                                   (:display-picture existing-feed)
+                                                   extracted-display)
+                                :updated-at (util/get-utc-timestamp-string)}})
         (run!
          (fn [post]
            (if (some #(= (:post-id post) (:post-id %)) existing-posts)
-             (services/update-incoming-post! ds {:where [:= :post-id (:post-id post)]
-                                                 :data post})
-             (services/insert-incoming-post! ds {:data post})))
+             (hon/update! ds {:tname :incoming-posts
+                              :where [:= :post-id (:post-id post)]
+                              :data post})
+             (hon/insert! ds {:tname :incoming-posts
+                              :data post})))
          extended-posts))
       (catch Exception _ :fail))))
 
-(defn update-bundle-job-id 
+(defn update-bundle-job-id
   "returns the job id of an update-bundle job with the given bundle id"
   [bundle-id]
   (str "bundle_" bundle-id))
@@ -109,7 +115,8 @@
             ids (mapv :post-id top-by-long-heuristics)
 
             ; get all incoming posts with the above id numbers
-            posts-in (services/incoming-posts ds {:where [:in :id ids]})
+            posts-in (hon/find ds {:tname :incoming-posts
+                                   :where [:in :id ids]})
             ; remove redacted posts
             outgoing-posts (reduce (fn [acc {:keys [redacted] :as post}]
                                      (if (:= redacted 0)
@@ -117,5 +124,6 @@
                                        acc))
                                    [] posts-in)]
         (when (seq posts-in)
-          (services/delete-outgoing-post! ds-bundle {})
-          (services/insert-outgoing-post! ds-bundle {:data outgoing-posts}))))))
+          (hon/delete! ds-bundle {:tname :outgoing-posts})
+          (hon/insert! ds-bundle {:tname :outgoing-posts
+                                  :data outgoing-posts}))))))
diff --git a/src/source/middleware/auth/core.clj b/src/source/middleware/auth/core.clj
index ab039670..e6db63b6 100644
--- a/src/source/middleware/auth/core.clj
+++ b/src/source/middleware/auth/core.clj
@@ -2,7 +2,6 @@
   (:require [source.middleware.auth.util :as util]
             [source.db.util :as db.util]
             [ring.util.response :as res]
-            [source.services.users :as users]
             [source.services.bundles :as bundles]
             [source.db.honey :as db]))
 
@@ -33,8 +32,9 @@
   (fn [request]
     (let [ds (db.util/conn :master)
           user-type (get-in request [:user :type])
-          expected-type (->> {:id (get-in request [:user :id])}
-                             (users/user ds)
+          expected-type (->> {:tname :users
+                              :where [:= :id (get-in request [:user :id])]}
+                             (db/find-one ds)
                              (:type))]
       (cond
         (not (some? required-type)) (handler request)
diff --git a/src/source/routes/admin.clj b/src/source/routes/admin.clj
index 98bd7e6b..78e2d955 100644
--- a/src/source/routes/admin.clj
+++ b/src/source/routes/admin.clj
@@ -1,8 +1,8 @@
 (ns source.routes.admin
-  (:require [source.services.users :as users]
-            [source.password :as pw]
+  (:require [source.password :as pw]
             [source.util :as util]
-            [ring.util.response :as res]))
+            [ring.util.response :as res]
+            [source.db.honey :as hon]))
 
 (defn post
   {:summary "registers an admin user"
@@ -17,7 +17,8 @@
   [{:keys [ds body] :as _request}]
 
   (let [{:keys [data error success]} (util/validate post body)
-        user (users/user ds {:where [:= :email (:email data)]})
+        user (hon/find-one ds {:tname :users
+                               :where [:= :email (:email data)]})
         {:keys [password confirm-password]} data]
     (cond
 
@@ -38,6 +39,6 @@
                                 :password pw
                                 :type "admin")
                          (dissoc :confirm-password))]
-        (users/insert-user! ds {:data new-user})
+        (hon/insert! ds {:tname :users
+                         :data new-user})
         (res/response {:message "successfully created user"})))))
-
diff --git a/src/source/routes/analytics/distributor/top.clj b/src/source/routes/analytics/distributor/top.clj
index a95bfd85..2ee5ff25 100644
--- a/src/source/routes/analytics/distributor/top.clj
+++ b/src/source/routes/analytics/distributor/top.clj
@@ -2,13 +2,15 @@
   (:require [source.services.analytics.interface :as analytics]
             [ring.util.response :as res]
             [clojure.set :as set]
-            [source.services.interface :as services]
-            [source.util :as utils]))
+            [source.util :as utils]
+            [source.db.honey :as hon]))
 
 (defn record-names [ds top-field ids]
   (if (= top-field :post-id)
-    (services/incoming-posts ds {:where [:in :id ids]})
-    (services/feeds ds {:where [:in :id ids]})))
+    (hon/find ds {:tname :incoming-posts
+                  :where [:in :id ids]})
+    (hon/find ds {:tname :feeds
+                  :where [:in :id ids]})))
 
 (defn get
   {:summary "Get the top n records with the highest number of impressions, clicks and views, in terms of the given top field. Optionally filtered by content type.
diff --git a/src/source/routes/approve_feed.clj b/src/source/routes/approve_feed.clj
index dc1746ec..90ead7c2 100644
--- a/src/source/routes/approve_feed.clj
+++ b/src/source/routes/approve_feed.clj
@@ -1,6 +1,5 @@
 (ns source.routes.approve-feed
-  (:require [source.services.interface :as services]
-            [source.email.gmail :as gmail]
+  (:require [source.email.gmail :as gmail]
             [source.email.templates :as templates]
             [ring.util.response :as res]
             [source.db.honey :as hon]))
@@ -16,8 +15,8 @@
   [{:keys [ds path-params] :as _request}]
   (let [{:keys [id user-id title]} (hon/find-one ds {:tname :feeds
                                                      :where [:= :id (:id path-params)]})
-        {:keys [email firstname]} (services/user ds {:tname :users
-                                                     :where [:= :id user-id]})]
+        {:keys [email firstname]} (hon/find-one ds {:tname :users
+                                                    :where [:= :id user-id]})]
     (hon/update! ds {:tname :feeds
                      :where [:= :id (:id path-params)]
                      :data {:state "live"}})
diff --git a/src/source/routes/feeds.clj b/src/source/routes/feeds.clj
index 34cb2f9f..b0dfa8dd 100644
--- a/src/source/routes/feeds.clj
+++ b/src/source/routes/feeds.clj
@@ -1,9 +1,6 @@
 (ns source.routes.feeds
-  (:require [source.services.interface :as services]
-            [source.util :as utils]
-            [source.workers.feeds :as feeds]
+  (:require [source.workers.feeds :as feeds]
             [congest.jobs :as congest]
-            [source.jobs.core :as jobs]
             [ring.util.response :as res]
             [source.db.honey :as hon]))
 
@@ -27,7 +24,8 @@
                              [:state [:enum "live" "not live" "pending"]]]]}}}
 
   [{:keys [ds user] :as _request}]
-  (-> (services/feeds ds {:where [:= :user-id (:id user)]})
+  (-> (hon/find ds {:tname :feeds
+                    :where [:= :user-id (:id user)]})
       (res/response)))
 
 (defn post
diff --git a/src/source/routes/provider.clj b/src/source/routes/provider.clj
index 6b3d1249..45854eac 100644
--- a/src/source/routes/provider.clj
+++ b/src/source/routes/provider.clj
@@ -16,8 +16,8 @@
                            [:placeholder-url [:maybe :string]]]}}}
 
   [{:keys [ds path-params] :as _request}]
-  (->> (services/provider ds {:tname :providers
-                              :where [:= :id (:id path-params)]})
+  (->> (hon/find-one ds {:tname :providers
+                         :where [:= :id (:id path-params)]})
        (res/response)))
 
 (defn post
diff --git a/src/source/routes/reject_feed.clj b/src/source/routes/reject_feed.clj
index b908d1dd..3b010a2e 100644
--- a/src/source/routes/reject_feed.clj
+++ b/src/source/routes/reject_feed.clj
@@ -1,6 +1,5 @@
 (ns source.routes.reject-feed
-  (:require [source.services.interface :as services]
-            [source.email.gmail :as gmail]
+  (:require [source.email.gmail :as gmail]
             [source.email.templates :as templates]
             [ring.util.response :as res]
             [source.db.honey :as hon]))
diff --git a/src/source/services/auth.clj b/src/source/services/auth.clj
index 92d31020..642d9098 100644
--- a/src/source/services/auth.clj
+++ b/src/source/services/auth.clj
@@ -1,7 +1,7 @@
 (ns source.services.auth
   (:require [source.password :as pw]
-            [source.services.users :as users]
-            [source.middleware.auth.core :as auth]))
+            [source.middleware.auth.core :as auth]
+            [source.db.honey :as hon]))
 
 (defn login [ds {:keys [user] :as _login}]
   (merge
@@ -9,10 +9,12 @@
    (auth/create-session (select-keys user [:id :type]))))
 
 (defn register [ds {:keys [email password] :as user}]
-  (users/insert-user! ds {:data (-> user
-                                    (dissoc :confirm-password)
-                                    (assoc :password (pw/hash-password password)))})
-  (let [user (users/user ds {:where [:= :email email]})]
+  (hon/insert! ds {:tname :users
+                   :data (-> user
+                             (dissoc :confirm-password)
+                             (assoc :password (pw/hash-password password)))})
+  (let [user (hon/find-one ds {:tname :users
+                               :where [:= :email email]})]
     (merge
      {:user (dissoc user :password)}
      (auth/create-session (select-keys user [:id :type])))))
diff --git a/src/source/services/bundles.clj b/src/source/services/bundles.clj
index 7f4ec533..1560a1ca 100644
--- a/src/source/services/bundles.clj
+++ b/src/source/services/bundles.clj
@@ -1,9 +1,9 @@
 (ns source.services.bundles
   (:require [source.db.interface :as db]
             [source.util :as utils]
-            [source.services.categories :as categories]
             [source.services.bundle-categories :as bundle-categories]
-            [source.db.util :as db.util]))
+            [source.db.util :as db.util]
+            [source.db.honey :as hon]))
 
 (defn insert-bundle! [ds {:keys [_values _ret] :as opts}]
   (->> {:tname :bundles}
@@ -49,4 +49,5 @@
   (with-open [bundle-ds (db.util/conn :bundle bundle-id)]
     (let [category-ids (bundle-categories/category-id bundle-ds {:bundle-id bundle-id})
           id-vec (mapv (fn [{:keys [category-id]}] category-id) category-ids)]
-      (categories/categories ds {:where [:in :id id-vec]}))))
+      (hon/find ds {:tname :categories
+                    :where [:in :id id-vec]}))))
diff --git a/src/source/services/categories.clj b/src/source/services/categories.clj
deleted file mode 100644
index b720e6b6..00000000
--- a/src/source/services/categories.clj
+++ /dev/null
@@ -1,26 +0,0 @@
-(ns source.services.categories
-  (:require [source.db.interface :as db]))
-
-(defn insert-category! [ds {:keys [data ret] :as opts}]
-  (->> {:tname :categories
-        :data data
-        :ret ret}
-       (merge opts)
-       (db/insert! ds)))
-
-(defn categories
-  ([ds] (categories ds {}))
-  ([ds {:keys [where] :as opts}]
-   (->> {:tname :categories
-         :where where
-         :ret :*}
-        (merge opts)
-        (db/find ds))))
-
-(comment 
-  (require '[source.db.util :as db.util])
-  (def ds (db.util/conn))
-
-  (categories ds)
-  (insert-category! ds {:data {:name "programming"}})
-  ())
diff --git a/src/source/services/event_categories.clj b/src/source/services/event_categories.clj
deleted file mode 100644
index 852a423e..00000000
--- a/src/source/services/event_categories.clj
+++ /dev/null
@@ -1,8 +0,0 @@
-(ns source.services.event-categories
-  (:require [source.db.interface :as db]))
-
-(defn insert-event-category! [ds {:keys [_values _ret] :as opts}]
-  (->> {:tname :event-categories}
-       (merge opts)
-       (db/insert! ds)))
-
diff --git a/src/source/services/feeds.clj b/src/source/services/feeds.clj
deleted file mode 100644
index 7f6e2047..00000000
--- a/src/source/services/feeds.clj
+++ /dev/null
@@ -1,28 +0,0 @@
-(ns source.services.feeds
-  (:require [source.db.interface :as db]))
-
-(defn update-feed! [ds {:keys [id data where] :as opts}]
-  (->> {:tname :feeds
-        :values data
-        :where (if (some? id) [:= :id id] where)
-        :ret :1}
-       (merge opts)
-       (db/update! ds)))
-
-(defn feeds
-  ([ds] (feeds ds {}))
-  ([ds {:keys [where] :as opts}]
-   (->> {:tname :feeds
-         :where where
-         :ret :*}
-        (merge opts)
-        (db/find ds))))
-
-(defn feed [ds {:keys [id where] :as opts}]
-  (->> {:tname :feeds
-        :where (if (some? id)
-                 [:= :id id]
-                 where)
-        :ret :1}
-       (merge opts)
-       (db/find ds)))
diff --git a/src/source/services/incoming_posts.clj b/src/source/services/incoming_posts.clj
index 0578d65c..d4df9b4a 100644
--- a/src/source/services/incoming_posts.clj
+++ b/src/source/services/incoming_posts.clj
@@ -2,21 +2,6 @@
   (:require [source.db.interface :as db]
             [source.db.honey :as hon]))
 
-(defn insert-incoming-post! [ds {:keys [data ret] :as opts}]
-  (->> {:tname :incoming-posts
-        :data data
-        :ret ret}
-       (merge opts)
-       (db/insert! ds)))
-
-(defn update-incoming-post! [ds {:keys [id data where] :as opts}]
-  (->> {:tname :incoming-posts
-        :data data
-        :where (if (some? id) [:= :id id] where)
-        :ret :1}
-       (merge opts)
-       (db/update! ds)))
-
 (defn incoming-posts
   ([ds] (incoming-posts ds {}))
   ([ds {:keys [where] :as opts}]
diff --git a/src/source/services/interface.clj b/src/source/services/interface.clj
index e5a59d4a..d2179cda 100644
--- a/src/source/services/interface.clj
+++ b/src/source/services/interface.clj
@@ -1,19 +1,12 @@
 (ns source.services.interface
-  (:require  [source.services.users :as users]
-             [source.services.auth :as auth]
-             [source.services.xml-schemas :as xml]
-             [source.services.bundles :as bundles]
-             [source.services.bundle-content-types :as bundle-content-types]
-             [source.services.post-heuristics :as post-heuristics]
-             [source.services.providers :as providers]
-             [source.services.feeds :as feeds]
-             [source.services.incoming-posts :as incoming-posts]
-             [source.services.outgoing-posts :as outgoing-posts]
-             [source.services.feed-categories :as feed-categories]
-             [source.services.jobs :as jobs]))
-
-(defn user [ds {:keys [_id] :as opts}]
-  (users/user ds opts))
+  (:require [source.services.auth :as auth]
+            [source.services.xml-schemas :as xml]
+            [source.services.bundles :as bundles]
+            [source.services.bundle-content-types :as bundle-content-types]
+            [source.services.post-heuristics :as post-heuristics]
+            [source.services.incoming-posts :as incoming-posts]
+            [source.services.feed-categories :as feed-categories]
+            [source.services.jobs :as jobs]))
 
 (defn register [ds user]
   (auth/register ds user))
@@ -47,12 +40,6 @@
 (defn top-posts-by-heuristic [ds {:keys [_select _limit _heuristic] :as opts}]
   (post-heuristics/top-posts-by-heuristic ds opts))
 
-(defn insert-outgoing-post! [ds {:keys [_values _ret] :as opts}]
-  (outgoing-posts/insert-outgoing-post! ds opts))
-
-(defn delete-outgoing-post! [ds {:keys [_id _where] :as opts}]
-  (outgoing-posts/delete-outgoing-post! ds opts))
-
 (defn selection-schemas
   ([ds]
    (selection-schemas ds {}))
@@ -77,27 +64,6 @@
 (defn insert-output-schema! [store schema]
   (xml/insert-output-schema! store schema))
 
-(defn provider [ds provider-id]
-  (providers/provider ds provider-id))
-
-(defn update-feed! [ds {:keys [_id _data _where] :as opts}]
-  (feeds/update-feed! ds opts))
-
-(defn feeds
-  ([ds]
-   (feeds/feeds ds))
-  ([ds {:keys [_where] :as opts}]
-   (feeds/feeds ds opts)))
-
-(defn feed [ds {:keys [_id] :as opts}]
-  (feeds/feed ds opts))
-
-(defn insert-incoming-post! [ds {:keys [_data _ret] :as opts}]
-  (incoming-posts/insert-incoming-post! ds opts))
-
-(defn update-incoming-post! [ds {:keys [_id _data _where] :as opts}]
-  (incoming-posts/update-incoming-post! ds opts))
-
 (defn incoming-posts
   ([ds]
    (incoming-posts/incoming-posts ds))
diff --git a/src/source/services/outgoing_posts.clj b/src/source/services/outgoing_posts.clj
deleted file mode 100644
index f45b89d3..00000000
--- a/src/source/services/outgoing_posts.clj
+++ /dev/null
@@ -1,25 +0,0 @@
-(ns source.services.outgoing-posts
-  (:require [source.db.interface :as db]))
-
-(defn outgoing-post [ds {:keys [id where] :as opts}]
-  (->> {:tname :outgoing-posts
-        :where (if (some? id)
-                 [:= :id id]
-                 where)
-        :ret :1}
-       (merge opts)
-       (db/find ds)))
-
-(defn insert-outgoing-post! [ds {:keys [_values _ret] :as opts}]
-  (->> {:tname :outgoing-posts}
-       (merge opts)
-       (db/insert! ds)))
-
-(defn delete-outgoing-post! [ds {:keys [id where] :as opts}]
-  (->> {:tname :outgoing-posts
-        :where (if (some? id)
-                 [:= :id id]
-                 where)
-        :ret :1}
-       (merge opts)
-       (db/delete! ds)))
diff --git a/src/source/services/providers.clj b/src/source/services/providers.clj
deleted file mode 100644
index df731c63..00000000
--- a/src/source/services/providers.clj
+++ /dev/null
@@ -1,11 +0,0 @@
-(ns source.services.providers
-  (:require [source.db.interface :as db]))
-
-(defn provider [ds {:keys [id where] :as opts}]
-  (->> {:tname :providers
-        :where (if (some? id)
-                 [:= :id id]
-                 where)
-        :ret :1}
-       (merge opts)
-       (db/find ds)))
diff --git a/src/source/services/users.clj b/src/source/services/users.clj
deleted file mode 100644
index 2c37cb1f..00000000
--- a/src/source/services/users.clj
+++ /dev/null
@@ -1,47 +0,0 @@
-(ns source.services.users
-  (:require [source.db.interface :as db]
-            [source.password :as pw]))
-
-(defn users
-  ([ds] (users ds {}))
-  ([ds opts]
-   (->> {:tname :users
-         :ret :*}
-        (merge opts)
-        (db/find ds)
-        (mapv #(dissoc % :password)))))
-
-(defn user [ds {:keys [id where] :as opts}]
-  (->> {:tname :users
-        :where (if (some? id)
-                 [:= :id id]
-                 where)
-        :ret :1}
-       (merge opts)
-       (db/find ds)))
-
-(defn insert-user! [ds {:keys [_values _ret] :as opts}]
-  (->> {:tname :users}
-       (merge opts)
-       (db/insert! ds)))
-
-(defn update-user! [ds {:keys [id values where] :as opts}]
-  (->> {:tname :users
-        :values values
-        :where (if (some? id) [:= :id id] where)}
-       (merge opts)
-       (db/update! ds)))
-
-(comment
-  (users (db/ds :master))
-  (insert-user! (db/ds :master) {:data {:email "chonkin@bonkin.com"
-                                        :password (pw/hash-password "test")
-                                        :firstname "merv"
-                                        :lastname "vaneck"
-                                        :type "creator"}
-                                 :ret :*})
-  (user (db/ds :master) {:id 5})
-  (update-user! (db/ds :master) {:id 5
-                                 :values {:firstname "kiigan"
-                                          :lastname "korinzu"}})
-  ())

From 7b422244a3b4f3b8ca5e4dd5fafd560134852b22 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 28 Jan 2026 15:33:24 +0200
Subject: [PATCH 241/391] fixed bugs

---
 src/source/db/honey.clj                      | 2 +-
 src/source/routes/bundle_feed_post.clj       | 6 ++++--
 src/source/routes/feed.clj                   | 4 ++--
 src/source/services/analytics/core.clj       | 9 +++++----
 src/source/services/bundle_content_types.clj | 3 ++-
 src/source/workers/feeds.clj                 | 3 ++-
 6 files changed, 16 insertions(+), 11 deletions(-)

diff --git a/src/source/db/honey.clj b/src/source/db/honey.clj
index 4691c200..f7fbc1cb 100644
--- a/src/source/db/honey.clj
+++ b/src/source/db/honey.clj
@@ -58,7 +58,7 @@
               (-> (hsql/insert-into (csk/->snake_case_keyword tname))
                   (hsql/values vals)
                   (hsql/returning :*))
-              :ret ret)))
+              :ret (or ret :1))))
 
 (defn delete!
   "deletes a record or set of records that match a predicate where clause. the where
diff --git a/src/source/routes/bundle_feed_post.clj b/src/source/routes/bundle_feed_post.clj
index ee3d6f9e..bdd234a1 100644
--- a/src/source/routes/bundle_feed_post.clj
+++ b/src/source/routes/bundle_feed_post.clj
@@ -1,7 +1,8 @@
 (ns source.routes.bundle-feed-post
   (:require [ring.util.response :as res]
             [source.db.honey :as hon]
-            [source.services.analytics.interface :as analytics]))
+            [source.services.analytics.interface :as analytics]
+            [source.db.util :as db.util]))
 
 (defn get
   {:summary "get post in outgoing feed for the associated uuid-authorized bundle by post id, updates click analytics"
@@ -28,6 +29,7 @@
 
   [{:keys [ds bundle-id path-params] :as _request}]
   (let [post (hon/find-one ds {:tname :incoming-posts
-                               :where [:= :id (:id path-params)]})]
+                               :where [:= :id (:post-id path-params)]
+                               :ret :1})]
     (analytics/insert-post-click! ds post bundle-id)
     (res/response post)))
diff --git a/src/source/routes/feed.clj b/src/source/routes/feed.clj
index b4ead652..7f107c3b 100644
--- a/src/source/routes/feed.clj
+++ b/src/source/routes/feed.clj
@@ -60,8 +60,8 @@
   (let [id (:id path-params)
         feed (hon/find-one ds {:tname :feeds
                                :where [:and
-                                       [:= :user-id (:id user)
-                                        := :id id]]})
+                                       [:= :user-id (:id user)]
+                                       [:= :id id]]})
         {:keys [email]} (hon/find-one ds {:tname :users
                                           :where [:= :id (:id user)]})]
     (if (some? feed)
diff --git a/src/source/services/analytics/core.clj b/src/source/services/analytics/core.clj
index fcdb06a7..149c79ff 100644
--- a/src/source/services/analytics/core.clj
+++ b/src/source/services/analytics/core.clj
@@ -192,7 +192,8 @@
                                             category-ids)))
                               (flatten)
                               (vec))]
-    (insert-event-categories! ds {:data event-categories})))
+    (when (seq event-categories)
+      (insert-event-categories! ds {:data event-categories}))))
 
 (defn insert-feed-impressions!
   "Given a list of feeds and a bundle id, inserts impression event reconds 
@@ -225,9 +226,9 @@
                         :creator-id creator-id
                         :bundle-id bundle-id
                         :distributor-id (:user-id bundle)}) posts)
-        events' (insert-event! ds {:data events
-                                   :ret :*})]
-    (insert-post-event-categories! ds events' posts)))
+        events' (when (seq posts) (insert-event! ds {:data events
+                                                     :ret :*}))]
+    (when (seq events') (insert-post-event-categories! ds events' posts))))
 
 (defn insert-feed-click!
   "Given a feed and a bundle id, inserts a click event record 
diff --git a/src/source/services/bundle_content_types.clj b/src/source/services/bundle_content_types.clj
index 38e972e8..5d26a177 100644
--- a/src/source/services/bundle_content_types.clj
+++ b/src/source/services/bundle_content_types.clj
@@ -37,4 +37,5 @@
                               {:bundle-id bundle-id
                                :content-type-id id}) content-types)]
     (delete-bundle-content-types! ds {:where [:= :bundle-id bundle-id]})
-    (insert-bundle-content-types! ds {:data content-types})))
+    (hon/insert! ds {:tname :bundle-content-types
+                     :data content-types})))
diff --git a/src/source/workers/feeds.clj b/src/source/workers/feeds.clj
index 53ce86b8..bf887d72 100644
--- a/src/source/workers/feeds.clj
+++ b/src/source/workers/feeds.clj
@@ -28,7 +28,8 @@
                                                :display-picture (get-in extracted [:feed :display-picture])
                                                :user-id user-id
                                                :created-at datetime
-                                               :state "pending"})})
+                                               :state "pending"})
+                   :ret :1})
         extended-posts (mapv (fn [post]
                                (merge post
                                       {:feed-id (:id new-feed)

From 4b99eda8b324c3bdca3e62cfd10025d7f1658b87 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Thu, 29 Jan 2026 10:50:44 +0200
Subject: [PATCH 242/391] removed unnecessary :*

---
 src/source/routes/cadences.clj | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/source/routes/cadences.clj b/src/source/routes/cadences.clj
index 47fad7c0..9fb98240 100644
--- a/src/source/routes/cadences.clj
+++ b/src/source/routes/cadences.clj
@@ -10,6 +10,5 @@
                             [:label :string]
                             [:days :int]]]}}}
   [{:keys [ds] :as _request}]
-  (->> (hon/find ds {:tname :cadences
-                     :ret :*})
+  (->> (hon/find ds {:tname :cadences})
        (res/response)))

From 2b8db1581d0072e2999bad4afb4037da6a09531b Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Thu, 29 Jan 2026 10:55:17 +0200
Subject: [PATCH 243/391] cleaned up integration services

---
 src/source/workers/integrations.clj | 38 ++++++++++++-----------------
 1 file changed, 16 insertions(+), 22 deletions(-)

diff --git a/src/source/workers/integrations.clj b/src/source/workers/integrations.clj
index 3a8910d2..e5888243 100644
--- a/src/source/workers/integrations.clj
+++ b/src/source/workers/integrations.clj
@@ -13,18 +13,18 @@
 
 (defn create-integration! [ds js store {:keys [user-id bundle-metadata categories content-types]}]
   (let [new-bundle (bundles/create-bundle! ds {:user-id user-id
-                                               :bundle-metadata bundle-metadata})
-        _ (migrate/migrate-bundle (:id new-bundle) ["up"])]
+                                               :bundle-metadata bundle-metadata})]
+    (migrate/migrate-bundle (:id new-bundle) ["up"])
 
     (with-open [bundle-ds (db.util/conn :bundle (:id new-bundle))]
-      (let [_ (bundle-categories/insert-bundle-categories! bundle-ds {:bundle-id (:id new-bundle)
-                                                                      :categories categories})
-            _ (bundle-content-types/insert-bundle-content-types! ds {:bundle-id (:id new-bundle)
-                                                                     :content-types content-types})
+      (bundle-categories/insert-bundle-categories! bundle-ds {:bundle-id (:id new-bundle)
+                                                              :categories categories})
+      (bundle-content-types/insert-bundle-content-types! ds {:bundle-id (:id new-bundle)
+                                                             :content-types content-types})
 
-            categories-by-bundle (bundles/categories-in-bundle ds (:id new-bundle))]
+      (let [categories-by-bundle (bundles/categories-in-bundle ds (:id new-bundle))]
 
-        ; service needed
+        ;TODO: service needed
         (->> (jobs/prepare-congest-metadata
               ds
               store
@@ -45,21 +45,15 @@
 
 (defn update-integration! [ds js store {:keys [bundle-id bundle-metadata categories content-types]}]
   (with-open [bundle-ds (db.util/conn :bundle bundle-id)]
-    (let [_ (bundles/update-bundle! ds {:id bundle-id
-                                        :data bundle-metadata})
-
-          job-id (str "bundle_" bundle-id)
-
-        ; update bundle categories
-          _ (bundle-categories/update-bundle-categories! bundle-ds {:bundle-id bundle-id
-                                                                    :categories categories})
-        ; update bundle content types
-          _ (bundle-content-types/update-bundle-content-types! ds {:bundle-id bundle-id
-                                                                   :content-types content-types})
-
+    (bundles/update-bundle! ds {:id bundle-id
+                                :data bundle-metadata})
+    (bundle-categories/update-bundle-categories! bundle-ds {:bundle-id bundle-id
+                                                            :categories categories})
+    (bundle-content-types/update-bundle-content-types! ds {:bundle-id bundle-id
+                                                           :content-types content-types})
+    (let [job-id (str "bundle_" bundle-id)
           categories-by-bundle (bundles/categories-in-bundle ds bundle-id)]
-
-      ; service needed
+      ;TODO: service needed
       (congest/deregister! js job-id)
       (->> (jobs/prepare-congest-metadata
             ds

From 17b099c3ebb903a57aa3f501b9ffd5b29ab8c62e Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Thu, 29 Jan 2026 14:41:22 +0200
Subject: [PATCH 244/391] moved job registration to handlers to avoid cyclic
 dependencies

---
 src/source/routes/feed.clj          |  8 +--
 src/source/routes/feeds.clj         | 33 ++++++++++--
 src/source/routes/integration.clj   | 45 ++++++++++++----
 src/source/routes/integrations.clj  | 36 ++++++++++---
 src/source/workers/feeds.clj        | 35 ++-----------
 src/source/workers/integrations.clj | 81 +++++++----------------------
 src/source/workers/users.clj        | 34 +++++++++++-
 7 files changed, 156 insertions(+), 116 deletions(-)

diff --git a/src/source/routes/feed.clj b/src/source/routes/feed.clj
index 7f107c3b..4b4f3ac4 100644
--- a/src/source/routes/feed.clj
+++ b/src/source/routes/feed.clj
@@ -1,7 +1,8 @@
 (ns source.routes.feed
   (:require [ring.util.response :as res]
             [source.db.honey :as hon]
-            [source.workers.feeds :as feeds]))
+            [source.workers.feeds :as feeds]
+            [source.jobs.handlers :as handlers]))
 
 (defn get
   {:summary "get feed by id"
@@ -63,10 +64,11 @@
                                        [:= :user-id (:id user)]
                                        [:= :id id]]})
         {:keys [email]} (hon/find-one ds {:tname :users
-                                          :where [:= :id (:id user)]})]
+                                          :where [:= :id (:id user)]})
+        job-id (handlers/update-feed-posts-job-id email id)]
     (if (some? feed)
       (do
-        (feeds/hard-delete-feed! ds js email id)
+        (feeds/hard-delete-feed! ds js job-id id)
         (res/response {:message "successfully deleted feed"}))
       (-> (res/response {:message "unauthorized"})
           (res/status 403)))))
diff --git a/src/source/routes/feeds.clj b/src/source/routes/feeds.clj
index b0dfa8dd..b424c0b9 100644
--- a/src/source/routes/feeds.clj
+++ b/src/source/routes/feeds.clj
@@ -2,7 +2,9 @@
   (:require [source.workers.feeds :as feeds]
             [congest.jobs :as congest]
             [ring.util.response :as res]
-            [source.db.honey :as hon]))
+            [source.db.honey :as hon]
+            [source.util :as util]
+            [source.jobs.core :as jobs]))
 
 (defn get
   {:summary "get all feeds"
@@ -63,10 +65,33 @@
       (-> (res/response {:message "There is already a feed with the given RSS feed"})
           (res/status 400))
 
-      (let [new-feed (feeds/create-feed! ds js store {:user-id (:id user)
-                                                      :feed-metadata body})]
+      (let [{:keys [provider-id rss-url content-type-id]} body
+            new-feed (feeds/create-feed! ds store {:user-id (:id user)
+                                                   :feed-metadata body})
+            {:keys [email]} (hon/find-one ds {:tname :users
+                                              :where [:= :id (:id user)]})]
         (if new-feed
-          (res/response new-feed)
+          (do
+            (->> (jobs/prepare-congest-metadata
+                  ds
+                  store
+                  {:id (str email "-" (:id new-feed))
+                   :initial-delay (* 1000 60 60 24)
+                   :auto-start true
+                   :stop-after-fail false,
+                   :interval (* 1000 60 60 24)
+                   :recurring? true
+                   :args {:feed-id (:id new-feed)
+                          :creator-id (:id user)
+                          :content-type-id content-type-id
+                          :provider-id provider-id
+                          :url rss-url}
+                   :handler :update-feed-posts
+                   :created-at (util/get-utc-timestamp-string)
+                   :sleep false})
+                 (congest/register! js))
+            (res/response new-feed))
+
           (-> (res/response {:message "Failed to parse RSS feed"})
               (res/status 422)))))))
 
diff --git a/src/source/routes/integration.clj b/src/source/routes/integration.clj
index 57de0f85..f74d2015 100644
--- a/src/source/routes/integration.clj
+++ b/src/source/routes/integration.clj
@@ -2,7 +2,11 @@
   (:require [ring.util.response :as res]
             [source.services.interface :as services]
             [source.services.bundles :as bundles]
-            [source.workers.integrations :as integrations]))
+            [source.workers.integrations :as integrations]
+            [congest.jobs :as congest]
+            [source.jobs.core :as jobs]
+            [source.util :as util]
+            [source.jobs.handlers :as handlers]))
 
 (defn get
   {:summary "get integration by id"
@@ -50,10 +54,31 @@
                403 {:body [:map [:message :string]]}}}
 
   [{:keys [js ds store path-params body] :as _request}]
-  (integrations/update-integration! ds js store {:bundle-id (:id path-params)
-                                                 :bundle-metadata (dissoc body :categories :content-types)
-                                                 :categories (:categories body)
-                                                 :content-types (:content-types body)})
+  (let [bundle-id (:id path-params)
+        job-id (str "bundle_" bundle-id)
+        categories-by-bundle (bundles/categories-in-bundle ds bundle-id)]
+    (integrations/update-integration! ds {:bundle-id bundle-id
+                                          :bundle-metadata (dissoc body :categories :content-types)
+                                          :categories (:categories body)
+                                          :content-types (:content-types body)})
+    ;TODO: service needed
+    (congest/deregister! js job-id)
+    (->> (jobs/prepare-congest-metadata
+          ds
+          store
+          {:id job-id
+           :initial-delay (* 1000 60 60 24)
+           :auto-start true
+           :stop-after-fail false,
+           :interval (* 1000 60 60 24)
+           :recurring? true
+           :ds ds
+           :args {:bundle-id bundle-id
+                  :categories categories-by-bundle}
+           :handler :update-bundle
+           :created-at (util/get-utc-timestamp-string)
+           :sleep false})
+         (congest/register! js)))
   (res/response {:message "successfully updated integration"}))
 
 (defn delete
@@ -64,12 +89,14 @@
                403 {:body [:map [:message :string]]}}}
 
   [{:keys [ds js user path-params] :as _request}]
-  (let [bundle (bundles/bundle ds {:where [:and
-                                           [:= :id (:id path-params)]
-                                           [:= :user-id (:id user)]]})]
+  (let [bundle-id (:id path-params)
+        bundle (bundles/bundle ds {:where [:and
+                                           [:= :id bundle-id]
+                                           [:= :user-id (:id user)]]})
+        job-id (handlers/update-bundle-job-id bundle-id)]
     (if (some? bundle)
       (do
-        (integrations/hard-delete-bundle! ds js (:id path-params))
+        (integrations/hard-delete-bundle! ds js job-id bundle-id)
         (res/response {:message "successfully deleted integration"}))
       (-> (res/response {:message "unauthorized"})
           (res/status 403)))))
diff --git a/src/source/routes/integrations.clj b/src/source/routes/integrations.clj
index dd5f5b95..b64008be 100644
--- a/src/source/routes/integrations.clj
+++ b/src/source/routes/integrations.clj
@@ -1,7 +1,11 @@
 (ns source.routes.integrations
   (:require [ring.util.response :as res]
             [source.services.bundles :as bundles]
-            [source.workers.integrations :as integrations]))
+            [source.workers.integrations :as integrations]
+            [source.util :as util]
+            [source.jobs.core :as jobs]
+            [source.jobs.handlers :as handlers]
+            [congest.jobs :as congest]))
 
 (defn get
   {:summary "get all integrations"
@@ -51,9 +55,27 @@
                403 {:body [:map [:message :string]]}}}
 
   [{:keys [js ds store user body] :as _request}]
-  (->> {:user-id (:id user)
-        :bundle-metadata (dissoc body :categories :content-types)
-        :categories (:categories body)
-        :content-types (:content-types body)}
-       (integrations/create-integration! ds js store)
-       (res/response)))
+  (let [new-bundle (->> {:user-id (:id user)
+                         :bundle-metadata (dissoc body :categories :content-types)
+                         :categories (:categories body)
+                         :content-types (:content-types body)}
+                        (integrations/create-integration! ds))
+        categories-by-bundle (bundles/categories-in-bundle ds (:id new-bundle))]
+    ;TODO: service needed
+    (->> (jobs/prepare-congest-metadata
+          ds
+          store
+          {:id (handlers/update-bundle-job-id (:id new-bundle))
+           :initial-delay 0
+           :auto-start true
+           :stop-after-fail false,
+           :interval (* 1000 60 60 24)
+           :recurring? true
+           :ds ds
+           :args {:bundle-id (:id new-bundle)
+                  :categories categories-by-bundle}
+           :handler :update-bundle
+           :created-at (util/get-utc-timestamp-string)
+           :sleep false})
+         (congest/register! js))
+    (res/response new-bundle)))
diff --git a/src/source/workers/feeds.clj b/src/source/workers/feeds.clj
index bf887d72..eb8dc4b8 100644
--- a/src/source/workers/feeds.clj
+++ b/src/source/workers/feeds.clj
@@ -1,14 +1,12 @@
 (ns source.workers.feeds
   (:require [source.util :as utils]
             [source.services.xml-schemas :as xml]
-            [source.jobs.core :as jobs]
             [congest.jobs :as congest]
-            [source.db.honey :as hon]
-            [source.jobs.handlers :as handlers]))
+            [source.db.honey :as hon]))
 
 (defn create-feed!
   "Creates feed with incoming posts pulled from RSS feed and starts associated job"
-  [ds js store {:keys [user-id feed-metadata]}]
+  [ds store {:keys [user-id feed-metadata]}]
   (let [{:keys [provider-id rss-url content-type-id]} feed-metadata
         datetime (utils/get-utc-timestamp-string)
         selection-schemas (->> [:= :provider-id provider-id]
@@ -36,33 +34,11 @@
                                        :creator-id user-id
                                        :content-type-id content-type-id
                                        :thumbnail (or (:thumbnail post) (:display-picture new-feed))}))
-                             extracted-posts)
-        {:keys [email]} (hon/find-one ds {:tname :users
-                                          :where [:= :id user-id]})]
-
+                             extracted-posts)]
     (if (some? extracted-posts)
       (do
         (hon/insert! ds {:tname :incoming-posts
                          :data extended-posts})
-
-        (->> (jobs/prepare-congest-metadata
-              ds
-              store
-              {:id (str email "-" (:id new-feed))
-               :initial-delay (* 1000 60 60 24)
-               :auto-start true
-               :stop-after-fail false,
-               :interval (* 1000 60 60 24)
-               :recurring? true
-               :args {:feed-id (:id new-feed)
-                      :creator-id user-id
-                      :content-type-id content-type-id
-                      :provider-id provider-id
-                      :url rss-url}
-               :handler :update-feed-posts
-               :created-at (utils/get-utc-timestamp-string)
-               :sleep false})
-             (congest/register! js))
         new-feed)
       false)))
 
@@ -72,9 +48,8 @@
                    :data feed-metadata
                    :ret :1}))
 
-(defn hard-delete-feed! [ds js creator-email feed-id]
-  (let [job-id (handlers/update-feed-posts-job-id creator-email feed-id)
-        post-ids (mapv :id (hon/find ds {:tname :incoming-posts
+(defn hard-delete-feed! [ds js job-id feed-id]
+  (let [post-ids (mapv :id (hon/find ds {:tname :incoming-posts
                                          :where [:= :feed-id feed-id]
                                          :ret :*}))]
     (hon/delete! ds {:tname :filtered-feeds
diff --git a/src/source/workers/integrations.clj b/src/source/workers/integrations.clj
index e5888243..cc651258 100644
--- a/src/source/workers/integrations.clj
+++ b/src/source/workers/integrations.clj
@@ -4,14 +4,12 @@
             [source.db.util :as db.util]
             [source.services.bundle-categories :as bundle-categories]
             [source.services.bundle-content-types :as bundle-content-types]
-            [source.jobs.core :as jobs]
-            [source.jobs.handlers :as handlers]
             [source.util :as utils]
-            [congest.jobs :as congest]
             [source.db.tables :as tables]
-            [source.db.honey :as hon]))
+            [source.db.honey :as hon]
+            [congest.jobs :as congest]))
 
-(defn create-integration! [ds js store {:keys [user-id bundle-metadata categories content-types]}]
+(defn create-integration! [ds {:keys [user-id bundle-metadata categories content-types]}]
   (let [new-bundle (bundles/create-bundle! ds {:user-id user-id
                                                :bundle-metadata bundle-metadata})]
     (migrate/migrate-bundle (:id new-bundle) ["up"])
@@ -20,72 +18,31 @@
       (bundle-categories/insert-bundle-categories! bundle-ds {:bundle-id (:id new-bundle)
                                                               :categories categories})
       (bundle-content-types/insert-bundle-content-types! ds {:bundle-id (:id new-bundle)
-                                                             :content-types content-types})
-
-      (let [categories-by-bundle (bundles/categories-in-bundle ds (:id new-bundle))]
-
-        ;TODO: service needed
-        (->> (jobs/prepare-congest-metadata
-              ds
-              store
-              {:id (handlers/update-bundle-job-id (:id new-bundle))
-               :initial-delay 0
-               :auto-start true
-               :stop-after-fail false,
-               :interval (* 1000 60 60 24)
-               :recurring? true
-               :ds ds
-               :args {:bundle-id (:id new-bundle)
-                      :categories categories-by-bundle}
-               :handler :update-bundle
-               :created-at (utils/get-utc-timestamp-string)
-               :sleep false})
-             (congest/register! js))))
+                                                             :content-types content-types}))
     new-bundle))
 
-(defn update-integration! [ds js store {:keys [bundle-id bundle-metadata categories content-types]}]
+(defn update-integration! [ds {:keys [bundle-id bundle-metadata categories content-types]}]
   (with-open [bundle-ds (db.util/conn :bundle bundle-id)]
     (bundles/update-bundle! ds {:id bundle-id
                                 :data bundle-metadata})
     (bundle-categories/update-bundle-categories! bundle-ds {:bundle-id bundle-id
                                                             :categories categories})
     (bundle-content-types/update-bundle-content-types! ds {:bundle-id bundle-id
-                                                           :content-types content-types})
-    (let [job-id (str "bundle_" bundle-id)
-          categories-by-bundle (bundles/categories-in-bundle ds bundle-id)]
-      ;TODO: service needed
-      (congest/deregister! js job-id)
-      (->> (jobs/prepare-congest-metadata
-            ds
-            store
-            {:id job-id
-             :initial-delay (* 1000 60 60 24)
-             :auto-start true
-             :stop-after-fail false,
-             :interval (* 1000 60 60 24)
-             :recurring? true
-             :ds ds
-             :args {:bundle-id bundle-id
-                    :categories categories-by-bundle}
-             :handler :update-bundle
-             :created-at (utils/get-utc-timestamp-string)
-             :sleep false})
-           (congest/register! js)))))
+                                                           :content-types content-types})))
 
-(defn hard-delete-bundle! [ds js bundle-id]
-  (let [job-id (handlers/update-bundle-job-id bundle-id)]
-    (hon/delete! ds {:tname :filtered-feeds
-                     :where [:= :bundle-id bundle-id]})
-    (hon/delete! ds {:tname :filtered-posts
-                     :where [:= :bundle-id bundle-id]})
-    (hon/delete! ds {:tname :bundle-content-types
-                     :where [:= :bundle-id bundle-id]})
-    (hon/delete! ds {:tname :events
-                     :where [:= :bundle-id bundle-id]})
-    (tables/drop-all-tables! (db.util/conn :bundle bundle-id))
-    (hon/delete! ds {:tname :bundles
-                     :where [:= :id bundle-id]})
-    (congest/deregister! js job-id)))
+(defn hard-delete-bundle! [ds js job-id bundle-id]
+  (hon/delete! ds {:tname :filtered-feeds
+                   :where [:= :bundle-id bundle-id]})
+  (hon/delete! ds {:tname :filtered-posts
+                   :where [:= :bundle-id bundle-id]})
+  (hon/delete! ds {:tname :bundle-content-types
+                   :where [:= :bundle-id bundle-id]})
+  (hon/delete! ds {:tname :events
+                   :where [:= :bundle-id bundle-id]})
+  (tables/drop-all-tables! (db.util/conn :bundle bundle-id))
+  (hon/delete! ds {:tname :bundles
+                   :where [:= :id bundle-id]})
+  (congest/deregister! js job-id))
 
 (defn generate-api-key! [ds user-id bundle-id]
   (let [uuid (utils/uuid)
diff --git a/src/source/workers/users.clj b/src/source/workers/users.clj
index 4ca1c0e6..c80cccf5 100644
--- a/src/source/workers/users.clj
+++ b/src/source/workers/users.clj
@@ -1,2 +1,34 @@
 (ns source.workers.users
-  (:require [source.workers.schemas :as schemas]))
+  (:require [source.workers.feeds :as feeds]
+            [source.workers.integrations :as integrations]
+            [source.db.honey :as hon]))
+
+(defn hard-delete-creator! [ds js user-id email]
+  (let [feed-ids (mapv :id (hon/find ds {:tname :feeds
+                                         :where [:= :user-id user-id]}))]
+    (run! #(feeds/hard-delete-feed! ds js (str email "-" %) %) feed-ids)
+    (hon/delete! ds {:tname :events
+                     :where [:= :creator-id user-id]})))
+
+(defn hard-delete-distributor! [ds js user-id]
+  (let [bundle-ids (mapv :id (hon/find ds {:tname :bundles
+                                           :where [:= :user-id user-id]}))]
+    (run! #(integrations/hard-delete-bundle! ds js (str "bundle_" %) %) bundle-ids)
+    (hon/delete! ds {:tname :events
+                     :where [:= :distributor-id user-id]})))
+
+(defn hard-delete-user! [ds js user-type user-id]
+  (let [{:keys [email business-id]} (hon/find-one ds {:tname :users
+                                                      :where [:= :id user-id]})]
+    (cond
+      (= user-type :creator)
+      (hard-delete-creator! ds js user-id email)
+      (= user-type :distributor)
+      (hard-delete-distributor! ds js user-id))
+
+    (hon/delete! ds {:tname :user-sectors
+                     :where [:= :user-id user-id]})
+    (when (some? business-id) (hon/delete! ds {:tname :businesses
+                                               :where [:= :id business-id]}))
+    (hon/delete! ds {:tname :users
+                     :where [:= :id user-id]})))

From 488505aaedba25a12fcc48d3bd8b4d7c92e44299 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Thu, 29 Jan 2026 14:41:55 +0200
Subject: [PATCH 245/391] implemented endpoint to schedule user deletion and
 endpoint to cancel user deletion

---
 src/source/jobs/handlers.clj | 16 ++++++++++++++-
 src/source/routes/me.clj     | 40 +++++++++++++++++++++++++++++++++++-
 src/source/routes/reitit.clj |  4 +++-
 3 files changed, 57 insertions(+), 3 deletions(-)

diff --git a/src/source/jobs/handlers.clj b/src/source/jobs/handlers.clj
index af20f91a..a0aadc6f 100644
--- a/src/source/jobs/handlers.clj
+++ b/src/source/jobs/handlers.clj
@@ -1,11 +1,13 @@
 (ns source.jobs.handlers
   (:require [source.services.interface :as services]
+            [source.workers.users :as users]
             [source.util :as util]
             [source.services.incoming-posts :as incoming-posts]
             [source.db.util :as db.util]
             [clojure.set :as set]
             [clojure.string :as string]
-            [source.db.honey :as hon]))
+            [source.db.honey :as hon]
+            [congest.jobs :as congest]))
 
 (defmulti handler
   (fn [opts]
@@ -127,3 +129,15 @@
           (hon/delete! ds-bundle {:tname :outgoing-posts})
           (hon/insert! ds-bundle {:tname :outgoing-posts
                                   :data outgoing-posts}))))))
+
+(defn user-deletion-job-id
+  "returns the job id of a user deletion job with the given user id"
+  [user-type user-id]
+  (str "delete_" user-type "_" user-id))
+
+(defmethod handler :delete-user [_]
+  (fn [{:keys [args ds store]}]
+    (try
+      (let [{:keys [user-type user-id]} args]
+        (users/hard-delete-user! ds store (keyword user-type) user-id))
+      (catch Exception e (println "Failed to delete user: " e) :fail))))
diff --git a/src/source/routes/me.clj b/src/source/routes/me.clj
index d544fa19..388e8ab6 100644
--- a/src/source/routes/me.clj
+++ b/src/source/routes/me.clj
@@ -1,7 +1,10 @@
 (ns source.routes.me
   (:require [ring.util.response :as res]
             [source.util :as util]
-            [source.db.honey :as hon]))
+            [source.db.honey :as hon]
+            [source.jobs.core :as jobs]
+            [source.jobs.handlers :as handlers]
+            [congest.jobs :as congest]))
 
 (defn get
   {:summary "get logged in user by access token"
@@ -47,3 +50,38 @@
                            :where [:= :id (:id user)]
                            :data data})
           (res/response {:message "successfully updated user"})))))
+
+(defn delete-user
+  {:summary "delete logged-in user by access token"
+   :responses {200 {:body [:map [:message :string]]}}}
+
+  [{:keys [ds js user] :as _request}]
+  (let [{:keys [id type]} user
+        job-id (handlers/user-deletion-job-id type id)]
+    (->> (jobs/prepare-congest-metadata
+          ds
+          js
+          {:id job-id
+           :initial-delay (* 1000 60) #_(* 1000 60 60 24 30)
+           :auto-start true
+           :stop-after-fail false,
+           :interval (* 1000 60) #_(* 1000 60 60 24 30)
+           :recurring? false
+           :kill-after 1
+           :args {:user-type type
+                  :user-id id}
+           :handler :delete-user
+           :created-at (util/get-utc-timestamp-string)
+           :sleep false})
+         (congest/register! js))
+    (res/response {:message "successfully scheduled user deletion"})))
+
+(defn cancel-deletion
+  {:summary "cancel deletion of logged-in user by access token"
+   :responses {200 {:body [:map [:message :string]]}}}
+
+  [{:keys [js user] :as _request}]
+  (let [{:keys [id type]} user
+        job-id (handlers/user-deletion-job-id type id)]
+    (congest/deregister! js job-id)
+    (res/response {:message "successfully cancelled user deletion"})))
diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index b857abd6..49fff57c 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -98,7 +98,9 @@
              :openapi {:security [{:bearerAuth []}]}}
 
       ["" (-> (get me/get)
-              (post me/post))]
+              (post me/post)
+              (delete me/delete-user))]
+      ["/deletion/cancel" (get me/cancel-deletion)]
       ["/business" (-> (get me-business/get)
                        (post me-business/post))]
       ["/sectors" (-> (get me-sectors/get)

From 8b915c11f17dff549e24e6d64915fc342c902177 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Thu, 29 Jan 2026 14:46:39 +0200
Subject: [PATCH 246/391] fixed deletion time to 30 days

---
 src/source/routes/me.clj | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/source/routes/me.clj b/src/source/routes/me.clj
index 388e8ab6..aff4e228 100644
--- a/src/source/routes/me.clj
+++ b/src/source/routes/me.clj
@@ -62,10 +62,10 @@
           ds
           js
           {:id job-id
-           :initial-delay (* 1000 60) #_(* 1000 60 60 24 30)
+           :initial-delay (* 1000 60 60 24 30)
            :auto-start true
            :stop-after-fail false,
-           :interval (* 1000 60) #_(* 1000 60 60 24 30)
+           :interval (* 1000 60 60 24 30)
            :recurring? false
            :kill-after 1
            :args {:user-type type

From b5b2aded4ddf80f1f0220097c065150341282f65 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Thu, 29 Jan 2026 15:18:32 +0200
Subject: [PATCH 247/391] added todos for extracting services

---
 src/source/routes/feeds.clj | 1 +
 src/source/routes/me.clj    | 1 +
 2 files changed, 2 insertions(+)

diff --git a/src/source/routes/feeds.clj b/src/source/routes/feeds.clj
index b424c0b9..c0de6f34 100644
--- a/src/source/routes/feeds.clj
+++ b/src/source/routes/feeds.clj
@@ -72,6 +72,7 @@
                                               :where [:= :id (:id user)]})]
         (if new-feed
           (do
+            ;TODO: service needed
             (->> (jobs/prepare-congest-metadata
                   ds
                   store
diff --git a/src/source/routes/me.clj b/src/source/routes/me.clj
index aff4e228..bb1c30c5 100644
--- a/src/source/routes/me.clj
+++ b/src/source/routes/me.clj
@@ -58,6 +58,7 @@
   [{:keys [ds js user] :as _request}]
   (let [{:keys [id type]} user
         job-id (handlers/user-deletion-job-id type id)]
+    ; TODO: service needed
     (->> (jobs/prepare-congest-metadata
           ds
           js

From 90a6c1aa3ed68921141d08327f28db5507048235 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 2 Feb 2026 09:07:28 +0200
Subject: [PATCH 248/391] added soft user deletion

---
 src/source/db/master.clj                   |  4 +-
 src/source/jobs/handlers.clj               | 98 ++++++++++++----------
 src/source/middleware/auth/core.clj        | 14 ++--
 src/source/migrations/009_user_removed.clj | 15 ++++
 src/source/routes/me.clj                   | 10 ++-
 src/source/services/analytics/core.clj     |  2 +-
 6 files changed, 89 insertions(+), 54 deletions(-)
 create mode 100644 src/source/migrations/009_user_removed.clj

diff --git a/src/source/db/master.clj b/src/source/db/master.clj
index 251933f5..269bc0d1 100644
--- a/src/source/db/master.clj
+++ b/src/source/db/master.clj
@@ -1,5 +1,7 @@
 (ns source.db.master
-  (:require [source.db.tables :as tables]))
+  (:require [source.db.tables :as tables]
+            [source.db.honey :as hon]
+            [source.db.util :as db.util]))
 
 (def users
   (tables/create-table-sql
diff --git a/src/source/jobs/handlers.clj b/src/source/jobs/handlers.clj
index a0aadc6f..d1daa78c 100644
--- a/src/source/jobs/handlers.clj
+++ b/src/source/jobs/handlers.clj
@@ -29,51 +29,56 @@
 (defmethod handler :update-feed-posts [_]
   (fn [{:keys [args ds store]}]
     (try
-      (let [{:keys [feed-id creator-id content-type-id provider-id url]} args
-            selection-schemas (->> [:= :provider-id provider-id]
-                                   (assoc {} :where)
-                                   (services/selection-schemas ds))
-            latest-ss (->> selection-schemas
-                           (reduce (fn [acc {:keys [id]}]
-                                     (conj acc id)) [])
-                           (apply max -1))
-            extracted (services/extract-data store {:schema-id latest-ss
-                                                    :url url})
-            extracted-posts (get-in extracted [:feed :posts])
-            extracted-display (get-in extracted [:feed :display-picture])
-            extended-posts (mapv (fn [{:keys [posted-at thumbnail] :as post}]
-                                   (merge post
-                                          {:feed-id feed-id
-                                           :creator-id creator-id
-                                           :content-type-id content-type-id
-                                           :posted-at (util/format-rss-date posted-at)
-                                           :thumbnail (if (and thumbnail
-                                                               (seq thumbnail)
-                                                               (not (string/includes? thumbnail ".mp3")))
-                                                        thumbnail
-                                                        extracted-display)}))
-                                 extracted-posts)
-            existing-posts (hon/find ds {:tname :incoming-posts
-                                         :where [:= :creator-id creator-id]})
-            existing-feed (hon/find-one ds {:tname :feeds
-                                            :where [:= :id feed-id]})]
-        (hon/update! ds {:tname :feeds
-                         :where [:= :id feed-id]
-                         :data {:title (get-in extracted [:feed :title])
-                                :display-picture (if (and (:display-picture existing-feed)
-                                                          (seq (:display-picture existing-feed)))
-                                                   (:display-picture existing-feed)
-                                                   extracted-display)
-                                :updated-at (util/get-utc-timestamp-string)}})
-        (run!
-         (fn [post]
-           (if (some #(= (:post-id post) (:post-id %)) existing-posts)
-             (hon/update! ds {:tname :incoming-posts
-                              :where [:= :post-id (:post-id post)]
-                              :data post})
-             (hon/insert! ds {:tname :incoming-posts
-                              :data post})))
-         extended-posts))
+      (when (-> (hon/find ds {:tname :users
+                              :where [:= :id (:creator-id args)]})
+                (:removed)
+                (= 0))
+        (let [{:keys [feed-id creator-id content-type-id provider-id url]} args
+              selection-schemas (->> [:= :provider-id provider-id]
+                                     (assoc {} :where)
+                                     (services/selection-schemas ds))
+              latest-ss (->> selection-schemas
+                             (reduce (fn [acc {:keys [id]}]
+                                       (conj acc id)) [])
+                             (apply max -1))
+              extracted (services/extract-data store {:schema-id latest-ss
+                                                      :url url})
+              extracted-posts (get-in extracted [:feed :posts])
+              extracted-display (get-in extracted [:feed :display-picture])
+              extended-posts (mapv (fn [{:keys [posted-at thumbnail] :as post}]
+                                     (merge post
+                                            {:feed-id feed-id
+                                             :creator-id creator-id
+                                             :content-type-id content-type-id
+                                             :posted-at (util/format-rss-date posted-at)
+                                             :thumbnail (if (and thumbnail
+                                                                 (seq thumbnail)
+                                                                 (not (string/includes? thumbnail ".mp3")))
+                                                          thumbnail
+                                                          extracted-display)}))
+                                   extracted-posts)
+              existing-posts (hon/find ds {:tname :incoming-posts
+                                           :where [:= :creator-id creator-id]})
+              existing-feed (hon/find-one ds {:tname :feeds
+                                              :where [:= :id feed-id]})]
+          (hon/update! ds {:tname :feeds
+                           :where [:= :id feed-id]
+                           :data {:title (get-in extracted [:feed :title])
+                                  :display-picture (if (and (:display-picture existing-feed)
+                                                            (seq (:display-picture existing-feed)))
+                                                     (:display-picture existing-feed)
+                                                     extracted-display)
+                                  :updated-at (util/get-utc-timestamp-string)}})
+          (run!
+           (fn [post]
+             (if (some #(= (:post-id post) (:post-id %)) existing-posts)
+               (hon/update! ds {:tname :incoming-posts
+                                :where [:= :post-id (:post-id post)]
+                                :data post})
+               (hon/insert! ds {:tname :incoming-posts
+                                :data post})))
+           extended-posts)))
+
       (catch Exception _ :fail))))
 
 (defn update-bundle-job-id
@@ -119,9 +124,10 @@
             ; get all incoming posts with the above id numbers
             posts-in (hon/find ds {:tname :incoming-posts
                                    :where [:in :id ids]})
+
             ; remove redacted posts
             outgoing-posts (reduce (fn [acc {:keys [redacted] :as post}]
-                                     (if (:= redacted 0)
+                                     (if (= redacted 0)
                                        (conj acc (dissoc post :redacted))
                                        acc))
                                    [] posts-in)]
diff --git a/src/source/middleware/auth/core.clj b/src/source/middleware/auth/core.clj
index e6db63b6..b49b41f2 100644
--- a/src/source/middleware/auth/core.clj
+++ b/src/source/middleware/auth/core.clj
@@ -50,9 +50,11 @@
   (fn [request]
     (let [ds (db.util/conn :master)
           bundle-uuid (get-in request [:query-params :uuid])
-          {:keys [id]} (db/find-one ds {:tname :bundles
-                                        :where [:= :uuid bundle-uuid]})]
-      (if (some? id)
+          {:keys [id user-id]} (db/find-one ds {:tname :bundles
+                                                :where [:= :uuid bundle-uuid]})
+          {:keys [removed]} (db/find-one ds {:tname :users
+                                             :where [:= :id user-id]})]
+      (if (and (some? id) (= removed 0))
         (-> request
             (assoc :bundle-id id)
             (handler))
@@ -67,8 +69,10 @@
   (fn [request]
     (let [ds (db.util/conn :master)
           token (util/auth-token request)
-          existing-bundle (bundles/bundle ds {:where [:= :hash token]})]
-      (if (some? existing-bundle)
+          {:keys [user-id] :as existing-bundle} (bundles/bundle ds {:where [:= :hash token]})
+          {:keys [removed]} (db/find-one ds {:tname :users
+                                             :where [:= :id user-id]})]
+      (if (and (some? existing-bundle) (= removed 0))
         (-> request
             (assoc :bundle-id (:id existing-bundle))
             (handler))
diff --git a/src/source/migrations/009_user_removed.clj b/src/source/migrations/009_user_removed.clj
new file mode 100644
index 00000000..062f6197
--- /dev/null
+++ b/src/source/migrations/009_user_removed.clj
@@ -0,0 +1,15 @@
+(ns source.migrations.009-user-removed
+  (:require [source.db.master]
+            [source.db.honey :as hon]))
+
+(defn run-up! [context]
+  (let [ds-master (:db-master context)]
+    (->> {:alter-table :users
+          :add-column [:removed :integer [:default 0]]}
+         (hon/execute! ds-master))))
+
+(defn run-down! [context]
+  (let [ds-master (:db-master context)]
+    (->> {:alter-table :users
+          :drop-column :removed}
+         (hon/execute! ds-master))))
diff --git a/src/source/routes/me.clj b/src/source/routes/me.clj
index bb1c30c5..bcbacbd6 100644
--- a/src/source/routes/me.clj
+++ b/src/source/routes/me.clj
@@ -58,6 +58,11 @@
   [{:keys [ds js user] :as _request}]
   (let [{:keys [id type]} user
         job-id (handlers/user-deletion-job-id type id)]
+
+    (hon/update! ds {:tname :users
+                     :where [:= :id id]
+                     :data {:removed 1}})
+
     ; TODO: service needed
     (->> (jobs/prepare-congest-metadata
           ds
@@ -81,8 +86,11 @@
   {:summary "cancel deletion of logged-in user by access token"
    :responses {200 {:body [:map [:message :string]]}}}
 
-  [{:keys [js user] :as _request}]
+  [{:keys [ds js user] :as _request}]
   (let [{:keys [id type]} user
         job-id (handlers/user-deletion-job-id type id)]
+    (hon/update! ds {:tname :users
+                     :where [:= :id id]
+                     :data {:removed 0}})
     (congest/deregister! js job-id)
     (res/response {:message "successfully cancelled user deletion"})))
diff --git a/src/source/services/analytics/core.clj b/src/source/services/analytics/core.clj
index 149c79ff..f6586e5b 100644
--- a/src/source/services/analytics/core.clj
+++ b/src/source/services/analytics/core.clj
@@ -169,7 +169,7 @@
                                             category-ids)))
                               (flatten)
                               (vec))]
-    (insert-event-categories! ds {:data event-categories})))
+    (when (seq event-categories) (insert-event-categories! ds {:data event-categories}))))
 
 (defn insert-post-event-categories!
   "Given a list of events and a list of posts (or a single event/post),

From 893e3f9b6c075339ebe259d448080d68e1c7cfde Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 2 Feb 2026 12:42:03 +0200
Subject: [PATCH 249/391] created workers for user soft deletion and used the
 approach of redacting all incoming posts

---
 src/source/db/master.clj     | 34 ++++++++++++++++------------------
 src/source/jobs/handlers.clj |  3 +--
 src/source/routes/me.clj     | 12 ++++--------
 src/source/workers/users.clj | 18 ++++++++++++++++++
 4 files changed, 39 insertions(+), 28 deletions(-)

diff --git a/src/source/db/master.clj b/src/source/db/master.clj
index 269bc0d1..70fb5b7f 100644
--- a/src/source/db/master.clj
+++ b/src/source/db/master.clj
@@ -1,7 +1,5 @@
 (ns source.db.master
-  (:require [source.db.tables :as tables]
-            [source.db.honey :as hon]
-            [source.db.util :as db.util]))
+  (:require [source.db.tables :as tables]))
 
 (def users
   (tables/create-table-sql
@@ -80,21 +78,21 @@
 
 (def filtered-feeds
   (tables/create-table-sql
-    :filtered-feeds
-    (tables/table-id)
-    [:feed-id :integer :not nil]
-    [:bundle-id :integer :not nil]
-    (tables/foreign-key :feed-id :feeds :id)
-    (tables/foreign-key :bundle-id :bundles :id)))
+   :filtered-feeds
+   (tables/table-id)
+   [:feed-id :integer :not nil]
+   [:bundle-id :integer :not nil]
+   (tables/foreign-key :feed-id :feeds :id)
+   (tables/foreign-key :bundle-id :bundles :id)))
 
 (def filtered-posts
   (tables/create-table-sql
-    :filtered-posts
-    (tables/table-id)
-    [:post-id :integer :not nil]
-    [:bundle-id :integer :not nil]
-    (tables/foreign-key :post-id :incoming-posts :id)
-    (tables/foreign-key :bundle-id :bundles :id)))
+   :filtered-posts
+   (tables/table-id)
+   [:post-id :integer :not nil]
+   [:bundle-id :integer :not nil]
+   (tables/foreign-key :post-id :incoming-posts :id)
+   (tables/foreign-key :bundle-id :bundles :id)))
 
 (def feeds
   (tables/create-table-sql
@@ -252,9 +250,9 @@
 
 (def business-types
   (tables/create-table-sql
-    :business-types 
-    (tables/table-id)
-    [:name :text :not nil]))
+   :business-types
+   (tables/table-id)
+   [:name :text :not nil]))
 
 (comment
   (require '[honey.sql :as sql])
diff --git a/src/source/jobs/handlers.clj b/src/source/jobs/handlers.clj
index d1daa78c..826d05e9 100644
--- a/src/source/jobs/handlers.clj
+++ b/src/source/jobs/handlers.clj
@@ -6,8 +6,7 @@
             [source.db.util :as db.util]
             [clojure.set :as set]
             [clojure.string :as string]
-            [source.db.honey :as hon]
-            [congest.jobs :as congest]))
+            [source.db.honey :as hon]))
 
 (defmulti handler
   (fn [opts]
diff --git a/src/source/routes/me.clj b/src/source/routes/me.clj
index bcbacbd6..eae27771 100644
--- a/src/source/routes/me.clj
+++ b/src/source/routes/me.clj
@@ -4,7 +4,8 @@
             [source.db.honey :as hon]
             [source.jobs.core :as jobs]
             [source.jobs.handlers :as handlers]
-            [congest.jobs :as congest]))
+            [congest.jobs :as congest]
+            [source.workers.users :as users]))
 
 (defn get
   {:summary "get logged in user by access token"
@@ -58,10 +59,7 @@
   [{:keys [ds js user] :as _request}]
   (let [{:keys [id type]} user
         job-id (handlers/user-deletion-job-id type id)]
-
-    (hon/update! ds {:tname :users
-                     :where [:= :id id]
-                     :data {:removed 1}})
+    (users/soft-delete-user! ds (keyword type) id)
 
     ; TODO: service needed
     (->> (jobs/prepare-congest-metadata
@@ -89,8 +87,6 @@
   [{:keys [ds js user] :as _request}]
   (let [{:keys [id type]} user
         job-id (handlers/user-deletion-job-id type id)]
-    (hon/update! ds {:tname :users
-                     :where [:= :id id]
-                     :data {:removed 0}})
+    (users/cancel-soft-user-deletion! ds (keyword type) id)
     (congest/deregister! js job-id)
     (res/response {:message "successfully cancelled user deletion"})))
diff --git a/src/source/workers/users.clj b/src/source/workers/users.clj
index c80cccf5..dc49f06d 100644
--- a/src/source/workers/users.clj
+++ b/src/source/workers/users.clj
@@ -32,3 +32,21 @@
                                                :where [:= :id business-id]}))
     (hon/delete! ds {:tname :users
                      :where [:= :id user-id]})))
+
+(defn soft-delete-user! [ds user-type user-id]
+  (hon/update! ds {:tname :users
+                   :where [:= :id user-id]
+                   :data {:removed true}})
+  (when (= user-type :creator)
+    (hon/update! ds {:tname :incoming-posts
+                     :where [:= :creator-id user-id]
+                     :data {:redacted true}})))
+
+(defn cancel-soft-user-deletion! [ds user-type user-id]
+  (hon/update! ds {:tname :users
+                   :where [:= :id user-id]
+                   :data {:removed false}})
+  (when (= user-type :creator)
+    (hon/update! ds {:tname :incoming-posts
+                     :where [:= :creator-id user-id]
+                     :data {:redacted false}})))

From ecda03c7b115793402f03fa9ef2360ce68f62e44 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 2 Feb 2026 13:21:40 +0200
Subject: [PATCH 250/391] fixed handler bug for bundles

---
 src/source/jobs/handlers.clj | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/source/jobs/handlers.clj b/src/source/jobs/handlers.clj
index 826d05e9..fd357479 100644
--- a/src/source/jobs/handlers.clj
+++ b/src/source/jobs/handlers.clj
@@ -126,7 +126,7 @@
 
             ; remove redacted posts
             outgoing-posts (reduce (fn [acc {:keys [redacted] :as post}]
-                                     (if (= redacted 0)
+                                     (if (:= redacted 0)
                                        (conj acc (dissoc post :redacted))
                                        acc))
                                    [] posts-in)]

From 142a89e38c2ffe3df19b7663b0d06720b685722c Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Tue, 3 Feb 2026 11:07:58 +0200
Subject: [PATCH 251/391] updated jobs to filter out removed user posts

---
 src/source/jobs/handlers.clj | 16 ++++++++++------
 src/source/routes/me.clj     |  4 ++--
 src/source/workers/users.clj | 22 ++++++++++------------
 3 files changed, 22 insertions(+), 20 deletions(-)

diff --git a/src/source/jobs/handlers.clj b/src/source/jobs/handlers.clj
index fd357479..3174dc1c 100644
--- a/src/source/jobs/handlers.clj
+++ b/src/source/jobs/handlers.clj
@@ -28,10 +28,7 @@
 (defmethod handler :update-feed-posts [_]
   (fn [{:keys [args ds store]}]
     (try
-      (when (-> (hon/find ds {:tname :users
-                              :where [:= :id (:creator-id args)]})
-                (:removed)
-                (= 0))
+      (when (users/removed? ds (:creator-id args))
         (let [{:keys [feed-id creator-id content-type-id provider-id url]} args
               selection-schemas (->> [:= :provider-id provider-id]
                                      (assoc {} :where)
@@ -124,9 +121,16 @@
             posts-in (hon/find ds {:tname :incoming-posts
                                    :where [:in :id ids]})
 
+            creator-ids (mapv :creator-id posts-in)
+            active-creator-ids (->> (hon/find ds {:tname :users
+                                                  :where [:in :id creator-ids]})
+                                    (filterv #(or (nil? (:removed %)) (= (:removed %) 0)))
+                                    (mapv :id))
+
             ; remove redacted posts
-            outgoing-posts (reduce (fn [acc {:keys [redacted] :as post}]
-                                     (if (:= redacted 0)
+            outgoing-posts (reduce (fn [acc {:keys [redacted creator-id] :as post}]
+                                     (if (and (or (nil? redacted) (= redacted 0))
+                                              (some #{creator-id} active-creator-ids))
                                        (conj acc (dissoc post :redacted))
                                        acc))
                                    [] posts-in)]
diff --git a/src/source/routes/me.clj b/src/source/routes/me.clj
index eae27771..74c68a5f 100644
--- a/src/source/routes/me.clj
+++ b/src/source/routes/me.clj
@@ -59,7 +59,7 @@
   [{:keys [ds js user] :as _request}]
   (let [{:keys [id type]} user
         job-id (handlers/user-deletion-job-id type id)]
-    (users/soft-delete-user! ds (keyword type) id)
+    (users/soft-delete-user! ds id)
 
     ; TODO: service needed
     (->> (jobs/prepare-congest-metadata
@@ -87,6 +87,6 @@
   [{:keys [ds js user] :as _request}]
   (let [{:keys [id type]} user
         job-id (handlers/user-deletion-job-id type id)]
-    (users/cancel-soft-user-deletion! ds (keyword type) id)
+    (users/cancel-soft-user-deletion! ds id)
     (congest/deregister! js job-id)
     (res/response {:message "successfully cancelled user deletion"})))
diff --git a/src/source/workers/users.clj b/src/source/workers/users.clj
index dc49f06d..83a8cd0e 100644
--- a/src/source/workers/users.clj
+++ b/src/source/workers/users.clj
@@ -33,20 +33,18 @@
     (hon/delete! ds {:tname :users
                      :where [:= :id user-id]})))
 
-(defn soft-delete-user! [ds user-type user-id]
+(defn soft-delete-user! [ds user-id]
   (hon/update! ds {:tname :users
                    :where [:= :id user-id]
-                   :data {:removed true}})
-  (when (= user-type :creator)
-    (hon/update! ds {:tname :incoming-posts
-                     :where [:= :creator-id user-id]
-                     :data {:redacted true}})))
+                   :data {:removed true}}))
 
-(defn cancel-soft-user-deletion! [ds user-type user-id]
+(defn cancel-soft-user-deletion! [ds user-id]
   (hon/update! ds {:tname :users
                    :where [:= :id user-id]
-                   :data {:removed false}})
-  (when (= user-type :creator)
-    (hon/update! ds {:tname :incoming-posts
-                     :where [:= :creator-id user-id]
-                     :data {:redacted false}})))
+                   :data {:removed false}}))
+
+(defn removed? [ds user-id]
+  (-> (hon/find ds {:tname :users
+                    :where [:= :id user-id]})
+      (:removed)
+      (= 0)))

From c8027f51a51504d231ce2cd2ebdcc221bf989664 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 4 Feb 2026 21:37:21 +0200
Subject: [PATCH 252/391] initial postgres refactor work

---
 deps.edn                                      |  2 +
 resources/admins_encrypted.json               |  2 +-
 resources/config.edn                          |  2 +-
 .../bundle_migrations/001_init_bundle_db.clj  |  5 ++-
 src/source/db/bundle.clj                      |  9 +---
 src/source/db/honey.clj                       |  5 +++
 src/source/db/master.clj                      |  4 +-
 src/source/db/tables.clj                      | 18 ++++----
 src/source/db/util.clj                        | 16 +++++--
 src/source/middleware/core.clj                | 17 +++++--
 src/source/migrate.clj                        | 22 +++++++---
 src/source/migrations/001_init_master_db.clj  | 13 +++---
 src/source/migrations/006_events.clj          |  1 -
 .../008_provider_rss_instructions.clj         |  4 +-
 src/source/routes/admin.clj                   | 14 ++----
 .../routes/analytics/distributor/top.clj      | 43 ++++++++----------
 src/source/routes/business.clj                | 26 +++--------
 src/source/routes/login.clj                   | 15 +++----
 src/source/routes/me.clj                      | 12 ++---
 src/source/routes/me_business.clj             | 36 +++++++--------
 src/source/routes/provider.clj                |  1 +
 src/source/routes/register.clj                |  9 +---
 src/source/routes/user.clj                    | 15 ++-----
 src/source/routes/util.clj                    |  8 ++--
 src/source/util.clj                           | 44 ++++++++++---------
 src/source/workers/integrations.clj           |  4 +-
 26 files changed, 167 insertions(+), 180 deletions(-)

diff --git a/deps.edn b/deps.edn
index 2b1bc78f..79348cad 100644
--- a/deps.edn
+++ b/deps.edn
@@ -16,6 +16,7 @@
         ring-cors/ring-cors {:mvn/version "0.1.13"}
         io.github.modulr-software/congest {:mvn/version "0.1.7"}
         com.github.seancorfield/next.jdbc {:mvn/version "1.3.1070"}
+        org.postgresql/postgresql {:mvn/version "42.2.10"}
         org.xerial/sqlite-jdbc {:mvn/version "3.51.0.0"}
         buddy/buddy-core {:mvn/version "1.12.0-430"}
         buddy/buddy-sign {:mvn/version "3.5.351"}
@@ -27,6 +28,7 @@
         com.gfredericks/test.chuck {:mvn/version "0.2.14"}
         com.kepler16/mallard {:mvn/version "3.2.1"}
         com.kepler16/mallard-sqlite-store {:mvn/version "3.2.1"}
+        com.kepler16/mallard-postgres-store {:mvn/version "3.2.5"}
         com.github.seancorfield/honeysql {:mvn/version "2.7.1310"}
         io.replikativ/datahike {:mvn/version "0.6.1599"}
         hickory/hickory {:mvn/version "0.7.1"}
diff --git a/resources/admins_encrypted.json b/resources/admins_encrypted.json
index 03531385..1418c113 100644
--- a/resources/admins_encrypted.json
+++ b/resources/admins_encrypted.json
@@ -1 +1 @@
-1f849fdbcaaa4d5cf82b56107541fa49174399ba9cc32f67fd7eceb7f0847b8050f85973837676f7d7f8d92fa4d6cacf9e175a8fd6fff154212d1187c2888ec3bd091e838d7719d1d920ea70d9c230806642de10bfedcc2b4512c127f839eae51cf766a96a7a9fbc9e3f17680e077c5eb408459ad866d7f6ed37f7ec6a5fd87757b9265552d481b087becef00af7cdadf8d4da15f0e4365b4b6ce3e7e6ea7717b0fe10240e36bafb1ec29fc45495076b502573651b1a025b718a09121c009d4c7dcc53386ba6a133d736dc4fc35ed01e681cfce6c1fab6212e5442b3c4430cae496390e7fdbc67e5a7d0154f81d805bdd1396c217c5a4f2d4d75ca920dd5ed0022d3776a4df151e2af3f22183fe815df3e429fce7bfb4f8885656863e841b4ba239d35682dd4e0e5d4e1f9ecf0a975c7833f60fc6511021b4c571f42907e559553093ec9f7cfe8ba88af64eaba1a408e7ae83cc2569cfb2ccfab05fbcf3350a929ec4dd85bb95d7f66ab5149d4d589f4d7cd7f4962ab0e046a88eceb0e3f3fb583b606c2bdde0b90539e41fde19aa7bd93a995ca6a6ed2dbd448e72e880332f10c4179863749308e6468a252c08206683af33ee5b7bc1f306af72c1a31495347478957a8925ff07d1a175d9ef83fd1c98a369f0ab8a06997b27527f645484a113eaf23f709a8594514ea72d04fe4cc307787030e14f9aad2df63bc8cd36917a3
\ No newline at end of file
+1f849fdbcaaa4d5cf82b56107541fa49174399ba9cc32f67fd7eceb7f0847b8050f85973837676f7d7f8d92fa4d6cacf9e175a8fd6fff154212d1187c2888ec39e5d813d2a247e571991dc61e3ab567e796a92e0fbb7a4ee4ff62567a2c405d3008ce6fa89e6b85f7515aff0420cf59ed7e6112e232dec6952b2165f46a167d875b46ec1d1fa4f2a035112fe739607040da8f90bc8abd8f40b798020b6046535a4415e373bc1be463b447cb3218d88419b05d47d99cd053e16f985610cbe9565db3192db3c2810958176f2ebcbd8808a55e2c7e970f2f12b201ce39babdfd1ffe1cf22910395aeb8ff16a5af98896765046a0514c23b1f4fee34bc1a7ad293406151deba329de10c71b919038c165f0c848b60de52a4737ed0f27793c4bc85e7a3c1373e8982280fd53edddad4fca12eeaf212633a55c63ef457185887477ab6068957502a76932f9a267afb03f18ccacf53e52faea2fc56527fedc89570261403b9ade0a6e43e2244223a998338ca623e291ed714b133913e870f1fc07dff801e4450a0a67ac328b6b22f18ffd3ac4704a8e8d6e63dd1940a1c02b6f9c4183358585dc600857fdc0b8a72309e4edb41d1cd2fa2f450b4c6ff4dca70a2a09b9fcafac195493dc43d4d25062bee0b7b29e4fb89c592c7d0eee06724b8fd60c501a5ed49cde6a25679f86c207527c6cbc52c6c7f4215a2d1c857a4de92d14d016b
\ No newline at end of file
diff --git a/resources/config.edn b/resources/config.edn
index d6d97e5e..f919d455 100644
--- a/resources/config.edn
+++ b/resources/config.edn
@@ -4,7 +4,7 @@
  :cors-origin #or [#env CORS_ORIGIN "http://localhost:8080"]
  :env #or [#env ENV "dev"]
  :database {:dir #or [#env DATABASE_DIR ".db/"]
-            :type "sqlite"}
+            :type "postgresql"}
  :email {:address #or [#env SUPPORT_ADDRESS "lee@wearesource.earth"]
          :username #env EMAIL_USERNAME
          :password #env EMAIL_PASSWORD}
diff --git a/src/source/bundle_migrations/001_init_bundle_db.clj b/src/source/bundle_migrations/001_init_bundle_db.clj
index d3bcf91c..9462b5be 100644
--- a/src/source/bundle_migrations/001_init_bundle_db.clj
+++ b/src/source/bundle_migrations/001_init_bundle_db.clj
@@ -1,9 +1,12 @@
 (ns source.bundle-migrations.001-init-bundle-db
   (:require [source.db.bundle]
-            [source.db.tables :as tables]))
+            [source.db.tables :as tables]
+            [next.jdbc :as jdbc]))
 
 (defn run-up! [context]
   (let [ds-bundle (:db-bundle context)]
+    (try (jdbc/execute! ds-bundle ["CREATE DOMAIN DATETIME TEXT"]) (catch Exception _))
+
     (tables/create-tables!
      ds-bundle
      :source.db.bundle
diff --git a/src/source/db/bundle.clj b/src/source/db/bundle.clj
index 3f930a37..9e4fcd33 100644
--- a/src/source/db/bundle.clj
+++ b/src/source/db/bundle.clj
@@ -25,19 +25,14 @@
    [:season :integer]
    [:episode :integer]
    [:content-type-id :integer :not nil]
-   [:posted-at :datetime]
-   (tables/foreign-key :feed-id :feeds :id)
-   (tables/foreign-key :creator-id :users :id)
-   (tables/foreign-key :content-type-id :content-types :id)))
+   [:posted-at :datetime]))
 
 (def bundle-categories
   (tables/create-table-sql
    :bundle-categories
    (tables/table-id)
    [:bundle-id :int :not nil]
-   [:category-id :int :not nil]
-   (tables/foreign-key :bundle-id :bundles :id)
-   (tables/foreign-key :category-id :categories :id)))
+   [:category-id :int :not nil]))
 
 (def post-heuristics
   (tables/create-table-sql
diff --git a/src/source/db/honey.clj b/src/source/db/honey.clj
index f7fbc1cb..113f0d66 100644
--- a/src/source/db/honey.clj
+++ b/src/source/db/honey.clj
@@ -100,6 +100,11 @@
 
   (def ds (db.util/conn :master))
 
+  (-> (hsql/select :*)
+      (hsql/from :providers)
+      (hsql/where [:= :id [:cast "2" :int]])
+      (sql/format))
+
   (find ds {:tname :incoming-posts
             :limit 5
             :order-by [[:id :asc]]
diff --git a/src/source/db/master.clj b/src/source/db/master.clj
index 70fb5b7f..30c30d73 100644
--- a/src/source/db/master.clj
+++ b/src/source/db/master.clj
@@ -228,8 +228,8 @@
    [:event :text [:check [:in :event ["impression" "click" "view"]]]]
    [:feed-id :integer :not nil]
    [:post-id :integer]
-   [:content-type-id :not nil]
-   [:creator-id :not nil]
+   [:content-type-id :integer :not nil]
+   [:creator-id :integer :not nil]
    [:bundle-id :integer :not nil]
    [:distributor-id :integer :not nil]
    (tables/foreign-key :feed-id :feeds :id)
diff --git a/src/source/db/tables.clj b/src/source/db/tables.clj
index 267c1741..3203ab3e 100644
--- a/src/source/db/tables.clj
+++ b/src/source/db/tables.clj
@@ -1,6 +1,8 @@
 (ns source.db.tables
   (:require [source.db.honey :as hon]
-            [honey.sql.helpers :as hsql]))
+            [honey.sql.helpers :as hsql]
+            [source.db.util :as db.util]
+            [honey.sql :as sql]))
 
 (defn create-table-sql
   "returns a honey data DSL structure for creating a table tname
@@ -43,20 +45,20 @@
         tables))
 
 (defn tables
-  "returns all current tables in a sqlite datasource"
+  "returns all current tables in a postgres datasource"
   [ds]
-  (->> {:tname :sqlite-master
-        :where [:and [:= :type "table"] [:<> :name "sqlite_sequence"]]
+  (->> {:tname :information_schema.tables
+        :where [:and [:= :table-schema "public"] [:= :table-type "BASE TABLE"]]
         :ret :*}
        (hon/find ds)))
 
 (defn table-name
   "return the name of a table record"
   [table]
-  (:name table))
+  (:table-name table))
 
 (defn table-names
-  "retrieves and returns all table names for an sqlite datasource"
+  "retrieves and returns all table names for an postgres datasource"
   [ds]
   (->> (tables ds)
        (mapv table-name)))
@@ -66,12 +68,12 @@
   with defaults: integer primary key autoincrement. Can be used
   with (create-table-sql) to simplify creating table ids."
   []
-  [:id :integer [:primary-key] :autoincrement])
+  [:id :integer :generated :by :default :as :identity :primary :key])
 
 (defn drop-table-sql
   "returns a honey data DSL structure for dropping a table tname"
   [tname]
-  (hsql/drop-table tname))
+  {:drop-table [tname [:cascade]]})
 
 (defn drop-table!
   "given a table tname, this function drops the table tname from 
diff --git a/src/source/db/util.clj b/src/source/db/util.clj
index 7d55805c..457354a7 100644
--- a/src/source/db/util.clj
+++ b/src/source/db/util.clj
@@ -18,11 +18,14 @@
    (str (name type) "_" id)))
 
 (defn- -conn [dbname]
-  (let [conn (-> {:dbtype (conf/read-value :database :type)}
-                 (merge {:dbname (db-path dbname)})
+  (let [conn (-> {:dbtype (conf/read-value :database :type)
+                  :user "postgres"
+                  :password "55589783"
+                  :host "localhost"
+                  :port 5432}
+                 (merge {:dbname (db-name dbname)})
                  (jdbc/get-connection))]
-    (jdbc/execute! conn ["PRAGMA journal_mode = WAL;"])
-    (jdbc/execute! conn ["PRAGMA synchronous = NORMAL;"])
+    (try (jdbc/execute! conn ["CREATE DOMAIN DATETIME TEXT"]) (catch Exception _))
     (jdbc/with-options conn {:builder-fn rs/as-unqualified-lower-maps})
     conn))
 
@@ -35,3 +38,8 @@
   ([db-type id]
    (assert (or (= db-type :bundle) (= db-type :creator)))
    (-conn (db-name db-type id))))
+
+(comment
+  (def ds (conn :bundle 1))
+  (jdbc/execute! ds ["DELETE FROM providers;"])
+  ())
diff --git a/src/source/middleware/core.clj b/src/source/middleware/core.clj
index cdc5bf24..6c4f7fdc 100644
--- a/src/source/middleware/core.clj
+++ b/src/source/middleware/core.clj
@@ -65,13 +65,24 @@
                                           (util/validate schema))]
     (->> (when error
            (str "In " (name param-type) ":\n" error))
-         (assoc validated :error))))
+         (assoc validated :param-type param-type :error))))
+
+(defn- attach-validations [request validations]
+  (reduce (fn [acc {:keys [data param-type]}]
+            (cond
+              (= param-type :body) (assoc acc :body data)
+              (= param-type :path) (assoc acc :path-params data)
+              (= param-type :query) (assoc acc :query-params data)
+              :else acc)) request validations))
 
 (defn wrap-input-validation [handler openapi-meta]
   (fn [request]
-    (let [errors (->> (mapv (partial validate-param request) (:parameters openapi-meta))
+    (let [validations (->> (mapv (partial validate-param request) (:parameters openapi-meta)))
+          errors (->> validations
                       (filter #(:error %))
-                      (mapv :error))]
+                      (mapv :error))
+          request (->> validations
+                       (attach-validations request))]
       (if (seq errors)
         (-> (res/response {:message (string/join "\n" errors)})
             (res/status 400))
diff --git a/src/source/migrate.clj b/src/source/migrate.clj
index 09134c9a..8a28575e 100644
--- a/src/source/migrate.clj
+++ b/src/source/migrate.clj
@@ -1,6 +1,6 @@
 (ns source.migrate
   (:require [k16.mallard :as mallard]
-            [k16.mallard.store.sqlite :as store]
+            [k16.mallard.store.postgres :as store]
             [k16.mallard.loader.fs :as loader.fs]
             [next.jdbc :as jdbc]
             [source.db.util :as db.util]
@@ -22,10 +22,16 @@
   (loader.fs/load! "src/source/bundle_migrations"))
 
 (defn run-migrations [args]
-  (let [context {:db-master (jdbc/get-datasource {:dbname (db.util/db-path "master") :dbtype "sqlite"})}
-        db-migrate (jdbc/get-datasource {:dbname (db.util/db-path "migrate") :dbtype "sqlite"})
+  (let [context {:db-master (jdbc/get-datasource {:dbname (db.util/db-name "master")
+                                                  :user "postgres"
+                                                  :password "55589783"
+                                                  :dbtype "postgresql"})}
+        db-migrate (jdbc/get-datasource {:dbname (db.util/db-name "migrate")
+                                         :user "postgres"
+                                         :password "55589783"
+                                         :dbtype "postgresql"})
         datastore (store/create-datastore
-                   {:db db-migrate
+                   {:ds db-migrate
                     :table-name "migrations"})]
     (mallard/run {:context context
                   :store datastore
@@ -34,10 +40,12 @@
 
 (defn migrate-bundle [bundle-id args]
   (let [db-name (db.util/db-name :bundle bundle-id)
-        context {:db-bundle (jdbc/get-datasource {:dbname (db.util/db-path db-name)
-                                                  :dbtype "sqlite"})}
+        context {:db-bundle (jdbc/get-datasource {:dbname (db.util/db-name db-name)
+                                                  :user "postgres"
+                                                  :password "55589783"
+                                                  :dbtype "postgresql"})}
         datastore (store/create-datastore
-                   {:db (:db-bundle context)
+                   {:ds (:db-bundle context)
                     :table-name "migrations"})]
     (mallard/run {:context context
                   :store datastore
diff --git a/src/source/migrations/001_init_master_db.clj b/src/source/migrations/001_init_master_db.clj
index 68ea8a2e..f83f74c7 100644
--- a/src/source/migrations/001_init_master_db.clj
+++ b/src/source/migrations/001_init_master_db.clj
@@ -65,29 +65,28 @@
 
 (defn run-up! [context]
   (let [ds-master (:db-master context)]
-
     (ds/create-datastore :datahike)
 
     (tables/create-tables!
      ds-master
      :source.db.master
-     [:users
+     [:businesses
+      :users
+      :content-types
+      :providers
       :sectors
       :categories
-      :content-types
       :cadences
       :baselines
       :bundles
       :feeds
       :feed-categories
-      :providers
-      :businesses
       :user-sectors
       :feed-sectors
       :selection-schemas
       :incoming-posts
-      :jobs
-      :job-metadata])
+      :job-metadata
+      :jobs])
 
     (db/insert! ds-master baselines-seed)
     (db/insert! ds-master cadences-seed)
diff --git a/src/source/migrations/006_events.clj b/src/source/migrations/006_events.clj
index 567319d6..f8defec0 100644
--- a/src/source/migrations/006_events.clj
+++ b/src/source/migrations/006_events.clj
@@ -9,7 +9,6 @@
      :source.db.master
      [:events
       :event-categories])))
-
 (defn run-down! [context]
   (let [ds-master (:db-master context)]
     (tables/drop-tables!
diff --git a/src/source/migrations/008_provider_rss_instructions.clj b/src/source/migrations/008_provider_rss_instructions.clj
index b39b3ff6..c83ec9c1 100644
--- a/src/source/migrations/008_provider_rss_instructions.clj
+++ b/src/source/migrations/008_provider_rss_instructions.clj
@@ -8,12 +8,12 @@
     (hon/execute!
      ds-master
      (-> (hsql/alter-table :providers)
-         (hsql/add-column :instructions :string)))
+         (hsql/add-column :instructions :text)))
 
     (hon/execute!
      ds-master
      (-> (hsql/alter-table :providers)
-         (hsql/add-column :placeholder-url :string)))))
+         (hsql/add-column :placeholder-url :text)))))
 
 (defn run-down! [context]
   (let [ds-master (:db-master context)]
diff --git a/src/source/routes/admin.clj b/src/source/routes/admin.clj
index 78e2d955..af3aacab 100644
--- a/src/source/routes/admin.clj
+++ b/src/source/routes/admin.clj
@@ -1,6 +1,5 @@
 (ns source.routes.admin
   (:require [source.password :as pw]
-            [source.util :as util]
             [ring.util.response :as res]
             [source.db.honey :as hon]))
 
@@ -16,15 +15,10 @@
 
   [{:keys [ds body] :as _request}]
 
-  (let [{:keys [data error success]} (util/validate post body)
-        user (hon/find-one ds {:tname :users
-                               :where [:= :email (:email data)]})
-        {:keys [password confirm-password]} data]
+  (let [user (hon/find-one ds {:tname :users
+                               :where [:= :email (:email body)]})
+        {:keys [password confirm-password]} body]
     (cond
-
-      (not success) (-> (res/response error)
-                        (res/status 400))
-
       (not (= password confirm-password))
       (-> (res/response {:message "passwords do not match!"})
           (res/status 400))
@@ -35,7 +29,7 @@
 
       :else
       (let [pw (pw/hash-password password)
-            new-user (-> (assoc data
+            new-user (-> (assoc body
                                 :password pw
                                 :type "admin")
                          (dissoc :confirm-password))]
diff --git a/src/source/routes/analytics/distributor/top.clj b/src/source/routes/analytics/distributor/top.clj
index 2ee5ff25..9a1f036e 100644
--- a/src/source/routes/analytics/distributor/top.clj
+++ b/src/source/routes/analytics/distributor/top.clj
@@ -29,27 +29,22 @@
                             [:views :int]]]}}}
 
   [{:keys [ds user query-params] :as _request}]
-  (let [{:keys [data success error]} (utils/validate get query-params :query)
-        {:keys [n mindate maxdate top contenttype]} data
-        top-field (if (= top "post") :post-id :feed-id)]
-    (if success
-      (let [results (->> {:distributor-id (:id user)
-                          :content-type-id contenttype}
-                         (analytics/top-statistics-query ds mindate maxdate n top-field)
-                         (mapv (fn [result]
-                                 (set/rename-keys result {top-field :top}))))
-            ids (mapv :top results)
-            names (record-names ds top-field ids)
-            juxted (->> names
-                        (mapv (fn [{:keys [id title]}]
-                                {:id id
-                                 :name title}))
-                        (mapv (juxt :id :name))
-                        (into {}))
-            named-results (mapv (fn [{:keys [top] :as r}]
-                                  (assoc r :top (clojure.core/get juxted top (str top))))
-                                results)]
-        (res/response named-results))
-
-      (-> (res/response error)
-          (res/status 400)))))
+  (let [{:keys [n mindate maxdate top contenttype]} query-params
+        top-field (if (= top "post") :post-id :feed-id)
+        results (->> {:distributor-id (:id user)
+                      :content-type-id contenttype}
+                     (analytics/top-statistics-query ds mindate maxdate n top-field)
+                     (mapv (fn [result]
+                             (set/rename-keys result {top-field :top}))))
+        ids (mapv :top results)
+        names (record-names ds top-field ids)
+        juxted (->> names
+                    (mapv (fn [{:keys [id title]}]
+                            {:id id
+                             :name title}))
+                    (mapv (juxt :id :name))
+                    (into {}))
+        named-results (mapv (fn [{:keys [top] :as r}]
+                              (assoc r :top (clojure.core/get juxted top (str top))))
+                            results)]
+    (res/response named-results)))
diff --git a/src/source/routes/business.clj b/src/source/routes/business.clj
index c89bfa0d..3730f73a 100644
--- a/src/source/routes/business.clj
+++ b/src/source/routes/business.clj
@@ -14,14 +14,9 @@
    :responses {201 {:body [:map [:message :string]]}}}
 
   [{:keys [ds body] :as _request}]
-
-  (let [{:keys [data error success]} (utils/validate post body)]
-    (if (not success) (-> (res/response error)
-                          (res/status 400))
-
-        (do (hon/insert! ds {:tname :businesses
-                             :data data})
-            (res/response {:message "successfully added business"})))))
+  (hon/insert! ds {:tname :businesses
+                   :data body})
+  (res/response {:message "successfully added business"}))
 
 (defn patch
   {:summary "update a business by id"
@@ -36,17 +31,10 @@
    :responses {200 {:body [:map [:message :string]]}}}
 
   [{:keys [ds body path-params] :as _request}]
-
-  (let [{:keys [data error success]} (utils/validate patch body)]
-    (if (not success)
-
-      (-> (res/response error)
-          (res/status 400))
-
-      (do (hon/update! ds {:tname :businesses
-                           :where  [:= :id (:id path-params)]
-                           :data data})
-          (res/response {:message "successfully updated business"})))))
+  (hon/update! ds {:tname :businesses
+                   :where  [:= :id (:id path-params)]
+                   :data body})
+  (res/response {:message "successfully updated business"}))
 
 (comment
   (require '[source.db.util :as db.util])
diff --git a/src/source/routes/login.clj b/src/source/routes/login.clj
index beab23b8..5ba54030 100644
--- a/src/source/routes/login.clj
+++ b/src/source/routes/login.clj
@@ -2,7 +2,6 @@
   (:require [source.services.auth :as auth]
             [ring.util.response :as res]
             [source.password :as pw]
-            [source.util :as util]
             [source.db.honey :as hon]))
 
 (defn post
@@ -29,20 +28,16 @@
 
   [{:keys [ds body] :as _request}]
 
-  (let [{:keys [data error success]} (util/validate post body)
-        {:keys [email password]} data
+  (let [{:keys [email password]} body
         user (hon/find-one ds {:tname :users
                                :where [:= :email email]})]
 
-    (cond
-      (not success) (-> (res/response error)
-                        (res/status 400))
-
-      (or (not (some? user))
-          (not (pw/verify-password password (:password user))))
+    (if
+     (or (not (some? user))
+         (not (pw/verify-password password (:password user))))
       {:status 401 :body {:message "Invalid username or password!"}}
 
-      :else (res/response (auth/login ds {:user user})))))
+      (res/response (auth/login ds {:user user})))))
 
 (comment
   (require '[source.db.interface :as db])
diff --git a/src/source/routes/me.clj b/src/source/routes/me.clj
index 74c68a5f..55fa80c3 100644
--- a/src/source/routes/me.clj
+++ b/src/source/routes/me.clj
@@ -43,14 +43,10 @@
                400 {:body [:map [:message :string]]}}}
 
   [{:keys [ds user body] :as _request}]
-  (let [{:keys [data error success]} (util/validate post body)]
-    (if (not success)
-      (-> (res/response {:message error})
-          (res/status 400))
-      (do (hon/update! ds {:tname :users
-                           :where [:= :id (:id user)]
-                           :data data})
-          (res/response {:message "successfully updated user"})))))
+  (hon/update! ds {:tname :users
+                   :where [:= :id (:id user)]
+                   :data body})
+  (res/response {:message "successfully updated user"}))
 
 (defn delete-user
   {:summary "delete logged-in user by access token"
diff --git a/src/source/routes/me_business.clj b/src/source/routes/me_business.clj
index 76567c98..84cdc4e6 100644
--- a/src/source/routes/me_business.clj
+++ b/src/source/routes/me_business.clj
@@ -1,6 +1,5 @@
 (ns source.routes.me-business
-  (:require [source.util :as util]
-            [ring.util.response :as res]
+  (:require [ring.util.response :as res]
             [source.db.honey :as hon]))
 
 (defn get
@@ -40,23 +39,18 @@
                400 {:body [:map [:message :string]]}}}
 
   [{:keys [ds user body] :as _request}]
-  (let [{:keys [data error success]} (util/validate post body)]
-    (if (not success)
-      (-> (res/response {:message error})
-          (res/status 400))
-
-      (let [{:keys [business-id]} (hon/find-one ds {:tname :users
-                                                    :where [:= :id (:id user)]})
-            business (when (nil? business-id)
-                       (hon/insert! ds {:tname :businesses
-                                        :data data
-                                        :ret :1}))]
-        (if (nil? business-id)
-          (hon/update! ds {:tname :users
-                           :where [:= :id (:id user)]
-                           :data {:business-id (:id business)}})
-          (hon/update! ds {:tname :businesses
-                           :where  [:= :id business-id]
-                           :data data}))
+  (let [{:keys [business-id]} (hon/find-one ds {:tname :users
+                                                :where [:= :id (:id user)]})
+        business (when (nil? business-id)
+                   (hon/insert! ds {:tname :businesses
+                                    :data body
+                                    :ret :1}))]
+    (if (nil? business-id)
+      (hon/update! ds {:tname :users
+                       :where [:= :id (:id user)]
+                       :data {:business-id (:id business)}})
+      (hon/update! ds {:tname :businesses
+                       :where  [:= :id business-id]
+                       :data body}))
 
-        (res/response {:message "successfully added or updated business"})))))
+    (res/response {:message "successfully added or updated business"})))
diff --git a/src/source/routes/provider.clj b/src/source/routes/provider.clj
index 45854eac..e097abed 100644
--- a/src/source/routes/provider.clj
+++ b/src/source/routes/provider.clj
@@ -16,6 +16,7 @@
                            [:placeholder-url [:maybe :string]]]}}}
 
   [{:keys [ds path-params] :as _request}]
+  (println (:id path-params))
   (->> (hon/find-one ds {:tname :providers
                          :where [:= :id (:id path-params)]})
        (res/response)))
diff --git a/src/source/routes/register.clj b/src/source/routes/register.clj
index 86557566..f616b0f9 100644
--- a/src/source/routes/register.clj
+++ b/src/source/routes/register.clj
@@ -29,15 +29,10 @@
 
   [{:keys [ds body] :as _request}]
 
-  (let [{:keys [data error success]} (util/validate post body)
-        {:keys [email password confirm-password]} data
+  (let [{:keys [email password confirm-password]} body
         existing-user (hon/find-one ds {:tname :users
                                         :where [:= :email email]})]
     (cond
-
-      (not success) (-> (res/response error)
-                        (res/status 400))
-
       (not (= password confirm-password))
       (-> (res/response {:error "Passwords do not match!"}))
 
@@ -45,7 +40,7 @@
       (-> (res/response {:error "An account for this email already exists!"}))
 
       :else
-      (-> (services/register ds data)
+      (-> (services/register ds body)
           (res/response)))))
 
 (comment
diff --git a/src/source/routes/user.clj b/src/source/routes/user.clj
index 78fe1bdc..b24f1b39 100644
--- a/src/source/routes/user.clj
+++ b/src/source/routes/user.clj
@@ -61,17 +61,10 @@
                403 {:body [:map [:message :string]]}}}
 
   [{:keys [ds body path-params] :as _request}]
-
-  (let [{:keys [data error success]} (util/validate patch body)]
-    (if (not success)
-
-      (-> (res/response error)
-          (res/status 400))
-
-      (do (hon/update! ds {:tname :users
-                           :where [:= :id (:id path-params)]
-                           :data data})
-          (res/response {:message "successfully updated user"})))))
+  (hon/update! ds {:tname :users
+                   :where [:= :id (:id path-params)]
+                   :data body})
+  (res/response {:message "successfully updated user"}))
 
 (comment
   (require '[source.db.interface :as db])
diff --git a/src/source/routes/util.clj b/src/source/routes/util.clj
index 8cb18850..7e42b0ae 100644
--- a/src/source/routes/util.clj
+++ b/src/source/routes/util.clj
@@ -14,10 +14,10 @@
 
 (defn- attach-handler [handler validation-mw openapi-meta]
   (cond-> openapi-meta
-    #_(some? (:parameters openapi-meta))
-    #_(assoc :middleware [[validation-mw openapi-meta]]
-             :responses (merge (:responses openapi-meta)
-                               (api/bad-request)))
+    (some? (:parameters openapi-meta))
+    (assoc :middleware [[validation-mw openapi-meta]]
+           :responses (merge (:responses openapi-meta)
+                             (api/bad-request)))
     true (assoc :handler handler)))
 
 (defn- merge-route-map [validation-mw acc [method handler]]
diff --git a/src/source/util.clj b/src/source/util.clj
index e515bf44..68692274 100644
--- a/src/source/util.clj
+++ b/src/source/util.clj
@@ -94,28 +94,28 @@
 (defn humanise [{:keys [errors] :as _error}]
   (reduce append-humanised-error "" errors))
 
-; (defn validate
-;   [data schema]
-;   (let [transformed (m/decode schema data mt/string-transformer)
-;         success (m/validate schema transformed)]
-;     {:data (when success transformed)
-;      :success success
-;      :error (when-not success (->> transformed
-;                                    (m/explain schema)
-;                                    (humanise)))}))
-
 (defn validate
-  ([handler data]
-   (validate handler data :body))
-  ([handler data schema-type]
-   (let [schema (get-in (metadata handler) [:parameters schema-type])
-         transformed (m/decode schema data mt/string-transformer)
-         success (m/validate schema transformed)]
-     {:data (when success transformed)
-      :success success
-      :error (when-not success (->> transformed
-                                    (m/explain schema)
-                                    (me/humanize)))})))
+  [data schema]
+  (let [transformed (m/decode schema data mt/string-transformer)
+        success (m/validate schema transformed)]
+    {:data (when success transformed)
+     :success success
+     :error (when-not success (->> transformed
+                                   (m/explain schema)
+                                   (humanise)))}))
+
+; (defn validate
+;   ([handler data]
+;    (validate handler data :body))
+;   ([handler data schema-type]
+;    (let [schema (get-in (metadata handler) [:parameters schema-type])
+;          transformed (m/decode schema data mt/string-transformer)
+;          success (m/validate schema transformed)]
+;      {:data (when success transformed)
+;       :success success
+;       :error (when-not success (->> transformed
+;                                     (m/explain schema)
+;                                     (me/humanize)))})))
 
 (defn format-rss-date
   "Takes a date as a string in RFC 1123 format and returns it in a format that meets ISO 8601 standards for SQLite.
@@ -131,6 +131,8 @@
 
 (comment
   (sha256 "1")
+  (validate {:a "1"} [:map [:a {:title "aoeu"
+                                :description "aoeu"} :int]])
 
   (validate {:message "yeet"
              :b 1
diff --git a/src/source/workers/integrations.clj b/src/source/workers/integrations.clj
index cc651258..affd473c 100644
--- a/src/source/workers/integrations.clj
+++ b/src/source/workers/integrations.clj
@@ -7,11 +7,13 @@
             [source.util :as utils]
             [source.db.tables :as tables]
             [source.db.honey :as hon]
-            [congest.jobs :as congest]))
+            [congest.jobs :as congest]
+            [next.jdbc :as jdbc]))
 
 (defn create-integration! [ds {:keys [user-id bundle-metadata categories content-types]}]
   (let [new-bundle (bundles/create-bundle! ds {:user-id user-id
                                                :bundle-metadata bundle-metadata})]
+    (jdbc/execute! ds [(str "CREATE DATABASE bundle_" (:id new-bundle))])
     (migrate/migrate-bundle (:id new-bundle) ["up"])
 
     (with-open [bundle-ds (db.util/conn :bundle (:id new-bundle))]

From 14b51fbb461a1ec2305231d6c577af19d0b54eb4 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Fri, 6 Feb 2026 13:24:51 +0200
Subject: [PATCH 253/391] changes to connection setup

---
 resources/config.edn    |  3 +++
 src/source/db/honey.clj | 36 +++++++++++++++---------------------
 src/source/db/util.clj  | 18 +++++++++++-------
 3 files changed, 29 insertions(+), 28 deletions(-)

diff --git a/resources/config.edn b/resources/config.edn
index f919d455..0894fac2 100644
--- a/resources/config.edn
+++ b/resources/config.edn
@@ -4,6 +4,9 @@
  :cors-origin #or [#env CORS_ORIGIN "http://localhost:8080"]
  :env #or [#env ENV "dev"]
  :database {:dir #or [#env DATABASE_DIR ".db/"]
+            :user #or [#env POSTGRES_USER "postgres"]
+            :password #env POSTGRES_PASSWORD
+            :host #or [#env POSTGRES_HOST "localhost"]
             :type "postgresql"}
  :email {:address #or [#env SUPPORT_ADDRESS "lee@wearesource.earth"]
          :username #env EMAIL_USERNAME
diff --git a/src/source/db/honey.clj b/src/source/db/honey.clj
index 113f0d66..2953786c 100644
--- a/src/source/db/honey.clj
+++ b/src/source/db/honey.clj
@@ -12,17 +12,18 @@
   or select all. returns results as unqualified lower maps by default."
   [ds sqlmap & {:keys [ret exec-opts]}]
   (assert (and (some? ds) (some? sqlmap) (or (some? ret) (nil? ret))))
-  (let [ps (sql/format sqlmap)
-        exec-opts' (merge
-                    {:builder-fn rs/as-unqualified-lower-maps}
-                    exec-opts)
-        result (cske/transform-keys
-                csk/->kebab-case-keyword
-                (jdbc/execute! ds ps exec-opts'))]
-    (cond
-      (= ret :1) (first result)
-      (= ret :*) result
-      :else nil)))
+  (with-open [conn (db.util/get-connection ds)]
+    (let [ps (sql/format sqlmap)
+          exec-opts' (merge
+                      {:builder-fn rs/as-unqualified-lower-maps}
+                      exec-opts)
+          result (cske/transform-keys
+                  csk/->kebab-case-keyword
+                  (jdbc/execute! conn ps exec-opts'))]
+      (cond
+        (= ret :1) (first result)
+        (= ret :*) result
+        :else nil))))
 
 (defn find
   "does find one or find all for a given table name and where clause. The where
@@ -58,7 +59,7 @@
               (-> (hsql/insert-into (csk/->snake_case_keyword tname))
                   (hsql/values vals)
                   (hsql/returning :*))
-              :ret (or ret :1))))
+              :ret (or ret nil))))
 
 (defn delete!
   "deletes a record or set of records that match a predicate where clause. the where
@@ -98,16 +99,9 @@
 (comment
   (hsql/where :or [:= :id 1] [:= :id 2])
 
-  (def ds (db.util/conn :master))
+  (def ds (db.util/conn :bundle 5))
 
-  (-> (hsql/select :*)
-      (hsql/from :providers)
-      (hsql/where [:= :id [:cast "2" :int]])
-      (sql/format))
-
-  (find ds {:tname :incoming-posts
-            :limit 5
-            :order-by [[:id :asc]]
+  (find ds {:tname :outgoing-posts
             :ret :*})
 
   (insert! ds {:tname :sectors
diff --git a/src/source/db/util.clj b/src/source/db/util.clj
index 457354a7..1141b184 100644
--- a/src/source/db/util.clj
+++ b/src/source/db/util.clj
@@ -17,18 +17,22 @@
   ([type id]
    (str (name type) "_" id)))
 
-(defn- -conn [dbname]
-  (let [conn (-> {:dbtype (conf/read-value :database :type)
-                  :user "postgres"
-                  :password "55589783"
-                  :host "localhost"
-                  :port 5432}
-                 (merge {:dbname (db-name dbname)})
+(defn get-connection [ds]
+  (let [conn (-> ds
                  (jdbc/get-connection))]
     (try (jdbc/execute! conn ["CREATE DOMAIN DATETIME TEXT"]) (catch Exception _))
     (jdbc/with-options conn {:builder-fn rs/as-unqualified-lower-maps})
     conn))
 
+(defn- -conn [dbname]
+  (-> {:dbtype (conf/read-value :database :type)
+       :user (conf/read-value :database :user)
+       :password (conf/read-value :database :password)
+       :host (conf/read-value :database :host)
+       :maximum-pool-size 10
+       :port 5432}
+      (merge {:dbname (db-name dbname)})))
+
 (defn conn
   ([]
    (conn :master))

From 59b6a5bfa7caab9c0fd18ca15111d0b0624ea2f7 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Fri, 6 Feb 2026 13:25:39 +0200
Subject: [PATCH 254/391] endpoint fixes and schema validation changes for
 postgres

---
 src/source/routes/bundle_feeds.clj           |   2 +-
 src/source/routes/bundle_post.clj            |  12 +-
 src/source/routes/integration_categories.clj |   2 +-
 src/source/routes/provider.clj               |   1 -
 src/source/services/bundles.clj              |  10 +-
 src/source/workers/bundles.clj               | 152 +++++++++----------
 src/source/workers/integrations.clj          |   4 +-
 7 files changed, 90 insertions(+), 93 deletions(-)

diff --git a/src/source/routes/bundle_feeds.clj b/src/source/routes/bundle_feeds.clj
index bc91ce41..3e0aba00 100644
--- a/src/source/routes/bundle_feeds.clj
+++ b/src/source/routes/bundle_feeds.clj
@@ -32,7 +32,7 @@
                404 {:body [:map [:message :string]]}}}
 
   [{:keys [ds bundle-id query-params body] :as _request}]
-  (let [{:keys [type latest nonfiltered]} (walk/keywordize-keys query-params)
+  (let [{:keys [type latest nonfiltered]} query-params
         feeds (->> {:bundle-id bundle-id
                     :type type
                     :latest latest
diff --git a/src/source/routes/bundle_post.clj b/src/source/routes/bundle_post.clj
index 71a6270f..ec62cfb3 100644
--- a/src/source/routes/bundle_post.clj
+++ b/src/source/routes/bundle_post.clj
@@ -27,9 +27,9 @@
                404 {:body [:map [:message :string]]}}}
 
   [{:keys [ds bundle-id path-params] :as _request}]
-  (with-open [bundle-ds (db.util/conn :bundle bundle-id)]
-    (let [post (hon/find-one bundle-ds {:tname :outgoing-posts
-                                        :where [:= :id (:id path-params)]
-                                        :ret :1})]
-      (analytics/insert-post-click! ds post bundle-id)
-      (res/response post))))
+  (let [bundle-ds (db.util/conn :bundle bundle-id)
+        post (hon/find-one bundle-ds {:tname :outgoing-posts
+                                      :where [:= :id (:id path-params)]
+                                      :ret :1})]
+    (analytics/insert-post-click! ds post bundle-id)
+    (res/response post)))
diff --git a/src/source/routes/integration_categories.clj b/src/source/routes/integration_categories.clj
index b35d511d..6087f03d 100644
--- a/src/source/routes/integration_categories.clj
+++ b/src/source/routes/integration_categories.clj
@@ -28,7 +28,7 @@
    :responses {200 {:body [:map [:message :string]]}}}
 
   [{:keys [path-params body] :as _request}]
-  (with-open [bundle-ds (db.util/conn :bundle (:id path-params))]
+  (let [bundle-ds (db.util/conn :bundle (:id path-params))]
     (bundle-categories/update-bundle-categories! bundle-ds {:bundle-id (:id path-params)
                                                             :categories body})
     (res/response {:message "successfully updated integration categories"})))
diff --git a/src/source/routes/provider.clj b/src/source/routes/provider.clj
index e097abed..45854eac 100644
--- a/src/source/routes/provider.clj
+++ b/src/source/routes/provider.clj
@@ -16,7 +16,6 @@
                            [:placeholder-url [:maybe :string]]]}}}
 
   [{:keys [ds path-params] :as _request}]
-  (println (:id path-params))
   (->> (hon/find-one ds {:tname :providers
                          :where [:= :id (:id path-params)]})
        (res/response)))
diff --git a/src/source/services/bundles.clj b/src/source/services/bundles.clj
index 1560a1ca..4f77aa5d 100644
--- a/src/source/services/bundles.clj
+++ b/src/source/services/bundles.clj
@@ -46,8 +46,8 @@
 
 ;;NEW
 (defn categories-in-bundle [ds bundle-id]
-  (with-open [bundle-ds (db.util/conn :bundle bundle-id)]
-    (let [category-ids (bundle-categories/category-id bundle-ds {:bundle-id bundle-id})
-          id-vec (mapv (fn [{:keys [category-id]}] category-id) category-ids)]
-      (hon/find ds {:tname :categories
-                    :where [:in :id id-vec]}))))
+  (let [bundle-ds (db.util/conn :bundle bundle-id)
+        category-ids (bundle-categories/category-id bundle-ds {:bundle-id bundle-id})
+        id-vec (mapv (fn [{:keys [category-id]}] category-id) category-ids)]
+    (hon/find ds {:tname :categories
+                  :where [:in :id id-vec]})))
diff --git a/src/source/workers/bundles.clj b/src/source/workers/bundles.clj
index 254f954e..64abecb2 100644
--- a/src/source/workers/bundles.clj
+++ b/src/source/workers/bundles.clj
@@ -8,93 +8,91 @@
 (defn get-bundle-categories
   "Get all categories for feeds/posts in bundle"
   [ds bundle-id]
-  (with-open [bundle-ds (db.util/conn :bundle bundle-id)]
-    (let [feed-ids (->> (hon/find bundle-ds {:tname :outgoing-posts
-                                             :ret :*})
-                        (mapv :feed-id))
-          category-ids (->> (hon/find ds {:tname :feed-categories
-                                          :where [:in :feed-id feed-ids]
-                                          :ret :*})
-                            (mapv :category-id))]
-      (hon/find ds {:tname :categories
-                    :where [:in :id category-ids]
-                    :ret :*}))))
+  (let [bundle-ds (db.util/conn :bundle bundle-id)
+        feed-ids (->> (hon/find bundle-ds {:tname :outgoing-posts
+                                           :ret :*})
+                      (mapv :feed-id))
+        category-ids (->> (hon/find ds {:tname :feed-categories
+                                        :where [:in :feed-id feed-ids]
+                                        :ret :*})
+                          (mapv :category-id))]
+    (hon/find ds {:tname :categories
+                  :where [:in :id category-ids]
+                  :ret :*})))
 
 (defn get-outgoing-feeds
   "Gets a filtered list of outgoing feeds for the associated bundle."
   [ds {:keys [bundle-id type latest category-ids nonfiltered]}]
-  (with-open [bundle-ds (db.util/conn :bundle bundle-id)]
-    (let [feed-ids (mapv :feed-id (hon/find bundle-ds {:tname :outgoing-posts
-                                                       :ret :*}))
-          category-filtered-feed-ids (if (empty? category-ids)
-                                       feed-ids
-                                       (->> (hsql/where
-                                             [:in :feed-id feed-ids]
-                                             [:in :category-id category-ids])
-                                            (merge {:tname :feed-categories
-                                                    :ret :*})
-                                            (hon/find ds)
-                                            (mapv :feed-id)))
-          blocked-feed-ids (if (some? nonfiltered)
-                             []
-                             (mapv :feed-id (hon/find ds {:tname :filtered-feeds
-                                                          :where [:= :bundle-id bundle-id]
-                                                          :ret :*})))
-          query (-> (when type [:= :content-type-id type])
-                    (hsql/where [:in :id category-filtered-feed-ids]
-                                [:not [:in :id blocked-feed-ids]])
-                    (hsql/order-by (when latest [:created-at :desc]))
-                    (merge {:tname :feeds
-                            :ret :*}))
-          type-filtered (hon/find ds query)]
-      type-filtered)))
+  (let [bundle-ds (db.util/conn :bundle bundle-id)
+        feed-ids (mapv :feed-id (hon/find bundle-ds {:tname :outgoing-posts
+                                                     :ret :*}))
+        category-filtered-feed-ids (if (empty? category-ids)
+                                     feed-ids
+                                     (->> (hsql/where
+                                           [:in :feed-id feed-ids]
+                                           [:in :category-id category-ids])
+                                          (merge {:tname :feed-categories
+                                                  :ret :*})
+                                          (hon/find ds)
+                                          (mapv :feed-id)))
+        blocked-feed-ids (if (some? nonfiltered)
+                           []
+                           (mapv :feed-id (hon/find ds {:tname :filtered-feeds
+                                                        :where [:= :bundle-id bundle-id]
+                                                        :ret :*})))
+        query (-> (hsql/where (when type [:= :content-type-id type])
+                              [:in :id category-filtered-feed-ids]
+                              (when (seq blocked-feed-ids) [:not [:in :id blocked-feed-ids]]))
+                  (assoc :order-by (when latest [[:created-at :desc]]))
+                  (merge {:tname :feeds
+                          :ret :*}))
+        type-filtered (hon/find ds query)]
+    type-filtered))
 
 (defn get-outgoing-posts
   "Get outgoing posts based on short heuristics and update analytics impressions"
   [ds {:keys [bundle-id limit start type latest category-ids]}]
-  (with-open [bundle-ds (db.util/conn :bundle bundle-id)]
-    (let [start (when start (try (Integer/parseInt start) (catch Exception _)))
-          limit (when limit (try (Integer/parseInt limit) (catch Exception _)))
-
-          all-feed-ids (mapv :id (hon/find ds {:tname :feeds
-                                               :ret :*}))
-          blocked-feed-ids (mapv :feed-id (hon/find ds {:tname :filtered-feeds
-                                                        :where [:= :bundle-id bundle-id]
-                                                        :ret :*}))
-          available-feed-ids (vec (remove (set blocked-feed-ids) all-feed-ids))
+  (let [bundle-ds (db.util/conn :bundle bundle-id)
+        all-feed-ids (mapv :id (hon/find ds {:tname :feeds
+                                             :ret :*}))
+        blocked-feed-ids (mapv :feed-id (hon/find ds {:tname :filtered-feeds
+                                                      :where [:= :bundle-id bundle-id]
+                                                      :ret :*}))
+        available-feed-ids (vec (remove (set blocked-feed-ids) all-feed-ids))
 
-          blocked-post-ids (mapv :post-id (hon/find ds {:tname :filtered-posts
-                                                        :where [:= :bundle-id bundle-id]
-                                                        :ret :*}))
+        blocked-post-ids (mapv :post-id (hon/find ds {:tname :filtered-posts
+                                                      :where [:= :bundle-id bundle-id]
+                                                      :ret :*}))
 
-          filtered-posts (hon/find bundle-ds (-> (hsql/where (when type [:= :content-type-id type])
-                                                             [:not [:in :id blocked-post-ids]]
-                                                             [:in :feed-id available-feed-ids])
-                                                 (hsql/order-by (when (= latest "true") [[:posted-at :desc]]))
-                                                 (merge {:tname :outgoing-posts
-                                                         :ret :*})))
+        filtered-posts (hon/find bundle-ds (-> (hsql/where (when type [:= :content-type-id type])
+                                                           (when (seq blocked-post-ids) [:not [:in :id blocked-post-ids]])
+                                                           [:in :feed-id available-feed-ids])
+                                               (assoc :order-by (when (= latest "true") [[[:posted-at :desc]]]))
+                                               (merge {:tname :outgoing-posts
+                                                       :ret :*})))
 
-          categorised-posts (vec
-                             (if (seq category-ids)
-                               (->> filtered-posts
-                                    (mapv
-                                     (fn [post]
-                                       (when (seq (set/intersection
-                                                   (set category-ids)
-                                                   (->> {:feed-id (:feed-id post)}
-                                                        (feed-categories/categories-by-feed ds)
-                                                        (mapv :id)
-                                                        (set))))
-                                         post)))
-                                    (remove nil?))
-                               filtered-posts))
+        categorised-posts (vec
+                           (if (seq category-ids)
+                             (->> filtered-posts
+                                  (mapv
+                                   (fn [post]
+                                     (when (seq (set/intersection
+                                                 (set category-ids)
+                                                 (->> {:feed-id (:feed-id post)}
+                                                      (feed-categories/categories-by-feed ds)
+                                                      (mapv :id)
+                                                      (set))))
+                                       post)))
+                                  (remove nil?))
+                             filtered-posts))
 
-          valid-start? (and (some? start) (>= start 0) (< start (count categorised-posts)))
-          started-posts (if valid-start?
-                          (subvec categorised-posts start)
-                          categorised-posts)
+        valid-start? (and (some? start) (>= start 0) (< start (count categorised-posts)))
+        started-posts (if valid-start?
+                        (subvec categorised-posts start)
+                        categorised-posts)
 
-          limited-posts (if (and (some? limit) (> (count started-posts) limit))
-                          (subvec started-posts 0 limit)
-                          started-posts)]
-      limited-posts)))
+        valid-limit? (and (some? limit) (> (count started-posts) limit))
+        limited-posts (if valid-limit?
+                        (subvec started-posts 0 limit)
+                        started-posts)]
+    limited-posts))
diff --git a/src/source/workers/integrations.clj b/src/source/workers/integrations.clj
index affd473c..47a61aca 100644
--- a/src/source/workers/integrations.clj
+++ b/src/source/workers/integrations.clj
@@ -16,7 +16,7 @@
     (jdbc/execute! ds [(str "CREATE DATABASE bundle_" (:id new-bundle))])
     (migrate/migrate-bundle (:id new-bundle) ["up"])
 
-    (with-open [bundle-ds (db.util/conn :bundle (:id new-bundle))]
+    (let [bundle-ds (db.util/conn :bundle (:id new-bundle))]
       (bundle-categories/insert-bundle-categories! bundle-ds {:bundle-id (:id new-bundle)
                                                               :categories categories})
       (bundle-content-types/insert-bundle-content-types! ds {:bundle-id (:id new-bundle)
@@ -24,7 +24,7 @@
     new-bundle))
 
 (defn update-integration! [ds {:keys [bundle-id bundle-metadata categories content-types]}]
-  (with-open [bundle-ds (db.util/conn :bundle bundle-id)]
+  (let [bundle-ds (db.util/conn :bundle bundle-id)]
     (bundles/update-bundle! ds {:id bundle-id
                                 :data bundle-metadata})
     (bundle-categories/update-bundle-categories! bundle-ds {:bundle-id bundle-id

From 016ff3c72dda6aa773bbf2b29c850e798b5a6960 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Fri, 6 Feb 2026 14:07:40 +0200
Subject: [PATCH 255/391] conn cleanup

---
 src/source/migrate.clj | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/src/source/migrate.clj b/src/source/migrate.clj
index 8a28575e..03416799 100644
--- a/src/source/migrate.clj
+++ b/src/source/migrate.clj
@@ -5,7 +5,8 @@
             [next.jdbc :as jdbc]
             [source.db.util :as db.util]
             [source.db.honey :as db]
-            [source.db.tables :as tables]))
+            [source.db.tables :as tables]
+            [source.config :as conf]))
 
 ;; This is our interface for running migrations.
 ;;
@@ -15,6 +16,11 @@
 ;; - seed the appropriate tables with data.
 ;; - (TODO) generate malli schemas to match the affected db schemas 
 
+(def ^:private postgres-ds
+  {:user (conf/read-value :database :user)
+   :password (conf/read-value :database :password)
+   :dbtype (conf/read-value :database :type)})
+
 (def ^:private migrations
   (loader.fs/load! "src/source/migrations"))
 
@@ -22,14 +28,10 @@
   (loader.fs/load! "src/source/bundle_migrations"))
 
 (defn run-migrations [args]
-  (let [context {:db-master (jdbc/get-datasource {:dbname (db.util/db-name "master")
-                                                  :user "postgres"
-                                                  :password "55589783"
-                                                  :dbtype "postgresql"})}
-        db-migrate (jdbc/get-datasource {:dbname (db.util/db-name "migrate")
-                                         :user "postgres"
-                                         :password "55589783"
-                                         :dbtype "postgresql"})
+  (let [context {:db-master (jdbc/get-datasource (-> {:dbname (db.util/db-name "master")}
+                                                     (merge postgres-ds)))}
+        db-migrate (jdbc/get-datasource (-> {:dbname (db.util/db-name "migrate")}
+                                            (merge postgres-ds)))
         datastore (store/create-datastore
                    {:ds db-migrate
                     :table-name "migrations"})]
@@ -40,10 +42,8 @@
 
 (defn migrate-bundle [bundle-id args]
   (let [db-name (db.util/db-name :bundle bundle-id)
-        context {:db-bundle (jdbc/get-datasource {:dbname (db.util/db-name db-name)
-                                                  :user "postgres"
-                                                  :password "55589783"
-                                                  :dbtype "postgresql"})}
+        context {:db-bundle (jdbc/get-datasource (-> {:dbname (db.util/db-name db-name)}
+                                                     (merge postgres-ds)))}
         datastore (store/create-datastore
                    {:ds (:db-bundle context)
                     :table-name "migrations"})]

From a510ac138cc841dc0ff17232ca3201dc49c534ae Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Fri, 6 Feb 2026 14:42:55 +0200
Subject: [PATCH 256/391] set up bundle migrations to create tenanted tables on
 master instead

---
 .../bundle_migrations/001_init_bundle_db.clj  | 37 +++++++++----------
 .../bundle_migrations/002_outgoing_posts.clj  | 16 ++++----
 src/source/db/bundle.clj                      |  7 ++++
 src/source/migrate.clj                        |  9 +++--
 4 files changed, 39 insertions(+), 30 deletions(-)

diff --git a/src/source/bundle_migrations/001_init_bundle_db.clj b/src/source/bundle_migrations/001_init_bundle_db.clj
index 9462b5be..7c13522a 100644
--- a/src/source/bundle_migrations/001_init_bundle_db.clj
+++ b/src/source/bundle_migrations/001_init_bundle_db.clj
@@ -1,27 +1,26 @@
 (ns source.bundle-migrations.001-init-bundle-db
-  (:require [source.db.bundle]
-            [source.db.tables :as tables]
-            [next.jdbc :as jdbc]))
+  (:require [source.db.bundle :as bundle]
+            [source.db.tables :as tables]))
 
 (defn run-up! [context]
-  (let [ds-bundle (:db-bundle context)]
-    (try (jdbc/execute! ds-bundle ["CREATE DOMAIN DATETIME TEXT"]) (catch Exception _))
-
+  (let [{:keys [ds-master bundle-id]} context]
     (tables/create-tables!
-     ds-bundle
+     ds-master
      :source.db.bundle
-     [:outgoing-posts
-      :bundle-categories
-      :post-heuristics
-      :analytics
-      :event-categories])))
+     (-> [:outgoing-posts
+          :bundle-categories
+          :post-heuristics
+          :analytics
+          :event-categories]
+         (bundle/tnames bundle-id)))))
 
 (defn run-down! [context]
-  (let [ds-bundle (:db-bundle context)]
+  (let [{:keys [ds-master bundle-id]} context]
     (tables/drop-tables!
-     ds-bundle
-     [:outgoing-posts
-      :bundle-categories
-      :post-heuristics
-      :analytics
-      :event-categories])))
+     ds-master
+     (-> [:outgoing-posts
+          :bundle-categories
+          :post-heuristics
+          :analytics
+          :event-categories]
+         (bundle/tnames bundle-id)))))
diff --git a/src/source/bundle_migrations/002_outgoing_posts.clj b/src/source/bundle_migrations/002_outgoing_posts.clj
index fce2ecd3..3f4d726c 100644
--- a/src/source/bundle_migrations/002_outgoing_posts.clj
+++ b/src/source/bundle_migrations/002_outgoing_posts.clj
@@ -1,16 +1,18 @@
 (ns source.bundle-migrations.002-outgoing-posts
-  (:require [source.db.bundle]
+  (:require [source.db.bundle :as bundle]
             [source.db.tables :as tables]))
 
 (defn run-up! [context]
-  (let [ds-bundle (:db-bundle context)]
+  (let [{:keys [ds-master bundle-id]} context]
     (tables/create-tables!
-     ds-bundle
+     ds-master
      :source.db.bundle
-     [:outgoing-posts])))
+     (-> [:outgoing-posts]
+         (bundle/tnames bundle-id)))))
 
 (defn run-down! [context]
-  (let [ds-bundle (:db-bundle context)]
+  (let [{:keys [ds-master bundle-id]} context]
     (tables/drop-tables!
-     ds-bundle
-     [:outgoing-posts])))
+     ds-master
+     (-> [:outgoing-posts]
+         (bundle/tnames bundle-id)))))
diff --git a/src/source/db/bundle.clj b/src/source/db/bundle.clj
index 9e4fcd33..281225d9 100644
--- a/src/source/db/bundle.clj
+++ b/src/source/db/bundle.clj
@@ -51,6 +51,13 @@
    [:event-type :text :not nil]
    [:timestamp :text :not nil]))
 
+(defn tname [tname id]
+  (-> (str (name tname) "-" id)
+      (keyword)))
+
+(defn tnames [tnames id]
+  (mapv #(tname % id) tnames))
+
 (comment
   (sql/format event-categories)
   (sql/format outgoing-posts)
diff --git a/src/source/migrate.clj b/src/source/migrate.clj
index 03416799..3306d7ca 100644
--- a/src/source/migrate.clj
+++ b/src/source/migrate.clj
@@ -35,17 +35,18 @@
         datastore (store/create-datastore
                    {:ds db-migrate
                     :table-name "migrations"})]
+    (try (jdbc/execute! (:db-master context) ["CREATE DOMAIN DATETIME TEXT"]) (catch Exception _))
     (mallard/run {:context context
                   :store datastore
                   :operations migrations}
                  args)))
 
 (defn migrate-bundle [bundle-id args]
-  (let [db-name (db.util/db-name :bundle bundle-id)
-        context {:db-bundle (jdbc/get-datasource (-> {:dbname (db.util/db-name db-name)}
-                                                     (merge postgres-ds)))}
+  (let [context {:db-master (jdbc/get-datasource (-> {:dbname (db.util/db-name "master")}
+                                                     (merge postgres-ds)))
+                 :bundle-id bundle-id}
         datastore (store/create-datastore
-                   {:ds (:db-bundle context)
+                   {:ds (:db-master context)
                     :table-name "migrations"})]
     (mallard/run {:context context
                   :store datastore

From ea35c9747c2bf6a2dbeea567f0f11320feac541a Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 9 Feb 2026 11:04:40 +0200
Subject: [PATCH 257/391] refactored more bundle-ds calls to use tables
 instead, fixed migrations

---
 .../bundle_migrations/001_init_bundle_db.clj  | 16 ++++-----
 .../bundle_migrations/002_outgoing_posts.clj  |  6 ++--
 src/source/db/bundle.clj                      |  3 +-
 src/source/db/event.clj                       | 28 ++++++++--------
 src/source/db/honey.clj                       |  5 +--
 src/source/db/tables.clj                      | 17 ++++++----
 src/source/jobs/handlers.clj                  | 20 ++++++-----
 src/source/migrate.clj                        |  6 ++--
 src/source/routes/bundle_post.clj             | 12 +++----
 src/source/routes/integration_categories.clj  | 13 +++-----
 src/source/services/bundle_categories.clj     | 26 +++++----------
 src/source/services/bundles.clj               | 12 +++----
 src/source/services/post_heuristics.clj       | 14 ++++----
 src/source/workers/bundles.clj                | 33 +++++++++----------
 src/source/workers/integrations.clj           | 26 +++++++--------
 15 files changed, 107 insertions(+), 130 deletions(-)

diff --git a/src/source/bundle_migrations/001_init_bundle_db.clj b/src/source/bundle_migrations/001_init_bundle_db.clj
index 7c13522a..e998a21f 100644
--- a/src/source/bundle_migrations/001_init_bundle_db.clj
+++ b/src/source/bundle_migrations/001_init_bundle_db.clj
@@ -3,15 +3,15 @@
             [source.db.tables :as tables]))
 
 (defn run-up! [context]
-  (let [{:keys [ds-master bundle-id]} context]
+  (let [{:keys [ds-master bundle-id]} context
+        tables [:outgoing-posts
+                :bundle-categories
+                :post-heuristics]]
     (tables/create-tables!
      ds-master
      :source.db.bundle
-     (-> [:outgoing-posts
-          :bundle-categories
-          :post-heuristics
-          :analytics
-          :event-categories]
+     tables
+     (-> tables
          (bundle/tnames bundle-id)))))
 
 (defn run-down! [context]
@@ -20,7 +20,5 @@
      ds-master
      (-> [:outgoing-posts
           :bundle-categories
-          :post-heuristics
-          :analytics
-          :event-categories]
+          :post-heuristics]
          (bundle/tnames bundle-id)))))
diff --git a/src/source/bundle_migrations/002_outgoing_posts.clj b/src/source/bundle_migrations/002_outgoing_posts.clj
index 3f4d726c..c71036b8 100644
--- a/src/source/bundle_migrations/002_outgoing_posts.clj
+++ b/src/source/bundle_migrations/002_outgoing_posts.clj
@@ -3,11 +3,13 @@
             [source.db.tables :as tables]))
 
 (defn run-up! [context]
-  (let [{:keys [ds-master bundle-id]} context]
+  (let [{:keys [ds-master bundle-id]} context
+        tables [:outgoing-posts]]
     (tables/create-tables!
      ds-master
      :source.db.bundle
-     (-> [:outgoing-posts]
+     tables
+     (-> tables
          (bundle/tnames bundle-id)))))
 
 (defn run-down! [context]
diff --git a/src/source/db/bundle.clj b/src/source/db/bundle.clj
index 281225d9..53a1d6b6 100644
--- a/src/source/db/bundle.clj
+++ b/src/source/db/bundle.clj
@@ -7,8 +7,7 @@
    :event-categories
    (tables/table-id)
    [:event-id :integer :not nil]
-   [:category-id :text :not nil]
-   (tables/foreign-key :event-id :analytics :id)))
+   [:category-id :text :not nil]))
 
 (def outgoing-posts
   (tables/create-table-sql
diff --git a/src/source/db/event.clj b/src/source/db/event.clj
index 33ce7e13..06886117 100644
--- a/src/source/db/event.clj
+++ b/src/source/db/event.clj
@@ -3,34 +3,32 @@
             [source.services.feed-categories :as feed-categories]
             [source.db.util :as db.util]
             [source.util :as util]
-            [source.db.honey :as db]))
+            [source.db.honey :as db]
+            [source.db.bundle :as bundle]))
 
-(defn get-post-categories [bundle-ds ds post-id]
-  (let [feed-id (-> (db/find-one bundle-ds {:tname :outgoing-posts
-                                            :where [:= :id post-id]})
+(defn get-post-categories [ds post-id bundle-id]
+  (let [feed-id (-> (db/find-one ds {:tname (bundle/tname :outgoing-posts bundle-id)
+                                     :where [:= :id post-id]})
                     (:feed-id))]
     (feed-categories/category-id ds {:feed-id feed-id})))
 
 (defn log! [{:keys [post-id bundle-id type]}]
   (let [ds (db.util/conn :master)
         timestamp (util/get-utc-timestamp-string)
-        bundle-ds (->> bundle-id
-                       (db.util/db-name :bundle)
-                       (db.util/conn))
         creator-ds (->> {:post-id post-id}
                         (db/find ds)
                         (:creator-id)
                         (db.util/db-name :creator)
                         (db.util/conn))
-        categories (get-post-categories bundle-ds ds post-id)]
-    (let [event-id (-> (analytics/insert-event! bundle-ds {:data {:post_id post-id
-                                                                  :event_type type
-                                                                  :timestamp timestamp}
-                                                           :ret :*})
+        categories (get-post-categories ds post-id bundle-id)]
+    (let [event-id (-> (analytics/insert-event! ds {:data {:post_id post-id
+                                                           :event_type type
+                                                           :timestamp timestamp}
+                                                    :ret :*})
                        (first))]
-      (db/insert! bundle-ds {:tname :event-categories
-                             :data {:event-id event-id
-                                    :category-id (:category-id categories)}}))
+      (db/insert! ds {:tname (bundle/tname :event-categories bundle-id)
+                      :data {:event-id event-id
+                             :category-id (:category-id categories)}}))
     (let [event-id (-> (analytics/insert-event! creator-ds {:data {:post_id post-id
                                                                    :event_type type
                                                                    :timestamp timestamp}
diff --git a/src/source/db/honey.clj b/src/source/db/honey.clj
index 2953786c..b9137dd7 100644
--- a/src/source/db/honey.clj
+++ b/src/source/db/honey.clj
@@ -99,10 +99,7 @@
 (comment
   (hsql/where :or [:= :id 1] [:= :id 2])
 
-  (def ds (db.util/conn :bundle 5))
-
-  (find ds {:tname :outgoing-posts
-            :ret :*})
+  (def ds (db.util/conn))
 
   (insert! ds {:tname :sectors
                :values {:name "something"}
diff --git a/src/source/db/tables.clj b/src/source/db/tables.clj
index 3203ab3e..3b11460d 100644
--- a/src/source/db/tables.clj
+++ b/src/source/db/tables.clj
@@ -33,16 +33,21 @@
   resolvable keyword, resolves the symbol to retrieve defined
   sql create table honey statements, prepares jdbc statements from them,
   and executes with next.jdbc, returning the result of the execution."
-  [ds ns tname]
-  (->> (resolve-sql-def ns tname)
-       (hon/execute! ds)))
+  ([ds ns tname]
+   (create-table! ds ns tname tname))
+  ([ds ns tname new-tname]
+   (let [table-stmt (-> (resolve-sql-def ns tname)
+                        (assoc :create-table [new-tname :if-not-exists]))]
+     (hon/execute! ds table-stmt))))
 
 (defn create-tables!
   "Like create-table! but accepts a vector of keywords for table names
   and runs create-table with ns on every table name keyword in the vector."
-  [ds ns tables]
-  (mapv #(create-table! ds ns %)
-        tables))
+  ([ds ns tables]
+   (create-tables! ds ns tables tables))
+  ([ds ns tables new-tnames]
+   (mapv (fn [t nt]
+           (create-table! ds ns t nt)) tables new-tnames)))
 
 (defn tables
   "returns all current tables in a postgres datasource"
diff --git a/src/source/jobs/handlers.clj b/src/source/jobs/handlers.clj
index 3174dc1c..bf3529a7 100644
--- a/src/source/jobs/handlers.clj
+++ b/src/source/jobs/handlers.clj
@@ -6,7 +6,8 @@
             [source.db.util :as db.util]
             [clojure.set :as set]
             [clojure.string :as string]
-            [source.db.honey :as hon]))
+            [source.db.honey :as hon]
+            [source.db.bundle :as bundle]))
 
 (defmulti handler
   (fn [opts]
@@ -86,7 +87,6 @@
 (defmethod handler :update-bundle [_]
   (fn [{:keys [args ds]}]
     (let [{:keys [bundle-id categories]} args
-          ds-bundle (db.util/conn :bundle bundle-id)
           incoming-posts (services/incoming-posts-with-feeds ds {:where [:= :feeds.state "live"]})
           posts-categories (incoming-posts/categories-by-posts ds {:where [:= :state "live"]})]
       (run!
@@ -105,15 +105,17 @@
                matches (count (set/intersection (set post-categories-vec)
                                                 (set match-categories-vec)))]
             ; use matches as a score and upsert long-heuristic for this post
-           (services/upsert-post-heuristics! ds-bundle {:data [{:post-id (:id post)
-                                                                :long-heuristic matches}]})))
+           (services/upsert-post-heuristics! ds {:bundle-id bundle-id
+                                                 :data [{:post-id (:id post)
+                                                         :long-heuristic matches}]})))
        incoming-posts)
 
       ; pull highest scored posts by long heuristics into outgoing posts
             ; top 1000 post-heuristics records ordered by long heuristic in descending order
-      (let [top-by-long-heuristics (services/top-posts-by-heuristic ds-bundle
+      (let [top-by-long-heuristics (services/top-posts-by-heuristic ds
                                                                     {:heuristic :long-heuristic
-                                                                     :limit 1000})
+                                                                     :limit 1000
+                                                                     :bundle-id bundle-id})
             ; convert into a vector of id numbers
             ids (mapv :post-id top-by-long-heuristics)
 
@@ -135,9 +137,9 @@
                                        acc))
                                    [] posts-in)]
         (when (seq posts-in)
-          (hon/delete! ds-bundle {:tname :outgoing-posts})
-          (hon/insert! ds-bundle {:tname :outgoing-posts
-                                  :data outgoing-posts}))))))
+          (hon/delete! ds {:tname (bundle/tname :outgoing-posts bundle-id)})
+          (hon/insert! ds {:tname (bundle/tname :outgoing-posts bundle-id)
+                           :data outgoing-posts}))))))
 
 (defn user-deletion-job-id
   "returns the job id of a user deletion job with the given user id"
diff --git a/src/source/migrate.clj b/src/source/migrate.clj
index 3306d7ca..2313ab29 100644
--- a/src/source/migrate.clj
+++ b/src/source/migrate.clj
@@ -42,12 +42,12 @@
                  args)))
 
 (defn migrate-bundle [bundle-id args]
-  (let [context {:db-master (jdbc/get-datasource (-> {:dbname (db.util/db-name "master")}
+  (let [context {:ds-master (jdbc/get-datasource (-> {:dbname (db.util/db-name "master")}
                                                      (merge postgres-ds)))
                  :bundle-id bundle-id}
         datastore (store/create-datastore
-                   {:ds (:db-master context)
-                    :table-name "migrations"})]
+                   {:ds (:ds-master context)
+                    :table-name (str "migrations_" bundle-id)})]
     (mallard/run {:context context
                   :store datastore
                   :operations bundle-migrations}
diff --git a/src/source/routes/bundle_post.clj b/src/source/routes/bundle_post.clj
index ec62cfb3..50f87082 100644
--- a/src/source/routes/bundle_post.clj
+++ b/src/source/routes/bundle_post.clj
@@ -1,8 +1,8 @@
 (ns source.routes.bundle-post
-  (:require [source.db.util :as db.util]
-            [ring.util.response :as res]
+  (:require [ring.util.response :as res]
             [source.services.analytics.interface :as analytics]
-            [source.db.honey :as hon]))
+            [source.db.honey :as hon]
+            [source.db.bundle :as bundle]))
 
 (defn get
   {:summary "get a single outgoing post in the uuid-authorized bundle by post id, updates click analytics"
@@ -27,9 +27,7 @@
                404 {:body [:map [:message :string]]}}}
 
   [{:keys [ds bundle-id path-params] :as _request}]
-  (let [bundle-ds (db.util/conn :bundle bundle-id)
-        post (hon/find-one bundle-ds {:tname :outgoing-posts
-                                      :where [:= :id (:id path-params)]
-                                      :ret :1})]
+  (let [post (hon/find-one ds {:tname (bundle/tname :outgoing-posts bundle-id)
+                               :where [:= :id (:id path-params)]})]
     (analytics/insert-post-click! ds post bundle-id)
     (res/response post)))
diff --git a/src/source/routes/integration_categories.clj b/src/source/routes/integration_categories.clj
index 6087f03d..d5fd0812 100644
--- a/src/source/routes/integration_categories.clj
+++ b/src/source/routes/integration_categories.clj
@@ -1,7 +1,5 @@
 (ns source.routes.integration-categories
-  (:require [source.services.interface :as services]
-            [ring.util.response :as res]
-            [source.db.util :as db.util]
+  (:require [ring.util.response :as res]
             [source.services.bundles :as bundles]
             [source.services.bundle-categories :as bundle-categories]))
 
@@ -27,8 +25,7 @@
                         [:name :string]]]}
    :responses {200 {:body [:map [:message :string]]}}}
 
-  [{:keys [path-params body] :as _request}]
-  (let [bundle-ds (db.util/conn :bundle (:id path-params))]
-    (bundle-categories/update-bundle-categories! bundle-ds {:bundle-id (:id path-params)
-                                                            :categories body})
-    (res/response {:message "successfully updated integration categories"})))
+  [{:keys [ds path-params body] :as _request}]
+  (bundle-categories/update-bundle-categories! ds {:bundle-id (:id path-params)
+                                                   :categories body})
+  (res/response {:message "successfully updated integration categories"}))
diff --git a/src/source/services/bundle_categories.clj b/src/source/services/bundle_categories.clj
index 0c4c004c..6d5905be 100644
--- a/src/source/services/bundle_categories.clj
+++ b/src/source/services/bundle_categories.clj
@@ -1,19 +1,6 @@
 (ns source.services.bundle-categories
-  (:require [source.db.interface :as db]))
-
-(defn insert-bundle-category! [ds {:keys [_data _ret] :as opts}]
-  (->> {:tname :bundle-categories}
-       (merge opts)
-       (db/insert! ds)))
-
-(defn delete-bundle-category! [ds {:keys [id where] :as opts}]
-  (->> {:tname :bundle-categories
-        :where (if (some? id)
-                 [:= :id id]
-                 where)
-        :ret :1}
-       (merge opts)
-       (db/delete! ds)))
+  (:require [source.db.interface :as db]
+            [source.db.bundle :as bundle]))
 
 (defn category-id [ds {:keys [bundle-id where] :as opts}]
   (->> {:tname :bundle-categories
@@ -28,11 +15,14 @@
   (let [bundle-categories (mapv (fn [{:keys [id]}]
                                   {:bundle-id bundle-id
                                    :category-id id}) categories)]
-    (insert-bundle-category! ds {:data bundle-categories})))
+    (db/insert! ds {:tname (bundle/tname :bundle-categories bundle-id)
+                    :data bundle-categories})))
 
 (defn update-bundle-categories! [ds {:keys [bundle-id categories]}]
   (let [bundle-categories (mapv (fn [{:keys [id]}]
                                   {:bundle-id bundle-id
                                    :category-id id}) categories)]
-    (delete-bundle-category! ds {:where [:= :bundle-id bundle-id]})
-    (insert-bundle-category! ds {:data bundle-categories})))
+    (db/delete! ds {:tname (bundle/tname :bundle-categories bundle-id)
+                    :where [:= :bundle-id bundle-id]})
+    (db/insert! ds {:tname (bundle/tname :bundle-categories bundle-id)
+                    :data bundle-categories})))
diff --git a/src/source/services/bundles.clj b/src/source/services/bundles.clj
index 4f77aa5d..6ffb23f8 100644
--- a/src/source/services/bundles.clj
+++ b/src/source/services/bundles.clj
@@ -1,9 +1,7 @@
 (ns source.services.bundles
   (:require [source.db.interface :as db]
             [source.util :as utils]
-            [source.services.bundle-categories :as bundle-categories]
-            [source.db.util :as db.util]
-            [source.db.honey :as hon]))
+            [source.db.bundle :as bundle]))
 
 (defn insert-bundle! [ds {:keys [_values _ret] :as opts}]
   (->> {:tname :bundles}
@@ -46,8 +44,8 @@
 
 ;;NEW
 (defn categories-in-bundle [ds bundle-id]
-  (let [bundle-ds (db.util/conn :bundle bundle-id)
-        category-ids (bundle-categories/category-id bundle-ds {:bundle-id bundle-id})
+  (let [category-ids (db/find-one ds {:tname (bundle/tname :bundle-categories bundle-id)
+                                      :where [:= :bundle-id bundle-id]})
         id-vec (mapv (fn [{:keys [category-id]}] category-id) category-ids)]
-    (hon/find ds {:tname :categories
-                  :where [:in :id id-vec]})))
+    (db/find ds {:tname :categories
+                 :where [:in :id id-vec]})))
diff --git a/src/source/services/post_heuristics.clj b/src/source/services/post_heuristics.clj
index b682efa7..fba7f20d 100644
--- a/src/source/services/post_heuristics.clj
+++ b/src/source/services/post_heuristics.clj
@@ -1,21 +1,21 @@
 (ns source.services.post-heuristics
-  (:require [source.db.interface :as db]
-            [source.db.honey :as hon]
-            [honey.sql.helpers :as hsql]))
+  (:require [source.db.honey :as hon]
+            [honey.sql.helpers :as hsql]
+            [source.db.bundle :as bundle]))
 
-(defn upsert-post-heuristics! [ds {:keys [data]}]
+(defn upsert-post-heuristics! [ds {:keys [bundle-id data]}]
   (hon/execute!
    ds
-   (-> (hsql/insert-into :post-heuristics)
+   (-> (hsql/insert-into (bundle/tname :post-heuristics bundle-id))
        (hsql/values data)
        (assoc :on-conflict [:post-id])
        (assoc :do-update-set {:long-heuristic :excluded.long-heuristic
                               :short-heuristic :excluded.short-heuristic}))))
 
-(defn top-posts-by-heuristic [ds {:keys [select limit heuristic] :as _opts}]
+(defn top-posts-by-heuristic [ds {:keys [select limit heuristic bundle-id] :as _opts}]
   (hon/execute! ds
                 (merge {:select (or select :*)
-                        :from :post-heuristics
+                        :from (bundle/tname :post-heuristics bundle-id)
                         :order-by [[heuristic :desc]]
                         :limit limit})
                 {:ret :*}))
diff --git a/src/source/workers/bundles.clj b/src/source/workers/bundles.clj
index 64abecb2..38c1f126 100644
--- a/src/source/workers/bundles.clj
+++ b/src/source/workers/bundles.clj
@@ -1,16 +1,15 @@
 (ns source.workers.bundles
-  (:require [source.db.util :as db.util]
-            [source.db.honey :as hon]
+  (:require [source.db.honey :as hon]
             [honey.sql.helpers :as hsql]
             [clojure.set :as set]
-            [source.services.feed-categories :as feed-categories]))
+            [source.services.feed-categories :as feed-categories]
+            [source.db.bundle :as bundle]))
 
 (defn get-bundle-categories
   "Get all categories for feeds/posts in bundle"
   [ds bundle-id]
-  (let [bundle-ds (db.util/conn :bundle bundle-id)
-        feed-ids (->> (hon/find bundle-ds {:tname :outgoing-posts
-                                           :ret :*})
+  (let [feed-ids (->> (hon/find ds {:tname (bundle/tname :outgoing-posts bundle-id)
+                                    :ret :*})
                       (mapv :feed-id))
         category-ids (->> (hon/find ds {:tname :feed-categories
                                         :where [:in :feed-id feed-ids]
@@ -23,9 +22,8 @@
 (defn get-outgoing-feeds
   "Gets a filtered list of outgoing feeds for the associated bundle."
   [ds {:keys [bundle-id type latest category-ids nonfiltered]}]
-  (let [bundle-ds (db.util/conn :bundle bundle-id)
-        feed-ids (mapv :feed-id (hon/find bundle-ds {:tname :outgoing-posts
-                                                     :ret :*}))
+  (let [feed-ids (mapv :feed-id (hon/find ds {:tname (bundle/tname :outgoing-posts bundle-id)
+                                              :ret :*}))
         category-filtered-feed-ids (if (empty? category-ids)
                                      feed-ids
                                      (->> (hsql/where
@@ -41,7 +39,7 @@
                                                         :where [:= :bundle-id bundle-id]
                                                         :ret :*})))
         query (-> (hsql/where (when type [:= :content-type-id type])
-                              [:in :id category-filtered-feed-ids]
+                              (when (seq category-filtered-feed-ids) [:in :id category-filtered-feed-ids])
                               (when (seq blocked-feed-ids) [:not [:in :id blocked-feed-ids]]))
                   (assoc :order-by (when latest [[:created-at :desc]]))
                   (merge {:tname :feeds
@@ -52,8 +50,7 @@
 (defn get-outgoing-posts
   "Get outgoing posts based on short heuristics and update analytics impressions"
   [ds {:keys [bundle-id limit start type latest category-ids]}]
-  (let [bundle-ds (db.util/conn :bundle bundle-id)
-        all-feed-ids (mapv :id (hon/find ds {:tname :feeds
+  (let [all-feed-ids (mapv :id (hon/find ds {:tname :feeds
                                              :ret :*}))
         blocked-feed-ids (mapv :feed-id (hon/find ds {:tname :filtered-feeds
                                                       :where [:= :bundle-id bundle-id]
@@ -64,12 +61,12 @@
                                                       :where [:= :bundle-id bundle-id]
                                                       :ret :*}))
 
-        filtered-posts (hon/find bundle-ds (-> (hsql/where (when type [:= :content-type-id type])
-                                                           (when (seq blocked-post-ids) [:not [:in :id blocked-post-ids]])
-                                                           [:in :feed-id available-feed-ids])
-                                               (assoc :order-by (when (= latest "true") [[[:posted-at :desc]]]))
-                                               (merge {:tname :outgoing-posts
-                                                       :ret :*})))
+        filtered-posts (hon/find ds (-> (hsql/where (when type [:= :content-type-id type])
+                                                    (when (seq blocked-post-ids) [:not [:in :id blocked-post-ids]])
+                                                    [:in :feed-id available-feed-ids])
+                                        (assoc :order-by (when (= latest "true") [[[:posted-at :desc]]]))
+                                        (merge {:tname (bundle/tname :outgoing-posts bundle-id)
+                                                :ret :*})))
 
         categorised-posts (vec
                            (if (seq category-ids)
diff --git a/src/source/workers/integrations.clj b/src/source/workers/integrations.clj
index 47a61aca..ed156f29 100644
--- a/src/source/workers/integrations.clj
+++ b/src/source/workers/integrations.clj
@@ -7,30 +7,26 @@
             [source.util :as utils]
             [source.db.tables :as tables]
             [source.db.honey :as hon]
-            [congest.jobs :as congest]
-            [next.jdbc :as jdbc]))
+            [congest.jobs :as congest]))
 
 (defn create-integration! [ds {:keys [user-id bundle-metadata categories content-types]}]
   (let [new-bundle (bundles/create-bundle! ds {:user-id user-id
                                                :bundle-metadata bundle-metadata})]
-    (jdbc/execute! ds [(str "CREATE DATABASE bundle_" (:id new-bundle))])
     (migrate/migrate-bundle (:id new-bundle) ["up"])
 
-    (let [bundle-ds (db.util/conn :bundle (:id new-bundle))]
-      (bundle-categories/insert-bundle-categories! bundle-ds {:bundle-id (:id new-bundle)
-                                                              :categories categories})
-      (bundle-content-types/insert-bundle-content-types! ds {:bundle-id (:id new-bundle)
-                                                             :content-types content-types}))
+    (bundle-categories/insert-bundle-categories! ds {:bundle-id (:id new-bundle)
+                                                     :categories categories})
+    (bundle-content-types/insert-bundle-content-types! ds {:bundle-id (:id new-bundle)
+                                                           :content-types content-types})
     new-bundle))
 
 (defn update-integration! [ds {:keys [bundle-id bundle-metadata categories content-types]}]
-  (let [bundle-ds (db.util/conn :bundle bundle-id)]
-    (bundles/update-bundle! ds {:id bundle-id
-                                :data bundle-metadata})
-    (bundle-categories/update-bundle-categories! bundle-ds {:bundle-id bundle-id
-                                                            :categories categories})
-    (bundle-content-types/update-bundle-content-types! ds {:bundle-id bundle-id
-                                                           :content-types content-types})))
+  (bundles/update-bundle! ds {:id bundle-id
+                              :data bundle-metadata})
+  (bundle-categories/update-bundle-categories! ds {:bundle-id bundle-id
+                                                   :categories categories})
+  (bundle-content-types/update-bundle-content-types! ds {:bundle-id bundle-id
+                                                         :content-types content-types}))
 
 (defn hard-delete-bundle! [ds js job-id bundle-id]
   (hon/delete! ds {:tname :filtered-feeds

From bd1c685a4bf3606cda487d493e8cb2dbe3ecae07 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 9 Feb 2026 12:11:12 +0200
Subject: [PATCH 258/391] fixed integration deletion to use tables instead of
 bundle ds

---
 src/source/workers/integrations.clj | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/source/workers/integrations.clj b/src/source/workers/integrations.clj
index ed156f29..f7d3cc92 100644
--- a/src/source/workers/integrations.clj
+++ b/src/source/workers/integrations.clj
@@ -7,7 +7,8 @@
             [source.util :as utils]
             [source.db.tables :as tables]
             [source.db.honey :as hon]
-            [congest.jobs :as congest]))
+            [congest.jobs :as congest]
+            [source.db.bundle :as bundle]))
 
 (defn create-integration! [ds {:keys [user-id bundle-metadata categories content-types]}]
   (let [new-bundle (bundles/create-bundle! ds {:user-id user-id
@@ -37,7 +38,10 @@
                    :where [:= :bundle-id bundle-id]})
   (hon/delete! ds {:tname :events
                    :where [:= :bundle-id bundle-id]})
-  (tables/drop-all-tables! (db.util/conn :bundle bundle-id))
+  (tables/drop-tables! ds (bundle/tnames [:outgoing-posts
+                                          :bundle-categories
+                                          :post-heuristics]
+                                         bundle-id))
   (hon/delete! ds {:tname :bundles
                    :where [:= :id bundle-id]})
   (congest/deregister! js job-id))

From 218086013adc076dcd245eef3b459cd64b1a4252 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 9 Feb 2026 13:58:30 +0200
Subject: [PATCH 259/391] refactored tname function to return {:tname tname}
 for threadability

---
 .../bundle_migrations/001_init_bundle_db.clj  |  5 ++--
 .../bundle_migrations/002_outgoing_posts.clj  | 12 ++++----
 src/source/db/event.clj                       | 13 ++++----
 src/source/db/tables.clj                      | 30 +++++++++++--------
 src/source/db/util.clj                        | 12 ++++++++
 src/source/jobs/handlers.clj                  |  6 ++--
 src/source/routes/bundle_post.clj             |  8 +++--
 src/source/services/bundle_categories.clj     | 13 ++++----
 src/source/services/bundles.clj               |  7 +++--
 src/source/services/post_heuristics.clj       |  6 ++--
 src/source/workers/bundles.clj                | 12 ++++----
 src/source/workers/integrations.clj           |  8 ++---
 12 files changed, 76 insertions(+), 56 deletions(-)

diff --git a/src/source/bundle_migrations/001_init_bundle_db.clj b/src/source/bundle_migrations/001_init_bundle_db.clj
index e998a21f..91f3e05b 100644
--- a/src/source/bundle_migrations/001_init_bundle_db.clj
+++ b/src/source/bundle_migrations/001_init_bundle_db.clj
@@ -7,12 +7,11 @@
         tables [:outgoing-posts
                 :bundle-categories
                 :post-heuristics]]
-    (tables/create-tables!
+    (tables/create-bundle-tables!
      ds-master
      :source.db.bundle
      tables
-     (-> tables
-         (bundle/tnames bundle-id)))))
+     bundle-id)))
 
 (defn run-down! [context]
   (let [{:keys [ds-master bundle-id]} context]
diff --git a/src/source/bundle_migrations/002_outgoing_posts.clj b/src/source/bundle_migrations/002_outgoing_posts.clj
index c71036b8..180348d3 100644
--- a/src/source/bundle_migrations/002_outgoing_posts.clj
+++ b/src/source/bundle_migrations/002_outgoing_posts.clj
@@ -1,20 +1,20 @@
 (ns source.bundle-migrations.002-outgoing-posts
-  (:require [source.db.bundle :as bundle]
-            [source.db.tables :as tables]))
+  (:require [source.db.bundle]
+            [source.db.tables :as tables]
+            [source.db.util :as db.util]))
 
 (defn run-up! [context]
   (let [{:keys [ds-master bundle-id]} context
         tables [:outgoing-posts]]
-    (tables/create-tables!
+    (tables/create-bundle-tables!
      ds-master
      :source.db.bundle
      tables
-     (-> tables
-         (bundle/tnames bundle-id)))))
+     bundle-id)))
 
 (defn run-down! [context]
   (let [{:keys [ds-master bundle-id]} context]
     (tables/drop-tables!
      ds-master
      (-> [:outgoing-posts]
-         (bundle/tnames bundle-id)))))
+         (db.util/tnames bundle-id)))))
diff --git a/src/source/db/event.clj b/src/source/db/event.clj
index 06886117..80594636 100644
--- a/src/source/db/event.clj
+++ b/src/source/db/event.clj
@@ -4,11 +4,12 @@
             [source.db.util :as db.util]
             [source.util :as util]
             [source.db.honey :as db]
-            [source.db.bundle :as bundle]))
+            [source.db.bundle :as bundle]
+            [honey.sql.helpers :as hsql]))
 
 (defn get-post-categories [ds post-id bundle-id]
-  (let [feed-id (-> (db/find-one ds {:tname (bundle/tname :outgoing-posts bundle-id)
-                                     :where [:= :id post-id]})
+  (let [feed-id (-> (db/find-one ds (-> (db.util/tname :outgoing-posts bundle-id)
+                                        (hsql/where [:= :id post-id])))
                     (:feed-id))]
     (feed-categories/category-id ds {:feed-id feed-id})))
 
@@ -26,9 +27,9 @@
                                                            :timestamp timestamp}
                                                     :ret :*})
                        (first))]
-      (db/insert! ds {:tname (bundle/tname :event-categories bundle-id)
-                      :data {:event-id event-id
-                             :category-id (:category-id categories)}}))
+      (db/insert! ds (-> (db.util/tname :event-categories bundle-id)
+                         (assoc :data {:event-id event-id
+                                       :category-id (:category-id categories)}))))
     (let [event-id (-> (analytics/insert-event! creator-ds {:data {:post_id post-id
                                                                    :event_type type
                                                                    :timestamp timestamp}
diff --git a/src/source/db/tables.clj b/src/source/db/tables.clj
index 3b11460d..a3863f17 100644
--- a/src/source/db/tables.clj
+++ b/src/source/db/tables.clj
@@ -32,22 +32,28 @@
   "Given keywords ns and table, parses the keywords into a
   resolvable keyword, resolves the symbol to retrieve defined
   sql create table honey statements, prepares jdbc statements from them,
-  and executes with next.jdbc, returning the result of the execution."
-  ([ds ns tname]
-   (create-table! ds ns tname tname))
-  ([ds ns tname new-tname]
-   (let [table-stmt (-> (resolve-sql-def ns tname)
-                        (assoc :create-table [new-tname :if-not-exists]))]
-     (hon/execute! ds table-stmt))))
+  and executes with next.jdbc, returning the result of the execution.
+  tname can either be a resolvable var keyword, or a vector containing
+  a resolvable var keyword and a table name to be assigned"
+  [ds ns tname]
+  (let [multi? (vector? tname)
+        tname' (if multi? (first tname) tname)
+        new-tname (if multi? (last tname) tname)
+        table-stmt (-> (resolve-sql-def ns tname')
+                       (assoc :create-table [new-tname :if-not-exists]))]
+    (hon/execute! ds table-stmt)))
 
 (defn create-tables!
   "Like create-table! but accepts a vector of keywords for table names
   and runs create-table with ns on every table name keyword in the vector."
-  ([ds ns tables]
-   (create-tables! ds ns tables tables))
-  ([ds ns tables new-tnames]
-   (mapv (fn [t nt]
-           (create-table! ds ns t nt)) tables new-tnames)))
+  [ds ns tables]
+  (mapv #(create-table! ds ns %) tables))
+
+(defn create-bundle-tables!
+  [ds ns tables bundle-id]
+  (->> (mapv (fn [t] [t (-> (db.util/tname t bundle-id)
+                            :tname)]) tables)
+       (mapv #(create-table! ds ns %))))
 
 (defn tables
   "returns all current tables in a postgres datasource"
diff --git a/src/source/db/util.clj b/src/source/db/util.clj
index 1141b184..96dea558 100644
--- a/src/source/db/util.clj
+++ b/src/source/db/util.clj
@@ -43,6 +43,18 @@
    (assert (or (= db-type :bundle) (= db-type :creator)))
    (-conn (db-name db-type id))))
 
+(defn tname
+  ([tname id]
+   {:tname (->> (str (name tname) "-" id)
+                (keyword))})
+  ([data-map tname id]
+   (->> (str (name tname) "-" id)
+        (keyword)
+        (assoc data-map :tname))))
+
+(defn tnames [tnames id]
+  (mapv #(tname % id) tnames))
+
 (comment
   (def ds (conn :bundle 1))
   (jdbc/execute! ds ["DELETE FROM providers;"])
diff --git a/src/source/jobs/handlers.clj b/src/source/jobs/handlers.clj
index bf3529a7..40a96ce7 100644
--- a/src/source/jobs/handlers.clj
+++ b/src/source/jobs/handlers.clj
@@ -137,9 +137,9 @@
                                        acc))
                                    [] posts-in)]
         (when (seq posts-in)
-          (hon/delete! ds {:tname (bundle/tname :outgoing-posts bundle-id)})
-          (hon/insert! ds {:tname (bundle/tname :outgoing-posts bundle-id)
-                           :data outgoing-posts}))))))
+          (hon/delete! ds (db.util/tname :outgoing-posts bundle-id))
+          (hon/insert! ds (-> (db.util/tname :outgoing-posts bundle-id)
+                              (assoc :data outgoing-posts))))))))
 
 (defn user-deletion-job-id
   "returns the job id of a user deletion job with the given user id"
diff --git a/src/source/routes/bundle_post.clj b/src/source/routes/bundle_post.clj
index 50f87082..44e720a2 100644
--- a/src/source/routes/bundle_post.clj
+++ b/src/source/routes/bundle_post.clj
@@ -2,7 +2,9 @@
   (:require [ring.util.response :as res]
             [source.services.analytics.interface :as analytics]
             [source.db.honey :as hon]
-            [source.db.bundle :as bundle]))
+            [source.db.bundle :as bundle]
+            [source.db.util :as db.util]
+            [honey.sql.helpers :as hsql]))
 
 (defn get
   {:summary "get a single outgoing post in the uuid-authorized bundle by post id, updates click analytics"
@@ -27,7 +29,7 @@
                404 {:body [:map [:message :string]]}}}
 
   [{:keys [ds bundle-id path-params] :as _request}]
-  (let [post (hon/find-one ds {:tname (bundle/tname :outgoing-posts bundle-id)
-                               :where [:= :id (:id path-params)]})]
+  (let [post (hon/find-one ds (-> (db.util/tname :outgoing-posts bundle-id)
+                                  (hsql/where [:= :id (:id path-params)])))]
     (analytics/insert-post-click! ds post bundle-id)
     (res/response post)))
diff --git a/src/source/services/bundle_categories.clj b/src/source/services/bundle_categories.clj
index 6d5905be..f2317020 100644
--- a/src/source/services/bundle_categories.clj
+++ b/src/source/services/bundle_categories.clj
@@ -1,6 +1,7 @@
 (ns source.services.bundle-categories
   (:require [source.db.interface :as db]
-            [source.db.bundle :as bundle]))
+            [source.db.bundle :as bundle]
+            [source.db.util :as db.util]))
 
 (defn category-id [ds {:keys [bundle-id where] :as opts}]
   (->> {:tname :bundle-categories
@@ -15,14 +16,14 @@
   (let [bundle-categories (mapv (fn [{:keys [id]}]
                                   {:bundle-id bundle-id
                                    :category-id id}) categories)]
-    (db/insert! ds {:tname (bundle/tname :bundle-categories bundle-id)
-                    :data bundle-categories})))
+    (db/insert! ds (-> (db.util/tname :bundle-categories bundle-id)
+                       (assoc :data bundle-categories)))))
 
 (defn update-bundle-categories! [ds {:keys [bundle-id categories]}]
   (let [bundle-categories (mapv (fn [{:keys [id]}]
                                   {:bundle-id bundle-id
                                    :category-id id}) categories)]
-    (db/delete! ds {:tname (bundle/tname :bundle-categories bundle-id)
+    (db/delete! ds {:tname (db.util/tname :bundle-categories bundle-id)
                     :where [:= :bundle-id bundle-id]})
-    (db/insert! ds {:tname (bundle/tname :bundle-categories bundle-id)
-                    :data bundle-categories})))
+    (db/insert! ds (-> (db.util/tname :bundle-categories bundle-id)
+                       (assoc :data bundle-categories)))))
diff --git a/src/source/services/bundles.clj b/src/source/services/bundles.clj
index 6ffb23f8..f234f4ae 100644
--- a/src/source/services/bundles.clj
+++ b/src/source/services/bundles.clj
@@ -1,7 +1,8 @@
 (ns source.services.bundles
   (:require [source.db.interface :as db]
             [source.util :as utils]
-            [source.db.bundle :as bundle]))
+            [source.db.util :as db.util]
+            [honey.sql.helpers :as hsql]))
 
 (defn insert-bundle! [ds {:keys [_values _ret] :as opts}]
   (->> {:tname :bundles}
@@ -44,8 +45,8 @@
 
 ;;NEW
 (defn categories-in-bundle [ds bundle-id]
-  (let [category-ids (db/find-one ds {:tname (bundle/tname :bundle-categories bundle-id)
-                                      :where [:= :bundle-id bundle-id]})
+  (let [category-ids (db/find-one ds (-> (db.util/tname :bundle-categories bundle-id)
+                                         (hsql/where [:= :bundle-id bundle-id])))
         id-vec (mapv (fn [{:keys [category-id]}] category-id) category-ids)]
     (db/find ds {:tname :categories
                  :where [:in :id id-vec]})))
diff --git a/src/source/services/post_heuristics.clj b/src/source/services/post_heuristics.clj
index fba7f20d..d295a04c 100644
--- a/src/source/services/post_heuristics.clj
+++ b/src/source/services/post_heuristics.clj
@@ -1,12 +1,12 @@
 (ns source.services.post-heuristics
   (:require [source.db.honey :as hon]
             [honey.sql.helpers :as hsql]
-            [source.db.bundle :as bundle]))
+            [source.db.util :as db.util]))
 
 (defn upsert-post-heuristics! [ds {:keys [bundle-id data]}]
   (hon/execute!
    ds
-   (-> (hsql/insert-into (bundle/tname :post-heuristics bundle-id))
+   (-> (hsql/insert-into (:tname (db.util/tname :post-heuristics bundle-id)))
        (hsql/values data)
        (assoc :on-conflict [:post-id])
        (assoc :do-update-set {:long-heuristic :excluded.long-heuristic
@@ -15,7 +15,7 @@
 (defn top-posts-by-heuristic [ds {:keys [select limit heuristic bundle-id] :as _opts}]
   (hon/execute! ds
                 (merge {:select (or select :*)
-                        :from (bundle/tname :post-heuristics bundle-id)
+                        :from (:tname (db.util/tname :post-heuristics bundle-id))
                         :order-by [[heuristic :desc]]
                         :limit limit})
                 {:ret :*}))
diff --git a/src/source/workers/bundles.clj b/src/source/workers/bundles.clj
index 38c1f126..9fa441d4 100644
--- a/src/source/workers/bundles.clj
+++ b/src/source/workers/bundles.clj
@@ -3,13 +3,13 @@
             [honey.sql.helpers :as hsql]
             [clojure.set :as set]
             [source.services.feed-categories :as feed-categories]
-            [source.db.bundle :as bundle]))
+            [source.db.bundle :as bundle]
+            [source.db.util :as db.util]))
 
 (defn get-bundle-categories
   "Get all categories for feeds/posts in bundle"
   [ds bundle-id]
-  (let [feed-ids (->> (hon/find ds {:tname (bundle/tname :outgoing-posts bundle-id)
-                                    :ret :*})
+  (let [feed-ids (->> (hon/find ds (db.util/tname :outgoing-posts bundle-id))
                       (mapv :feed-id))
         category-ids (->> (hon/find ds {:tname :feed-categories
                                         :where [:in :feed-id feed-ids]
@@ -22,8 +22,7 @@
 (defn get-outgoing-feeds
   "Gets a filtered list of outgoing feeds for the associated bundle."
   [ds {:keys [bundle-id type latest category-ids nonfiltered]}]
-  (let [feed-ids (mapv :feed-id (hon/find ds {:tname (bundle/tname :outgoing-posts bundle-id)
-                                              :ret :*}))
+  (let [feed-ids (mapv :feed-id (hon/find ds (db.util/tname :outgoing-posts bundle-id)))
         category-filtered-feed-ids (if (empty? category-ids)
                                      feed-ids
                                      (->> (hsql/where
@@ -65,8 +64,7 @@
                                                     (when (seq blocked-post-ids) [:not [:in :id blocked-post-ids]])
                                                     [:in :feed-id available-feed-ids])
                                         (assoc :order-by (when (= latest "true") [[[:posted-at :desc]]]))
-                                        (merge {:tname (bundle/tname :outgoing-posts bundle-id)
-                                                :ret :*})))
+                                        (merge (db.util/tname :outgoing-posts bundle-id))))
 
         categorised-posts (vec
                            (if (seq category-ids)
diff --git a/src/source/workers/integrations.clj b/src/source/workers/integrations.clj
index f7d3cc92..40b5e332 100644
--- a/src/source/workers/integrations.clj
+++ b/src/source/workers/integrations.clj
@@ -38,10 +38,10 @@
                    :where [:= :bundle-id bundle-id]})
   (hon/delete! ds {:tname :events
                    :where [:= :bundle-id bundle-id]})
-  (tables/drop-tables! ds (bundle/tnames [:outgoing-posts
-                                          :bundle-categories
-                                          :post-heuristics]
-                                         bundle-id))
+  (tables/drop-tables! ds (db.util/tnames [:outgoing-posts
+                                           :bundle-categories
+                                           :post-heuristics]
+                                          bundle-id))
   (hon/delete! ds {:tname :bundles
                    :where [:= :id bundle-id]})
   (congest/deregister! js job-id))

From 510c039db0a9da31d4d7ac67e7d0652dd2b696f1 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 9 Feb 2026 14:00:11 +0200
Subject: [PATCH 260/391] updated migrations to use refactored tname

---
 src/source/bundle_migrations/001_init_bundle_db.clj | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/source/bundle_migrations/001_init_bundle_db.clj b/src/source/bundle_migrations/001_init_bundle_db.clj
index 91f3e05b..b372c460 100644
--- a/src/source/bundle_migrations/001_init_bundle_db.clj
+++ b/src/source/bundle_migrations/001_init_bundle_db.clj
@@ -1,6 +1,7 @@
 (ns source.bundle-migrations.001-init-bundle-db
   (:require [source.db.bundle :as bundle]
-            [source.db.tables :as tables]))
+            [source.db.tables :as tables]
+            [source.db.util :as db.util]))
 
 (defn run-up! [context]
   (let [{:keys [ds-master bundle-id]} context
@@ -20,4 +21,4 @@
      (-> [:outgoing-posts
           :bundle-categories
           :post-heuristics]
-         (bundle/tnames bundle-id)))))
+         (db.util/tnames bundle-id)))))

From 3faad005a612d53b2d580a89b1227b8ad5fe7350 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 9 Feb 2026 14:01:20 +0200
Subject: [PATCH 261/391] removed unnecessary dependency

---
 src/source/bundle_migrations/001_init_bundle_db.clj | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/source/bundle_migrations/001_init_bundle_db.clj b/src/source/bundle_migrations/001_init_bundle_db.clj
index b372c460..80377dbd 100644
--- a/src/source/bundle_migrations/001_init_bundle_db.clj
+++ b/src/source/bundle_migrations/001_init_bundle_db.clj
@@ -1,5 +1,5 @@
 (ns source.bundle-migrations.001-init-bundle-db
-  (:require [source.db.bundle :as bundle]
+  (:require [source.db.bundle]
             [source.db.tables :as tables]
             [source.db.util :as db.util]))
 

From c6faf6012f7165407af5e327671a8d4011566b8f Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 9 Feb 2026 15:28:59 +0200
Subject: [PATCH 262/391] updated config to use database url instead of
 separate fields

---
 resources/config.edn   | 4 +---
 src/source/db/util.clj | 8 ++------
 2 files changed, 3 insertions(+), 9 deletions(-)

diff --git a/resources/config.edn b/resources/config.edn
index 0894fac2..6efcbc40 100644
--- a/resources/config.edn
+++ b/resources/config.edn
@@ -4,9 +4,7 @@
  :cors-origin #or [#env CORS_ORIGIN "http://localhost:8080"]
  :env #or [#env ENV "dev"]
  :database {:dir #or [#env DATABASE_DIR ".db/"]
-            :user #or [#env POSTGRES_USER "postgres"]
-            :password #env POSTGRES_PASSWORD
-            :host #or [#env POSTGRES_HOST "localhost"]
+            :url #or [#env DATABASE_URL "postgres://localhost:5432/"]
             :type "postgresql"}
  :email {:address #or [#env SUPPORT_ADDRESS "lee@wearesource.earth"]
          :username #env EMAIL_USERNAME
diff --git a/src/source/db/util.clj b/src/source/db/util.clj
index 96dea558..106eb70e 100644
--- a/src/source/db/util.clj
+++ b/src/source/db/util.clj
@@ -26,12 +26,8 @@
 
 (defn- -conn [dbname]
   (-> {:dbtype (conf/read-value :database :type)
-       :user (conf/read-value :database :user)
-       :password (conf/read-value :database :password)
-       :host (conf/read-value :database :host)
-       :maximum-pool-size 10
-       :port 5432}
-      (merge {:dbname (db-name dbname)})))
+       :jdbcUrl (str "jdbc:" (conf/read-value :database :url) "/" dbname)
+       :maximum-pool-size 10}))
 
 (defn conn
   ([]

From f5ae330cc080de0ccd1510e5bac8aaad259f6231 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 9 Feb 2026 17:45:36 +0200
Subject: [PATCH 263/391] updated config

---
 fly.dev.toml          | 1 -
 resources/config.edn  | 3 +--
 src/source/config.clj | 2 +-
 3 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/fly.dev.toml b/fly.dev.toml
index ea05eff3..78e4a46e 100644
--- a/fly.dev.toml
+++ b/fly.dev.toml
@@ -13,7 +13,6 @@ primary_region = 'jnb'
   ENV = 'prod'
   GOOGLE_CLIENT_ID = '449412212863-h1rre1gdmfgq96jt160fdd30bfatjiqb.apps.googleusercontent.com'
   GOOGLE_REDIRECT_URI = 'https://source-be-staging.fly.dev/oauth2/google/callback'
-  DATABASE_DIR = '/data'
   SUPPORT_ADDRESS="keaganncollins@gmail.com"
   EMAIL_USERNAME="merveillevaneck@gmail.com"
 
diff --git a/resources/config.edn b/resources/config.edn
index 6efcbc40..bebfa295 100644
--- a/resources/config.edn
+++ b/resources/config.edn
@@ -3,8 +3,7 @@
  :admins-encrypted-path "resources/admins_encrypted.json"
  :cors-origin #or [#env CORS_ORIGIN "http://localhost:8080"]
  :env #or [#env ENV "dev"]
- :database {:dir #or [#env DATABASE_DIR ".db/"]
-            :url #or [#env DATABASE_URL "postgres://localhost:5432/"]
+ :database {:url #or [#env DATABASE_URL "postgres://localhost:5432/"]
             :type "postgresql"}
  :email {:address #or [#env SUPPORT_ADDRESS "lee@wearesource.earth"]
          :username #env EMAIL_USERNAME
diff --git a/src/source/config.clj b/src/source/config.clj
index f6397c59..196a1793 100644
--- a/src/source/config.clj
+++ b/src/source/config.clj
@@ -24,7 +24,7 @@
    [:cors-origin :string]
    [:env :string]
    [:database [:map
-               [:dir :string]
+               [:url :string]
                [:type :string]]]
    [:oauth2 [:map-of keyword? oauth2-provider-schema]]])
 

From d30101f8465ca4382e16d5d1c7e6e7fede161969 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Tue, 10 Feb 2026 09:08:43 +0200
Subject: [PATCH 264/391] changed region from johannesburg to frankfurt

---
 fly.dev.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fly.dev.toml b/fly.dev.toml
index 78e4a46e..50b5c039 100644
--- a/fly.dev.toml
+++ b/fly.dev.toml
@@ -4,7 +4,7 @@
 #
 
 app = 'source-be-staging'
-primary_region = 'jnb'
+primary_region = 'fra'
 
 [build]
 

From 7b28ccd4fddf9ce1208093c55252c223ad2c429c Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Tue, 10 Feb 2026 10:39:32 +0200
Subject: [PATCH 265/391] updated region

---
 fly.dev.toml          | 5 +++--
 resources/config.edn  | 3 ++-
 src/source/config.clj | 1 +
 3 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/fly.dev.toml b/fly.dev.toml
index 50b5c039..78dc6fb9 100644
--- a/fly.dev.toml
+++ b/fly.dev.toml
@@ -11,10 +11,11 @@ primary_region = 'fra'
 [env]
   CORS_ORIGIN = 'https://source-staging.fly.dev'
   ENV = 'prod'
+  DATABASE_DIR = '/data'
   GOOGLE_CLIENT_ID = '449412212863-h1rre1gdmfgq96jt160fdd30bfatjiqb.apps.googleusercontent.com'
   GOOGLE_REDIRECT_URI = 'https://source-be-staging.fly.dev/oauth2/google/callback'
-  SUPPORT_ADDRESS="keaganncollins@gmail.com"
-  EMAIL_USERNAME="merveillevaneck@gmail.com"
+  SUPPORT_ADDRESS = 'keaganncollins@gmail.com'
+  EMAIL_USERNAME = 'merveillevaneck@gmail.com'
 
 [http_service]
   internal_port = 3000
diff --git a/resources/config.edn b/resources/config.edn
index bebfa295..6efcbc40 100644
--- a/resources/config.edn
+++ b/resources/config.edn
@@ -3,7 +3,8 @@
  :admins-encrypted-path "resources/admins_encrypted.json"
  :cors-origin #or [#env CORS_ORIGIN "http://localhost:8080"]
  :env #or [#env ENV "dev"]
- :database {:url #or [#env DATABASE_URL "postgres://localhost:5432/"]
+ :database {:dir #or [#env DATABASE_DIR ".db/"]
+            :url #or [#env DATABASE_URL "postgres://localhost:5432/"]
             :type "postgresql"}
  :email {:address #or [#env SUPPORT_ADDRESS "lee@wearesource.earth"]
          :username #env EMAIL_USERNAME
diff --git a/src/source/config.clj b/src/source/config.clj
index 196a1793..ed0a58a4 100644
--- a/src/source/config.clj
+++ b/src/source/config.clj
@@ -24,6 +24,7 @@
    [:cors-origin :string]
    [:env :string]
    [:database [:map
+               [:dir :string]
                [:url :string]
                [:type :string]]]
    [:oauth2 [:map-of keyword? oauth2-provider-schema]]])

From 3d2ebf2b7d40dfff8eaec880d8575826486b7ba9 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 11 Feb 2026 13:03:21 +0200
Subject: [PATCH 266/391] removed extra pair of brackets

---
 src/source/workers/bundles.clj | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/source/workers/bundles.clj b/src/source/workers/bundles.clj
index 254f954e..1b3b0571 100644
--- a/src/source/workers/bundles.clj
+++ b/src/source/workers/bundles.clj
@@ -70,7 +70,7 @@
           filtered-posts (hon/find bundle-ds (-> (hsql/where (when type [:= :content-type-id type])
                                                              [:not [:in :id blocked-post-ids]]
                                                              [:in :feed-id available-feed-ids])
-                                                 (hsql/order-by (when (= latest "true") [[:posted-at :desc]]))
+                                                 (hsql/order-by (when (= latest "true") [:posted-at :desc]))
                                                  (merge {:tname :outgoing-posts
                                                          :ret :*})))
 

From ca7c8ebfe220c55e7832d931f71ba8d451096874 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 11 Feb 2026 17:01:50 +0200
Subject: [PATCH 267/391] optimised bundle job handler, staggered job
 executions

---
 fly.dev.toml                            | 18 +++----
 resources/config.edn                    |  2 +-
 src/source/db/honey.clj                 | 23 +++++----
 src/source/db/util.clj                  | 15 ++----
 src/source/jobs/core.clj                | 21 ++++++---
 src/source/jobs/handlers.clj            | 62 ++++++++++++++-----------
 src/source/middleware/auth/core.clj     | 16 +++----
 src/source/migrate.clj                  | 17 ++-----
 src/source/routes/bundle_feed.clj       |  4 +-
 src/source/routes/bundle_feed_post.clj  |  7 +--
 src/source/routes/bundle_feed_posts.clj |  4 +-
 src/source/routes/bundle_feeds.clj      |  5 +-
 src/source/routes/bundle_post.clj       |  5 +-
 src/source/routes/bundle_posts.clj      |  4 +-
 src/source/routes/integrations.clj      | 17 +++++--
 src/source/services/bundles.clj         |  4 +-
 src/source/workers/bundles.clj          |  7 ++-
 src/source/workers/users.clj            |  8 ++--
 18 files changed, 126 insertions(+), 113 deletions(-)

diff --git a/fly.dev.toml b/fly.dev.toml
index 78dc6fb9..93667d56 100644
--- a/fly.dev.toml
+++ b/fly.dev.toml
@@ -1,4 +1,4 @@
-# fly.toml app configuration file generated for source-be-staging on 2025-07-08T14:36:46+02:00
+# fly.toml app configuration file generated for source-be-staging on 2026-02-10T10:41:37+02:00
 #
 # See https://fly.io/docs/reference/configuration/ for information about how to use this file.
 #
@@ -10,12 +10,18 @@ primary_region = 'fra'
 
 [env]
   CORS_ORIGIN = 'https://source-staging.fly.dev'
-  ENV = 'prod'
   DATABASE_DIR = '/data'
+  EMAIL_USERNAME = 'merveillevaneck@gmail.com'
+  ENV = 'prod'
   GOOGLE_CLIENT_ID = '449412212863-h1rre1gdmfgq96jt160fdd30bfatjiqb.apps.googleusercontent.com'
   GOOGLE_REDIRECT_URI = 'https://source-be-staging.fly.dev/oauth2/google/callback'
   SUPPORT_ADDRESS = 'keaganncollins@gmail.com'
-  EMAIL_USERNAME = 'merveillevaneck@gmail.com'
+
+[[mounts]]
+  source = 'source_storage_staging'
+  destination = '/data'
+  initial_size = '50gb'
+  processes = ['app']
 
 [http_service]
   internal_port = 3000
@@ -28,9 +34,3 @@ primary_region = 'fra'
   memory = '2gb'
   cpu_kind = 'shared'
   cpus = 2
-
-[mounts]
-  source = 'source_storage_staging'
-  destination = '/data'
-  processes = ['app']
-  initial_size = '50gb'
diff --git a/resources/config.edn b/resources/config.edn
index 6efcbc40..fe166c79 100644
--- a/resources/config.edn
+++ b/resources/config.edn
@@ -4,7 +4,7 @@
  :cors-origin #or [#env CORS_ORIGIN "http://localhost:8080"]
  :env #or [#env ENV "dev"]
  :database {:dir #or [#env DATABASE_DIR ".db/"]
-            :url #or [#env DATABASE_URL "postgres://localhost:5432/"]
+            :url #or [#env DATABASE_URL "postgres://localhost"]
             :type "postgresql"}
  :email {:address #or [#env SUPPORT_ADDRESS "lee@wearesource.earth"]
          :username #env EMAIL_USERNAME
diff --git a/src/source/db/honey.clj b/src/source/db/honey.clj
index b9137dd7..b863c80f 100644
--- a/src/source/db/honey.clj
+++ b/src/source/db/honey.clj
@@ -12,18 +12,17 @@
   or select all. returns results as unqualified lower maps by default."
   [ds sqlmap & {:keys [ret exec-opts]}]
   (assert (and (some? ds) (some? sqlmap) (or (some? ret) (nil? ret))))
-  (with-open [conn (db.util/get-connection ds)]
-    (let [ps (sql/format sqlmap)
-          exec-opts' (merge
-                      {:builder-fn rs/as-unqualified-lower-maps}
-                      exec-opts)
-          result (cske/transform-keys
-                  csk/->kebab-case-keyword
-                  (jdbc/execute! conn ps exec-opts'))]
-      (cond
-        (= ret :1) (first result)
-        (= ret :*) result
-        :else nil))))
+  (let [ps (sql/format sqlmap)
+        exec-opts' (merge
+                    {:builder-fn rs/as-unqualified-lower-maps}
+                    exec-opts)
+        result (cske/transform-keys
+                csk/->kebab-case-keyword
+                (jdbc/execute! ds ps exec-opts'))]
+    (cond
+      (= ret :1) (first result)
+      (= ret :*) result
+      :else nil)))
 
 (defn find
   "does find one or find all for a given table name and where clause. The where
diff --git a/src/source/db/util.clj b/src/source/db/util.clj
index 106eb70e..39c43ac7 100644
--- a/src/source/db/util.clj
+++ b/src/source/db/util.clj
@@ -26,18 +26,14 @@
 
 (defn- -conn [dbname]
   (-> {:dbtype (conf/read-value :database :type)
-       :jdbcUrl (str "jdbc:" (conf/read-value :database :url) "/" dbname)
-       :maximum-pool-size 10}))
+       :jdbcUrl (str "jdbc:" (conf/read-value :database :url) ":5432/" dbname)}))
 
 (defn conn
   ([]
    (conn :master))
   ([db-type]
-   (assert (= db-type :master))
-   (-conn (db-name db-type)))
-  ([db-type id]
-   (assert (or (= db-type :bundle) (= db-type :creator)))
-   (-conn (db-name db-type id))))
+   (assert (or (= db-type :master) (= db-type :migrate)))
+   (-conn (db-name db-type))))
 
 (defn tname
   ([tname id]
@@ -50,8 +46,3 @@
 
 (defn tnames [tnames id]
   (mapv #(tname % id) tnames))
-
-(comment
-  (def ds (conn :bundle 1))
-  (jdbc/execute! ds ["DELETE FROM providers;"])
-  ())
diff --git a/src/source/jobs/core.clj b/src/source/jobs/core.clj
index 0b25c1d4..4e25bfc2 100644
--- a/src/source/jobs/core.clj
+++ b/src/source/jobs/core.clj
@@ -40,13 +40,20 @@
   "Get congest-ready metadata of all jobs marked as running"
   [ds store]
   (let [jobs (services/jobs ds)]
-    (reduce (fn [acc {:keys [job-id job-metadata-id args handler status]}]
-              (when (= status "running")
-                (let [metadata (-> (services/job-metadata ds {:id job-metadata-id})
-                                   (assoc :id job-id
-                                          :args args
-                                          :handler handler))]
-                  (conj acc (prepare-congest-metadata ds store metadata))))) [] jobs)))
+    (mapv (fn [{:keys [job-id job-metadata-id args handler status]} i]
+            (when (= status "running")
+              (let [{:keys [initial-delay
+                            interval]
+                     :as m} (-> (services/job-metadata ds {:id job-metadata-id})
+                                (assoc :id job-id
+                                       :args args
+                                       :handler handler))
+                    metadata (assoc m
+                                    :initial-delay (+ initial-delay (* 1000 5 i))
+                                    :interval (+ interval (* 1000 5 i)))]
+                (prepare-congest-metadata ds store metadata))))
+          jobs
+          (-> jobs count inc range))))
 
 (comment
   (require '[source.db.util :as db.util]
diff --git a/src/source/jobs/handlers.clj b/src/source/jobs/handlers.clj
index 40a96ce7..de19bccc 100644
--- a/src/source/jobs/handlers.clj
+++ b/src/source/jobs/handlers.clj
@@ -6,8 +6,7 @@
             [source.db.util :as db.util]
             [clojure.set :as set]
             [clojure.string :as string]
-            [source.db.honey :as hon]
-            [source.db.bundle :as bundle]))
+            [source.db.honey :as hon]))
 
 (defmulti handler
   (fn [opts]
@@ -31,6 +30,7 @@
     (try
       (when (users/removed? ds (:creator-id args))
         (let [{:keys [feed-id creator-id content-type-id provider-id url]} args
+              _ (println "feed" feed-id "job started")
               selection-schemas (->> [:= :provider-id provider-id]
                                      (assoc {} :where)
                                      (services/selection-schemas ds))
@@ -74,41 +74,49 @@
                                 :data post})
                (hon/insert! ds {:tname :incoming-posts
                                 :data post})))
-           extended-posts)))
+           extended-posts)
+          (println "feed" feed-id "job finished")))
 
-      (catch Exception _ :fail))))
+      (catch Exception e (println "feed job failed: " e) :fail))))
 
 (defn update-bundle-job-id
   "returns the job id of an update-bundle job with the given bundle id"
   [bundle-id]
   (str "bundle_" bundle-id))
 
+(defn determine-post-score [post posts-categories categories]
+  ; calculate score for post
+  ; determine number of categories matched
+  ; get vector of category ids in the given post, e.g. [1 3]
+  (let [post-categories-vec (->> posts-categories
+                                 (mapv (fn [{:keys [post-id id]}]
+                                         (when (= post-id (:id post)) id)))
+                                 (filterv identity))
+        ; get vector of category ids in categories to match, e.g. [1 2 3 4]
+        match-categories-vec (reduce (fn [acc {:keys [id]}]
+                                       (conj acc id)) [] categories)
+        ; get number of matches between the 2 vectors, e.g. #{1 3} intersect #{1 2 3 4} -> (count #{1 3}) -> 2
+        matches (count (set/intersection (set post-categories-vec)
+                                         (set match-categories-vec)))]
+    ; use matches as a score to upsert long-heuristic for this post
+    {:post-id (:id post)
+     :long-heuristic matches}))
+
 ; run long heuristics and pull the highest scoring incoming posts into the bundle's outgoing posts
 (defmethod handler :update-bundle [_]
   (fn [{:keys [args ds]}]
     (let [{:keys [bundle-id categories]} args
+          _ (println "starting bundle" bundle-id "job")
           incoming-posts (services/incoming-posts-with-feeds ds {:where [:= :feeds.state "live"]})
-          posts-categories (incoming-posts/categories-by-posts ds {:where [:= :state "live"]})]
-      (run!
-       (fn [post]
-          ; calculate score for post
-          ; determine number of categories matched
-              ; get vector of category ids in the given post, e.g. [1 3]
-         (let [post-categories-vec (->> posts-categories
-                                        (mapv (fn [{:keys [post-id id]}]
-                                                (when (= post-id (:id post)) id)))
-                                        (filterv identity))
-               ; get vector of category ids in categories to match, e.g. [1 2 3 4]
-               match-categories-vec (reduce (fn [acc {:keys [id]}]
-                                              (conj acc id)) [] categories)
-               ; get number of matches between the 2 vectors, e.g. #{1 3} intersect #{1 2 3 4} -> (count #{1 3}) -> 2
-               matches (count (set/intersection (set post-categories-vec)
-                                                (set match-categories-vec)))]
-            ; use matches as a score and upsert long-heuristic for this post
-           (services/upsert-post-heuristics! ds {:bundle-id bundle-id
-                                                 :data [{:post-id (:id post)
-                                                         :long-heuristic matches}]})))
-       incoming-posts)
+          posts-categories (incoming-posts/categories-by-posts ds {:where [:= :state "live"]})
+          heuristics (mapv
+                      #(determine-post-score % posts-categories categories)
+                      incoming-posts)]
+      ; use precalculated heuristics and insert this data to the database
+      (try
+        (services/upsert-post-heuristics! ds {:bundle-id bundle-id
+                                              :data heuristics})
+        (catch Exception e (println "bundle" bundle-id "upserting post heuristics failed: " (.getMessage e))))
 
       ; pull highest scored posts by long heuristics into outgoing posts
             ; top 1000 post-heuristics records ordered by long heuristic in descending order
@@ -139,7 +147,9 @@
         (when (seq posts-in)
           (hon/delete! ds (db.util/tname :outgoing-posts bundle-id))
           (hon/insert! ds (-> (db.util/tname :outgoing-posts bundle-id)
-                              (assoc :data outgoing-posts))))))))
+                              (assoc :data outgoing-posts))))
+
+        (println "bundle" bundle-id "job done")))))
 
 (defn user-deletion-job-id
   "returns the job id of a user deletion job with the given user id"
diff --git a/src/source/middleware/auth/core.clj b/src/source/middleware/auth/core.clj
index b49b41f2..1519e6ab 100644
--- a/src/source/middleware/auth/core.clj
+++ b/src/source/middleware/auth/core.clj
@@ -33,7 +33,7 @@
     (let [ds (db.util/conn :master)
           user-type (get-in request [:user :type])
           expected-type (->> {:tname :users
-                              :where [:= :id (get-in request [:user :id])]}
+                              :id (get-in request [:user :id])}
                              (db/find-one ds)
                              (:type))]
       (cond
@@ -50,11 +50,9 @@
   (fn [request]
     (let [ds (db.util/conn :master)
           bundle-uuid (get-in request [:query-params :uuid])
-          {:keys [id user-id]} (db/find-one ds {:tname :bundles
-                                                :where [:= :uuid bundle-uuid]})
-          {:keys [removed]} (db/find-one ds {:tname :users
-                                             :where [:= :id user-id]})]
-      (if (and (some? id) (= removed 0))
+          {:keys [id]} (db/find-one ds {:tname :bundles
+                                        :where [:= :uuid bundle-uuid]})]
+      (if (some? id)
         (-> request
             (assoc :bundle-id id)
             (handler))
@@ -69,10 +67,8 @@
   (fn [request]
     (let [ds (db.util/conn :master)
           token (util/auth-token request)
-          {:keys [user-id] :as existing-bundle} (bundles/bundle ds {:where [:= :hash token]})
-          {:keys [removed]} (db/find-one ds {:tname :users
-                                             :where [:= :id user-id]})]
-      (if (and (some? existing-bundle) (= removed 0))
+          existing-bundle (bundles/bundle ds {:where [:= :hash token]})]
+      (if (some? existing-bundle)
         (-> request
             (assoc :bundle-id (:id existing-bundle))
             (handler))
diff --git a/src/source/migrate.clj b/src/source/migrate.clj
index 2313ab29..3faa2f8b 100644
--- a/src/source/migrate.clj
+++ b/src/source/migrate.clj
@@ -5,8 +5,7 @@
             [next.jdbc :as jdbc]
             [source.db.util :as db.util]
             [source.db.honey :as db]
-            [source.db.tables :as tables]
-            [source.config :as conf]))
+            [source.db.tables :as tables]))
 
 ;; This is our interface for running migrations.
 ;;
@@ -16,11 +15,6 @@
 ;; - seed the appropriate tables with data.
 ;; - (TODO) generate malli schemas to match the affected db schemas 
 
-(def ^:private postgres-ds
-  {:user (conf/read-value :database :user)
-   :password (conf/read-value :database :password)
-   :dbtype (conf/read-value :database :type)})
-
 (def ^:private migrations
   (loader.fs/load! "src/source/migrations"))
 
@@ -28,10 +22,8 @@
   (loader.fs/load! "src/source/bundle_migrations"))
 
 (defn run-migrations [args]
-  (let [context {:db-master (jdbc/get-datasource (-> {:dbname (db.util/db-name "master")}
-                                                     (merge postgres-ds)))}
-        db-migrate (jdbc/get-datasource (-> {:dbname (db.util/db-name "migrate")}
-                                            (merge postgres-ds)))
+  (let [context {:db-master (jdbc/get-datasource (db.util/conn))}
+        db-migrate (jdbc/get-datasource (db.util/conn :migrate))
         datastore (store/create-datastore
                    {:ds db-migrate
                     :table-name "migrations"})]
@@ -42,8 +34,7 @@
                  args)))
 
 (defn migrate-bundle [bundle-id args]
-  (let [context {:ds-master (jdbc/get-datasource (-> {:dbname (db.util/db-name "master")}
-                                                     (merge postgres-ds)))
+  (let [context {:ds-master (jdbc/get-datasource (db.util/conn))
                  :bundle-id bundle-id}
         datastore (store/create-datastore
                    {:ds (:ds-master context)
diff --git a/src/source/routes/bundle_feed.clj b/src/source/routes/bundle_feed.clj
index 23233eff..d499857b 100644
--- a/src/source/routes/bundle_feed.clj
+++ b/src/source/routes/bundle_feed.clj
@@ -28,5 +28,7 @@
   [{:keys [ds bundle-id path-params] :as _request}]
   (let [feed (hon/find-one ds {:tname :feeds
                                :where [:= :id (:id path-params)]})]
-    (analytics/insert-feed-click! ds feed bundle-id)
+    (try
+      (analytics/insert-feed-click! ds feed bundle-id)
+      (catch Exception e (println (.getMessage e))))
     (res/response feed)))
diff --git a/src/source/routes/bundle_feed_post.clj b/src/source/routes/bundle_feed_post.clj
index bdd234a1..5c2fdb91 100644
--- a/src/source/routes/bundle_feed_post.clj
+++ b/src/source/routes/bundle_feed_post.clj
@@ -1,8 +1,7 @@
 (ns source.routes.bundle-feed-post
   (:require [ring.util.response :as res]
             [source.db.honey :as hon]
-            [source.services.analytics.interface :as analytics]
-            [source.db.util :as db.util]))
+            [source.services.analytics.interface :as analytics]))
 
 (defn get
   {:summary "get post in outgoing feed for the associated uuid-authorized bundle by post id, updates click analytics"
@@ -31,5 +30,7 @@
   (let [post (hon/find-one ds {:tname :incoming-posts
                                :where [:= :id (:post-id path-params)]
                                :ret :1})]
-    (analytics/insert-post-click! ds post bundle-id)
+    (try
+      (analytics/insert-post-click! ds post bundle-id)
+      (catch Exception e (println (.getMessage e))))
     (res/response post)))
diff --git a/src/source/routes/bundle_feed_posts.clj b/src/source/routes/bundle_feed_posts.clj
index cae4d957..49f468e2 100644
--- a/src/source/routes/bundle_feed_posts.clj
+++ b/src/source/routes/bundle_feed_posts.clj
@@ -31,5 +31,7 @@
   (let [posts (hon/find ds {:tname :incoming-posts
                             :where [:= :feed-id (:id path-params)]
                             :ret :*})]
-    (analytics/insert-post-impressions! ds posts bundle-id)
+    (try
+      (analytics/insert-post-impressions! ds posts bundle-id)
+      (catch Exception e (println (.getMessage e))))
     (res/response posts)))
diff --git a/src/source/routes/bundle_feeds.clj b/src/source/routes/bundle_feeds.clj
index 3e0aba00..b0762dff 100644
--- a/src/source/routes/bundle_feeds.clj
+++ b/src/source/routes/bundle_feeds.clj
@@ -1,7 +1,6 @@
 (ns source.routes.bundle-feeds
   (:require [ring.util.response :as res]
             [source.db.util :as db.util]
-            [clojure.walk :as walk]
             [source.workers.bundles :as bundles]
             [source.services.analytics.interface :as analytics]))
 
@@ -39,7 +38,9 @@
                     :category-ids (:category-ids body)
                     :nonfiltered nonfiltered}
                    (bundles/get-outgoing-feeds ds))]
-    (analytics/insert-feed-impressions! ds feeds bundle-id)
+    (try
+      (analytics/insert-feed-impressions! ds feeds bundle-id)
+      (catch Exception e (println (.getMessage e))))
     (res/response feeds)))
 
 (comment
diff --git a/src/source/routes/bundle_post.clj b/src/source/routes/bundle_post.clj
index 44e720a2..ab789d58 100644
--- a/src/source/routes/bundle_post.clj
+++ b/src/source/routes/bundle_post.clj
@@ -2,7 +2,6 @@
   (:require [ring.util.response :as res]
             [source.services.analytics.interface :as analytics]
             [source.db.honey :as hon]
-            [source.db.bundle :as bundle]
             [source.db.util :as db.util]
             [honey.sql.helpers :as hsql]))
 
@@ -31,5 +30,7 @@
   [{:keys [ds bundle-id path-params] :as _request}]
   (let [post (hon/find-one ds (-> (db.util/tname :outgoing-posts bundle-id)
                                   (hsql/where [:= :id (:id path-params)])))]
-    (analytics/insert-post-click! ds post bundle-id)
+    (try
+      (analytics/insert-post-click! ds post bundle-id)
+      (catch Exception e (println (.getMessage e))))
     (res/response post)))
diff --git a/src/source/routes/bundle_posts.clj b/src/source/routes/bundle_posts.clj
index b4b8c5c6..09002faa 100644
--- a/src/source/routes/bundle_posts.clj
+++ b/src/source/routes/bundle_posts.clj
@@ -40,5 +40,7 @@
                     :latest latest
                     :category-ids (:category-ids body)}
                    (bundles/get-outgoing-posts ds))]
-    (analytics/insert-post-impressions! ds posts bundle-id)
+    (try
+      (analytics/insert-post-impressions! ds posts bundle-id)
+      (catch Exception e (println (.getMessage e))))
     (res/response posts)))
diff --git a/src/source/routes/integrations.clj b/src/source/routes/integrations.clj
index b64008be..5d485fef 100644
--- a/src/source/routes/integrations.clj
+++ b/src/source/routes/integrations.clj
@@ -5,7 +5,9 @@
             [source.util :as util]
             [source.jobs.core :as jobs]
             [source.jobs.handlers :as handlers]
-            [congest.jobs :as congest]))
+            [congest.jobs :as congest]
+            [source.db.honey :as hon]
+            [source.db.util :as db.util]))
 
 (defn get
   {:summary "get all integrations"
@@ -60,7 +62,17 @@
                          :categories (:categories body)
                          :content-types (:content-types body)}
                         (integrations/create-integration! ds))
-        categories-by-bundle (bundles/categories-in-bundle ds (:id new-bundle))]
+        categories-by-bundle (bundles/categories-in-bundle ds (:id new-bundle))
+
+        ; TEMPORARY BLOCK
+        ; TODO: delete this and fix the problem in jobs that causes outgoing posts to be empty
+       ;posts (->> (hon/find ds {:tname :incoming-posts
+       ;                         :limit 2000})
+       ;           (mapv #(dissoc % :redacted)))
+       ;_ (hon/insert! ds (-> (db.util/tname :outgoing-posts (:id new-bundle))
+       ;                      (assoc :data posts)))
+        ; END OF TEMPORARY BLOCK
+        ]
     ;TODO: service needed
     (->> (jobs/prepare-congest-metadata
           ds
@@ -71,7 +83,6 @@
            :stop-after-fail false,
            :interval (* 1000 60 60 24)
            :recurring? true
-           :ds ds
            :args {:bundle-id (:id new-bundle)
                   :categories categories-by-bundle}
            :handler :update-bundle
diff --git a/src/source/services/bundles.clj b/src/source/services/bundles.clj
index f234f4ae..0180dec4 100644
--- a/src/source/services/bundles.clj
+++ b/src/source/services/bundles.clj
@@ -45,8 +45,8 @@
 
 ;;NEW
 (defn categories-in-bundle [ds bundle-id]
-  (let [category-ids (db/find-one ds (-> (db.util/tname :bundle-categories bundle-id)
-                                         (hsql/where [:= :bundle-id bundle-id])))
+  (let [category-ids (db/find ds (-> (db.util/tname :bundle-categories bundle-id)
+                                     (hsql/where [:= :bundle-id bundle-id])))
         id-vec (mapv (fn [{:keys [category-id]}] category-id) category-ids)]
     (db/find ds {:tname :categories
                  :where [:in :id id-vec]})))
diff --git a/src/source/workers/bundles.clj b/src/source/workers/bundles.clj
index 9fa441d4..ce3b37f3 100644
--- a/src/source/workers/bundles.clj
+++ b/src/source/workers/bundles.clj
@@ -3,7 +3,6 @@
             [honey.sql.helpers :as hsql]
             [clojure.set :as set]
             [source.services.feed-categories :as feed-categories]
-            [source.db.bundle :as bundle]
             [source.db.util :as db.util]))
 
 (defn get-bundle-categories
@@ -12,11 +11,11 @@
   (let [feed-ids (->> (hon/find ds (db.util/tname :outgoing-posts bundle-id))
                       (mapv :feed-id))
         category-ids (->> (hon/find ds {:tname :feed-categories
-                                        :where [:in :feed-id feed-ids]
+                                        :where (when (seq feed-ids) [:in :feed-id feed-ids])
                                         :ret :*})
                           (mapv :category-id))]
     (hon/find ds {:tname :categories
-                  :where [:in :id category-ids]
+                  :where (when (seq category-ids) [:in :id category-ids])
                   :ret :*})))
 
 (defn get-outgoing-feeds
@@ -63,7 +62,7 @@
         filtered-posts (hon/find ds (-> (hsql/where (when type [:= :content-type-id type])
                                                     (when (seq blocked-post-ids) [:not [:in :id blocked-post-ids]])
                                                     [:in :feed-id available-feed-ids])
-                                        (assoc :order-by (when (= latest "true") [[[:posted-at :desc]]]))
+                                        (assoc :order-by (when (= latest "true") [[:posted-at :desc]]))
                                         (merge (db.util/tname :outgoing-posts bundle-id))))
 
         categorised-posts (vec
diff --git a/src/source/workers/users.clj b/src/source/workers/users.clj
index 83a8cd0e..206774aa 100644
--- a/src/source/workers/users.clj
+++ b/src/source/workers/users.clj
@@ -44,7 +44,7 @@
                    :data {:removed false}}))
 
 (defn removed? [ds user-id]
-  (-> (hon/find ds {:tname :users
-                    :where [:= :id user-id]})
-      (:removed)
-      (= 0)))
+  (let [removed? (-> (hon/find ds {:tname :users
+                                   :where [:= :id user-id]})
+                     (:removed))]
+    (when (or (nil? removed?) (= removed? 0)) true)))

From c66e97688f80f379c0cdadf727121b0a8efda62e Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Thu, 12 Feb 2026 11:31:54 +0200
Subject: [PATCH 268/391] added used categories endpoint for getting all
 categories for which feeds exist

---
 src/source/routes/categories.clj  | 13 ++++++++++++-
 src/source/routes/reitit.clj      |  9 +++++----
 src/source/workers/categories.clj | 18 ++++++++++++++++--
 3 files changed, 33 insertions(+), 7 deletions(-)

diff --git a/src/source/routes/categories.clj b/src/source/routes/categories.clj
index d77c1ac9..2f83e7a4 100644
--- a/src/source/routes/categories.clj
+++ b/src/source/routes/categories.clj
@@ -1,6 +1,7 @@
 (ns source.routes.categories
   (:require [ring.util.response :as res]
-            [source.db.honey :as hon]))
+            [source.db.honey :as hon]
+            [source.workers.categories :as categories]))
 
 (defn get
   {:summary "get all categories"
@@ -12,3 +13,13 @@
   [{:keys [ds] :as _request}]
   (->> (hon/find ds {:tname :categories})
        (res/response)))
+
+(defn used-categories
+  {:summary "get all categories for which feeds and posts exist in the system"
+   :responses {200 {:body [:vector
+                           [:map
+                            [:id :int]
+                            [:name :string]
+                            [:display-picture {:optional true} [:maybe :string]]]]}}}
+  [{:keys [ds] :as _request}]
+  (res/response (categories/used-categories ds)))
diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index 49fff57c..2f15fb9b 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -157,8 +157,9 @@
       ["" (get cadences/get)]]
 
      ["/categories" {:tags #{"categories"}}
-      ["" (get categories/get)]
-      ["/:id" (get category/get)]]
+      ["/all" (get categories/get)]
+      ["/all/:id" (get category/get)]
+      ["/used" (get categories/used-categories)]]
 
      ["/baselines" {:tags #{"baselines"}}
       ["" (get baselines/get)]]
@@ -170,8 +171,8 @@
      ["/integrations" {:middleware [[mw/apply-auth]]
                        :tags #{"integrations"}}
 
-      ["" (->  (get integrations/get)
-               (post integrations/post))]
+      ["" (-> (get integrations/get)
+              (post integrations/post))]
       ["/:id" (-> (get integration/get)
                   (post integration/post)
                   (delete integration/delete))]
diff --git a/src/source/workers/categories.clj b/src/source/workers/categories.clj
index e4cd7905..0fb886f3 100644
--- a/src/source/workers/categories.clj
+++ b/src/source/workers/categories.clj
@@ -1,4 +1,18 @@
 (ns source.workers.categories
-  (:require [source.workers.schemas :as schemas]))
+  (:require [source.db.honey :as hon]))
 
-;; Service schemas
+(defn used-categories
+  "returns all categories for which feeds exist."
+  [ds]
+  (let [category-ids (->> (hon/find ds {:tname :feed-categories})
+                          (mapv :category-id))]
+    (hon/find ds {:tname :categories
+                  :where (if (seq category-ids)
+                           [:in :id category-ids]
+                           [:= :id -1])})))
+
+(comment
+  (require '[source.db.util :as db.util])
+
+  (used-categories (db.util/conn))
+  ())

From 97024519b18234db3ab0d08b2398a7e6ad785199 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Thu, 12 Feb 2026 11:32:53 +0200
Subject: [PATCH 269/391] fixed regression bug in bundle content types
 migration and readded post shuffling

---
 src/source/migrations/003_bundle_content_types.clj | 2 +-
 src/source/workers/bundles.clj                     | 9 ++++++---
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/src/source/migrations/003_bundle_content_types.clj b/src/source/migrations/003_bundle_content_types.clj
index 0ac8785f..cc798ee0 100644
--- a/src/source/migrations/003_bundle_content_types.clj
+++ b/src/source/migrations/003_bundle_content_types.clj
@@ -15,7 +15,7 @@
     (run! (fn [{:keys [id content-type-id]}]
             (when (some? content-type-id)
               (services/insert-bundle-content-types! ds-master {:data {:bundle-id id
-                                                                       :content-type-id content-type-id}})))
+                                                                       :content-types [{:id content-type-id}]}})))
           bundles)))
 
 (defn run-down! [context]
diff --git a/src/source/workers/bundles.clj b/src/source/workers/bundles.clj
index ce3b37f3..1865bf7c 100644
--- a/src/source/workers/bundles.clj
+++ b/src/source/workers/bundles.clj
@@ -3,7 +3,8 @@
             [honey.sql.helpers :as hsql]
             [clojure.set :as set]
             [source.services.feed-categories :as feed-categories]
-            [source.db.util :as db.util]))
+            [source.db.util :as db.util]
+            [honey.sql :as sql]))
 
 (defn get-bundle-categories
   "Get all categories for feeds/posts in bundle"
@@ -65,9 +66,11 @@
                                         (assoc :order-by (when (= latest "true") [[:posted-at :desc]]))
                                         (merge (db.util/tname :outgoing-posts bundle-id))))
 
+        shuffled (shuffle filtered-posts)
+
         categorised-posts (vec
                            (if (seq category-ids)
-                             (->> filtered-posts
+                             (->> shuffled
                                   (mapv
                                    (fn [post]
                                      (when (seq (set/intersection
@@ -78,7 +81,7 @@
                                                       (set))))
                                        post)))
                                   (remove nil?))
-                             filtered-posts))
+                             shuffled))
 
         valid-start? (and (some? start) (>= start 0) (< start (count categorised-posts)))
         started-posts (if valid-start?

From 003ab2ea56f08ef47b2ff9535c04738469ee9ccd Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Thu, 12 Feb 2026 12:46:16 +0200
Subject: [PATCH 270/391] removed services and middleware for api keys,
 including the endpoints that use them

---
 src/source/middleware/auth/core.clj   | 15 ---------------
 src/source/middleware/core.clj        |  4 ----
 src/source/middleware/interface.clj   |  3 ---
 src/source/routes/integration_key.clj | 16 ----------------
 src/source/routes/reitit.clj          | 27 ++++-----------------------
 src/source/workers/integrations.clj   | 12 +-----------
 6 files changed, 5 insertions(+), 72 deletions(-)
 delete mode 100644 src/source/routes/integration_key.clj

diff --git a/src/source/middleware/auth/core.clj b/src/source/middleware/auth/core.clj
index 1519e6ab..0441adcd 100644
--- a/src/source/middleware/auth/core.clj
+++ b/src/source/middleware/auth/core.clj
@@ -60,21 +60,6 @@
          (res/response {:message "The bundle you are looking for does not exist."})
          (res/status 404))))))
 
-(defn wrap-auth-api-key
-  "validates the api key from the Authorization header for unauthenticated 
-  users and attaches the bundle-id to the request"
-  [handler]
-  (fn [request]
-    (let [ds (db.util/conn :master)
-          token (util/auth-token request)
-          existing-bundle (bundles/bundle ds {:where [:= :hash token]})]
-      (if (some? existing-bundle)
-        (-> request
-            (assoc :bundle-id (:id existing-bundle))
-            (handler))
-        (-> (res/response {:message "The bundle you are looking for does not exist."})
-            (res/status 404))))))
-
 (comment
   (let [authed-request {:headers {"Authorization"
                                   (str
diff --git a/src/source/middleware/core.clj b/src/source/middleware/core.clj
index 6c4f7fdc..bd067200 100644
--- a/src/source/middleware/core.clj
+++ b/src/source/middleware/core.clj
@@ -137,7 +137,3 @@
 (defn apply-bundle [app]
   (-> app
       (auth/wrap-bundle-id)))
-
-(defn apply-api-key [app]
-  (-> app
-      (auth/wrap-auth-api-key)))
diff --git a/src/source/middleware/interface.clj b/src/source/middleware/interface.clj
index 636aaa27..ca5bb98d 100644
--- a/src/source/middleware/interface.clj
+++ b/src/source/middleware/interface.clj
@@ -12,8 +12,5 @@
 (defn apply-bundle [app]
   (mw/apply-bundle app))
 
-(defn apply-api-key [app]
-  (mw/apply-api-key app))
-
 (defn apply-validation [app openapi-meta]
   (mw/wrap-input-validation app openapi-meta))
diff --git a/src/source/routes/integration_key.clj b/src/source/routes/integration_key.clj
deleted file mode 100644
index 6521eaaf..00000000
--- a/src/source/routes/integration_key.clj
+++ /dev/null
@@ -1,16 +0,0 @@
-(ns source.routes.integration-key
-  (:require [ring.util.response :as res]
-            [source.workers.integrations :as integrations]))
-
-(defn post
-  {:summary "generate an API key for the integration with the given id"
-   :parameters {:path [:map [:id {:title "id"
-                                  :description "integration id"} :int]]}
-   :responses {200 {:body [:map [:key :string]]}
-               401 {:body [:map [:message :string]]}
-               403 {:body [:map [:message :string]]}}}
-
-  [{:keys [ds user path-params] :as _request}]
-  (->> (integrations/generate-api-key! ds (:id user) (:id path-params))
-       (assoc {} :key)
-       (res/response)))
diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index 49fff57c..4f6c2e8c 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -47,7 +47,6 @@
             [source.routes.analytics.bundle.posts.-id-.views :as analytics-bundle-posts-id-views]
             [source.routes.integrations :as integrations]
             [source.routes.integration :as integration]
-            [source.routes.integration-key :as integration-key]
             [source.routes.integration-categories :as integration-categories]
             [source.routes.integration-filter-feeds :as integration-filter-feeds]
             [source.routes.integration-filter-feed :as integration-filter-feed]
@@ -175,12 +174,11 @@
       ["/:id" (-> (get integration/get)
                   (post integration/post)
                   (delete integration/delete))]
-      ["/:id/key" (post integration-key/post)]
-      ["/:id/categories" (->  (get integration-categories/get)
-                              (post integration-categories/post))]
+      ["/:id/categories" (-> (get integration-categories/get)
+                             (post integration-categories/post))]
       ["/:id/filter/feeds" (get integration-filter-feeds/get)]
-      ["/:id/filter/feeds/:feed-id" (->  (get integration-filter-feed/get)
-                                         (post integration-filter-feed/post))]
+      ["/:id/filter/feeds/:feed-id" (-> (get integration-filter-feed/get)
+                                        (post integration-filter-feed/post))]
       ["/:id/filter/posts" (get integration-filter-posts/get)]
       ["/:id/filter/posts/:post-id" (-> (get integration-filter-post/get)
                                         (post integration-filter-post/post))]]
@@ -227,23 +225,6 @@
       ["/posts" (post bundle-posts/post)]
       ["/posts/:id" (get bundle-post/get)]]
 
-     ["/api" {:middleware [[mw/apply-api-key]]
-              :tags #{"api"}
-              :swagger {:security [{"apiKey" []}]}
-              :openapi {:security [{:apiKey []}]}}
-
-      ["/bundle" {:middleware [[mw/apply-bundle]]}
-       ["" (get bundle/get)]
-       ["/categories" (get bundle-categories/get)]
-       ["/feeds" (post bundle-feeds/post)]
-       ["/feeds/:id" (get bundle-feed/get)]
-       ["/feeds/:id/posts" (get bundle-feed-posts/get)]
-       ["/feeds/:id/posts/:post-id" (get bundle-feed-post/get)]
-       ["/posts" (post bundle-posts/post)]
-       ["/posts/:id" (get bundle-post/get)]]
-      ["/categories" (get categories/get)]
-      ["/categories/:id" (get category/get)]]
-
      ["/admin" {:middleware [[mw/apply-auth {:required-type :admin}]]
                 :tags #{"admin"}
                 :swagger {:security [{"auth" []}]}
diff --git a/src/source/workers/integrations.clj b/src/source/workers/integrations.clj
index 40b5e332..760778ee 100644
--- a/src/source/workers/integrations.clj
+++ b/src/source/workers/integrations.clj
@@ -4,11 +4,9 @@
             [source.db.util :as db.util]
             [source.services.bundle-categories :as bundle-categories]
             [source.services.bundle-content-types :as bundle-content-types]
-            [source.util :as utils]
             [source.db.tables :as tables]
             [source.db.honey :as hon]
-            [congest.jobs :as congest]
-            [source.db.bundle :as bundle]))
+            [congest.jobs :as congest]))
 
 (defn create-integration! [ds {:keys [user-id bundle-metadata categories content-types]}]
   (let [new-bundle (bundles/create-bundle! ds {:user-id user-id
@@ -46,14 +44,6 @@
                    :where [:= :id bundle-id]})
   (congest/deregister! js job-id))
 
-(defn generate-api-key! [ds user-id bundle-id]
-  (let [uuid (utils/uuid)
-        api-key (utils/sha256 (str user-id bundle-id uuid))]
-    (hon/update! ds {:tname :bundles
-                     :where [:= :id bundle-id]
-                     :data {:hash api-key}})
-    api-key))
-
 (defn update-filtered-feeds! [ds {:keys [filtered bundle-id feed-id]}]
   (if filtered
     (hon/insert! ds {:tname :filtered-feeds

From c87ca13a2ad661c31f387b1a6e1cc44aac80be60 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Fri, 13 Feb 2026 10:59:02 +0200
Subject: [PATCH 271/391] fixed bug in integrations updating

---
 src/source/jobs/oplog.clj                 | 4 ++--
 src/source/routes/integrations.clj        | 4 +---
 src/source/services/bundle_categories.clj | 7 ++++---
 src/source/workers/integrations.clj       | 3 +--
 4 files changed, 8 insertions(+), 10 deletions(-)

diff --git a/src/source/jobs/oplog.clj b/src/source/jobs/oplog.clj
index 60c0a92b..694b25f7 100644
--- a/src/source/jobs/oplog.clj
+++ b/src/source/jobs/oplog.clj
@@ -7,8 +7,8 @@
   "Delete job record and metadata associated therewith"
   [{:keys [ds id]}]
   (let [{:keys [job-metadata-id id]} (services/job ds {:where [:= :job-id id]})]
-    (services/delete-job-metadata! ds {:id job-metadata-id})
-    (services/delete-job! ds {:id id})))
+    (services/delete-job! ds {:id id})
+    (services/delete-job-metadata! ds {:id job-metadata-id})))
 
 (defn update-job!
   "Update job status and last heartbeat"
diff --git a/src/source/routes/integrations.clj b/src/source/routes/integrations.clj
index 5d485fef..99a153f2 100644
--- a/src/source/routes/integrations.clj
+++ b/src/source/routes/integrations.clj
@@ -5,9 +5,7 @@
             [source.util :as util]
             [source.jobs.core :as jobs]
             [source.jobs.handlers :as handlers]
-            [congest.jobs :as congest]
-            [source.db.honey :as hon]
-            [source.db.util :as db.util]))
+            [congest.jobs :as congest]))
 
 (defn get
   {:summary "get all integrations"
diff --git a/src/source/services/bundle_categories.clj b/src/source/services/bundle_categories.clj
index f2317020..293c4baa 100644
--- a/src/source/services/bundle_categories.clj
+++ b/src/source/services/bundle_categories.clj
@@ -1,7 +1,8 @@
 (ns source.services.bundle-categories
   (:require [source.db.interface :as db]
             [source.db.bundle :as bundle]
-            [source.db.util :as db.util]))
+            [source.db.util :as db.util]
+            [honey.sql.helpers :as hsql]))
 
 (defn category-id [ds {:keys [bundle-id where] :as opts}]
   (->> {:tname :bundle-categories
@@ -23,7 +24,7 @@
   (let [bundle-categories (mapv (fn [{:keys [id]}]
                                   {:bundle-id bundle-id
                                    :category-id id}) categories)]
-    (db/delete! ds {:tname (db.util/tname :bundle-categories bundle-id)
-                    :where [:= :bundle-id bundle-id]})
+    (db/delete! ds (-> (db.util/tname :bundle-categories bundle-id)
+                       (hsql/where [:= :bundle-id bundle-id])))
     (db/insert! ds (-> (db.util/tname :bundle-categories bundle-id)
                        (assoc :data bundle-categories)))))
diff --git a/src/source/workers/integrations.clj b/src/source/workers/integrations.clj
index 40b5e332..a135e182 100644
--- a/src/source/workers/integrations.clj
+++ b/src/source/workers/integrations.clj
@@ -7,8 +7,7 @@
             [source.util :as utils]
             [source.db.tables :as tables]
             [source.db.honey :as hon]
-            [congest.jobs :as congest]
-            [source.db.bundle :as bundle]))
+            [congest.jobs :as congest]))
 
 (defn create-integration! [ds {:keys [user-id bundle-metadata categories content-types]}]
   (let [new-bundle (bundles/create-bundle! ds {:user-id user-id

From fce5df8ddf3a00235349558146e1c6b68071f849 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Fri, 13 Feb 2026 14:54:49 +0200
Subject: [PATCH 272/391] updated post shuffling to be seeded using
 seeded-shuffle from prandom.core

---
 src/source/workers/bundles.clj | 19 +++++++++++++++----
 1 file changed, 15 insertions(+), 4 deletions(-)

diff --git a/src/source/workers/bundles.clj b/src/source/workers/bundles.clj
index 1865bf7c..d109655b 100644
--- a/src/source/workers/bundles.clj
+++ b/src/source/workers/bundles.clj
@@ -4,7 +4,9 @@
             [clojure.set :as set]
             [source.services.feed-categories :as feed-categories]
             [source.db.util :as db.util]
-            [honey.sql :as sql]))
+            [source.prandom.core :as prandom])
+  (:import [java.time LocalDateTime]
+           [java.time.format DateTimeFormatter]))
 
 (defn get-bundle-categories
   "Get all categories for feeds/posts in bundle"
@@ -66,11 +68,20 @@
                                         (assoc :order-by (when (= latest "true") [[:posted-at :desc]]))
                                         (merge (db.util/tname :outgoing-posts bundle-id))))
 
-        shuffled (shuffle filtered-posts)
+        order-map (->> (.format (LocalDateTime/now) (DateTimeFormatter/ofPattern "yyyy-MM-dd HH"))
+                       (prandom/seeded-shuffle (count filtered-posts))
+                       (map-indexed (fn [i item] [item i]))
+                       (into {}))
+
+        shuffled-posts (if (= latest "true")
+                         filtered-posts
+                         (->> (zipmap (-> filtered-posts count inc range) filtered-posts)
+                              (sort-by #(get order-map (first %)))
+                              (mapv last)))
 
         categorised-posts (vec
                            (if (seq category-ids)
-                             (->> shuffled
+                             (->> shuffled-posts
                                   (mapv
                                    (fn [post]
                                      (when (seq (set/intersection
@@ -81,7 +92,7 @@
                                                       (set))))
                                        post)))
                                   (remove nil?))
-                             shuffled))
+                             shuffled-posts))
 
         valid-start? (and (some? start) (>= start 0) (< start (count categorised-posts)))
         started-posts (if valid-start?

From 3346d88f396161e4a82fbf788e731278ddccd290 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 16 Feb 2026 09:50:06 +0200
Subject: [PATCH 273/391] removed temporary fix to outgoing posts

---
 src/source/routes/integrations.clj | 16 ++--------------
 1 file changed, 2 insertions(+), 14 deletions(-)

diff --git a/src/source/routes/integrations.clj b/src/source/routes/integrations.clj
index 5d485fef..32164668 100644
--- a/src/source/routes/integrations.clj
+++ b/src/source/routes/integrations.clj
@@ -5,9 +5,7 @@
             [source.util :as util]
             [source.jobs.core :as jobs]
             [source.jobs.handlers :as handlers]
-            [congest.jobs :as congest]
-            [source.db.honey :as hon]
-            [source.db.util :as db.util]))
+            [congest.jobs :as congest]))
 
 (defn get
   {:summary "get all integrations"
@@ -62,17 +60,7 @@
                          :categories (:categories body)
                          :content-types (:content-types body)}
                         (integrations/create-integration! ds))
-        categories-by-bundle (bundles/categories-in-bundle ds (:id new-bundle))
-
-        ; TEMPORARY BLOCK
-        ; TODO: delete this and fix the problem in jobs that causes outgoing posts to be empty
-       ;posts (->> (hon/find ds {:tname :incoming-posts
-       ;                         :limit 2000})
-       ;           (mapv #(dissoc % :redacted)))
-       ;_ (hon/insert! ds (-> (db.util/tname :outgoing-posts (:id new-bundle))
-       ;                      (assoc :data posts)))
-        ; END OF TEMPORARY BLOCK
-        ]
+        categories-by-bundle (bundles/categories-in-bundle ds (:id new-bundle))]
     ;TODO: service needed
     (->> (jobs/prepare-congest-metadata
           ds

From 7b414329f59133aab1e3f62a1b321a5a8bed069d Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 16 Feb 2026 11:25:05 +0200
Subject: [PATCH 274/391] Added thorough documentation for all bundle endpoints

---
 src/source/routes/bundle.clj            |  4 ++--
 src/source/routes/bundle_categories.clj |  4 ++--
 src/source/routes/bundle_feed.clj       |  7 ++++---
 src/source/routes/bundle_feed_post.clj  |  9 +++++----
 src/source/routes/bundle_feed_posts.clj |  7 ++++---
 src/source/routes/bundle_feeds.clj      | 14 +++++++++-----
 src/source/routes/bundle_post.clj       |  8 +++++---
 src/source/routes/bundle_posts.clj      | 23 +++++++++++++++++------
 src/source/routes/integrations.clj      |  6 ++----
 src/source/routes/util.clj              | 12 ++++++++++--
 10 files changed, 60 insertions(+), 34 deletions(-)

diff --git a/src/source/routes/bundle.clj b/src/source/routes/bundle.clj
index 68b93e3d..4fcabf7f 100644
--- a/src/source/routes/bundle.clj
+++ b/src/source/routes/bundle.clj
@@ -3,8 +3,8 @@
             [source.db.honey :as hon]))
 
 (defn get
-  {:summary "get metadata for the associated uuid-authorized bundle"
-   :parameters {:query [:map [:uuid :string]]}
+  {:summary "Get metadata for the associated uuid-authorized bundle"
+   :parameters {:query [:map [:uuid {:description "Bundle UUID"} :string]]}
    :responses {200 {:body [:map
                            [:id :int]
                            [:name :string]
diff --git a/src/source/routes/bundle_categories.clj b/src/source/routes/bundle_categories.clj
index ada7ba21..477bfb17 100644
--- a/src/source/routes/bundle_categories.clj
+++ b/src/source/routes/bundle_categories.clj
@@ -3,8 +3,8 @@
             [ring.util.response :as res]))
 
 (defn get
-  {:summary "get all categories for content present in the uuid-authorized bundle"
-   :parameters {:query [:map [:uuid :string]]}
+  {:summary "Get all categories for which content is present in the uuid-authorized bundle (RSS feeds / posts)"
+   :parameters {:query [:map [:uuid {:description "Bundle UUID"} :string]]}
    :responses {200 {:body [:vector
                            [:map
                             [:id :int]
diff --git a/src/source/routes/bundle_feed.clj b/src/source/routes/bundle_feed.clj
index d499857b..4e7c58ea 100644
--- a/src/source/routes/bundle_feed.clj
+++ b/src/source/routes/bundle_feed.clj
@@ -4,10 +4,11 @@
             [source.services.analytics.interface :as analytics]))
 
 (defn get
-  {:summary "get feed associated with the uuid-authorized bundle by id, updates click analytics for the given feed"
-   :parameters {:query [:map [:uuid :string]]
+  {:summary "Get a single RSS feed by id from RSS feeds within the uuid-authorized bundle. 
+   This endpoint will update click analytics for the returned feed."
+   :parameters {:query [:map [:uuid {:description "Bundle UUID"} :string]]
                 :path [:map [:id {:title "id"
-                                  :description "feed id"} :int]]}
+                                  :description "Feed ID"} :int]]}
    :responses  {200 {:body [:map
                             [:id :int]
                             [:title :string]
diff --git a/src/source/routes/bundle_feed_post.clj b/src/source/routes/bundle_feed_post.clj
index 5c2fdb91..995de187 100644
--- a/src/source/routes/bundle_feed_post.clj
+++ b/src/source/routes/bundle_feed_post.clj
@@ -4,10 +4,11 @@
             [source.services.analytics.interface :as analytics]))
 
 (defn get
-  {:summary "get post in outgoing feed for the associated uuid-authorized bundle by post id, updates click analytics"
-   :parameters {:query [:map [:uuid :string]]
-                :path [:map [:post-id {:title "post-id"
-                                       :description "post id"} :int]]}
+  {:summary "Get a single post by post id belonging to a feed in the associated uuid-authorized bundle. 
+   This endpoint updates click analytics for the returned post."
+   :parameters {:query [:map [:uuid {:description "Bundle UUID"} :string]]
+                :path [:map [:post-id {:title "postId"
+                                       :description "Post ID"} :int]]}
    :responses {200 {:body
                     [:map
                      [:id :int]
diff --git a/src/source/routes/bundle_feed_posts.clj b/src/source/routes/bundle_feed_posts.clj
index 49f468e2..38cea80a 100644
--- a/src/source/routes/bundle_feed_posts.clj
+++ b/src/source/routes/bundle_feed_posts.clj
@@ -4,10 +4,11 @@
             [source.services.analytics.interface :as analytics]))
 
 (defn get
-  {:summary "get all posts in the outgoing feed for the associated uuid-authorized bundle by feed id"
-   :parameters {:query [:map [:uuid :string]]
+  {:summary "Get all posts present within a given RSS feed by feed id, within the uuid-authorized bundle.
+   This endpoint will update impressions analytics for the returned posts."
+   :parameters {:query [:map [:uuid {:description "Bundle UUID"} :string]]
                 :path [:map [:id {:title "id"
-                                  :description "feed id"} :int]]}
+                                  :description "Feed ID"} :int]]}
    :responses {200 {:body [:vector
                            [:map
                             [:id :int]
diff --git a/src/source/routes/bundle_feeds.clj b/src/source/routes/bundle_feeds.clj
index b0762dff..e93c745e 100644
--- a/src/source/routes/bundle_feeds.clj
+++ b/src/source/routes/bundle_feeds.clj
@@ -5,12 +5,16 @@
             [source.services.analytics.interface :as analytics]))
 
 (defn post
-  {:summary "get all feeds present in the bundle authorised by uuid, updating impression analytics."
+  {:summary "Get all RSS feeds present in the bundle authorised by uuid.
+   This endpoint will update impressions analytics for the returned feeds."
    :parameters {:query [:map
-                        [:uuid :string]
-                        [:type {:optional true} :int]
-                        [:latest {:optional true} :boolean]
-                        [:nonfiltered {:optional true} :boolean]]
+                        [:uuid {:description "Bundle UUID"} :string]
+                        [:type {:optional true
+                                :description "Filters by content type ID"} :int]
+                        [:latest {:optional true
+                                  :description "Filters by most recently uploaded feeds"} :boolean]
+                        [:nonfiltered {:optional true
+                                       :description "Marking this field as true will disable all filters"} :boolean]]
                 :body [:map [:category-ids [:vector :int]]]}
    :responses {200 {:body [:vector
                            [:map
diff --git a/src/source/routes/bundle_post.clj b/src/source/routes/bundle_post.clj
index ab789d58..0a1f4b95 100644
--- a/src/source/routes/bundle_post.clj
+++ b/src/source/routes/bundle_post.clj
@@ -6,10 +6,12 @@
             [honey.sql.helpers :as hsql]))
 
 (defn get
-  {:summary "get a single outgoing post in the uuid-authorized bundle by post id, updates click analytics"
-   :parameters {:query [:map [:uuid :string]]
+  {:summary "Get a single post by post id in the uuid-authorized bundle.
+   Used to return a single post present in the bundle.
+   This endpoint updates click analytics for the returned post."
+   :parameters {:query [:map [:uuid {:description "Bundle UUID"} :string]]
                 :path [:map [:id {:title "id"
-                                  :description "post id"} :int]]}
+                                  :description "Post ID"} :int]]}
    :responses {200 {:body [:map
                            [:id :int]
                            [:post-id :string]
diff --git a/src/source/routes/bundle_posts.clj b/src/source/routes/bundle_posts.clj
index 09002faa..292368c5 100644
--- a/src/source/routes/bundle_posts.clj
+++ b/src/source/routes/bundle_posts.clj
@@ -5,14 +5,25 @@
             [source.workers.bundles :as bundles]))
 
 (defn post
-  {:summary "get all (optionally filtered) outgoing posts in the uuid-authorized bundle, updates impression analytics"
+  {:summary "Get a list of posts in the uuid-authorized bundle, determined by analytics.
+   This endpoint updates impression analytics for the returned posts."
    :parameters {:body [:map [:category-ids [:vector :int]]]
                 :query [:map
-                        [:uuid :string]
-                        [:limit {:optional true} :int]
-                        [:start {:optional true} :int]
-                        [:type {:optional true} :int]
-                        [:latest {:optional true} [:enum "true" "false"]]]}
+                        [:uuid {:description "Bundle UUID"} :string]
+                        [:limit
+                         {:optional true
+                          :description "Used for pagination. Specifies a number of posts to be returned."}
+                         :int]
+                        [:start
+                         {:optional true
+                          :description "Used for pagination. Specifies the starting point for the returned posts, incremented by the limit."}
+                         :int]
+                        [:type {:optional true
+                                :description "Filters by content type ID"} :int]
+                        [:latest
+                         {:optional true
+                          :description "Filters by most recently uploaded posts, not determined by analytics"}
+                         [:enum "true" "false"]]]}
    :responses {200 {:body [:vector
                            [:map
                             [:id :int]
diff --git a/src/source/routes/integrations.clj b/src/source/routes/integrations.clj
index 5d485fef..c6e0bdbe 100644
--- a/src/source/routes/integrations.clj
+++ b/src/source/routes/integrations.clj
@@ -5,9 +5,7 @@
             [source.util :as util]
             [source.jobs.core :as jobs]
             [source.jobs.handlers :as handlers]
-            [congest.jobs :as congest]
-            [source.db.honey :as hon]
-            [source.db.util :as db.util]))
+            [congest.jobs :as congest]))
 
 (defn get
   {:summary "get all integrations"
@@ -30,7 +28,7 @@
   (res/response (bundles/bundles ds {:where [:= :user-id (:id user)]})))
 
 (defn post
-  {:summary "add an integration"
+  {:summary "creates an integration and the associated bundle in which content is stored"
    :parameters {:body [:map
                        [:name :string]
                        [:ts-and-cs {:optional true} :int]
diff --git a/src/source/routes/util.clj b/src/source/routes/util.clj
index 7e42b0ae..71226f27 100644
--- a/src/source/routes/util.clj
+++ b/src/source/routes/util.clj
@@ -68,11 +68,18 @@
                                                   :urls.primaryName "swagger"
                                                   :operationsSorter "alpha"}}))
 
+(defn tag-definitions []
+  [{:name "integrations"
+    :description "Management endpoints for integrations and the associated bundle"}
+   {:name "bundles"
+    :description "Content endpoints for the bundle associated with an integration"}])
+
 (defn swagger-route []
   ["/swagger.json" {:get {:no-doc true
                           :swagger {:info {:title "source-api"
                                            :description "swagger docs for source api with malli and reitit-ring"
-                                           :version "0.0.1"}
+                                           :version "0.0.2"}
+                                    :tags (tag-definitions)
                                     :securityDefinitions {"auth" {:type :apiKey
                                                                   :in :header
                                                                   :name "Authorization"}
@@ -85,7 +92,8 @@
   ["/openapi.json" {:get {:no-doc true
                           :openapi {:info {:title "source-api"
                                            :description "openapi3 docs for source api with malli and reitit-ring"
-                                           :version "0.0.1"}
+                                           :version "0.0.2"}
+                                    :tags (tag-definitions)
                                     :components {:securitySchemes {"bearerAuth" {:type :http
                                                                                  :scheme :bearer
                                                                                  :bearerFormat "JWT"

From 4802c3b5240f728f5f616e1193968de4c6421ecf Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 16 Feb 2026 12:11:15 +0200
Subject: [PATCH 275/391] added source backend postman collection

---
 Source.postman_collection.json | 1255 ++++++++++++++++++++++++++++++++
 1 file changed, 1255 insertions(+)
 create mode 100644 Source.postman_collection.json

diff --git a/Source.postman_collection.json b/Source.postman_collection.json
new file mode 100644
index 00000000..1e31d12c
--- /dev/null
+++ b/Source.postman_collection.json
@@ -0,0 +1,1255 @@
+{
+	"info": {
+		"_postman_id": "5858dc0c-cdd1-4c48-a7b8-5e7bb0519be5",
+		"name": "Source",
+		"schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json",
+		"_exporter_id": "23875094"
+	},
+	"item": [
+		{
+			"name": "Home",
+			"request": {
+				"auth": {
+					"type": "noauth"
+				},
+				"method": "GET",
+				"header": [],
+				"url": {
+					"raw": "http://localhost:3000/",
+					"protocol": "http",
+					"host": [
+						"localhost"
+					],
+					"port": "3000",
+					"path": [
+						""
+					]
+				}
+			},
+			"response": []
+		},
+		{
+			"name": "Get Users",
+			"request": {
+				"auth": {
+					"type": "bearer",
+					"bearer": [
+						{
+							"key": "token",
+							"value": "eyJhbGciOiJkaXIiLCJlbmMiOiJBMTI4Q0JDLUhTMjU2In0..1n1VNQ-I1Z7VhgW8IWfiPg.YkGk31r1DG073Q5uyW6I0rGbm9YNKB3r_8D_yWUfG60.suZYrgS9-JXGbRF05iRA0g",
+							"type": "string"
+						}
+					]
+				},
+				"method": "GET",
+				"header": [],
+				"url": {
+					"raw": "http://localhost:3000/users",
+					"protocol": "http",
+					"host": [
+						"localhost"
+					],
+					"port": "3000",
+					"path": [
+						"users"
+					]
+				}
+			},
+			"response": []
+		},
+		{
+			"name": "Register",
+			"request": {
+				"auth": {
+					"type": "noauth"
+				},
+				"method": "POST",
+				"header": [
+					{
+						"key": "",
+						"value": "yeet",
+						"type": "text"
+					}
+				],
+				"body": {
+					"mode": "raw",
+					"raw": "{\n    \"email\": \"toast@toast.com\",\n    \"password\": \"Pa55w.rd\",\n    \"confirm-password\": \"Pa55w.rd\",\n    \"firstname\": \"toast\",\n    \"lastname\": \"toast\",\n    \"type\": \"creator\"\n}",
+					"options": {
+						"raw": {
+							"language": "json"
+						}
+					}
+				},
+				"url": {
+					"raw": "http://localhost:3000/register",
+					"protocol": "http",
+					"host": [
+						"localhost"
+					],
+					"port": "3000",
+					"path": [
+						"register"
+					]
+				}
+			},
+			"response": []
+		},
+		{
+			"name": "Login",
+			"request": {
+				"auth": {
+					"type": "noauth"
+				},
+				"method": "POST",
+				"header": [],
+				"body": {
+					"mode": "raw",
+					"raw": "{\n    \"email\": \"keaganncollins@gmail.com\",\n    \"password\": \"Pa55w.rd\"\n}",
+					"options": {
+						"raw": {
+							"language": "json"
+						}
+					}
+				},
+				"url": {
+					"raw": "http://localhost:3000/login",
+					"protocol": "http",
+					"host": [
+						"localhost"
+					],
+					"port": "3000",
+					"path": [
+						"login"
+					]
+				}
+			},
+			"response": []
+		},
+		{
+			"name": "Update User",
+			"request": {
+				"auth": {
+					"type": "noauth"
+				},
+				"method": "PATCH",
+				"header": [],
+				"body": {
+					"mode": "raw",
+					"raw": "{\n    \"address\": \"Cape Town\",\n    \"lastname\": \"Collins\"\n}",
+					"options": {
+						"raw": {
+							"language": "json"
+						}
+					}
+				},
+				"url": {
+					"raw": "http://localhost:3000/users/3",
+					"protocol": "http",
+					"host": [
+						"localhost"
+					],
+					"port": "3000",
+					"path": [
+						"users",
+						"3"
+					]
+				}
+			},
+			"response": []
+		},
+		{
+			"name": "Add Admin",
+			"request": {
+				"auth": {
+					"type": "bearer",
+					"bearer": [
+						{
+							"key": "token",
+							"value": "eyJhbGciOiJkaXIiLCJlbmMiOiJBMTI4Q0JDLUhTMjU2In0..k38ZDx6t9xOhPKwi37gmxQ.C-DvEva5l913IgKY7tlyhQ.z0DD0yqQF-vQ2A3sPRt9yw",
+							"type": "string"
+						}
+					]
+				},
+				"method": "POST",
+				"header": [],
+				"body": {
+					"mode": "raw",
+					"raw": "{\n    \"email\": \"toast@toast.com\",\n    \"password\": \"Pa55w.rd\",\n    \"confirm-password\": \"Pa55w.rd\",\n    \"firstname\": \"toast\",\n    \"lastname\": \"toast\"\n}",
+					"options": {
+						"raw": {
+							"language": "json"
+						}
+					}
+				},
+				"url": {
+					"raw": "http://localhost:3000/add-admin",
+					"protocol": "http",
+					"host": [
+						"localhost"
+					],
+					"port": "3000",
+					"path": [
+						"add-admin"
+					]
+				}
+			},
+			"response": []
+		},
+		{
+			"name": "Get Businesses",
+			"protocolProfileBehavior": {
+				"disableBodyPruning": true
+			},
+			"request": {
+				"auth": {
+					"type": "bearer",
+					"bearer": [
+						{
+							"key": "token",
+							"value": "eyJhbGciOiJkaXIiLCJlbmMiOiJBMTI4Q0JDLUhTMjU2In0..VeiUSfpzZQEfRiCUnTWfqw.seHlV7WJPNHxdwbzxGQ-NZzTp2RYYxmhDI2mPx4jsd0.TCN3XRVFi6jiwa6a6a4ttA",
+							"type": "string"
+						}
+					]
+				},
+				"method": "GET",
+				"header": [
+					{
+						"key": "Content-Type",
+						"value": "application/json",
+						"type": "text"
+					}
+				],
+				"body": {
+					"mode": "raw",
+					"raw": "{\n    \"name\": \"modulr\",\n    \"url\": \"https://modulr.com\"\n}",
+					"options": {
+						"raw": {
+							"language": "json"
+						}
+					}
+				},
+				"url": {
+					"raw": "http://localhost:3000/businesses",
+					"protocol": "http",
+					"host": [
+						"localhost"
+					],
+					"port": "3000",
+					"path": [
+						"businesses"
+					]
+				}
+			},
+			"response": []
+		},
+		{
+			"name": "Add Business",
+			"request": {
+				"auth": {
+					"type": "bearer",
+					"bearer": [
+						{
+							"key": "token",
+							"value": "eyJhbGciOiJkaXIiLCJlbmMiOiJBMTI4Q0JDLUhTMjU2In0..VeiUSfpzZQEfRiCUnTWfqw.seHlV7WJPNHxdwbzxGQ-NZzTp2RYYxmhDI2mPx4jsd0.TCN3XRVFi6jiwa6a6a4ttA",
+							"type": "string"
+						}
+					]
+				},
+				"method": "POST",
+				"header": [],
+				"body": {
+					"mode": "raw",
+					"raw": "{\n    \"name\": \"yeet\",\n    \"url\": \"https://yeet.com\"\n}",
+					"options": {
+						"raw": {
+							"language": "json"
+						}
+					}
+				},
+				"url": {
+					"raw": "http://localhost:3000/businesses",
+					"protocol": "http",
+					"host": [
+						"localhost"
+					],
+					"port": "3000",
+					"path": [
+						"businesses"
+					]
+				}
+			},
+			"response": []
+		},
+		{
+			"name": "Get Selection Schemas",
+			"protocolProfileBehavior": {
+				"disableBodyPruning": true
+			},
+			"request": {
+				"auth": {
+					"type": "bearer",
+					"bearer": [
+						{
+							"key": "token",
+							"value": "eyJhbGciOiJkaXIiLCJlbmMiOiJBMTI4Q0JDLUhTMjU2In0..1n1VNQ-I1Z7VhgW8IWfiPg.YkGk31r1DG073Q5uyW6I0rGbm9YNKB3r_8D_yWUfG60.suZYrgS9-JXGbRF05iRA0g",
+							"type": "string"
+						}
+					]
+				},
+				"method": "GET",
+				"header": [],
+				"body": {
+					"mode": "raw",
+					"raw": "{\n    \"record\": {\n        \"providerId\": 1,\n        \"outputSchemaId\": 1\n    },\n    \"schema\": {\n        \"title\": {\n            \"path\": [\"tag/body\", \"tag/feed\", \"tag/title\", \"content/0\"]\n        }\n    }\n}",
+					"options": {
+						"raw": {
+							"language": "json"
+						}
+					}
+				},
+				"url": {
+					"raw": "http://localhost:3000/admin/selection-schemas",
+					"protocol": "http",
+					"host": [
+						"localhost"
+					],
+					"port": "3000",
+					"path": [
+						"admin",
+						"selection-schemas"
+					]
+				}
+			},
+			"response": []
+		},
+		{
+			"name": "Get Selection Schemas By Provider",
+			"request": {
+				"auth": {
+					"type": "bearer",
+					"bearer": [
+						{
+							"key": "token",
+							"value": "eyJhbGciOiJkaXIiLCJlbmMiOiJBMTI4Q0JDLUhTMjU2In0..1n1VNQ-I1Z7VhgW8IWfiPg.YkGk31r1DG073Q5uyW6I0rGbm9YNKB3r_8D_yWUfG60.suZYrgS9-JXGbRF05iRA0g",
+							"type": "string"
+						}
+					]
+				},
+				"method": "GET",
+				"header": [],
+				"url": {
+					"raw": "http://localhost:3000/admin/selection-schemas/providers/1",
+					"protocol": "http",
+					"host": [
+						"localhost"
+					],
+					"port": "3000",
+					"path": [
+						"admin",
+						"selection-schemas",
+						"providers",
+						"1"
+					]
+				}
+			},
+			"response": []
+		},
+		{
+			"name": "Add Selection Schema",
+			"request": {
+				"auth": {
+					"type": "bearer",
+					"bearer": [
+						{
+							"key": "token",
+							"value": "eyJhbGciOiJkaXIiLCJlbmMiOiJBMTI4Q0JDLUhTMjU2In0..1n1VNQ-I1Z7VhgW8IWfiPg.YkGk31r1DG073Q5uyW6I0rGbm9YNKB3r_8D_yWUfG60.suZYrgS9-JXGbRF05iRA0g",
+							"type": "string"
+						}
+					]
+				},
+				"method": "POST",
+				"header": [],
+				"body": {
+					"mode": "raw",
+					"raw": "{\n    \"record\": {\n        \"providerId\": 1,\n        \"outputSchemaId\": 1\n    },\n    \"schema\": {\n        \"title\": {\n            \"path\": [\"tag/body\", \"tag/feed\", \"tag/title\", \"content/0\"]\n        }\n    }\n}",
+					"options": {
+						"raw": {
+							"language": "json"
+						}
+					}
+				},
+				"url": {
+					"raw": "http://localhost:3000/admin/selection-schemas",
+					"protocol": "http",
+					"host": [
+						"localhost"
+					],
+					"port": "3000",
+					"path": [
+						"admin",
+						"selection-schemas"
+					]
+				}
+			},
+			"response": []
+		},
+		{
+			"name": "Get Output Schemas",
+			"request": {
+				"auth": {
+					"type": "bearer",
+					"bearer": [
+						{
+							"key": "token",
+							"value": "eyJhbGciOiJkaXIiLCJlbmMiOiJBMTI4Q0JDLUhTMjU2In0..1n1VNQ-I1Z7VhgW8IWfiPg.YkGk31r1DG073Q5uyW6I0rGbm9YNKB3r_8D_yWUfG60.suZYrgS9-JXGbRF05iRA0g",
+							"type": "string"
+						}
+					]
+				},
+				"method": "GET",
+				"header": [],
+				"url": {
+					"raw": "http://localhost:3000/admin/output-schemas",
+					"protocol": "http",
+					"host": [
+						"localhost"
+					],
+					"port": "3000",
+					"path": [
+						"admin",
+						"output-schemas"
+					]
+				}
+			},
+			"response": []
+		},
+		{
+			"name": "Add Output Schema",
+			"request": {
+				"auth": {
+					"type": "bearer",
+					"bearer": [
+						{
+							"key": "token",
+							"value": "eyJhbGciOiJkaXIiLCJlbmMiOiJBMTI4Q0JDLUhTMjU2In0..1n1VNQ-I1Z7VhgW8IWfiPg.YkGk31r1DG073Q5uyW6I0rGbm9YNKB3r_8D_yWUfG60.suZYrgS9-JXGbRF05iRA0g",
+							"type": "string"
+						}
+					]
+				},
+				"method": "POST",
+				"header": [],
+				"body": {
+					"mode": "raw",
+					"raw": "{\n    \"schema\": {\n        \"feed\": {\n            \"type\": \"map\",\n            \"schema\": {\n                \"title\": {\n                    \"type\": \"string\"\n                },\n                \"posts\": {\n                    \"type\": \"vector\",\n                    \"schema\": {\n                        \"post-id\": {\n                            \"type\": \"string\"\n                        },\n                        \"title\": {\n                            \"type\": \"string\"\n                        },\n                        \"stream-url\": {\n                            \"type\": \"string\"\n                        }\n                    }\n                }\n            }\n        }\n    }\n}",
+					"options": {
+						"raw": {
+							"language": "json"
+						}
+					}
+				},
+				"url": {
+					"raw": "http://localhost:3000/admin/output-schemas",
+					"protocol": "http",
+					"host": [
+						"localhost"
+					],
+					"port": "3000",
+					"path": [
+						"admin",
+						"output-schemas"
+					]
+				}
+			},
+			"response": []
+		},
+		{
+			"name": "Extract Data",
+			"request": {
+				"auth": {
+					"type": "bearer",
+					"bearer": [
+						{
+							"key": "token",
+							"value": "eyJhbGciOiJkaXIiLCJlbmMiOiJBMTI4Q0JDLUhTMjU2In0..1n1VNQ-I1Z7VhgW8IWfiPg.YkGk31r1DG073Q5uyW6I0rGbm9YNKB3r_8D_yWUfG60.suZYrgS9-JXGbRF05iRA0g",
+							"type": "string"
+						}
+					]
+				},
+				"method": "POST",
+				"header": [],
+				"body": {
+					"mode": "raw",
+					"raw": "{\n    \"schemaId\": 3,\n    \"url\": \"https://www.youtube.com/feeds/videos.xml?channel_id=UCUyeluBRhGPCW4rPe_UvBZQ\"\n}",
+					"options": {
+						"raw": {
+							"language": "json"
+						}
+					}
+				},
+				"url": {
+					"raw": "http://localhost:3000/admin/extract-data",
+					"protocol": "http",
+					"host": [
+						"localhost"
+					],
+					"port": "3000",
+					"path": [
+						"admin",
+						"extract-data"
+					]
+				}
+			},
+			"response": []
+		},
+		{
+			"name": "Get Providers",
+			"request": {
+				"auth": {
+					"type": "bearer",
+					"bearer": [
+						{
+							"key": "token",
+							"value": "eyJhbGciOiJkaXIiLCJlbmMiOiJBMTI4Q0JDLUhTMjU2In0..1n1VNQ-I1Z7VhgW8IWfiPg.YkGk31r1DG073Q5uyW6I0rGbm9YNKB3r_8D_yWUfG60.suZYrgS9-JXGbRF05iRA0g",
+							"type": "string"
+						}
+					]
+				},
+				"method": "GET",
+				"header": [],
+				"url": {
+					"raw": "http://localhost:3000/providers",
+					"protocol": "http",
+					"host": [
+						"localhost"
+					],
+					"port": "3000",
+					"path": [
+						"providers"
+					]
+				}
+			},
+			"response": []
+		},
+		{
+			"name": "Add Provider",
+			"request": {
+				"auth": {
+					"type": "bearer",
+					"bearer": [
+						{
+							"key": "token",
+							"value": "eyJhbGciOiJkaXIiLCJlbmMiOiJBMTI4Q0JDLUhTMjU2In0..1n1VNQ-I1Z7VhgW8IWfiPg.YkGk31r1DG073Q5uyW6I0rGbm9YNKB3r_8D_yWUfG60.suZYrgS9-JXGbRF05iRA0g",
+							"type": "string"
+						}
+					]
+				},
+				"method": "POST",
+				"header": [],
+				"body": {
+					"mode": "raw",
+					"raw": "{\n    \"name\": \"Spotify\"\n}",
+					"options": {
+						"raw": {
+							"language": "json"
+						}
+					}
+				},
+				"url": {
+					"raw": "http://localhost:3000/admin/providers",
+					"protocol": "http",
+					"host": [
+						"localhost"
+					],
+					"port": "3000",
+					"path": [
+						"admin",
+						"providers"
+					]
+				}
+			},
+			"response": []
+		},
+		{
+			"name": "Get Jobs",
+			"request": {
+				"auth": {
+					"type": "bearer",
+					"bearer": [
+						{
+							"key": "token",
+							"value": "eyJhbGciOiJkaXIiLCJlbmMiOiJBMTI4Q0JDLUhTMjU2In0..1n1VNQ-I1Z7VhgW8IWfiPg.YkGk31r1DG073Q5uyW6I0rGbm9YNKB3r_8D_yWUfG60.suZYrgS9-JXGbRF05iRA0g",
+							"type": "string"
+						}
+					]
+				},
+				"method": "GET",
+				"header": [],
+				"url": {
+					"raw": "http://localhost:3000/admin/jobs",
+					"protocol": "http",
+					"host": [
+						"localhost"
+					],
+					"port": "3000",
+					"path": [
+						"admin",
+						"jobs"
+					]
+				}
+			},
+			"response": []
+		},
+		{
+			"name": "Get Job",
+			"request": {
+				"auth": {
+					"type": "bearer",
+					"bearer": [
+						{
+							"key": "token",
+							"value": "eyJhbGciOiJkaXIiLCJlbmMiOiJBMTI4Q0JDLUhTMjU2In0..1n1VNQ-I1Z7VhgW8IWfiPg.YkGk31r1DG073Q5uyW6I0rGbm9YNKB3r_8D_yWUfG60.suZYrgS9-JXGbRF05iRA0g",
+							"type": "string"
+						}
+					]
+				},
+				"method": "GET",
+				"header": [],
+				"url": {
+					"raw": "http://localhost:3000/admin/jobs/1",
+					"protocol": "http",
+					"host": [
+						"localhost"
+					],
+					"port": "3000",
+					"path": [
+						"admin",
+						"jobs",
+						"1"
+					]
+				}
+			},
+			"response": []
+		},
+		{
+			"name": "Get Feeds",
+			"request": {
+				"auth": {
+					"type": "bearer",
+					"bearer": [
+						{
+							"key": "token",
+							"value": "eyJhbGciOiJkaXIiLCJlbmMiOiJBMTI4Q0JDLUhTMjU2In0..WnRJPqICwpp9h10Sr3w2cg.2luYqAPIVBDcUZSM_661Iqza0TCkP3lhg_-HbACQzdE.DG0evgm8d9aMn4ZcV8oHnQ",
+							"type": "string"
+						}
+					]
+				},
+				"method": "GET",
+				"header": [],
+				"url": {
+					"raw": "http://localhost:3000/feeds",
+					"protocol": "http",
+					"host": [
+						"localhost"
+					],
+					"port": "3000",
+					"path": [
+						"feeds"
+					]
+				}
+			},
+			"response": []
+		},
+		{
+			"name": "Add Feed",
+			"request": {
+				"auth": {
+					"type": "bearer",
+					"bearer": [
+						{
+							"key": "token",
+							"value": "eyJhbGciOiJkaXIiLCJlbmMiOiJBMTI4Q0JDLUhTMjU2In0..WnRJPqICwpp9h10Sr3w2cg.2luYqAPIVBDcUZSM_661Iqza0TCkP3lhg_-HbACQzdE.DG0evgm8d9aMn4ZcV8oHnQ",
+							"type": "string"
+						}
+					]
+				},
+				"method": "POST",
+				"header": [],
+				"body": {
+					"mode": "raw",
+					"raw": "{\n    \"rssUrl\": \"https://www.youtube.com/feeds/videos.xml?channel_id=UCUyeluBRhGPCW4rPe_UvBZQ\",\n    \"providerId\": 1,\n    \"contentTypeId\": 1,\n    \"cadenceId\": 1,\n    \"baselineId\": 1\n}",
+					"options": {
+						"raw": {
+							"language": "json"
+						}
+					}
+				},
+				"url": {
+					"raw": "http://localhost:3000/feeds",
+					"protocol": "http",
+					"host": [
+						"localhost"
+					],
+					"port": "3000",
+					"path": [
+						"feeds"
+					]
+				}
+			},
+			"response": []
+		},
+		{
+			"name": "Bundle Categories",
+			"request": {
+				"auth": {
+					"type": "bearer",
+					"bearer": [
+						{
+							"key": "token",
+							"value": "4ba928b9604e00be952642e9ce7d210648dd713c9330df26ecfadbe900c72fac",
+							"type": "string"
+						}
+					]
+				},
+				"method": "GET",
+				"header": [],
+				"url": {
+					"raw": "http://localhost:3000/api/bundle/categories?uuid=1c1953c7cbeeefd2",
+					"protocol": "http",
+					"host": [
+						"localhost"
+					],
+					"port": "3000",
+					"path": [
+						"api",
+						"bundle",
+						"categories"
+					],
+					"query": [
+						{
+							"key": "uuid",
+							"value": "1c1953c7cbeeefd2"
+						}
+					]
+				}
+			},
+			"response": []
+		},
+		{
+			"name": "Bundle Feeds",
+			"request": {
+				"auth": {
+					"type": "bearer",
+					"bearer": [
+						{
+							"key": "token",
+							"value": "4ba928b9604e00be952642e9ce7d210648dd713c9330df26ecfadbe900c72fac",
+							"type": "string"
+						}
+					]
+				},
+				"method": "POST",
+				"header": [],
+				"body": {
+					"mode": "raw",
+					"raw": "{\n    \"categoryIds\": [14, 15, 16]\n}",
+					"options": {
+						"raw": {
+							"language": "json"
+						}
+					}
+				},
+				"url": {
+					"raw": "http://localhost:3000/api/bundle/feeds?uuid=1c1953c7cbeeefd2&type=2",
+					"protocol": "http",
+					"host": [
+						"localhost"
+					],
+					"port": "3000",
+					"path": [
+						"api",
+						"bundle",
+						"feeds"
+					],
+					"query": [
+						{
+							"key": "uuid",
+							"value": "1c1953c7cbeeefd2"
+						},
+						{
+							"key": "type",
+							"value": "2"
+						}
+					]
+				}
+			},
+			"response": []
+		},
+		{
+			"name": "Bundle Feed",
+			"request": {
+				"auth": {
+					"type": "bearer",
+					"bearer": [
+						{
+							"key": "token",
+							"value": "4ba928b9604e00be952642e9ce7d210648dd713c9330df26ecfadbe900c72fac",
+							"type": "string"
+						}
+					]
+				},
+				"method": "GET",
+				"header": [],
+				"url": {
+					"raw": "http://localhost:3000/bundle/feeds/15?uuid=d5a87ac8f4e90682",
+					"protocol": "http",
+					"host": [
+						"localhost"
+					],
+					"port": "3000",
+					"path": [
+						"bundle",
+						"feeds",
+						"15"
+					],
+					"query": [
+						{
+							"key": "uuid",
+							"value": "d5a87ac8f4e90682"
+						}
+					]
+				}
+			},
+			"response": []
+		},
+		{
+			"name": "Bundle Feed Posts",
+			"request": {
+				"auth": {
+					"type": "bearer",
+					"bearer": [
+						{
+							"key": "token",
+							"value": "4ba928b9604e00be952642e9ce7d210648dd713c9330df26ecfadbe900c72fac",
+							"type": "string"
+						}
+					]
+				},
+				"method": "GET",
+				"header": [],
+				"url": {
+					"raw": "http://localhost:3000/bundle/feeds/15/posts?uuid=d5a87ac8f4e90682",
+					"protocol": "http",
+					"host": [
+						"localhost"
+					],
+					"port": "3000",
+					"path": [
+						"bundle",
+						"feeds",
+						"15",
+						"posts"
+					],
+					"query": [
+						{
+							"key": "uuid",
+							"value": "d5a87ac8f4e90682"
+						}
+					]
+				}
+			},
+			"response": []
+		},
+		{
+			"name": "Bundle Feed Post",
+			"request": {
+				"auth": {
+					"type": "bearer",
+					"bearer": [
+						{
+							"key": "token",
+							"value": "4ba928b9604e00be952642e9ce7d210648dd713c9330df26ecfadbe900c72fac",
+							"type": "string"
+						}
+					]
+				},
+				"method": "GET",
+				"header": [],
+				"url": {
+					"raw": "http://localhost:3000/api/bundle/feeds/57/posts/4073?uuid=d5a87ac8f4e90682",
+					"protocol": "http",
+					"host": [
+						"localhost"
+					],
+					"port": "3000",
+					"path": [
+						"api",
+						"bundle",
+						"feeds",
+						"57",
+						"posts",
+						"4073"
+					],
+					"query": [
+						{
+							"key": "uuid",
+							"value": "d5a87ac8f4e90682"
+						}
+					]
+				}
+			},
+			"response": []
+		},
+		{
+			"name": "Bundle Posts",
+			"request": {
+				"auth": {
+					"type": "bearer",
+					"bearer": [
+						{
+							"key": "token",
+							"value": "4ba928b9604e00be952642e9ce7d210648dd713c9330df26ecfadbe900c72fac",
+							"type": "string"
+						}
+					]
+				},
+				"method": "POST",
+				"header": [],
+				"body": {
+					"mode": "raw",
+					"raw": "{\n    \"categoryIds\": [14]\n}",
+					"options": {
+						"raw": {
+							"language": "json"
+						}
+					}
+				},
+				"url": {
+					"raw": "http://localhost:3000/api/bundle/posts?uuid=1c1953c7cbeeefd2&limit=6&start=0&type=2",
+					"protocol": "http",
+					"host": [
+						"localhost"
+					],
+					"port": "3000",
+					"path": [
+						"api",
+						"bundle",
+						"posts"
+					],
+					"query": [
+						{
+							"key": "uuid",
+							"value": "1c1953c7cbeeefd2"
+						},
+						{
+							"key": "limit",
+							"value": "6"
+						},
+						{
+							"key": "start",
+							"value": "0"
+						},
+						{
+							"key": "type",
+							"value": "2"
+						}
+					]
+				}
+			},
+			"response": []
+		},
+		{
+			"name": "Bundle Post",
+			"request": {
+				"auth": {
+					"type": "bearer",
+					"bearer": [
+						{
+							"key": "token",
+							"value": "4ba928b9604e00be952642e9ce7d210648dd713c9330df26ecfadbe900c72fac",
+							"type": "string"
+						}
+					]
+				},
+				"method": "GET",
+				"header": [],
+				"url": {
+					"raw": "http://localhost:3000/bundle/posts/190?uuid=d5a87ac8f4e90682",
+					"protocol": "http",
+					"host": [
+						"localhost"
+					],
+					"port": "3000",
+					"path": [
+						"bundle",
+						"posts",
+						"190"
+					],
+					"query": [
+						{
+							"key": "uuid",
+							"value": "d5a87ac8f4e90682"
+						}
+					]
+				}
+			},
+			"response": []
+		},
+		{
+			"name": "Categories",
+			"request": {
+				"auth": {
+					"type": "bearer",
+					"bearer": [
+						{
+							"key": "token",
+							"value": "35d38319c1fb1a1f10d1cb83090ae296112a0c606862261b79b8a4f39a5941ab",
+							"type": "string"
+						}
+					]
+				},
+				"method": "GET",
+				"header": [],
+				"url": {
+					"raw": "http://localhost:3000/api/categories",
+					"protocol": "http",
+					"host": [
+						"localhost"
+					],
+					"port": "3000",
+					"path": [
+						"api",
+						"categories"
+					]
+				}
+			},
+			"response": []
+		},
+		{
+			"name": "Category",
+			"request": {
+				"auth": {
+					"type": "bearer",
+					"bearer": [
+						{
+							"key": "token",
+							"value": "4ba928b9604e00be952642e9ce7d210648dd713c9330df26ecfadbe900c72fac",
+							"type": "string"
+						}
+					]
+				},
+				"method": "GET",
+				"header": [],
+				"url": {
+					"raw": "http://localhost:3000/api/categories/11",
+					"protocol": "http",
+					"host": [
+						"localhost"
+					],
+					"port": "3000",
+					"path": [
+						"api",
+						"categories",
+						"11"
+					]
+				}
+			},
+			"response": []
+		},
+		{
+			"name": "Create Integration",
+			"request": {
+				"auth": {
+					"type": "bearer",
+					"bearer": [
+						{
+							"key": "token",
+							"value": "eyJhbGciOiJkaXIiLCJlbmMiOiJBMTI4Q0JDLUhTMjU2In0..avx8YJcoRzOQjg3PRUS6Rg.rTtvfTfHZCExbyV31miHRrHidmNUWbp79hkyWJdyOBA.0HOnXs8Ang1bofzTF5B68A",
+							"type": "string"
+						}
+					]
+				},
+				"method": "POST",
+				"header": [],
+				"body": {
+					"mode": "raw",
+					"raw": "{\n    \"name\": \"poop\",\n    \"tsAndCs\": 1,\n    \"contentTypes\": [{\"id\": 1}],\n    \"categories\": [{\"id\": 9}, {\"id\": 10}]\n}",
+					"options": {
+						"raw": {
+							"language": "json"
+						}
+					}
+				},
+				"url": {
+					"raw": "https://source-be-staging.fly.dev/integrations",
+					"protocol": "https",
+					"host": [
+						"source-be-staging",
+						"fly",
+						"dev"
+					],
+					"path": [
+						"integrations"
+					]
+				}
+			},
+			"response": []
+		},
+		{
+			"name": "Delete Feed",
+			"request": {
+				"auth": {
+					"type": "bearer",
+					"bearer": [
+						{
+							"key": "token",
+							"value": "eyJhbGciOiJkaXIiLCJlbmMiOiJBMTI4Q0JDLUhTMjU2In0..qHwUsdiT1O9yEQ7PvmDGRg.VMc4H5xe4T_6_pItihZJUi8UJQpnX4glob4fgmzKeBc.5EQrSl_mPb_2WXhsG3dH1g",
+							"type": "string"
+						}
+					]
+				},
+				"method": "DELETE",
+				"header": [],
+				"url": {
+					"raw": "http://localhost:3000/feeds/18",
+					"protocol": "http",
+					"host": [
+						"localhost"
+					],
+					"port": "3000",
+					"path": [
+						"feeds",
+						"18"
+					]
+				}
+			},
+			"response": []
+		},
+		{
+			"name": "Delete Integration",
+			"request": {
+				"auth": {
+					"type": "bearer",
+					"bearer": [
+						{
+							"key": "token",
+							"value": "eyJhbGciOiJkaXIiLCJlbmMiOiJBMTI4Q0JDLUhTMjU2In0..qHwUsdiT1O9yEQ7PvmDGRg.VMc4H5xe4T_6_pItihZJUi8UJQpnX4glob4fgmzKeBc.5EQrSl_mPb_2WXhsG3dH1g",
+							"type": "string"
+						}
+					]
+				},
+				"method": "DELETE",
+				"header": [],
+				"url": {
+					"raw": "http://localhost:3000/integrations/2",
+					"protocol": "http",
+					"host": [
+						"localhost"
+					],
+					"port": "3000",
+					"path": [
+						"integrations",
+						"2"
+					]
+				}
+			},
+			"response": []
+		},
+		{
+			"name": "Delete User",
+			"request": {
+				"auth": {
+					"type": "bearer",
+					"bearer": [
+						{
+							"key": "token",
+							"value": "eyJhbGciOiJkaXIiLCJlbmMiOiJBMTI4Q0JDLUhTMjU2In0..x0B7HOqbRp8kU2WhSt_vFg.J316r0lA44pCWJYtR7D51_yOhuAIhDZFe--rK6UMA34.wCp-1HYrbfg5OJ3QKaDJkQ",
+							"type": "string"
+						}
+					]
+				},
+				"method": "DELETE",
+				"header": [],
+				"url": {
+					"raw": "http://localhost:3000/me",
+					"protocol": "http",
+					"host": [
+						"localhost"
+					],
+					"port": "3000",
+					"path": [
+						"me"
+					]
+				}
+			},
+			"response": []
+		},
+		{
+			"name": "Cancel Delete User",
+			"request": {
+				"auth": {
+					"type": "bearer",
+					"bearer": [
+						{
+							"key": "token",
+							"value": "eyJhbGciOiJkaXIiLCJlbmMiOiJBMTI4Q0JDLUhTMjU2In0..qHwUsdiT1O9yEQ7PvmDGRg.VMc4H5xe4T_6_pItihZJUi8UJQpnX4glob4fgmzKeBc.5EQrSl_mPb_2WXhsG3dH1g",
+							"type": "string"
+						}
+					]
+				},
+				"method": "DELETE",
+				"header": [],
+				"url": {
+					"raw": "http://localhost:3000/integrations/2",
+					"protocol": "http",
+					"host": [
+						"localhost"
+					],
+					"port": "3000",
+					"path": [
+						"integrations",
+						"2"
+					]
+				}
+			},
+			"response": []
+		},
+		{
+			"name": "Admin All Feeds",
+			"request": {
+				"auth": {
+					"type": "bearer",
+					"bearer": [
+						{
+							"key": "token",
+							"value": "eyJhbGciOiJkaXIiLCJlbmMiOiJBMTI4Q0JDLUhTMjU2In0..TQO_QiitHYM0ydOwsQJVKg.8HnU2p4O-6OrbfE-wZDbIzVWeEt6fgk3IxMiC53pfa8.MhXN4L8Csj5uYHBd9mAMXg",
+							"type": "string"
+						}
+					]
+				},
+				"method": "GET",
+				"header": [],
+				"url": {
+					"raw": "http://localhost:3000/admin/feeds",
+					"protocol": "http",
+					"host": [
+						"localhost"
+					],
+					"port": "3000",
+					"path": [
+						"admin",
+						"feeds"
+					]
+				}
+			},
+			"response": []
+		},
+		{
+			"name": "Get All Categories",
+			"request": {
+				"method": "GET",
+				"header": []
+			},
+			"response": []
+		}
+	]
+}
\ No newline at end of file

From c4328df5296ccea15058adb8972a128de2d93cc5 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 16 Feb 2026 12:14:34 +0200
Subject: [PATCH 276/391] referenced the postman collection in our docs

---
 src/source/routes/util.clj | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/source/routes/util.clj b/src/source/routes/util.clj
index 71226f27..34914ed5 100644
--- a/src/source/routes/util.clj
+++ b/src/source/routes/util.clj
@@ -77,7 +77,7 @@
 (defn swagger-route []
   ["/swagger.json" {:get {:no-doc true
                           :swagger {:info {:title "source-api"
-                                           :description "swagger docs for source api with malli and reitit-ring"
+                                           :description "Swagger docs for Source API with Malli and Reitit-Ring. For usage examples, check out our postman collection on our GitHub Repository: https://github.com/modulr-software/source-be"
                                            :version "0.0.2"}
                                     :tags (tag-definitions)
                                     :securityDefinitions {"auth" {:type :apiKey

From 55493cf7da7debac58e959a8170f34ce085abd88 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 16 Feb 2026 12:16:11 +0200
Subject: [PATCH 277/391] renamed feeds to RSS feeds in docs

---
 src/source/routes/bundle_feed.clj      | 2 +-
 src/source/routes/bundle_feed_post.clj | 2 +-
 src/source/routes/bundle_feeds.clj     | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/source/routes/bundle_feed.clj b/src/source/routes/bundle_feed.clj
index 4e7c58ea..d6d679ab 100644
--- a/src/source/routes/bundle_feed.clj
+++ b/src/source/routes/bundle_feed.clj
@@ -5,7 +5,7 @@
 
 (defn get
   {:summary "Get a single RSS feed by id from RSS feeds within the uuid-authorized bundle. 
-   This endpoint will update click analytics for the returned feed."
+   This endpoint will update click analytics for the returned RSS feed."
    :parameters {:query [:map [:uuid {:description "Bundle UUID"} :string]]
                 :path [:map [:id {:title "id"
                                   :description "Feed ID"} :int]]}
diff --git a/src/source/routes/bundle_feed_post.clj b/src/source/routes/bundle_feed_post.clj
index 995de187..04e5f265 100644
--- a/src/source/routes/bundle_feed_post.clj
+++ b/src/source/routes/bundle_feed_post.clj
@@ -4,7 +4,7 @@
             [source.services.analytics.interface :as analytics]))
 
 (defn get
-  {:summary "Get a single post by post id belonging to a feed in the associated uuid-authorized bundle. 
+  {:summary "Get a single post by post id belonging to an RSS feed in the associated uuid-authorized bundle. 
    This endpoint updates click analytics for the returned post."
    :parameters {:query [:map [:uuid {:description "Bundle UUID"} :string]]
                 :path [:map [:post-id {:title "postId"
diff --git a/src/source/routes/bundle_feeds.clj b/src/source/routes/bundle_feeds.clj
index e93c745e..04410dcc 100644
--- a/src/source/routes/bundle_feeds.clj
+++ b/src/source/routes/bundle_feeds.clj
@@ -6,7 +6,7 @@
 
 (defn post
   {:summary "Get all RSS feeds present in the bundle authorised by uuid.
-   This endpoint will update impressions analytics for the returned feeds."
+   This endpoint will update impressions analytics for the returned RSS feeds."
    :parameters {:query [:map
                         [:uuid {:description "Bundle UUID"} :string]
                         [:type {:optional true

From 2e5c8197e5cf495a476920de72235f1668b1cbf2 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 16 Feb 2026 12:21:02 +0200
Subject: [PATCH 278/391] added fullstops to docs

---
 src/source/routes/bundle.clj            | 2 +-
 src/source/routes/bundle_categories.clj | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/source/routes/bundle.clj b/src/source/routes/bundle.clj
index 4fcabf7f..491724f1 100644
--- a/src/source/routes/bundle.clj
+++ b/src/source/routes/bundle.clj
@@ -3,7 +3,7 @@
             [source.db.honey :as hon]))
 
 (defn get
-  {:summary "Get metadata for the associated uuid-authorized bundle"
+  {:summary "Get metadata for the associated uuid-authorized bundle."
    :parameters {:query [:map [:uuid {:description "Bundle UUID"} :string]]}
    :responses {200 {:body [:map
                            [:id :int]
diff --git a/src/source/routes/bundle_categories.clj b/src/source/routes/bundle_categories.clj
index 477bfb17..3ab27364 100644
--- a/src/source/routes/bundle_categories.clj
+++ b/src/source/routes/bundle_categories.clj
@@ -3,7 +3,7 @@
             [ring.util.response :as res]))
 
 (defn get
-  {:summary "Get all categories for which content is present in the uuid-authorized bundle (RSS feeds / posts)"
+  {:summary "Get all categories for which content is present in the uuid-authorized bundle (RSS feeds / posts)."
    :parameters {:query [:map [:uuid {:description "Bundle UUID"} :string]]}
    :responses {200 {:body [:vector
                            [:map

From a5dfd2707c62025588cb33c2e72789590daa44b4 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 16 Feb 2026 13:46:33 +0200
Subject: [PATCH 279/391] added optional seed query param

---
 src/source/routes/bundle_posts.clj | 6 ++++--
 src/source/workers/bundles.clj     | 6 ++++--
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/src/source/routes/bundle_posts.clj b/src/source/routes/bundle_posts.clj
index 292368c5..ef1a97a1 100644
--- a/src/source/routes/bundle_posts.clj
+++ b/src/source/routes/bundle_posts.clj
@@ -23,7 +23,8 @@
                         [:latest
                          {:optional true
                           :description "Filters by most recently uploaded posts, not determined by analytics"}
-                         [:enum "true" "false"]]]}
+                         [:enum "true" "false"]]
+                        [:seed {:optional true} [:maybe :string]]]}
    :responses {200 {:body [:vector
                            [:map
                             [:id :int]
@@ -43,12 +44,13 @@
                404 {:boy [:map [:message :string]]}}}
 
   [{:keys [ds bundle-id query-params body] :as _request}]
-  (let [{:keys [limit start type latest]} (walk/keywordize-keys query-params)
+  (let [{:keys [limit start type latest seed]} (walk/keywordize-keys query-params)
         posts (->> {:bundle-id bundle-id
                     :limit limit
                     :start start
                     :type type
                     :latest latest
+                    :seed seed
                     :category-ids (:category-ids body)}
                    (bundles/get-outgoing-posts ds))]
     (try
diff --git a/src/source/workers/bundles.clj b/src/source/workers/bundles.clj
index d109655b..5318f95c 100644
--- a/src/source/workers/bundles.clj
+++ b/src/source/workers/bundles.clj
@@ -50,7 +50,7 @@
 
 (defn get-outgoing-posts
   "Get outgoing posts based on short heuristics and update analytics impressions"
-  [ds {:keys [bundle-id limit start type latest category-ids]}]
+  [ds {:keys [bundle-id limit start type latest category-ids seed]}]
   (let [all-feed-ids (mapv :id (hon/find ds {:tname :feeds
                                              :ret :*}))
         blocked-feed-ids (mapv :feed-id (hon/find ds {:tname :filtered-feeds
@@ -68,7 +68,9 @@
                                         (assoc :order-by (when (= latest "true") [[:posted-at :desc]]))
                                         (merge (db.util/tname :outgoing-posts bundle-id))))
 
-        order-map (->> (.format (LocalDateTime/now) (DateTimeFormatter/ofPattern "yyyy-MM-dd HH"))
+        order-map (->> (if (or (nil? seed) (= seed ""))
+                         (.format (LocalDateTime/now) (DateTimeFormatter/ofPattern "yyyy-MM-dd HH"))
+                         seed)
                        (prandom/seeded-shuffle (count filtered-posts))
                        (map-indexed (fn [i item] [item i]))
                        (into {}))

From 2c0518c63237e1be7bffdddb5a54e0ca73b29b02 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Tue, 17 Feb 2026 12:28:34 +0200
Subject: [PATCH 280/391] added migration for postgres xml schemas and new xml
 schema services

---
 .../migrations/010_selection_schema.clj       | 23 +++++++
 src/source/workers/xml_schemas.clj            | 67 +++++++++++++++++++
 2 files changed, 90 insertions(+)
 create mode 100644 src/source/migrations/010_selection_schema.clj
 create mode 100644 src/source/workers/xml_schemas.clj

diff --git a/src/source/migrations/010_selection_schema.clj b/src/source/migrations/010_selection_schema.clj
new file mode 100644
index 00000000..921bcd00
--- /dev/null
+++ b/src/source/migrations/010_selection_schema.clj
@@ -0,0 +1,23 @@
+(ns source.migrations.010-selection-schema
+  (:require [source.db.master]
+            [source.db.honey :as hon]
+            [source.db.tables :as tables]))
+
+(defn run-up! [context]
+  (let [ds-master (:db-master context)]
+    (tables/create-table!
+     ds-master
+     :source.db.master
+     :output-schemas)
+
+    (->> {:alter-table :selection-schemas
+          :add-column [:schema :text]}
+         (hon/execute! ds-master))))
+
+(defn run-down! [context]
+  (let [ds-master (:db-master context)]
+    (->> {:alter-table :selection-schemas
+          :drop-column :schema}
+         (hon/execute! ds-master))
+
+    (tables/drop-table! ds-master :output-schemas)))
diff --git a/src/source/workers/xml_schemas.clj b/src/source/workers/xml_schemas.clj
new file mode 100644
index 00000000..39573bfe
--- /dev/null
+++ b/src/source/workers/xml_schemas.clj
@@ -0,0 +1,67 @@
+(ns source.workers.xml-schemas
+  (:require [clojure.data.json :as json]
+            [source.db.honey :as hon]
+            [source.rss.core :as rss]))
+
+(defn insert-output-schema! [ds schema]
+  (hon/insert! ds {:tname :output-schemas
+                   :data {:schema (json/write-str schema)}}))
+
+(defn output-schemas [ds]
+  (->> (hon/find ds {:tname :output-schemas})
+       (mapv #(assoc % :schema (json/read-str (:schema %) {:key-fn keyword})))))
+
+(defn output-schema [ds id]
+  (let [{:keys [schema] :as os} (hon/find-one ds {:tname :output-schemas
+                                                  :where [:= :id id]})]
+    (assoc os :schema (json/read-str (or schema "{}") {:key-fn keyword}))))
+
+(defn insert-selection-schema!
+  [ds {:keys [schema record]}]
+  (let [{:keys [output-schema-id provider-id]} record]
+    (hon/insert! ds {:tname :selection-schemas
+                     :data {:schema (json/write-str schema)
+                            :output-schema-id output-schema-id
+                            :provider-id provider-id}
+                     :ret :1})))
+
+(defn delete-selection-schemas-by-provider!
+  [ds provider-id]
+  (hon/delete! ds {:tname :selection-schemas
+                   :where [:= :provider-id provider-id]}))
+
+(defn selection-schemas
+  ([ds] (selection-schemas ds {}))
+  ([ds opts]
+   (->> {:tname :selection-schemas
+         :ret :*}
+        (merge opts)
+        (hon/find ds)
+        (mapv #(assoc % :schema (json/read-str (or (:schema %) "{}") {:key-fn keyword}))))))
+
+(defn selection-schema [ds {:keys [id] :as opts}]
+  (let [{:keys [schema] :as selection-schema} (->> {:tname :selection-schemas
+                                                    :where [:= :id id]}
+                                                   (merge opts)
+                                                   (hon/find-one ds))
+        schema' (json/read-str (or schema "{}") {:key-fn keyword})]
+    (assoc selection-schema :schema schema')))
+
+(defn ast
+  [url]
+  (-> url
+      (slurp)
+      (rss/get-ast)
+      (rss/collect-leaf-paths)))
+
+(defn extract-data
+  [ds schema-id url]
+  (let [json (->> {:tname :selection-schemas
+                   :where [:= :id schema-id]}
+                  (hon/find-one ds)
+                  (:schema))
+        schema (json/read-str (or json "{}") {:key-fn keyword})]
+    (->> url
+         (slurp)
+         (rss/get-ast)
+         (rss/extract-data schema))))

From 59fb7d56dec230c7198852b51cf7e0f8e9e786ca Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Tue, 17 Feb 2026 12:30:10 +0200
Subject: [PATCH 281/391] removed old services and datahike utils

---
 src/source/datastore/config.clj     |  60 ------------
 src/source/datastore/datahike.clj   | 140 ----------------------------
 src/source/datastore/interface.clj  |  36 -------
 src/source/datastore/util.clj       |  32 -------
 src/source/services/interface.clj   |  31 ------
 src/source/services/xml_schemas.clj | 113 ----------------------
 6 files changed, 412 deletions(-)
 delete mode 100644 src/source/datastore/config.clj
 delete mode 100644 src/source/datastore/datahike.clj
 delete mode 100644 src/source/datastore/interface.clj
 delete mode 100644 src/source/datastore/util.clj
 delete mode 100644 src/source/services/xml_schemas.clj

diff --git a/src/source/datastore/config.clj b/src/source/datastore/config.clj
deleted file mode 100644
index 85fff782..00000000
--- a/src/source/datastore/config.clj
+++ /dev/null
@@ -1,60 +0,0 @@
-(ns source.datastore.config
-  (:require [datahike.api :as d]
-            [source.config :as conf]
-            [clojure.string :as string]))
-
-(defn absolute [path]
-  (-> (java.io.File. ".")
-      .getAbsolutePath
-      (clojure.string/replace "/." "/")
-      (str path)))
-
-(defn store-path
-  [store-name]
-  (let [db-dir (conf/read-value :database :dir)]
-    (str
-     db-dir
-     (when (not (= (last db-dir) \/))
-       "/")
-     store-name)))
-
-(defn config [store-name]
-  {:store {:backend :file
-           :path (store-path (name store-name))}
-   :schema-flexibility :read
-   :initial-tx [{:db/ident        :selection-schemas/id
-                 :db/valueType    :db.type/long
-                 :db/cardinality  :db.cardinality/one
-                 :db/unique       :db.unique/identity}
-                {:db/ident        :selection-schemas/schema
-                 :db/valueType    :db.type/any
-                 :db/cardinality  :db.cardinality/one}
-
-                {:db/ident        :output-schemas/id
-                 :db/valueType    :db.type/long
-                 :db/cardinality  :db.cardinality/one
-                 :db/unique       :db.unique/identity}
-                {:db/ident        :output-schemas/schema
-                 :db/valueType    :db.type/any
-                 :db/cardinality  :db.cardinality/one}
-                {:db/ident        :output-schemas/version
-                 :db/valueType    :db.type/long
-                 :db/cardinality  :db.cardinality/one}
-
-                {:db/ident        :providers/id
-                 :db/valueType    :db.type/long
-                 :db/cardinality  :db.cardinality/one
-                 :db/unique       :db.unique/identity}
-                {:db/ident        :providers/name
-                 :db/valueType    :db.type/string
-                 :db/cardinality  :db.cardinality/one}]})
-
-(defn create-datastore [store-name]
-  (when-not (d/database-exists? (config store-name))
-    (println "Creating datastore...")
-    (d/create-database (config store-name))))
-
-(defn delete-datastore [store-name]
-  (when (d/database-exists? (config store-name))
-    (println "Deleting datastore...")
-    (d/delete-database (config store-name))))
diff --git a/src/source/datastore/datahike.clj b/src/source/datastore/datahike.clj
deleted file mode 100644
index e590811d..00000000
--- a/src/source/datastore/datahike.clj
+++ /dev/null
@@ -1,140 +0,0 @@
-(ns source.datastore.datahike
-  (:require [datahike.api :as d]
-            [source.util :as util]))
-
-(defn entities
-  "Returns one or more entities associated with the provided entity id(s)"
-  [ds ids]
-  (let [multi? (vector? ids)
-        ids-vec (if multi? ids [ids])]
-    (mapv (fn [id]
-            (->> (d/entity @ds id)
-                 (into {}))) ids-vec)))
-
-(defn find
-  "Returns entity ids where the provided key matches the provided value"
-  [ds {:keys [key value]}]
-  (->> (d/q '[:find ?e
-              :in $ ?k ?v
-              :where [?e ?k ?v]]
-            @ds key value)
-       (vec)
-       (flatten)
-       (into [])))
-
-(defn find-entities
-  "Same as find, but returns a vector of entities"
-  [ds {:keys [_key _value] :as query}]
-  (->> query
-       (find ds)
-       (entities ds)))
-
-(defn lookup 
-  "Returns the first entity where the provided key matches the provided value"
-  [ds {:keys [_key _value] :as query}]
-  (->> query
-       (find-entities ds)
-       (first)))
-
-(defn entries
-  "Get eids of all entities in which the provided attribute is present"
-  [ds key]
-  (->> (d/q '[:find ?e
-              :in $ ?k
-              :where [?e ?k _]]
-            @ds key)
-       (vec)
-       (flatten)
-       (into [])))
-
-(defn entities-with
-  "Gets all entities in which the provided attribute is present"
-  [ds key]
-  (->> key
-       (entries ds)
-       (entities ds)))
-
-(defn delete!
-  "Accepts an entity id or a vec of entity ids and removes all the entities associated thereby"
-  [ds ids]
-  (let [multi? (vector? ids)
-        ids-vec (if multi? ids [ids])
-        query (mapv (fn [id] [:db/retractEntity id]) ids-vec)]
-    (d/transact ds query)))
-
-(defn insert!
-  "Inserts kv's into store. Returns the key-value pairs that were inserted."
-  [ds data]
-  (let [multi? (util/vectors? data)
-        input-kvs (if multi? data [data])]
-    (d/transact ds input-kvs)
-    input-kvs))
-
-(defn update!
-  "Update attributes and values for a given entity"
-  [ds id data]
-  (->> (merge {:db/add id} data)
-       (vec)
-       (flatten)
-       (conj [])
-       (d/transact ds)))
-
-(defn get-all
-  "Returns all key value pairs in table in kv-store."
-  [ds]
-  (->> (d/q '[:find ?e :where [?e _ _]] @ds)
-       (map (comp (partial into {}) (partial d/entity @ds) first))))
-
-(comment
-  (require '[source.datastore.util :as ds.util])
-
-  (def ds (ds.util/conn :datahike))
-  (get-all ds)
-  (entries ds :user/name)
-  (delete! ds (entries ds :output-schemas/id))
-  (update! ds 3 {:user/age 21})
-
-  (entities ds [6 7])
-  (find ds {:key :user/name
-            :value "Keagan"})
-  (find-entities ds {:key :user/age
-                     :value 23})
-  (insert! ds {:user/name "Shani"
-               :user/age 23})
-  (insert! ds {:providers/name "YouTube"})
-  (delete! ds (entries ds :providers/id))
-
-  (insert! ds {:selection-schemas/id 4
-               :selection-schemas/schema {:title {:path ["tag/body" "tag/feed" "tag/title" "content/0"]}}})
-  (get-all ds)
-  (find ds {:key :selection-schemas/id
-            :value 4})
-
-  (let [ds (ds.util/conn :datahike)
-        id :selection-schemas/id
-        k :selection-schemas/schema
-        v {:title {:path ["tag/body" "tag/feed" "tag/title" "content/0"]}}]
-    (println (get-all ds))
-    (println (entities-with ds k))
-    (println "Putting a value")
-    (insert! ds {id 1
-                 k v})
-    (assert (= v
-               (->> k
-                    (lookup ds)
-                    (k))))
-    (insert! ds {k v})
-    (println "1 " (get-all ds))
-    (println "2 " (find ds {:key id
-                            :value 1}))
-    (println "3 " (get-all ds))
-    (println "4 " (find ds {:key id
-                            :value 1}))
-    (println "Test passed")
-    (println "Deleting a value")
-    (delete! ds (entries ds k))
-    (assert (= []
-               (entries ds k)))
-    (println "Test passed"))
-
-  ())
diff --git a/src/source/datastore/interface.clj b/src/source/datastore/interface.clj
deleted file mode 100644
index 3c065890..00000000
--- a/src/source/datastore/interface.clj
+++ /dev/null
@@ -1,36 +0,0 @@
-(ns source.datastore.interface
-  (:require [source.datastore.datahike :as dh]
-            [source.datastore.util :as store.util]))
-
-(defn ds [store-name]
-  (store.util/conn store-name))
-
-(defn entities [ds eids]
-  (dh/entities ds eids))
-
-(defn find [ds {:keys [_key _value] :as opts}]
-  (dh/find ds opts))
-
-(defn find-entities [ds {:keys [_key _value] :as opts}]
-  (dh/find-entities ds opts))
-
-(defn lookup [ds {:keys [_key _value] :as opts}]
-  (dh/lookup ds opts))
-
-(defn insert! [ds data]
-  (dh/insert! ds data))
-
-(defn update! [ds eid data]
-  (dh/update! ds eid data))
-
-(defn get-all [ds]
-  (dh/get-all ds))
-
-(defn delete! [ds eids]
-  (dh/delete! ds eids))
-
-(defn entries [ds k]
-  (dh/entries ds k))
-
-(defn entities-with [ds k]
-  (dh/entities-with ds k))
diff --git a/src/source/datastore/util.clj b/src/source/datastore/util.clj
deleted file mode 100644
index c4151c2a..00000000
--- a/src/source/datastore/util.clj
+++ /dev/null
@@ -1,32 +0,0 @@
-(ns source.datastore.util
-  (:require [datahike.api :as d]
-            [source.datastore.config :as c]))
-
-(defn conn
-  "Open a connection to a datahike store"
-  [store-name]
-  (d/connect (c/config store-name)))
-
-(defn close
-  "Close a connection to a datahike store"
-  [conn]
-  (d/release conn))
-
-(defn store-name
-  ([type]
-   (name type))
-  ([type id]
-   (str (name type) "_" id)))
-
-(comment
-  (d/create-database (c/config :datahike))
-  (d/delete-database (c/config :datahike))
-
-  (defonce ds (conn :datahike))
-  (d/transact ds [{:user/name "Keagan"
-                   :user/age 23}])
-  (d/q '{:find [?e ?n ?a]
-         :where [[?e :user/name ?n]
-                 [?e :user/age ?a]]}
-       @ds)
-  ())
diff --git a/src/source/services/interface.clj b/src/source/services/interface.clj
index d2179cda..cfa3db3a 100644
--- a/src/source/services/interface.clj
+++ b/src/source/services/interface.clj
@@ -1,6 +1,5 @@
 (ns source.services.interface
   (:require [source.services.auth :as auth]
-            [source.services.xml-schemas :as xml]
             [source.services.bundles :as bundles]
             [source.services.bundle-content-types :as bundle-content-types]
             [source.services.post-heuristics :as post-heuristics]
@@ -11,12 +10,6 @@
 (defn register [ds user]
   (auth/register ds user))
 
-(defn insert-selection-schema! [store db {:keys [_schema _record] :as opts}]
-  (xml/insert-selection-schema! store db opts))
-
-(defn selection-schema [ds {:keys [_id] :as opts}]
-  (xml/selection-schema ds opts))
-
 (defn update-bundle! [ds {:keys [_id _data _where] :as opts}]
   (bundles/update-bundle! ds opts))
 
@@ -40,30 +33,6 @@
 (defn top-posts-by-heuristic [ds {:keys [_select _limit _heuristic] :as opts}]
   (post-heuristics/top-posts-by-heuristic ds opts))
 
-(defn selection-schemas
-  ([ds]
-   (selection-schemas ds {}))
-  ([ds opts]
-   (xml/selection-schemas ds opts)))
-
-(defn delete-selection-schemas-by-provider! [store db provider-id]
-  (xml/delete-selection-schemas-by-provider! store db provider-id))
-
-(defn ast [url]
-  (xml/ast url))
-
-(defn extract-data [store {:keys [schema-id url]}]
-  (xml/extract-data store schema-id url))
-
-(defn output-schemas [store]
-  (xml/output-schemas store))
-
-(defn output-schema [store output-schema-id]
-  (xml/output-schema store output-schema-id))
-
-(defn insert-output-schema! [store schema]
-  (xml/insert-output-schema! store schema))
-
 (defn incoming-posts
   ([ds]
    (incoming-posts/incoming-posts ds))
diff --git a/src/source/services/xml_schemas.clj b/src/source/services/xml_schemas.clj
deleted file mode 100644
index de22ff58..00000000
--- a/src/source/services/xml_schemas.clj
+++ /dev/null
@@ -1,113 +0,0 @@
-(ns source.services.xml-schemas
-  (:require [source.datastore.interface :as store]
-            [source.db.interface :as db]
-            [source.rss.core :as rss]))
-
-(defn insert-output-schema!
-  [store schema]
-  (let [id (-> (store/entries store :output-schemas/id)
-               (count)
-               (inc))]
-    (store/insert! store {:output-schemas/id id
-                          :output-schemas/schema schema})))
-
-(defn output-schemas
-  [store]
-  (store/entities-with store :output-schemas/id))
-
-(defn output-schema
-  [store output-schema-id]
-  (->> {:key :output-schemas/id
-        :value output-schema-id}
-       (store/lookup store)))
-
-(defn insert-selection-schema!
-  [store db {:keys [schema record]}]
-  (let [{:keys [output-schema-id provider-id]} record
-        db-result (db/insert! db {:tname :selection-schemas
-                                  :data {:output-schema-id output-schema-id
-                                         :provider-id provider-id}
-                                  :ret :1})]
-    (store/insert! store {:selection-schemas/id (:id db-result)
-                          :selection-schemas/schema schema})))
-
-(defn delete-selection-schemas-by-provider!
-  [store db provider-id]
-  (let [selection-schemas (db/find db {:tname :selection-schemas
-                                       :where [:= :provider-id provider-id]
-                                       :ret :*})]
-    (run! (fn [{:keys [id]}]
-            (->> (store/find store {:key :selection-schemas/id
-                                    :value id})
-                 (store/delete! store))) selection-schemas)
-    (db/delete! db {:tname :selection-schemas
-                    :where [:= :provider-id provider-id]})))
-
-(defn selection-schemas
-  ([ds] (selection-schemas ds {}))
-  ([ds opts]
-   (->> {:tname :selection-schemas
-         :ret :*}
-        (merge opts)
-        (db/find ds))))
-
-(defn selection-schema [ds {:keys [id where] :as opts}]
-  (->> {:tname :selection-schemas
-        :where (if (some? id)
-                 [:= :id id]
-                 where)
-        :ret :1}
-       (merge opts)
-       (db/find ds)))
-
-(defn ast
-  [url]
-  (-> url
-      (slurp)
-      (rss/get-ast)
-      (rss/collect-leaf-paths)))
-
-(defn extract-data
-  [store schema-id url]
-  (let [schema (->> {:key :selection-schemas/id
-                     :value schema-id}
-                    (store/lookup store)
-                    (:selection-schemas/schema))]
-    (->> url
-         (slurp)
-         (rss/get-ast)
-         (rss/extract-data schema))))
-
-(comment
-  (require '[source.db.util :as db.util])
-  (ast "https://www.youtube.com/feeds/videos.xml?channel_id=UCUyeluBRhGPCW4rPe_UvBZQ")
-
-  (reduce (fn [acc {:keys [id]}]
-            (conj acc id)) [] [{:id 2} {:id 3}])
-
-  (db/find (db.util/conn) {:tname :selection-schemas
-                           :where [:and
-                                   [:= :output-schema-id 1]
-                                   [:= :provider-id 1]]
-                           :ret :*})
-
-  (def ds (store/ds :datahike))
-
-  (count (store/entries ds :selection-schemas/id))
-  (store/entities-with ds :selection-schemas/id)
-  (store/find-entities ds {:key :selection-schemas/id
-                           :value 100})
-
-  (extract-data
-   ds
-   1
-   "https://www.youtube.com/feeds/videos.xml?channel_id=UCUyeluBRhGPCW4rPe_UvBZQ")
-
-  (insert-selection-schema!
-   ds
-   (db.util/conn)
-   {:record {:provider-id 1
-             :output-schema-id 1}
-    :schema {:title {:path ["tag/body" "tag/feed" "tag/title" "content/0"]}}})
-
-  ())

From 055190e2abbf826535a9b54bb59c111a2d0f6f0b Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Tue, 17 Feb 2026 12:30:54 +0200
Subject: [PATCH 282/391] removed db dir from config

---
 resources/config.edn  | 3 +--
 src/source/config.clj | 2 --
 2 files changed, 1 insertion(+), 4 deletions(-)

diff --git a/resources/config.edn b/resources/config.edn
index fe166c79..0a354152 100644
--- a/resources/config.edn
+++ b/resources/config.edn
@@ -3,8 +3,7 @@
  :admins-encrypted-path "resources/admins_encrypted.json"
  :cors-origin #or [#env CORS_ORIGIN "http://localhost:8080"]
  :env #or [#env ENV "dev"]
- :database {:dir #or [#env DATABASE_DIR ".db/"]
-            :url #or [#env DATABASE_URL "postgres://localhost"]
+ :database {:url #or [#env DATABASE_URL "postgres://localhost"]
             :type "postgresql"}
  :email {:address #or [#env SUPPORT_ADDRESS "lee@wearesource.earth"]
          :username #env EMAIL_USERNAME
diff --git a/src/source/config.clj b/src/source/config.clj
index ed0a58a4..45b6d597 100644
--- a/src/source/config.clj
+++ b/src/source/config.clj
@@ -24,7 +24,6 @@
    [:cors-origin :string]
    [:env :string]
    [:database [:map
-               [:dir :string]
                [:url :string]
                [:type :string]]]
    [:oauth2 [:map-of keyword? oauth2-provider-schema]]])
@@ -47,7 +46,6 @@
 
 (comment
   (read-value :supersecretkey)
-  (read-value :database :dir)
   (read-value :oauth2 :google)
   (read-value :cors-origin)
   (read-value :admins-path)

From b17421a0e8c9f68899d781830119520feff1a90f Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Tue, 17 Feb 2026 12:31:28 +0200
Subject: [PATCH 283/391] removed datahike dependency

---
 deps.edn | 1 -
 1 file changed, 1 deletion(-)

diff --git a/deps.edn b/deps.edn
index 79348cad..99442e04 100644
--- a/deps.edn
+++ b/deps.edn
@@ -30,7 +30,6 @@
         com.kepler16/mallard-sqlite-store {:mvn/version "3.2.1"}
         com.kepler16/mallard-postgres-store {:mvn/version "3.2.5"}
         com.github.seancorfield/honeysql {:mvn/version "2.7.1310"}
-        io.replikativ/datahike {:mvn/version "0.6.1599"}
         hickory/hickory {:mvn/version "0.7.1"}
         com.draines/postal {:mvn/version "2.0.5"}
         hiccup/hiccup {:mvn/version "2.0.0"}

From 38333545928595a772dba67bf14a748041993a03 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Tue, 17 Feb 2026 12:32:00 +0200
Subject: [PATCH 284/391] updated all endpoints and handlers to use new
 services and removed every use of store

---
 .../bundle_migrations/002_outgoing_posts.clj  |  4 ++--
 src/source/db/master.clj                      |  6 +++++
 src/source/db/util.clj                        |  8 -------
 src/source/jobs/core.clj                      | 23 ++++++++-----------
 src/source/jobs/handlers.clj                  | 12 +++++-----
 src/source/middleware/core.clj                | 13 +----------
 src/source/middleware/interface.clj           |  4 ++--
 src/source/migrations/001_init_master_db.clj  |  3 ---
 src/source/routes/data.clj                    | 15 ++++--------
 src/source/routes/feeds.clj                   | 11 ++++-----
 src/source/routes/integration.clj             |  3 +--
 src/source/routes/integrations.clj            | 15 ++----------
 src/source/routes/interface.clj               |  2 +-
 src/source/routes/job_start.clj               |  4 ++--
 src/source/routes/jobs.clj                    |  4 ++--
 src/source/routes/output_schema.clj           |  6 ++---
 src/source/routes/output_schemas.clj          | 10 ++++----
 src/source/routes/provider.clj                |  6 ++---
 .../routes/provider_selection_schemas.clj     | 12 +++++-----
 src/source/routes/reitit.clj                  | 15 ++----------
 src/source/routes/selection_schema.clj        | 12 ++++------
 src/source/routes/selection_schemas.clj       | 10 ++++----
 src/source/routes/xml.clj                     |  4 ++--
 src/source/server.clj                         | 20 +++++-----------
 src/source/workers/feeds.clj                  |  6 ++---
 25 files changed, 83 insertions(+), 145 deletions(-)

diff --git a/src/source/bundle_migrations/002_outgoing_posts.clj b/src/source/bundle_migrations/002_outgoing_posts.clj
index 180348d3..77535736 100644
--- a/src/source/bundle_migrations/002_outgoing_posts.clj
+++ b/src/source/bundle_migrations/002_outgoing_posts.clj
@@ -16,5 +16,5 @@
   (let [{:keys [ds-master bundle-id]} context]
     (tables/drop-tables!
      ds-master
-     (-> [:outgoing-posts]
-         (db.util/tnames bundle-id)))))
+     (mapv :tname (-> [:outgoing-posts]
+                      (db.util/tnames bundle-id))))))
diff --git a/src/source/db/master.clj b/src/source/db/master.clj
index 30c30d73..d32e95aa 100644
--- a/src/source/db/master.clj
+++ b/src/source/db/master.clj
@@ -172,6 +172,12 @@
    [:provider-id :integer :not nil]
    (tables/foreign-key :provider-id :providers :id)))
 
+(def output-schemas
+  (tables/create-table-sql
+   :output-schemas
+   (tables/table-id)
+   [:schema :text]))
+
 (def incoming-posts
   (tables/create-table-sql
    :incoming-posts
diff --git a/src/source/db/util.clj b/src/source/db/util.clj
index 39c43ac7..792c904a 100644
--- a/src/source/db/util.clj
+++ b/src/source/db/util.clj
@@ -3,14 +3,6 @@
             [next.jdbc :as jdbc]
             [next.jdbc.result-set :as rs]))
 
-(defn db-path [dbname]
-  (let [db-dir (conf/read-value :database :dir)]
-    (str
-     db-dir
-     (when (not (= (last db-dir) \/))
-       "/")
-     dbname)))
-
 (defn db-name
   ([type]
    (name type))
diff --git a/src/source/jobs/core.clj b/src/source/jobs/core.clj
index 4e25bfc2..c00cbd15 100644
--- a/src/source/jobs/core.clj
+++ b/src/source/jobs/core.clj
@@ -7,7 +7,7 @@
 
 (defn prepare-congest-metadata
   "given raw job metadata, returns extended metadata necessary for use with congest"
-  [ds store metadata]
+  [ds metadata]
   (let [i->b (fn [i] (if (integer? i)
                        (if (= i 1) true false)
                        i))
@@ -21,24 +21,23 @@
                :handler-name (:handler metadata)
                :handler (handlers/handler metadata)
                :logger oplog/operation-logger
-               :ds ds
-               :store store)
+               :ds ds)
         (dissoc :recurring))))
 
 (defn start!
   "given a job-id, re-registers an existing job from the database"
-  [js ds store job-id]
+  [js ds job-id]
   (let [{:keys [job-metadata-id args handler]} (services/job ds {:where [:= :job-id job-id]})
         metadata (-> (services/job-metadata ds {:id job-metadata-id})
                      (assoc :id job-id
                             :args args
                             :handler handler))]
     (congest/deregister! js job-id)
-    (congest/register! js (prepare-congest-metadata ds store metadata))))
+    (congest/register! js (prepare-congest-metadata ds metadata))))
 
 (defn interrupted-jobs
   "Get congest-ready metadata of all jobs marked as running"
-  [ds store]
+  [ds]
   (let [jobs (services/jobs ds)]
     (mapv (fn [{:keys [job-id job-metadata-id args handler status]} i]
             (when (= status "running")
@@ -51,16 +50,14 @@
                     metadata (assoc m
                                     :initial-delay (+ initial-delay (* 1000 5 i))
                                     :interval (+ interval (* 1000 5 i)))]
-                (prepare-congest-metadata ds store metadata))))
+                (prepare-congest-metadata ds metadata))))
           jobs
           (-> jobs count inc range))))
 
 (comment
-  (require '[source.db.util :as db.util]
-           '[source.datastore.util :as store.util])
+  (require '[source.db.util :as db.util])
 
   (def ds (db.util/conn))
-  (def store (store.util/conn :datahike))
 
   (def testjob {:id "test"
                 :initial-delay 10
@@ -80,10 +77,10 @@
   (services/delete-job-metadata! ds {})
 
   (def js (congest/create-job-service []))
-  (congest/register! js (prepare-congest-metadata ds store testjob))
+  (congest/register! js (prepare-congest-metadata ds testjob))
   (congest/deregister! js "test")
   (congest/stop! js "test" false)
-  (start! js ds store "test")
-  (interrupted-jobs ds store)
+  (start! js ds "test")
+  (interrupted-jobs ds)
 
   ())
diff --git a/src/source/jobs/handlers.clj b/src/source/jobs/handlers.clj
index de19bccc..55c6bf4c 100644
--- a/src/source/jobs/handlers.clj
+++ b/src/source/jobs/handlers.clj
@@ -1,5 +1,6 @@
 (ns source.jobs.handlers
   (:require [source.services.interface :as services]
+            [source.workers.xml-schemas :as xml]
             [source.workers.users :as users]
             [source.util :as util]
             [source.services.incoming-posts :as incoming-posts]
@@ -26,20 +27,19 @@
   (str email "-" feed-id))
 
 (defmethod handler :update-feed-posts [_]
-  (fn [{:keys [args ds store]}]
+  (fn [{:keys [args ds]}]
     (try
       (when (users/removed? ds (:creator-id args))
         (let [{:keys [feed-id creator-id content-type-id provider-id url]} args
               _ (println "feed" feed-id "job started")
               selection-schemas (->> [:= :provider-id provider-id]
                                      (assoc {} :where)
-                                     (services/selection-schemas ds))
+                                     (xml/selection-schemas ds))
               latest-ss (->> selection-schemas
                              (reduce (fn [acc {:keys [id]}]
                                        (conj acc id)) [])
                              (apply max -1))
-              extracted (services/extract-data store {:schema-id latest-ss
-                                                      :url url})
+              extracted (xml/extract-data ds latest-ss url)
               extracted-posts (get-in extracted [:feed :posts])
               extracted-display (get-in extracted [:feed :display-picture])
               extended-posts (mapv (fn [{:keys [posted-at thumbnail] :as post}]
@@ -157,8 +157,8 @@
   (str "delete_" user-type "_" user-id))
 
 (defmethod handler :delete-user [_]
-  (fn [{:keys [args ds store]}]
+  (fn [{:keys [args ds]}]
     (try
       (let [{:keys [user-type user-id]} args]
-        (users/hard-delete-user! ds store (keyword user-type) user-id))
+        (users/hard-delete-user! ds (keyword user-type) user-id))
       (catch Exception e (println "Failed to delete user: " e) :fail))))
diff --git a/src/source/middleware/core.clj b/src/source/middleware/core.clj
index bd067200..e5a461bf 100644
--- a/src/source/middleware/core.clj
+++ b/src/source/middleware/core.clj
@@ -20,12 +20,6 @@
         (assoc :ds ds)
         (handler))))
 
-(defn wrap-store [handler store]
-  (fn [request]
-    (-> request
-        (assoc :store store)
-        (handler))))
-
 (defn wrap-js
   "attaches the provided job service to the handler's request"
   [handler js]
@@ -38,10 +32,6 @@
   (-> app
       (wrap-ds ds)))
 
-(defn apply-store [app store]
-  (-> app
-      (wrap-store store)))
-
 (defn apply-js
   "middleware for attaching the job service to the request"
   [app js]
@@ -112,11 +102,10 @@
         (assoc :query-params (walk/keywordize-keys query-params))
         (handler))))
 
-(defn apply-generic [app & {:keys [ds store js]}]
+(defn apply-generic [app & {:keys [ds js]}]
   (-> app
       (wrap-exception-logger)
       (apply-ds ds)
-      (apply-store store)
       (apply-js js)
       (wrap-case-conversion)
       (wrap-query)
diff --git a/src/source/middleware/interface.clj b/src/source/middleware/interface.clj
index ca5bb98d..d8959b84 100644
--- a/src/source/middleware/interface.clj
+++ b/src/source/middleware/interface.clj
@@ -1,8 +1,8 @@
 (ns source.middleware.interface
   (:require [source.middleware.core :as mw]))
 
-(defn apply-generic [app & {:keys [ds store js]}]
-  (mw/apply-generic app :ds ds :store store :js js))
+(defn apply-generic [app & {:keys [ds js]}]
+  (mw/apply-generic app :ds ds :js js))
 
 (defn apply-auth
   "accepts required-type as an optional parameter to authorize the route only for the specified user type"
diff --git a/src/source/migrations/001_init_master_db.clj b/src/source/migrations/001_init_master_db.clj
index f83f74c7..2fd17daf 100644
--- a/src/source/migrations/001_init_master_db.clj
+++ b/src/source/migrations/001_init_master_db.clj
@@ -3,7 +3,6 @@
             [source.db.master]
             [source.db.honey :as db]
             [source.db.tables :as tables]
-            [source.datastore.config :as ds]
             [source.config :as conf]))
 
 (def baselines-seed
@@ -65,7 +64,6 @@
 
 (defn run-up! [context]
   (let [ds-master (:db-master context)]
-    (ds/create-datastore :datahike)
 
     (tables/create-tables!
      ds-master
@@ -109,7 +107,6 @@
 
 (defn run-down! [context]
   (let [ds-master (:db-master context)]
-    (ds/delete-datastore :datahike)
 
     (tables/drop-all-tables! ds-master)))
 
diff --git a/src/source/routes/data.clj b/src/source/routes/data.clj
index 124a26a5..8c70ac0a 100644
--- a/src/source/routes/data.clj
+++ b/src/source/routes/data.clj
@@ -1,21 +1,16 @@
 (ns source.routes.data
-  (:require [source.services.interface :as services]
+  (:require [source.workers.xml-schemas :as xml]
             [ring.util.response :as res]))
 
-(defn post [{:keys [store body] :as _request}]
-  (let [{:keys [_schema-id _url] :as opts} body]
-    (-> (services/extract-data store opts)
+(defn post [{:keys [ds body] :as _request}]
+  (let [{:keys [schema-id url]} body]
+    (-> (xml/extract-data ds schema-id url)
         (res/response))))
 
 (comment
-  (require '[source.rss.youtube :as yt]
-           '[source.datastore.interface :as store])
+  (require '[source.rss.youtube :as yt])
 
   (def url (->> "https://www.youtube.com/@ThePrimeTimeagen"
                 (yt/find-channel-id)
                 (str "https://www.youtube.com/feeds/videos.xml?channel_id=")))
-
-  (post {:store (store/ds :datahike)
-         :body {:schema-id 100
-                :url url}})
   ())
diff --git a/src/source/routes/feeds.clj b/src/source/routes/feeds.clj
index c0de6f34..64bb8c00 100644
--- a/src/source/routes/feeds.clj
+++ b/src/source/routes/feeds.clj
@@ -57,7 +57,7 @@
                            [:ts-and-cs [:maybe :int]]
                            [:state [:enum "live" "not live" "pending"]]]}}}
 
-  [{:keys [js ds store user body] :as _request}]
+  [{:keys [js ds user body] :as _request}]
   (let [exists (hon/exists? ds {:tname :feeds
                                 :where [:= :rss-url (:rss-url body)]
                                 :ret :1})]
@@ -66,8 +66,8 @@
           (res/status 400))
 
       (let [{:keys [provider-id rss-url content-type-id]} body
-            new-feed (feeds/create-feed! ds store {:user-id (:id user)
-                                                   :feed-metadata body})
+            new-feed (feeds/create-feed! ds {:user-id (:id user)
+                                             :feed-metadata body})
             {:keys [email]} (hon/find-one ds {:tname :users
                                               :where [:= :id (:id user)]})]
         (if new-feed
@@ -75,7 +75,6 @@
             ;TODO: service needed
             (->> (jobs/prepare-congest-metadata
                   ds
-                  store
                   {:id (str email "-" (:id new-feed))
                    :initial-delay (* 1000 60 60 24)
                    :auto-start true
@@ -97,13 +96,11 @@
               (res/status 422)))))))
 
 (comment
-  (require '[source.db.util :as db.util]
-           '[source.datastore.util :as store.util])
+  (require '[source.db.util :as db.util])
 
   (get {:ds (db.util/conn) :user {:id 3}})
   (post {:ds (db.util/conn)
          :js (congest/create-job-service [])
-         :store (store.util/conn :datahike)
          :user {:id 3}
          :body {:rss-url "https://www.youtube.com/feeds/videos.xml?channel_id=UCUyeluBRhGPCW4rPe_UvBZQ"
                 :provider-id 1
diff --git a/src/source/routes/integration.clj b/src/source/routes/integration.clj
index f74d2015..d9dbf0ae 100644
--- a/src/source/routes/integration.clj
+++ b/src/source/routes/integration.clj
@@ -53,7 +53,7 @@
                401 {:body [:map [:message :string]]}
                403 {:body [:map [:message :string]]}}}
 
-  [{:keys [js ds store path-params body] :as _request}]
+  [{:keys [js ds path-params body] :as _request}]
   (let [bundle-id (:id path-params)
         job-id (str "bundle_" bundle-id)
         categories-by-bundle (bundles/categories-in-bundle ds bundle-id)]
@@ -65,7 +65,6 @@
     (congest/deregister! js job-id)
     (->> (jobs/prepare-congest-metadata
           ds
-          store
           {:id job-id
            :initial-delay (* 1000 60 60 24)
            :auto-start true
diff --git a/src/source/routes/integrations.clj b/src/source/routes/integrations.clj
index c6e0bdbe..ac05c7e5 100644
--- a/src/source/routes/integrations.clj
+++ b/src/source/routes/integrations.clj
@@ -54,27 +54,16 @@
                401 {:body [:map [:message :string]]}
                403 {:body [:map [:message :string]]}}}
 
-  [{:keys [js ds store user body] :as _request}]
+  [{:keys [js ds user body] :as _request}]
   (let [new-bundle (->> {:user-id (:id user)
                          :bundle-metadata (dissoc body :categories :content-types)
                          :categories (:categories body)
                          :content-types (:content-types body)}
                         (integrations/create-integration! ds))
-        categories-by-bundle (bundles/categories-in-bundle ds (:id new-bundle))
-
-        ; TEMPORARY BLOCK
-        ; TODO: delete this and fix the problem in jobs that causes outgoing posts to be empty
-       ;posts (->> (hon/find ds {:tname :incoming-posts
-       ;                         :limit 2000})
-       ;           (mapv #(dissoc % :redacted)))
-       ;_ (hon/insert! ds (-> (db.util/tname :outgoing-posts (:id new-bundle))
-       ;                      (assoc :data posts)))
-        ; END OF TEMPORARY BLOCK
-        ]
+        categories-by-bundle (bundles/categories-in-bundle ds (:id new-bundle))]
     ;TODO: service needed
     (->> (jobs/prepare-congest-metadata
           ds
-          store
           {:id (handlers/update-bundle-job-id (:id new-bundle))
            :initial-delay 0
            :auto-start true
diff --git a/src/source/routes/interface.clj b/src/source/routes/interface.clj
index baec7cf9..86c8ccc5 100644
--- a/src/source/routes/interface.clj
+++ b/src/source/routes/interface.clj
@@ -1,6 +1,6 @@
 (ns source.routes.interface
   (:require [source.routes.reitit :as reitit]))
 
-(defn create-app [{:keys [ds store js] :as opts}]
+(defn create-app [{:keys [ds js] :as opts}]
   (reitit/create-app opts))
 
diff --git a/src/source/routes/job_start.clj b/src/source/routes/job_start.clj
index 7a3873fe..6a6fe08e 100644
--- a/src/source/routes/job_start.clj
+++ b/src/source/routes/job_start.clj
@@ -3,7 +3,7 @@
             [ring.util.response :as res]
             [source.services.interface :as services]))
 
-(defn get [{:keys [js ds store path-params]}]
+(defn get [{:keys [js ds path-params]}]
   (let [job (services/job ds path-params)]
-    (jobs/start! js ds store (:job-id job))
+    (jobs/start! js ds (:job-id job))
     (res/response {:message "successfully started job"})))
diff --git a/src/source/routes/jobs.clj b/src/source/routes/jobs.clj
index 14d19f81..7aa0a9fd 100644
--- a/src/source/routes/jobs.clj
+++ b/src/source/routes/jobs.clj
@@ -13,7 +13,7 @@
                     {:metadata metadata}))))
    (res/response)))
 
-(defn post [{:keys [js ds store body] :as _req}]
+(defn post [{:keys [js ds body] :as _req}]
   (let [{:keys [metadata]} body]
-    (congest/register! js (jobs/prepare-congest-metadata ds store metadata))
+    (congest/register! js (jobs/prepare-congest-metadata ds metadata))
     (res/response {:message "successfully registered job"})))
diff --git a/src/source/routes/output_schema.clj b/src/source/routes/output_schema.clj
index 0ff89175..9306a3e8 100644
--- a/src/source/routes/output_schema.clj
+++ b/src/source/routes/output_schema.clj
@@ -1,14 +1,14 @@
 (ns source.routes.output-schema
-  (:require [source.services.interface :as services]
+  (:require [source.workers.xml-schemas :as xml]
             [ring.util.response :as res]))
 
-(defn get [{:keys [store path-params] :as _request}]
+(defn get [{:keys [ds path-params] :as _request}]
   (let [id (try
              (Integer/parseInt (:id path-params))
              (catch Exception _ nil))]
     (if (some? id)
       (->> id
-           (services/output-schema store)
+           (xml/output-schema ds)
            (res/response))
       (-> (res/response {:message "invalid id"})
           (res/status 400)))))
diff --git a/src/source/routes/output_schemas.clj b/src/source/routes/output_schemas.clj
index ac6d7a0f..6b5691ae 100644
--- a/src/source/routes/output_schemas.clj
+++ b/src/source/routes/output_schemas.clj
@@ -1,12 +1,12 @@
 (ns source.routes.output-schemas
-  (:require [source.services.interface :as services]
+  (:require [source.workers.xml-schemas :as xml]
             [ring.util.response :as res]))
 
-(defn get [{:keys [store] :as _request}]
-  (-> (services/output-schemas store)
+(defn get [{:keys [ds] :as _request}]
+  (-> (xml/output-schemas ds)
       (res/response)))
 
-(defn post [{:keys [store body] :as _request}]
+(defn post [{:keys [ds body] :as _request}]
   (let [{:keys [schema]} body]
-    (services/insert-output-schema! store schema)
+    (xml/insert-output-schema! ds schema)
     (res/response {:message "successfully added output schema"})))
diff --git a/src/source/routes/provider.clj b/src/source/routes/provider.clj
index 45854eac..c228bc02 100644
--- a/src/source/routes/provider.clj
+++ b/src/source/routes/provider.clj
@@ -1,5 +1,5 @@
 (ns source.routes.provider
-  (:require [source.services.interface :as services]
+  (:require [source.workers.xml-schemas :as xml]
             [ring.util.response :as res]
             [source.db.honey :as hon]))
 
@@ -44,8 +44,8 @@
                                   :description "provider id"} :int]]}
    :responses {200 {:body [:map [:message :string]]}}}
 
-  [{:keys [store ds path-params] :as _request}]
+  [{:keys [ds path-params] :as _request}]
   (hon/delete! ds {:tname :providers
                    :where [:= :id (:id path-params)]})
-  (services/delete-selection-schemas-by-provider! store ds (:id path-params))
+  (xml/delete-selection-schemas-by-provider! ds (:id path-params))
   (res/response {:message "successfully deleted provider"}))
diff --git a/src/source/routes/provider_selection_schemas.clj b/src/source/routes/provider_selection_schemas.clj
index bc146c1d..59bb1f61 100644
--- a/src/source/routes/provider_selection_schemas.clj
+++ b/src/source/routes/provider_selection_schemas.clj
@@ -1,15 +1,15 @@
 (ns source.routes.provider-selection-schemas
   (:require [ring.util.response :as res]
-            [source.services.interface :as services]))
+            [source.workers.xml-schemas :as xml]))
 
-(defn get [{:keys [ds path-params store] :as _request}]
-  (let [selection-schemas (services/selection-schemas 
-                            ds 
-                            {:where [:= :provider-id (:id path-params)]})
+(defn get [{:keys [ds path-params] :as _request}]
+  (let [selection-schemas (xml/selection-schemas
+                           ds
+                           {:where [:= :provider-id (Integer/parseInt (:id path-params))]})
         results (mapv (fn [{:keys [output-schema-id] :as ss}]
                         (merge
                          ss
                          {:output-schema
-                          (services/output-schema store output-schema-id)}))
+                          (xml/output-schema ds output-schema-id)}))
                       selection-schemas)]
     (res/response results)))
diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index 4f6c2e8c..86a899c2 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -76,7 +76,7 @@
             [source.routes.approve-feed :as approve-feed]
             [source.routes.reject-feed :as reject-feed]))
 
-(defn create-app [{:keys [ds store js]}]
+(defn create-app [{:keys [ds js]}]
   (ring/ring-handler
    (ring/router
     [(rutil/swagger-route)
@@ -257,7 +257,7 @@
       ["/ast" (post xml/post)]
       ["/extract-data" (post data/post)]]]
 
-    (rutil/data-map [[mw/apply-generic :ds ds :store store :js js]]))
+    (rutil/data-map [[mw/apply-generic :ds ds :js js]]))
    (ring/routes
     (rutil/swagger-ui-handler)
     (ring/create-default-handler))))
@@ -266,11 +266,9 @@
   (require '[source.middleware.auth.util :as auth.util]
            '[source.db.util :as db.util]
            '[congest.jobs :as js]
-           '[source.datastore.interface :as store]
            '[source.rss.youtube :as yt])
 
   (def components {:ds (db.util/conn)
-                   :store (store/ds :datahike)
                    :js (js/create-job-service [])})
 
   (let [app (create-app components)
@@ -428,10 +426,8 @@
         (json/read-json {:key-fn keyword})))
 
   (let [app (create-app components)
-        store (store/ds :datahike)
         request {:uri "/admin/selection-schemas"
                  :request-method :post
-                 :store store
                  :body {:record {:provider-id 1
                                  :output-schema-id 1}
                         :schema {:title {:path ["tag/body" "tag/feed" "tag/title" "content/0"]}}}
@@ -451,9 +447,7 @@
         (json/read-json {:key-fn keyword})))
 
   (let [app (create-app components)
-        store (store/ds :datahike)
         request {:uri "/admin/selection-schemas"
-                 :store store
                  :request-method :get
                  :headers {"authorization" (str "Bearer " (auth.util/sign-jwt {:id 1 :type "admin"}))}}]
     (-> request
@@ -477,16 +471,11 @@
         (json/read-json {:key-fn keyword})))
 
   (let [app (create-app components)
-        store (store/ds :datahike)
         request {:uri "/admin/extract-data"
-                 :store store
                  :request-method :post
                  :body {:schema-id 1
                         :url (get-url)}
                  :headers {"authorization" (str "Bearer " (auth.util/sign-jwt {:id 1 :type "admin"}))}}]
-    (println (store/entities-with store :selection-schemas/id))
-    (println (store/find-entities store {:key :selection-schemas/id
-                                         :value 1}))
     (-> request
         app
         :body
diff --git a/src/source/routes/selection_schema.clj b/src/source/routes/selection_schema.clj
index 77d25d67..79387288 100644
--- a/src/source/routes/selection_schema.clj
+++ b/src/source/routes/selection_schema.clj
@@ -1,18 +1,16 @@
 (ns source.routes.selection-schema
   (:require [ring.util.response :as res]
-            [source.services.interface :as services]))
+            [source.workers.xml-schemas :as xml]))
 
-(defn get [{:keys [ds path-params store] :as _request}]
+(defn get [{:keys [ds path-params] :as _request}]
   (let [{:keys [output-schema-id] :as selection-schema}
-        (services/selection-schema ds path-params)
-        output-schema (services/output-schema store output-schema-id)]
+        (xml/selection-schema ds path-params)
+        output-schema (xml/output-schema ds output-schema-id)]
     (res/response (merge {:output-schema output-schema}
                          selection-schema))))
 
 (comment
-  (require '[source.db.util :as db.util]
-           '[source.datastore.util :as store.util])
+  (require '[source.db.util :as db.util])
   (get {:ds (db.util/conn)
-        :store (store.util/conn :datahike)
         :path-params {:id 1}})
   ())
diff --git a/src/source/routes/selection_schemas.clj b/src/source/routes/selection_schemas.clj
index f0a67534..43df3cd2 100644
--- a/src/source/routes/selection_schemas.clj
+++ b/src/source/routes/selection_schemas.clj
@@ -1,17 +1,15 @@
 (ns source.routes.selection-schemas
   (:require [ring.util.response :as res]
-            [source.services.interface :as services]))
+            [source.workers.xml-schemas :as xml]))
 
 (defn get [{:keys [ds] :as _request}]
-  (-> (services/selection-schemas ds)
+  (-> (xml/selection-schemas ds)
       (res/response)))
 
-(defn post [{:keys [store ds body] :as _request}]
-  (-> (services/insert-selection-schema! store ds body)
-      (first)
+(defn post [{:keys [ds body] :as _request}]
+  (-> (xml/insert-selection-schema! ds body)
       (res/response)))
 
-
 (comment
   (require '[source.db.util :as db.util])
   (get {:ds (db.util/conn)})
diff --git a/src/source/routes/xml.clj b/src/source/routes/xml.clj
index b2d944e9..37793cce 100644
--- a/src/source/routes/xml.clj
+++ b/src/source/routes/xml.clj
@@ -1,8 +1,8 @@
 (ns source.routes.xml
-  (:require [source.services.interface :as services]
+  (:require [source.workers.xml-schemas :as xml]
             [ring.util.response :as res]))
 
 (defn post [{:keys [body] :as _request}]
   (let [{:keys [url]} body]
-    (-> (services/ast url)
+    (-> (xml/ast url)
         (res/response))))
diff --git a/src/source/server.clj b/src/source/server.clj
index 0a84b7dd..49184673 100644
--- a/src/source/server.clj
+++ b/src/source/server.clj
@@ -1,7 +1,6 @@
 (ns source.server
   (:require [org.httpkit.server :as http]
             [source.db.interface :as db]
-            [source.datastore.interface :as store]
             [congest.jobs :as congest]
             [source.jobs.core :as jobs]
             [source.routes.interface :as routes]
@@ -9,15 +8,14 @@
 
 (defonce ^:private *components (atom nil))
 
-(defn initialise-server! [{:keys [ds store js]}]
+(defn initialise-server! [{:keys [ds js]}]
   (http/run-server
    (routes/create-app {:ds ds
-                       :store store
                        :js js})
    {:port 3000}))
 
-(defn initialise-job-service! [{:keys [ds store] :as _deps}]
-  (->> (jobs/interrupted-jobs ds store)
+(defn initialise-job-service! [{:keys [ds] :as _deps}]
+  (->> (jobs/interrupted-jobs ds)
        (congest/create-job-service)))
 
 (defn component-on? [component]
@@ -49,14 +47,11 @@
           (initialise-components! [{:name :ds
                                     :init-fn (fn [_deps] (db/ds :master))
                                     :deps []}
-                                   {:name :store
-                                    :init-fn (fn [_deps] (store/ds :datahike))
-                                    :deps []}
                                    {:name :js
-                                    :deps [:ds :store]
+                                    :deps [:ds]
                                     :init-fn initialise-job-service!}
                                    {:name :server
-                                    :deps [:ds :store :js]
+                                    :deps [:ds :js]
                                     :init-fn initialise-server!}]))
         :else
         (println "Server already running!")))
@@ -80,11 +75,8 @@
       (initialise-components! [{:name :ds
                                 :init-fn (fn [_deps] (db/ds :master))
                                 :deps []}
-                               {:name :store
-                                :init-fn (fn [_deps] (store/ds :datahike))
-                                :deps []}
                                {:name :server
-                                :deps [:ds :store :js]
+                                :deps [:ds :js]
                                 :init-fn initialise-server!}]))
     (do
       (stop-server)
diff --git a/src/source/workers/feeds.clj b/src/source/workers/feeds.clj
index eb8dc4b8..47a045f8 100644
--- a/src/source/workers/feeds.clj
+++ b/src/source/workers/feeds.clj
@@ -1,12 +1,12 @@
 (ns source.workers.feeds
   (:require [source.util :as utils]
-            [source.services.xml-schemas :as xml]
+            [source.workers.xml-schemas :as xml]
             [congest.jobs :as congest]
             [source.db.honey :as hon]))
 
 (defn create-feed!
   "Creates feed with incoming posts pulled from RSS feed and starts associated job"
-  [ds store {:keys [user-id feed-metadata]}]
+  [ds {:keys [user-id feed-metadata]}]
   (let [{:keys [provider-id rss-url content-type-id]} feed-metadata
         datetime (utils/get-utc-timestamp-string)
         selection-schemas (->> [:= :provider-id provider-id]
@@ -17,7 +17,7 @@
                                  (conj acc id)) [])
                        (apply max -1))
         extracted (when-not (= latest-ss -1)
-                    (xml/extract-data store latest-ss rss-url))
+                    (xml/extract-data ds latest-ss rss-url))
         extracted-posts (get-in extracted [:feed :posts])
         new-feed (hon/insert!
                   ds

From 9272db3bdd0e49ff783c65b7824eafc6fc222095 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Tue, 17 Feb 2026 17:25:58 +0200
Subject: [PATCH 285/391] switched from jdbc to pg2

---
 deps.edn                |  2 ++
 src/source/db/honey.clj | 24 +++++++++++-------------
 src/source/db/util.clj  | 17 ++++++++++++++---
 src/source/migrate.clj  | 14 +++++++++-----
 4 files changed, 36 insertions(+), 21 deletions(-)

diff --git a/deps.edn b/deps.edn
index 99442e04..1ce2e076 100644
--- a/deps.edn
+++ b/deps.edn
@@ -17,6 +17,8 @@
         io.github.modulr-software/congest {:mvn/version "0.1.7"}
         com.github.seancorfield/next.jdbc {:mvn/version "1.3.1070"}
         org.postgresql/postgresql {:mvn/version "42.2.10"}
+        com.github.igrishaev/pg2-core {:mvn/version "0.1.44"}
+        com.github.igrishaev/pg2-honey {:mvn/version "0.1.44"}
         org.xerial/sqlite-jdbc {:mvn/version "3.51.0.0"}
         buddy/buddy-core {:mvn/version "1.12.0-430"}
         buddy/buddy-sign {:mvn/version "3.5.351"}
diff --git a/src/source/db/honey.clj b/src/source/db/honey.clj
index b863c80f..8898c62d 100644
--- a/src/source/db/honey.clj
+++ b/src/source/db/honey.clj
@@ -1,28 +1,24 @@
 (ns source.db.honey
   (:require [camel-snake-kebab.core :as csk]
             [camel-snake-kebab.extras :as cske]
-            [honey.sql :as sql]
             [honey.sql.helpers :as hsql]
-            [next.jdbc :as jdbc]
-            [next.jdbc.result-set :as rs]
-            [source.db.util :as db.util]))
+            [pg.core :as pg]
+            [pg.honey :as pgh]
+            [source.db.util :as db.util]
+            [source.db.honey :as hon]))
 
 (defn execute!
   "computes a prepared statement for an sql map and executes select one
   or select all. returns results as unqualified lower maps by default."
-  [ds sqlmap & {:keys [ret exec-opts]}]
+  [ds sqlmap & {:keys [ret]}]
   (assert (and (some? ds) (some? sqlmap) (or (some? ret) (nil? ret))))
-  (let [ps (sql/format sqlmap)
-        exec-opts' (merge
-                    {:builder-fn rs/as-unqualified-lower-maps}
-                    exec-opts)
-        result (cske/transform-keys
+  (let [result (cske/transform-keys
                 csk/->kebab-case-keyword
-                (jdbc/execute! ds ps exec-opts'))]
+                (pg/with-conn [conn ds]
+                  (pgh/execute conn sqlmap)))]
     (cond
       (= ret :1) (first result)
-      (= ret :*) result
-      :else nil)))
+      :else result)))
 
 (defn find
   "does find one or find all for a given table name and where clause. The where
@@ -114,4 +110,6 @@
             :where [:= :id 3]
             :values {:type "creator"}})
 
+  (find ds {:tname :bundles})
+
   ())
diff --git a/src/source/db/util.clj b/src/source/db/util.clj
index 792c904a..942903d9 100644
--- a/src/source/db/util.clj
+++ b/src/source/db/util.clj
@@ -1,7 +1,8 @@
 (ns source.db.util
   (:require [source.config :as conf]
             [next.jdbc :as jdbc]
-            [next.jdbc.result-set :as rs]))
+            [next.jdbc.result-set :as rs]
+            [pg.core :as pg]))
 
 (defn db-name
   ([type]
@@ -17,8 +18,7 @@
     conn))
 
 (defn- -conn [dbname]
-  (-> {:dbtype (conf/read-value :database :type)
-       :jdbcUrl (str "jdbc:" (conf/read-value :database :url) ":5432/" dbname)}))
+  {:connection-uri (str (conf/read-value :database :url) ":5432/" dbname)})
 
 (defn conn
   ([]
@@ -38,3 +38,14 @@
 
 (defn tnames [tnames id]
   (mapv #(tname % id) tnames))
+
+(comment
+  (def q "SELECT * FROM events")
+
+  (time (pg/with-conn [conn {:connection-uri "postgresql://postgres:postgres@localhost:5432/master?ssl=false"}]
+          (pg/query conn q)))
+
+  (time (jdbc/execute! {:dbtype "postgresql"
+                        :jdbcUrl (str "jdbc:" (conf/read-value :database :url) ":5432/master")} [q]))
+
+  ())
diff --git a/src/source/migrate.clj b/src/source/migrate.clj
index 3faa2f8b..019b23c3 100644
--- a/src/source/migrate.clj
+++ b/src/source/migrate.clj
@@ -3,8 +3,10 @@
             [k16.mallard.store.postgres :as store]
             [k16.mallard.loader.fs :as loader.fs]
             [next.jdbc :as jdbc]
+            [pg.core :as pg]
             [source.db.util :as db.util]
             [source.db.honey :as db]
+            [source.config :as conf]
             [source.db.tables :as tables]))
 
 ;; This is our interface for running migrations.
@@ -22,22 +24,24 @@
   (loader.fs/load! "src/source/bundle_migrations"))
 
 (defn run-migrations [args]
-  (let [context {:db-master (jdbc/get-datasource (db.util/conn))}
-        db-migrate (jdbc/get-datasource (db.util/conn :migrate))
+  (let [context {:db-master (db.util/conn)}
+        db-migrate (jdbc/get-datasource {:dbtype "postgresql"
+                                         :jdbcUrl (str "jdbc:" (conf/read-value :database :url) ":5432/migrate")})
         datastore (store/create-datastore
                    {:ds db-migrate
                     :table-name "migrations"})]
-    (try (jdbc/execute! (:db-master context) ["CREATE DOMAIN DATETIME TEXT"]) (catch Exception _))
+    #_(try (jdbc/execute! (:db-master context) ["CREATE DOMAIN DATETIME TEXT"]) (catch Exception _))
     (mallard/run {:context context
                   :store datastore
                   :operations migrations}
                  args)))
 
 (defn migrate-bundle [bundle-id args]
-  (let [context {:ds-master (jdbc/get-datasource (db.util/conn))
+  (let [context {:ds-master (db.util/conn)
                  :bundle-id bundle-id}
         datastore (store/create-datastore
-                   {:ds (:ds-master context)
+                   {:ds (jdbc/get-datasource {:dbtype "postgresql"
+                                              :jdbcUrl (str "jdbc:" (conf/read-value :database :url) ":5432/master")})
                     :table-name (str "migrations_" bundle-id)})]
     (mallard/run {:context context
                   :store datastore

From b592bf853ff935e185025cccb471f545e46832c3 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Tue, 17 Feb 2026 17:41:53 +0200
Subject: [PATCH 286/391] removed circ dependency

---
 src/source/db/honey.clj | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/src/source/db/honey.clj b/src/source/db/honey.clj
index 8898c62d..029377ae 100644
--- a/src/source/db/honey.clj
+++ b/src/source/db/honey.clj
@@ -3,9 +3,7 @@
             [camel-snake-kebab.extras :as cske]
             [honey.sql.helpers :as hsql]
             [pg.core :as pg]
-            [pg.honey :as pgh]
-            [source.db.util :as db.util]
-            [source.db.honey :as hon]))
+            [pg.honey :as pgh]))
 
 (defn execute!
   "computes a prepared statement for an sql map and executes select one
@@ -93,8 +91,7 @@
 
 (comment
   (hsql/where :or [:= :id 1] [:= :id 2])
-
-  (def ds (db.util/conn))
+  (def ds {})
 
   (insert! ds {:tname :sectors
                :values {:name "something"}

From 330a96021bb382011c1aa42ba5db94abcd51741c Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 18 Feb 2026 10:05:31 +0200
Subject: [PATCH 287/391] fixed migration

---
 src/source/migrations/003_bundle_content_types.clj | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/source/migrations/003_bundle_content_types.clj b/src/source/migrations/003_bundle_content_types.clj
index cc798ee0..f2b55db7 100644
--- a/src/source/migrations/003_bundle_content_types.clj
+++ b/src/source/migrations/003_bundle_content_types.clj
@@ -14,8 +14,8 @@
 
     (run! (fn [{:keys [id content-type-id]}]
             (when (some? content-type-id)
-              (services/insert-bundle-content-types! ds-master {:data {:bundle-id id
-                                                                       :content-types [{:id content-type-id}]}})))
+              (services/insert-bundle-content-types! ds-master {:bundle-id id
+                                                                :content-types [{:id content-type-id}]})))
           bundles)))
 
 (defn run-down! [context]

From 24c28e598b6e96596252914f74e06b2c442dc5f2 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Thu, 19 Feb 2026 12:59:40 +0200
Subject: [PATCH 288/391] updated outgoing posts endpoint to use single query
 instead of multiple with clojure work

---
 src/source/workers/bundles.clj | 49 +++++++++++++++++-----------------
 1 file changed, 25 insertions(+), 24 deletions(-)

diff --git a/src/source/workers/bundles.clj b/src/source/workers/bundles.clj
index 5318f95c..e22d186f 100644
--- a/src/source/workers/bundles.clj
+++ b/src/source/workers/bundles.clj
@@ -4,7 +4,8 @@
             [clojure.set :as set]
             [source.services.feed-categories :as feed-categories]
             [source.db.util :as db.util]
-            [source.prandom.core :as prandom])
+            [source.prandom.core :as prandom]
+            [honey.sql :as sql])
   (:import [java.time LocalDateTime]
            [java.time.format DateTimeFormatter]))
 
@@ -62,11 +63,18 @@
                                                       :where [:= :bundle-id bundle-id]
                                                       :ret :*}))
 
-        filtered-posts (hon/find ds (-> (hsql/where (when type [:= :content-type-id type])
-                                                    (when (seq blocked-post-ids) [:not [:in :id blocked-post-ids]])
-                                                    [:in :feed-id available-feed-ids])
-                                        (assoc :order-by (when (= latest "true") [[:posted-at :desc]]))
-                                        (merge (db.util/tname :outgoing-posts bundle-id))))
+        filtered-posts (hon/execute!
+                        ds
+                        (-> (hsql/select :p.*)
+                            (hsql/from [(:tname (db.util/tname :outgoing-posts bundle-id)) :p])
+                            (hsql/join [:feed-categories :fc] [:= :p.feed-id :fc.feed-id])
+                            (hsql/join [:categories :c] [:= :fc.category-id :c.id])
+                            (hsql/where
+                             (when type [:= :content-type-id type])
+                             (when (seq blocked-post-ids) [:not [:in :p.id blocked-post-ids]])
+                             (when (seq available-feed-ids) [:in :p.feed-id available-feed-ids])
+                             (when (seq category-ids) [:in :c.id category-ids]))
+                            (hsql/order-by [:p.posted-at :desc])))
 
         order-map (->> (if (or (nil? seed) (= seed ""))
                          (.format (LocalDateTime/now) (DateTimeFormatter/ofPattern "yyyy-MM-dd HH"))
@@ -81,28 +89,21 @@
                               (sort-by #(get order-map (first %)))
                               (mapv last)))
 
-        categorised-posts (vec
-                           (if (seq category-ids)
-                             (->> shuffled-posts
-                                  (mapv
-                                   (fn [post]
-                                     (when (seq (set/intersection
-                                                 (set category-ids)
-                                                 (->> {:feed-id (:feed-id post)}
-                                                      (feed-categories/categories-by-feed ds)
-                                                      (mapv :id)
-                                                      (set))))
-                                       post)))
-                                  (remove nil?))
-                             shuffled-posts))
-
-        valid-start? (and (some? start) (>= start 0) (< start (count categorised-posts)))
+        valid-start? (and (some? start) (>= start 0) (< start (count shuffled-posts)))
         started-posts (if valid-start?
-                        (subvec categorised-posts start)
-                        categorised-posts)
+                        (subvec shuffled-posts start)
+                        shuffled-posts)
 
         valid-limit? (and (some? limit) (> (count started-posts) limit))
         limited-posts (if valid-limit?
                         (subvec started-posts 0 limit)
                         started-posts)]
     limited-posts))
+
+(comment 
+
+  (time (get-outgoing-posts (db.util/conn) {:bundle-id 14
+                                            :category-ids [50 52 54]
+                                            :latest "false"}))
+  
+  ())

From e520f5f5619ddaac8ed8b57f0d317df3bbed4e50 Mon Sep 17 00:00:00 2001
From: merveillevaneck <merveillevaneck@gmail.com>
Date: Thu, 19 Feb 2026 13:18:43 +0200
Subject: [PATCH 289/391] added deployment scripts

---
 deploy.sh         | 13 +++++++++++++
 server_startup.sh |  8 ++++++++
 start.sh          |  4 +++-
 3 files changed, 24 insertions(+), 1 deletion(-)
 create mode 100755 deploy.sh
 create mode 100755 server_startup.sh

diff --git a/deploy.sh b/deploy.sh
new file mode 100755
index 00000000..83065e1f
--- /dev/null
+++ b/deploy.sh
@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+systemctl stop source-be.service
+
+export $(grep '.*' .env | xargs)
+
+echo "Pulling latest changes..."
+git pull
+echo "Starting compilation..."
+export JAVA_CMD="/home/merv/.jenv/shims/java"
+clojure -T:build uber
+
+systemctl start source-be.service
diff --git a/server_startup.sh b/server_startup.sh
new file mode 100755
index 00000000..99d12bfa
--- /dev/null
+++ b/server_startup.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+echo "Starting tailscale funnel..."
+tailscale funnel 3000 &
+cd /home/merv/Developer/source-be
+
+echo "Starting server..."
+./start.sh
diff --git a/start.sh b/start.sh
index d16c0e08..2a0d33ba 100755
--- a/start.sh
+++ b/start.sh
@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
+export $(grep '.*' .env | xargs)
 
+export JAVA_CMD="/home/merv/.jenv/shims/java"
 clojure -M:migrate
 
-java -jar target/source-be-standalone.jar
+/home/merv/.jenv/shims/java -jar target/source-be-standalone.jar

From 59690e6c6e0affdfb6fa93e2c1c0909901408474 Mon Sep 17 00:00:00 2001
From: merveillevaneck <merveillevaneck@gmail.com>
Date: Thu, 19 Feb 2026 14:14:54 +0200
Subject: [PATCH 290/391] fixed start script

---
 merv_start.sh     | 7 +++++++
 server_startup.sh | 2 +-
 start.sh          | 4 +---
 3 files changed, 9 insertions(+), 4 deletions(-)
 create mode 100755 merv_start.sh

diff --git a/merv_start.sh b/merv_start.sh
new file mode 100755
index 00000000..2a0d33ba
--- /dev/null
+++ b/merv_start.sh
@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+export $(grep '.*' .env | xargs)
+
+export JAVA_CMD="/home/merv/.jenv/shims/java"
+clojure -M:migrate
+
+/home/merv/.jenv/shims/java -jar target/source-be-standalone.jar
diff --git a/server_startup.sh b/server_startup.sh
index 99d12bfa..9d4d8a19 100755
--- a/server_startup.sh
+++ b/server_startup.sh
@@ -5,4 +5,4 @@ tailscale funnel 3000 &
 cd /home/merv/Developer/source-be
 
 echo "Starting server..."
-./start.sh
+./merv_start.sh
diff --git a/start.sh b/start.sh
index 2a0d33ba..d16c0e08 100755
--- a/start.sh
+++ b/start.sh
@@ -1,7 +1,5 @@
 #!/usr/bin/env bash
-export $(grep '.*' .env | xargs)
 
-export JAVA_CMD="/home/merv/.jenv/shims/java"
 clojure -M:migrate
 
-/home/merv/.jenv/shims/java -jar target/source-be-standalone.jar
+java -jar target/source-be-standalone.jar

From ad161d88f0c4bbf96e945e925885a1e7207fb441 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Fri, 20 Feb 2026 11:14:09 +0200
Subject: [PATCH 291/391] optimised feeds and posts endpoints to both use a
 single SQL query

---
 src/source/routes/bundle_feed_post.clj |  9 ++--
 src/source/workers/bundles.clj         | 69 ++++++++++----------------
 2 files changed, 30 insertions(+), 48 deletions(-)

diff --git a/src/source/routes/bundle_feed_post.clj b/src/source/routes/bundle_feed_post.clj
index 04e5f265..ad27b255 100644
--- a/src/source/routes/bundle_feed_post.clj
+++ b/src/source/routes/bundle_feed_post.clj
@@ -1,7 +1,9 @@
 (ns source.routes.bundle-feed-post
   (:require [ring.util.response :as res]
             [source.db.honey :as hon]
-            [source.services.analytics.interface :as analytics]))
+            [source.services.analytics.interface :as analytics]
+            [source.db.util :as db.util]
+            [honey.sql.helpers :as hsql]))
 
 (defn get
   {:summary "Get a single post by post id belonging to an RSS feed in the associated uuid-authorized bundle. 
@@ -28,9 +30,8 @@
                404 {:body [:map [:message :string]]}}}
 
   [{:keys [ds bundle-id path-params] :as _request}]
-  (let [post (hon/find-one ds {:tname :incoming-posts
-                               :where [:= :id (:post-id path-params)]
-                               :ret :1})]
+  (let [post (hon/find-one ds (-> (db.util/tname :outgoing-posts bundle-id)
+                                  (hsql/where [:= :id (:post-id path-params)])))]
     (try
       (analytics/insert-post-click! ds post bundle-id)
       (catch Exception e (println (.getMessage e))))
diff --git a/src/source/workers/bundles.clj b/src/source/workers/bundles.clj
index e22d186f..949423e7 100644
--- a/src/source/workers/bundles.clj
+++ b/src/source/workers/bundles.clj
@@ -1,11 +1,8 @@
 (ns source.workers.bundles
   (:require [source.db.honey :as hon]
             [honey.sql.helpers :as hsql]
-            [clojure.set :as set]
-            [source.services.feed-categories :as feed-categories]
             [source.db.util :as db.util]
-            [source.prandom.core :as prandom]
-            [honey.sql :as sql])
+            [source.prandom.core :as prandom])
   (:import [java.time LocalDateTime]
            [java.time.format DateTimeFormatter]))
 
@@ -25,54 +22,38 @@
 (defn get-outgoing-feeds
   "Gets a filtered list of outgoing feeds for the associated bundle."
   [ds {:keys [bundle-id type latest category-ids nonfiltered]}]
-  (let [feed-ids (mapv :feed-id (hon/find ds (db.util/tname :outgoing-posts bundle-id)))
-        category-filtered-feed-ids (if (empty? category-ids)
-                                     feed-ids
-                                     (->> (hsql/where
-                                           [:in :feed-id feed-ids]
-                                           [:in :category-id category-ids])
-                                          (merge {:tname :feed-categories
-                                                  :ret :*})
-                                          (hon/find ds)
-                                          (mapv :feed-id)))
-        blocked-feed-ids (if (some? nonfiltered)
-                           []
-                           (mapv :feed-id (hon/find ds {:tname :filtered-feeds
-                                                        :where [:= :bundle-id bundle-id]
-                                                        :ret :*})))
-        query (-> (hsql/where (when type [:= :content-type-id type])
-                              (when (seq category-filtered-feed-ids) [:in :id category-filtered-feed-ids])
-                              (when (seq blocked-feed-ids) [:not [:in :id blocked-feed-ids]]))
-                  (assoc :order-by (when latest [[:created-at :desc]]))
-                  (merge {:tname :feeds
-                          :ret :*}))
-        type-filtered (hon/find ds query)]
-    type-filtered))
+  (let [filtered-query (-> (hsql/select-distinct :f.*)
+                           (hsql/from [:feeds :f])
+                           (hsql/join [(:tname (db.util/tname :outgoing-posts bundle-id)) :p] [:= :f.id :p.feed-id])
+                           (hsql/join [:feed-categories :fc] [:= :fc.feed-id :f.id])
+                           (hsql/where
+                            (when (nil? nonfiltered) [:not-in :f.id (-> (hsql/select :feed-id)
+                                                                        (hsql/from :filtered-feeds)
+                                                                        (hsql/where [:= :bundle-id bundle-id]))])
+                            (when type [:= :f.content-type-id type])
+                            (when (seq category-ids) [:in :fc.category-id category-ids])))
+        filtered-feeds (hon/execute! ds (if (some? latest)
+                                          (assoc filtered-query :order-by [[:created-at :desc]])
+                                          filtered-query))]
+    filtered-feeds))
 
 (defn get-outgoing-posts
   "Get outgoing posts based on short heuristics and update analytics impressions"
   [ds {:keys [bundle-id limit start type latest category-ids seed]}]
-  (let [all-feed-ids (mapv :id (hon/find ds {:tname :feeds
-                                             :ret :*}))
-        blocked-feed-ids (mapv :feed-id (hon/find ds {:tname :filtered-feeds
-                                                      :where [:= :bundle-id bundle-id]
-                                                      :ret :*}))
-        available-feed-ids (vec (remove (set blocked-feed-ids) all-feed-ids))
-
-        blocked-post-ids (mapv :post-id (hon/find ds {:tname :filtered-posts
-                                                      :where [:= :bundle-id bundle-id]
-                                                      :ret :*}))
-
-        filtered-posts (hon/execute!
+  (let [filtered-posts (hon/execute!
                         ds
-                        (-> (hsql/select :p.*)
+                        (-> (hsql/select-distinct :p.*)
                             (hsql/from [(:tname (db.util/tname :outgoing-posts bundle-id)) :p])
                             (hsql/join [:feed-categories :fc] [:= :p.feed-id :fc.feed-id])
                             (hsql/join [:categories :c] [:= :fc.category-id :c.id])
                             (hsql/where
                              (when type [:= :content-type-id type])
-                             (when (seq blocked-post-ids) [:not [:in :p.id blocked-post-ids]])
-                             (when (seq available-feed-ids) [:in :p.feed-id available-feed-ids])
+                             [:not-in :p.id (-> (hsql/select :post-id)
+                                                (hsql/from :filtered-posts)
+                                                (hsql/where [:= :bundle-id bundle-id]))]
+                             [:not-in :p.feed-id (-> (hsql/select :feed-id)
+                                                     (hsql/from :filtered-feeds)
+                                                     (hsql/where [:= :bundle-id bundle-id]))]
                              (when (seq category-ids) [:in :c.id category-ids]))
                             (hsql/order-by [:p.posted-at :desc])))
 
@@ -100,10 +81,10 @@
                         started-posts)]
     limited-posts))
 
-(comment 
+(comment
 
   (time (get-outgoing-posts (db.util/conn) {:bundle-id 14
                                             :category-ids [50 52 54]
                                             :latest "false"}))
-  
+
   ())

From 11c3a93b371c1bebef07b9ad15da27a0eb1b0acb Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Fri, 20 Feb 2026 15:01:11 +0200
Subject: [PATCH 292/391] added indices for a number of tables and foreign keys
 in migrations

---
 .../bundle_migrations/003_bundle_indices.clj  | 29 ++++++++++++
 src/source/migrations/011_indices.clj         | 47 +++++++++++++++++++
 2 files changed, 76 insertions(+)
 create mode 100644 src/source/bundle_migrations/003_bundle_indices.clj
 create mode 100644 src/source/migrations/011_indices.clj

diff --git a/src/source/bundle_migrations/003_bundle_indices.clj b/src/source/bundle_migrations/003_bundle_indices.clj
new file mode 100644
index 00000000..003b5ac8
--- /dev/null
+++ b/src/source/bundle_migrations/003_bundle_indices.clj
@@ -0,0 +1,29 @@
+(ns source.bundle-migrations.003-bundle-indices
+  (:require [source.db.bundle]
+            [pg.core :as pg]
+            [source.db.util :as db.util]
+            [camel-snake-kebab.core :as csk]))
+
+(defn run-up! [context]
+  (let [{:keys [ds-master bundle-id]} context
+        op (->> bundle-id
+                (db.util/tname :outgoing-posts)
+                (:tname)
+                (csk/->snake_case_string))]
+
+    (pg/with-connection [ds-master ds-master]
+      (pg/execute ds-master (str "CREATE INDEX idx_" op "_feed_id ON " op " (feed_id);"))
+      (pg/execute ds-master (str "CREATE INDEX idx_" op "_creator_id ON " op " (creator_id);"))
+      (pg/execute ds-master (str "CREATE INDEX idx_" op "_content_type_id ON " op " (content_type_id);")))))
+
+(defn run-down! [context]
+  (let [{:keys [ds-master bundle-id]} context
+        op (->> bundle-id
+                (db.util/tname :outgoing-posts)
+                (:tname)
+                (csk/->snake_case_string))]
+
+    (pg/with-connection [ds-master ds-master]
+      (pg/execute ds-master (str "DROP INDEX IF EXISTS idx_" op "_content_type_id"))
+      (pg/execute ds-master (str "DROP INDEX IF EXISTS idx_" op "_creator_id;"))
+      (pg/execute ds-master (str "DROP INDEX IF EXISTS idx_" op "_feed_id;")))))
diff --git a/src/source/migrations/011_indices.clj b/src/source/migrations/011_indices.clj
new file mode 100644
index 00000000..7726be96
--- /dev/null
+++ b/src/source/migrations/011_indices.clj
@@ -0,0 +1,47 @@
+(ns source.migrations.011-indices
+  (:require [source.db.master]
+            [pg.core :as pg]))
+
+(defn run-up! [context]
+  (let [ds-master (:db-master context)]
+    (pg/with-connection [ds-master ds-master]
+      (pg/execute ds-master "CREATE INDEX idx_incoming_posts_feed_id ON incoming_posts (feed_id);")
+      (pg/execute ds-master "CREATE INDEX idx_incoming_posts_creator_id ON incoming_posts (creator_id);")
+      (pg/execute ds-master "CREATE INDEX idx_incoming_posts_content_type_id ON incoming_posts (content_type_id);")
+
+      (pg/execute ds-master "CREATE INDEX idx_feeds_user_id ON feeds (user_id);")
+      (pg/execute ds-master "CREATE INDEX idx_feeds_content_type_id ON feeds (content_type_id);")
+
+      (pg/execute ds-master "CREATE INDEX idx_feed_categories_feed_id ON feed_categories (feed_id);")
+      (pg/execute ds-master "CREATE INDEX idx_feed_categories_category_id ON feed_categories (category_id);")
+
+      (pg/execute ds-master "CREATE INDEX idx_bundles_user_id ON bundles (user_id);")
+      (pg/execute ds-master "CREATE INDEX idx_bundles_content_type_id ON bundles (content_type_id);")
+
+      (pg/execute ds-master "CREATE INDEX idx_filtered_feeds_bundle_id ON filtered_feeds (bundle_id);")
+      (pg/execute ds-master "CREATE INDEX idx_filtered_feeds_feed_id ON filtered_feeds (feed_id);")
+
+      (pg/execute ds-master "CREATE INDEX idx_filtered_posts_bundle_id ON filtered_posts (bundle_id);")
+      (pg/execute ds-master "CREATE INDEX idx_filtered_posts_post_id ON filtered_posts (post_id);"))))
+
+(defn run-down! [context]
+  (let [ds-master (:db-master context)]
+    (pg/with-connection [ds-master ds-master]
+      (pg/execute ds-master "DROP INDEX IF EXISTS idx_incoming_posts_feed_id;")
+      (pg/execute ds-master "DROP INDEX IF EXISTS idx_incoming_posts_creator_id;")
+      (pg/execute ds-master "DROP INDEX IF EXISTS idx_incoming_posts_content_type_id;")
+
+      (pg/execute ds-master "DROP INDEX IF EXISTS idx_feeds_user_id;")
+      (pg/execute ds-master "DROP INDEX IF EXISTS idx_feeds_content_type_id;")
+
+      (pg/execute ds-master "DROP INDEX IF EXISTS idx_feed_categories_feed_id;")
+      (pg/execute ds-master "DROP INDEX IF EXISTS idx_feed_categories_category_id;")
+
+      (pg/execute ds-master "DROP INDEX IF EXISTS idx_bundles_user_id")
+      (pg/execute ds-master "DROP INDEX IF EXISTS idx_bundles_content_type_id")
+
+      (pg/execute ds-master "DROP INDEX IF EXISTS idx_filtered_feeds_bundle_id")
+      (pg/execute ds-master "DROP INDEX IF EXISTS idx_filtered_feeds_feed_id")
+
+      (pg/execute ds-master "DROP INDEX IF EXISTS idx_filtered_posts_bundle_id")
+      (pg/execute ds-master "DROP INDEX IF EXISTS idx_filtered_posts_post_id"))))

From 691cfbdf97b57f4b98ad0e69e32503a0220d9377 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 23 Feb 2026 10:18:09 +0200
Subject: [PATCH 293/391] fixed business schema to allow optional fields

---
 src/source/routes/me_business.clj | 29 +++++++++++++++--------------
 1 file changed, 15 insertions(+), 14 deletions(-)

diff --git a/src/source/routes/me_business.clj b/src/source/routes/me_business.clj
index 84cdc4e6..639f5b5e 100644
--- a/src/source/routes/me_business.clj
+++ b/src/source/routes/me_business.clj
@@ -1,29 +1,30 @@
 (ns source.routes.me-business
   (:require [ring.util.response :as res]
-            [source.db.honey :as hon]))
+            [source.db.honey :as hon]
+            [source.routes.openapi :as api]))
 
 (defn get
   {:summary "get business for logged-in user"
    :responses {200 {:body [:map
                            [:id :int]
-                           [:name [:maybe :string]]
-                           [:address [:maybe :string]]
-                           [:url [:maybe :string]]
-                           [:linkedin [:maybe :string]]
-                           [:twitter [:maybe :string]]
-                           [:registration [:maybe :string]]
-                           [:business-type-id [:maybe :int]]]}
+                           [:name :string]
+                           (api/sometimes :address :string)
+                           (api/sometimes :url :string)
+                           (api/sometimes :linkedin :string)
+                           (api/sometimes :twitter :string)
+                           (api/sometimes :registration :string)
+                           (api/sometimes :business-type-id :int)]}
+               404 {:body [:map]}
                401 {:body [:map [:message :string]]}
                403 {:body [:map [:message :string]]}}}
 
   [{:keys [ds user] :as _request}]
   (let [{:keys [business-id]} (hon/find-one ds {:tname :users
-                                                :where [:= :id (:id user)]})
-        business (if business-id
-                   (hon/find-one ds {:tname :businesses
-                                     :where [:= :id business-id]})
-                   {})]
-    (res/response business)))
+                                                :where [:= :id (:id user)]})]
+    (if business-id
+      (res/response (hon/find-one ds {:tname :businesses
+                                      :where [:= :id business-id]}))
+      (res/response {}))))
 
 (defn post
   {:summary "add or update business for logged-in user"

From 28dc1e0cd128c6cf157bc5778fb6c0c87ab7c690 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 23 Feb 2026 10:18:41 +0200
Subject: [PATCH 294/391] fixed event deletion in hard-delete-feed function

---
 src/source/workers/feeds.clj | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/source/workers/feeds.clj b/src/source/workers/feeds.clj
index 47a045f8..292771c9 100644
--- a/src/source/workers/feeds.clj
+++ b/src/source/workers/feeds.clj
@@ -50,8 +50,9 @@
 
 (defn hard-delete-feed! [ds js job-id feed-id]
   (let [post-ids (mapv :id (hon/find ds {:tname :incoming-posts
-                                         :where [:= :feed-id feed-id]
-                                         :ret :*}))]
+                                         :where [:= :feed-id feed-id]}))
+        event-ids (:mapv :id (hon/find ds {:tname :events
+                                           :where [:= :feed-id feed-id]}))]
     (hon/delete! ds {:tname :filtered-feeds
                      :where [:= :feed-id feed-id]})
     (hon/delete! ds {:tname :filtered-posts
@@ -60,6 +61,9 @@
                      :where [:= :feed-id feed-id]})
     (hon/delete! ds {:tname :feed-categories
                      :where [:= :feed-id feed-id]})
+    (if seq event-ids
+        (hon/delete! ds {:tname :event-categories
+                         :where [:in :event-id event-ids]}))
     (hon/delete! ds {:tname :events
                      :where [:= :feed-id feed-id]})
     (hon/delete! ds {:tname :feeds

From 65f5fda5c75b31dec53ef76bdb1a16b31da132d6 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 23 Feb 2026 10:22:19 +0200
Subject: [PATCH 295/391] added missing brackets in function call

---
 src/source/workers/feeds.clj | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/source/workers/feeds.clj b/src/source/workers/feeds.clj
index 292771c9..e9988cc5 100644
--- a/src/source/workers/feeds.clj
+++ b/src/source/workers/feeds.clj
@@ -61,9 +61,9 @@
                      :where [:= :feed-id feed-id]})
     (hon/delete! ds {:tname :feed-categories
                      :where [:= :feed-id feed-id]})
-    (if seq event-ids
-        (hon/delete! ds {:tname :event-categories
-                         :where [:in :event-id event-ids]}))
+    (when (seq event-ids)
+      (hon/delete! ds {:tname :event-categories
+                       :where [:in :event-id event-ids]}))
     (hon/delete! ds {:tname :events
                      :where [:= :feed-id feed-id]})
     (hon/delete! ds {:tname :feeds

From cdead2be5e8c65e322067bf67bb4b038c8aa00d5 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 23 Feb 2026 11:55:43 +0200
Subject: [PATCH 296/391] removed deprecated validation calls in analytics
 endpoints

---
 .../routes/analytics/creator/general.clj      |  7 ++--
 src/source/routes/analytics/creator/top.clj   | 41 ++++++++-----------
 .../routes/analytics/creator/top_average.clj  | 16 +++-----
 .../routes/analytics/distributor/general.clj  |  5 +--
 .../routes/analytics/distributor/top.clj      |  1 -
 .../analytics/distributor/top_average.clj     | 16 +++-----
 6 files changed, 33 insertions(+), 53 deletions(-)

diff --git a/src/source/routes/analytics/creator/general.clj b/src/source/routes/analytics/creator/general.clj
index 984084f3..d1f7a5b3 100644
--- a/src/source/routes/analytics/creator/general.clj
+++ b/src/source/routes/analytics/creator/general.clj
@@ -18,6 +18,7 @@
                             [:views :int]]]}}}
 
   [{:keys [ds user query-params] :as _request}]
-  (let [{:keys [mindate maxdate feed]} (w/keywordize-keys query-params)]
-    (res/response (analytics/interval-statistics-query ds :daily mindate maxdate {:creator-id (:id user)
-                                                                                  :feed-id feed}))))
+  (let [{:keys [mindate maxdate feed]} (w/keywordize-keys query-params)
+        res (analytics/interval-statistics-query ds :daily mindate maxdate {:creator-id (:id user)
+                                                                            :feed-id feed})]
+    (res/response res)))
diff --git a/src/source/routes/analytics/creator/top.clj b/src/source/routes/analytics/creator/top.clj
index 5040bece..1825d808 100644
--- a/src/source/routes/analytics/creator/top.clj
+++ b/src/source/routes/analytics/creator/top.clj
@@ -2,8 +2,7 @@
   (:require [source.services.analytics.interface :as analytics]
             [ring.util.response :as res]
             [clojure.set :as set]
-            [source.services.interface :as services]
-            [source.util :as utils]))
+            [source.services.interface :as services]))
 
 (defn record-names [ds top-field ids]
   (if (= top-field :post-id)
@@ -33,25 +32,19 @@
                             [:views :int]]]}}}
 
   [{:keys [ds user query-params] :as _request}]
-  (let [{:keys [data success error]} (utils/validate get query-params :query)
-        {:keys [n mindate maxdate top contenttype]} data
-        top-field (if (= top "post") :post-id :bundle-id)]
-    (if success
-      (let [results (->> {:creator-id (:id user)
-                          :content-type-id contenttype}
-                         (analytics/top-statistics-query ds mindate maxdate n top-field)
-                         (mapv (fn [result]
-                                 (set/rename-keys result {top-field :top}))))
-            ids (mapv :top results)
-            names (record-names ds top-field ids)
-            juxted (->> names
-                        (mapv (juxt :id :name))
-                        (into {}))
-            named-results (mapv (fn [{:keys [top] :as r}]
-                                  (assoc r :top (clojure.core/get juxted top (str top))))
-                                results)]
-
-        (res/response named-results))
-
-      (-> (res/response error)
-          (res/status 400)))))
+  (let [{:keys [n mindate maxdate top contenttype]} query-params
+        top-field (if (= top "post") :post-id :bundle-id)
+        results (->> {:creator-id (:id user)
+                      :content-type-id contenttype}
+                     (analytics/top-statistics-query ds mindate maxdate n top-field)
+                     (mapv (fn [result]
+                             (set/rename-keys result {top-field :top}))))
+        ids (mapv :top results)
+        names (record-names ds top-field ids)
+        juxted (->> names
+                    (mapv (juxt :id :name))
+                    (into {}))
+        named-results (mapv (fn [{:keys [top] :as r}]
+                              (assoc r :top (clojure.core/get juxted top (str top))))
+                            results)]
+    (res/response named-results)))
diff --git a/src/source/routes/analytics/creator/top_average.clj b/src/source/routes/analytics/creator/top_average.clj
index 4f50a396..8a63451a 100644
--- a/src/source/routes/analytics/creator/top_average.clj
+++ b/src/source/routes/analytics/creator/top_average.clj
@@ -1,7 +1,6 @@
 (ns source.routes.analytics.creator.top-average
   (:require [source.services.analytics.interface :as analytics]
-            [ring.util.response :as res]
-            [source.util :as utils]))
+            [ring.util.response :as res]))
 
 (defn get
   {:summary "Get the average engagement (clicks and views) for a creator, optionally filtered by content type.
@@ -13,12 +12,7 @@
    :responses {200 {:body [:map [:average :float]]}}}
 
   [{:keys [ds user query-params] :as _request}]
-  (let [{:keys [data success error]} (utils/validate get query-params :query)
-        {:keys [mindate maxdate contenttype]} data]
-    (if success
-      (let [result (analytics/average-engagement ds mindate maxdate {:creator-id (:id user)
-                                                                     :content-type-id contenttype})]
-        (res/response {:average result}))
-
-      (-> (res/response error)
-          (res/status 400)))))
+  (let [{:keys [mindate maxdate contenttype]} query-params
+        result (analytics/average-engagement ds mindate maxdate {:creator-id (:id user)
+                                                                 :content-type-id contenttype})]
+    (res/response {:average result})))
diff --git a/src/source/routes/analytics/distributor/general.clj b/src/source/routes/analytics/distributor/general.clj
index 77216a9a..8a0c8a68 100644
--- a/src/source/routes/analytics/distributor/general.clj
+++ b/src/source/routes/analytics/distributor/general.clj
@@ -1,7 +1,6 @@
 (ns source.routes.analytics.distributor.general
   (:require [source.services.analytics.interface :as analytics]
-            [ring.util.response :as res]
-            [clojure.walk :as w]))
+            [ring.util.response :as res]))
 
 (defn get
   {:summary "Gets the number of impressions, clicks and views per day for a distributor over the given time period. Optionally filtered by bundle.
@@ -18,6 +17,6 @@
                             [:views :int]]]}}}
 
   [{:keys [ds user query-params] :as _request}]
-  (let [{:keys [mindate maxdate bundle]} (w/keywordize-keys query-params)]
+  (let [{:keys [mindate maxdate bundle]} query-params]
     (res/response (analytics/interval-statistics-query ds :daily mindate maxdate {:distributor-id (:id user)
                                                                                   :bundle-id bundle}))))
diff --git a/src/source/routes/analytics/distributor/top.clj b/src/source/routes/analytics/distributor/top.clj
index 9a1f036e..6d3a5cd8 100644
--- a/src/source/routes/analytics/distributor/top.clj
+++ b/src/source/routes/analytics/distributor/top.clj
@@ -2,7 +2,6 @@
   (:require [source.services.analytics.interface :as analytics]
             [ring.util.response :as res]
             [clojure.set :as set]
-            [source.util :as utils]
             [source.db.honey :as hon]))
 
 (defn record-names [ds top-field ids]
diff --git a/src/source/routes/analytics/distributor/top_average.clj b/src/source/routes/analytics/distributor/top_average.clj
index e4f09f08..b0778dbd 100644
--- a/src/source/routes/analytics/distributor/top_average.clj
+++ b/src/source/routes/analytics/distributor/top_average.clj
@@ -1,7 +1,6 @@
 (ns source.routes.analytics.distributor.top-average
   (:require [source.services.analytics.interface :as analytics]
-            [ring.util.response :as res]
-            [source.util :as utils]))
+            [ring.util.response :as res]))
 
 (defn get
   {:summary "Get the average engagement (clicks and views) for a distributor, optionally filtered by content type.
@@ -13,12 +12,7 @@
    :responses {200 {:body [:map [:average :float]]}}}
 
   [{:keys [ds user query-params] :as _request}]
-  (let [{:keys [data success error]} (utils/validate get query-params :query)
-        {:keys [mindate maxdate contenttype]} data]
-    (if success
-      (let [result (analytics/average-engagement ds mindate maxdate {:distributor-id (:id user)
-                                                                     :content-type-id contenttype})]
-        (res/response {:average result}))
-
-      (-> (res/response error)
-          (res/status 400)))))
+  (let [{:keys [mindate maxdate contenttype]} query-params
+        result (analytics/average-engagement ds mindate maxdate {:distributor-id (:id user)
+                                                                 :content-type-id contenttype})]
+    (res/response {:average result})))

From c2d433b16c0a7bdf9d71b0d6d9a30d2e6e84ce33 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 23 Feb 2026 11:56:24 +0200
Subject: [PATCH 297/391] added function in analytics core to automatically
 convert all instances of java.time.LocalDate to string using postwalk

---
 src/source/services/analytics/core.clj | 22 ++++++++++++++++------
 1 file changed, 16 insertions(+), 6 deletions(-)

diff --git a/src/source/services/analytics/core.clj b/src/source/services/analytics/core.clj
index f6586e5b..6cc96b5b 100644
--- a/src/source/services/analytics/core.clj
+++ b/src/source/services/analytics/core.clj
@@ -4,7 +4,16 @@
             [source.services.bundles :as bundles]
             [source.util :as util]
             [source.services.feed-categories :as feed-categories]
-            [honey.sql :as sql]))
+            [honey.sql :as sql]
+            [clojure.walk :as walk]))
+
+(defn convert-all-datetimes-to-string
+  "Uses postwalk to convert all instances of java.time.LocalDate into string"
+  [m]
+  (walk/postwalk (fn [v]
+                   (if (instance? java.time.LocalDate v)
+                     (.toString v)
+                     v)) m))
 
 (defn metric-query
   "Generic select query function for returning analytics data from the events table"
@@ -28,11 +37,12 @@
                   (some? group-by) (merge group-by)
                   (seq category-ids) (-> (hsql/join [:event-categories :ec] [:= :events.id :ec.event-id])
                                          (hsql/where [:in :ec.category-id category-ids])))]
-    (hon/execute!
-     ds
-     (merge {:from [:events]}
-            clauses)
-     {:ret (if ret ret :*)})))
+    (-> ds
+        (hon/execute!
+         (merge {:from [:events]}
+                clauses)
+         {:ret (if ret ret :*)})
+        (convert-all-datetimes-to-string))))
 
 (defn statistics-query
   "Returns the number of impressions, clicks and views, filtered by any other arguments accepted by metric-query.

From 69a55e8cc2b98f9c65f5d9e8d37a2a21c5c12d75 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 23 Feb 2026 11:57:14 +0200
Subject: [PATCH 298/391] fixed bug in user type validation authorisation

---
 src/source/middleware/auth/core.clj | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/source/middleware/auth/core.clj b/src/source/middleware/auth/core.clj
index 0441adcd..751aa910 100644
--- a/src/source/middleware/auth/core.clj
+++ b/src/source/middleware/auth/core.clj
@@ -33,7 +33,7 @@
     (let [ds (db.util/conn :master)
           user-type (get-in request [:user :type])
           expected-type (->> {:tname :users
-                              :id (get-in request [:user :id])}
+                              :where [:= :id (get-in request [:user :id])]}
                              (db/find-one ds)
                              (:type))]
       (cond

From 91ca6ff773dc9d21876caa1cde72901c6ef78f0b Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 23 Feb 2026 11:58:09 +0200
Subject: [PATCH 299/391] updated postman collection to remove old local tokens
 and added analytics endpoints to it

---
 Source.postman_collection.json | 288 ++++++++++++++++++++++-----------
 1 file changed, 194 insertions(+), 94 deletions(-)

diff --git a/Source.postman_collection.json b/Source.postman_collection.json
index 1e31d12c..68ca22ef 100644
--- a/Source.postman_collection.json
+++ b/Source.postman_collection.json
@@ -31,16 +31,6 @@
 		{
 			"name": "Get Users",
 			"request": {
-				"auth": {
-					"type": "bearer",
-					"bearer": [
-						{
-							"key": "token",
-							"value": "eyJhbGciOiJkaXIiLCJlbmMiOiJBMTI4Q0JDLUhTMjU2In0..1n1VNQ-I1Z7VhgW8IWfiPg.YkGk31r1DG073Q5uyW6I0rGbm9YNKB3r_8D_yWUfG60.suZYrgS9-JXGbRF05iRA0g",
-							"type": "string"
-						}
-					]
-				},
 				"method": "GET",
 				"header": [],
 				"url": {
@@ -125,17 +115,22 @@
 			},
 			"response": []
 		},
+		{
+			"name": "Me",
+			"request": {
+				"method": "GET",
+				"header": []
+			},
+			"response": []
+		},
 		{
 			"name": "Update User",
 			"request": {
-				"auth": {
-					"type": "noauth"
-				},
 				"method": "PATCH",
 				"header": [],
 				"body": {
 					"mode": "raw",
-					"raw": "{\n    \"address\": \"Cape Town\",\n    \"lastname\": \"Collins\"\n}",
+					"raw": "{\n    \"email\": \"toast@toast.com\",\n    \"type\": \"creator\",\n    \"address\": \"Cape Town\",\n    \"firstname\": \"Keagan\",\n    \"lastname\": \"Collins\"\n}",
 					"options": {
 						"raw": {
 							"language": "json"
@@ -160,16 +155,6 @@
 		{
 			"name": "Add Admin",
 			"request": {
-				"auth": {
-					"type": "bearer",
-					"bearer": [
-						{
-							"key": "token",
-							"value": "eyJhbGciOiJkaXIiLCJlbmMiOiJBMTI4Q0JDLUhTMjU2In0..k38ZDx6t9xOhPKwi37gmxQ.C-DvEva5l913IgKY7tlyhQ.z0DD0yqQF-vQ2A3sPRt9yw",
-							"type": "string"
-						}
-					]
-				},
 				"method": "POST",
 				"header": [],
 				"body": {
@@ -286,16 +271,6 @@
 				"disableBodyPruning": true
 			},
 			"request": {
-				"auth": {
-					"type": "bearer",
-					"bearer": [
-						{
-							"key": "token",
-							"value": "eyJhbGciOiJkaXIiLCJlbmMiOiJBMTI4Q0JDLUhTMjU2In0..1n1VNQ-I1Z7VhgW8IWfiPg.YkGk31r1DG073Q5uyW6I0rGbm9YNKB3r_8D_yWUfG60.suZYrgS9-JXGbRF05iRA0g",
-							"type": "string"
-						}
-					]
-				},
 				"method": "GET",
 				"header": [],
 				"body": {
@@ -325,16 +300,6 @@
 		{
 			"name": "Get Selection Schemas By Provider",
 			"request": {
-				"auth": {
-					"type": "bearer",
-					"bearer": [
-						{
-							"key": "token",
-							"value": "eyJhbGciOiJkaXIiLCJlbmMiOiJBMTI4Q0JDLUhTMjU2In0..1n1VNQ-I1Z7VhgW8IWfiPg.YkGk31r1DG073Q5uyW6I0rGbm9YNKB3r_8D_yWUfG60.suZYrgS9-JXGbRF05iRA0g",
-							"type": "string"
-						}
-					]
-				},
 				"method": "GET",
 				"header": [],
 				"url": {
@@ -357,16 +322,6 @@
 		{
 			"name": "Add Selection Schema",
 			"request": {
-				"auth": {
-					"type": "bearer",
-					"bearer": [
-						{
-							"key": "token",
-							"value": "eyJhbGciOiJkaXIiLCJlbmMiOiJBMTI4Q0JDLUhTMjU2In0..1n1VNQ-I1Z7VhgW8IWfiPg.YkGk31r1DG073Q5uyW6I0rGbm9YNKB3r_8D_yWUfG60.suZYrgS9-JXGbRF05iRA0g",
-							"type": "string"
-						}
-					]
-				},
 				"method": "POST",
 				"header": [],
 				"body": {
@@ -396,25 +351,16 @@
 		{
 			"name": "Get Output Schemas",
 			"request": {
-				"auth": {
-					"type": "bearer",
-					"bearer": [
-						{
-							"key": "token",
-							"value": "eyJhbGciOiJkaXIiLCJlbmMiOiJBMTI4Q0JDLUhTMjU2In0..1n1VNQ-I1Z7VhgW8IWfiPg.YkGk31r1DG073Q5uyW6I0rGbm9YNKB3r_8D_yWUfG60.suZYrgS9-JXGbRF05iRA0g",
-							"type": "string"
-						}
-					]
-				},
 				"method": "GET",
 				"header": [],
 				"url": {
-					"raw": "http://localhost:3000/admin/output-schemas",
-					"protocol": "http",
+					"raw": "https://source-be-staging.fly.dev/admin/output-schemas",
+					"protocol": "https",
 					"host": [
-						"localhost"
+						"source-be-staging",
+						"fly",
+						"dev"
 					],
-					"port": "3000",
 					"path": [
 						"admin",
 						"output-schemas"
@@ -426,21 +372,11 @@
 		{
 			"name": "Add Output Schema",
 			"request": {
-				"auth": {
-					"type": "bearer",
-					"bearer": [
-						{
-							"key": "token",
-							"value": "eyJhbGciOiJkaXIiLCJlbmMiOiJBMTI4Q0JDLUhTMjU2In0..1n1VNQ-I1Z7VhgW8IWfiPg.YkGk31r1DG073Q5uyW6I0rGbm9YNKB3r_8D_yWUfG60.suZYrgS9-JXGbRF05iRA0g",
-							"type": "string"
-						}
-					]
-				},
 				"method": "POST",
 				"header": [],
 				"body": {
 					"mode": "raw",
-					"raw": "{\n    \"schema\": {\n        \"feed\": {\n            \"type\": \"map\",\n            \"schema\": {\n                \"title\": {\n                    \"type\": \"string\"\n                },\n                \"posts\": {\n                    \"type\": \"vector\",\n                    \"schema\": {\n                        \"post-id\": {\n                            \"type\": \"string\"\n                        },\n                        \"title\": {\n                            \"type\": \"string\"\n                        },\n                        \"stream-url\": {\n                            \"type\": \"string\"\n                        }\n                    }\n                }\n            }\n        }\n    }\n}",
+					"raw": "{\n    \"schema\": {\n        \"feed\": {\n            \"type\": \"map\",\n            \"schema\": {\n                \"title\": {\n                    \"type\": \"string\"\n                },\n                \"description\": {\n                    \"type\": \"string\"\n                },\n                \"display-picture\": {\n                    \"type\": \"string\"\n                },\n                \"posts\": {\n                    \"type\": \"vector\",\n                    \"schema\": {\n                        \"post-id\": {\n                            \"type\": \"string\"\n                        },\n                        \"title\": {\n                            \"type\": \"string\"\n                        },\n                        \"thumbnail\": {\n                            \"type\": \"string\"\n                        },\n                        \"info\": {\n                            \"type\": \"string\"\n                        },\n                        \"posted-at\": {\n                            \"type\": \"string\"\n                        },\n                        \"stream-url\": {\n                            \"type\": \"string\"\n                        }\n                    }\n                }\n            }\n        }\n    }\n}",
 					"options": {
 						"raw": {
 							"language": "json"
@@ -448,12 +384,13 @@
 					}
 				},
 				"url": {
-					"raw": "http://localhost:3000/admin/output-schemas",
-					"protocol": "http",
+					"raw": "https://source-be-staging.fly.dev/admin/output-schemas",
+					"protocol": "https",
 					"host": [
-						"localhost"
+						"source-be-staging",
+						"fly",
+						"dev"
 					],
-					"port": "3000",
 					"path": [
 						"admin",
 						"output-schemas"
@@ -465,16 +402,6 @@
 		{
 			"name": "Extract Data",
 			"request": {
-				"auth": {
-					"type": "bearer",
-					"bearer": [
-						{
-							"key": "token",
-							"value": "eyJhbGciOiJkaXIiLCJlbmMiOiJBMTI4Q0JDLUhTMjU2In0..1n1VNQ-I1Z7VhgW8IWfiPg.YkGk31r1DG073Q5uyW6I0rGbm9YNKB3r_8D_yWUfG60.suZYrgS9-JXGbRF05iRA0g",
-							"type": "string"
-						}
-					]
-				},
 				"method": "POST",
 				"header": [],
 				"body": {
@@ -1221,7 +1148,7 @@
 					"bearer": [
 						{
 							"key": "token",
-							"value": "eyJhbGciOiJkaXIiLCJlbmMiOiJBMTI4Q0JDLUhTMjU2In0..TQO_QiitHYM0ydOwsQJVKg.8HnU2p4O-6OrbfE-wZDbIzVWeEt6fgk3IxMiC53pfa8.MhXN4L8Csj5uYHBd9mAMXg",
+							"value": "",
 							"type": "string"
 						}
 					]
@@ -1250,6 +1177,179 @@
 				"header": []
 			},
 			"response": []
+		},
+		{
+			"name": "Analytics Creator General",
+			"request": {
+				"auth": {
+					"type": "bearer",
+					"bearer": [
+						{
+							"key": "token",
+							"value": "",
+							"type": "string"
+						}
+					]
+				},
+				"method": "GET",
+				"header": [],
+				"url": {
+					"raw": "http://localhost:3000/analytics/creator/general?mindate=2026-01-01&maxdate=2026-02-24",
+					"protocol": "http",
+					"host": [
+						"localhost"
+					],
+					"port": "3000",
+					"path": [
+						"analytics",
+						"creator",
+						"general"
+					],
+					"query": [
+						{
+							"key": "mindate",
+							"value": "2026-01-01"
+						},
+						{
+							"key": "maxdate",
+							"value": "2026-02-24"
+						}
+					]
+				}
+			},
+			"response": []
+		},
+		{
+			"name": "Analytics Creator Deltas",
+			"request": {
+				"auth": {
+					"type": "bearer",
+					"bearer": [
+						{
+							"key": "token",
+							"value": "",
+							"type": "string"
+						}
+					]
+				},
+				"method": "GET",
+				"header": [],
+				"url": {
+					"raw": "http://localhost:3000/analytics/creator/deltas?mindate=2026-01-01&maxdate=2026-02-24",
+					"protocol": "http",
+					"host": [
+						"localhost"
+					],
+					"port": "3000",
+					"path": [
+						"analytics",
+						"creator",
+						"deltas"
+					],
+					"query": [
+						{
+							"key": "mindate",
+							"value": "2026-01-01"
+						},
+						{
+							"key": "maxdate",
+							"value": "2026-02-24"
+						}
+					]
+				}
+			},
+			"response": []
+		},
+		{
+			"name": "Analytics Creator Top",
+			"request": {
+				"auth": {
+					"type": "bearer",
+					"bearer": [
+						{
+							"key": "token",
+							"value": "",
+							"type": "string"
+						}
+					]
+				},
+				"method": "GET",
+				"header": [],
+				"url": {
+					"raw": "http://localhost:3000/analytics/creator/top?mindate=2026-01-01&maxdate=2026-02-24&n=10&top=post",
+					"protocol": "http",
+					"host": [
+						"localhost"
+					],
+					"port": "3000",
+					"path": [
+						"analytics",
+						"creator",
+						"top"
+					],
+					"query": [
+						{
+							"key": "mindate",
+							"value": "2026-01-01"
+						},
+						{
+							"key": "maxdate",
+							"value": "2026-02-24"
+						},
+						{
+							"key": "n",
+							"value": "10"
+						},
+						{
+							"key": "top",
+							"value": "post"
+						}
+					]
+				}
+			},
+			"response": []
+		},
+		{
+			"name": "Analytics Creator Top Average",
+			"request": {
+				"auth": {
+					"type": "bearer",
+					"bearer": [
+						{
+							"key": "token",
+							"value": "",
+							"type": "string"
+						}
+					]
+				},
+				"method": "GET",
+				"header": [],
+				"url": {
+					"raw": "http://localhost:3000/analytics/creator/top/average?mindate=2026-01-01&maxdate=2026-02-24",
+					"protocol": "http",
+					"host": [
+						"localhost"
+					],
+					"port": "3000",
+					"path": [
+						"analytics",
+						"creator",
+						"top",
+						"average"
+					],
+					"query": [
+						{
+							"key": "mindate",
+							"value": "2026-01-01"
+						},
+						{
+							"key": "maxdate",
+							"value": "2026-02-24"
+						}
+					]
+				}
+			},
+			"response": []
 		}
 	]
 }
\ No newline at end of file

From c1157d86d15d416d108c5fe78edd7a6cb0e3d215 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 23 Feb 2026 12:03:08 +0200
Subject: [PATCH 300/391] removed unnecessary let binding

---
 src/source/routes/analytics/creator/general.clj | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/src/source/routes/analytics/creator/general.clj b/src/source/routes/analytics/creator/general.clj
index d1f7a5b3..984084f3 100644
--- a/src/source/routes/analytics/creator/general.clj
+++ b/src/source/routes/analytics/creator/general.clj
@@ -18,7 +18,6 @@
                             [:views :int]]]}}}
 
   [{:keys [ds user query-params] :as _request}]
-  (let [{:keys [mindate maxdate feed]} (w/keywordize-keys query-params)
-        res (analytics/interval-statistics-query ds :daily mindate maxdate {:creator-id (:id user)
-                                                                            :feed-id feed})]
-    (res/response res)))
+  (let [{:keys [mindate maxdate feed]} (w/keywordize-keys query-params)]
+    (res/response (analytics/interval-statistics-query ds :daily mindate maxdate {:creator-id (:id user)
+                                                                                  :feed-id feed}))))

From f821324029393f05634ba2266389bf7fc66c30b8 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 23 Feb 2026 13:26:06 +0200
Subject: [PATCH 301/391] improved documentation and added descriptions for
 bundle and integration endpoints

---
 src/source/routes/bundle_feed_post.clj         |  1 +
 src/source/routes/bundle_feed_posts.clj        |  1 +
 src/source/routes/bundle_feeds.clj             |  1 +
 src/source/routes/bundle_post.clj              |  1 +
 src/source/routes/bundle_posts.clj             |  3 +++
 src/source/routes/integration.clj              | 14 ++++++++------
 src/source/routes/integration_categories.clj   |  6 ++++--
 src/source/routes/integration_filter_feed.clj  | 17 +++++++++--------
 src/source/routes/integration_filter_feeds.clj |  3 ++-
 src/source/routes/integration_filter_post.clj  | 10 +++++-----
 src/source/routes/integration_filter_posts.clj |  5 +++--
 src/source/routes/integrations.clj             |  5 +++--
 12 files changed, 41 insertions(+), 26 deletions(-)

diff --git a/src/source/routes/bundle_feed_post.clj b/src/source/routes/bundle_feed_post.clj
index ad27b255..0b6f3ed8 100644
--- a/src/source/routes/bundle_feed_post.clj
+++ b/src/source/routes/bundle_feed_post.clj
@@ -8,6 +8,7 @@
 (defn get
   {:summary "Get a single post by post id belonging to an RSS feed in the associated uuid-authorized bundle. 
    This endpoint updates click analytics for the returned post."
+   :description "This endpoint will fetch a single post belonging to the given feed, regardless of whether the post made it into the bundle during post selection."
    :parameters {:query [:map [:uuid {:description "Bundle UUID"} :string]]
                 :path [:map [:post-id {:title "postId"
                                        :description "Post ID"} :int]]}
diff --git a/src/source/routes/bundle_feed_posts.clj b/src/source/routes/bundle_feed_posts.clj
index 38cea80a..eac89c13 100644
--- a/src/source/routes/bundle_feed_posts.clj
+++ b/src/source/routes/bundle_feed_posts.clj
@@ -6,6 +6,7 @@
 (defn get
   {:summary "Get all posts present within a given RSS feed by feed id, within the uuid-authorized bundle.
    This endpoint will update impressions analytics for the returned posts."
+   :description "This endpoint will fetch all posts within the given feed, regardless of whether these posts made it into this bundle during post selection."
    :parameters {:query [:map [:uuid {:description "Bundle UUID"} :string]]
                 :path [:map [:id {:title "id"
                                   :description "Feed ID"} :int]]}
diff --git a/src/source/routes/bundle_feeds.clj b/src/source/routes/bundle_feeds.clj
index 04410dcc..c6b8610e 100644
--- a/src/source/routes/bundle_feeds.clj
+++ b/src/source/routes/bundle_feeds.clj
@@ -7,6 +7,7 @@
 (defn post
   {:summary "Get all RSS feeds present in the bundle authorised by uuid.
    This endpoint will update impressions analytics for the returned RSS feeds."
+   :description "This endpoint can be filtered by content type ID, category IDs, or most recently added feeds."
    :parameters {:query [:map
                         [:uuid {:description "Bundle UUID"} :string]
                         [:type {:optional true
diff --git a/src/source/routes/bundle_post.clj b/src/source/routes/bundle_post.clj
index 0a1f4b95..7f0feee3 100644
--- a/src/source/routes/bundle_post.clj
+++ b/src/source/routes/bundle_post.clj
@@ -9,6 +9,7 @@
   {:summary "Get a single post by post id in the uuid-authorized bundle.
    Used to return a single post present in the bundle.
    This endpoint updates click analytics for the returned post."
+   :description "This endpoint will pull a single post by ID that made it into the bundle during post selection."
    :parameters {:query [:map [:uuid {:description "Bundle UUID"} :string]]
                 :path [:map [:id {:title "id"
                                   :description "Post ID"} :int]]}
diff --git a/src/source/routes/bundle_posts.clj b/src/source/routes/bundle_posts.clj
index ef1a97a1..65a6bd34 100644
--- a/src/source/routes/bundle_posts.clj
+++ b/src/source/routes/bundle_posts.clj
@@ -7,6 +7,9 @@
 (defn post
   {:summary "Get a list of posts in the uuid-authorized bundle, determined by analytics.
    This endpoint updates impression analytics for the returned posts."
+   :description "This endpoint pulls a curated list of content (determined by analytics) of the posts that made it into the bundle during post selection. This can be filtered by content type ID, category IDs, or latest (most recently added posts). If results are filtered by latest, they will not be curated by analytics.
+   
+   Results can be paginated using the `start` and `limit` query parameters."
    :parameters {:body [:map [:category-ids [:vector :int]]]
                 :query [:map
                         [:uuid {:description "Bundle UUID"} :string]
diff --git a/src/source/routes/integration.clj b/src/source/routes/integration.clj
index d9dbf0ae..956ce7e8 100644
--- a/src/source/routes/integration.clj
+++ b/src/source/routes/integration.clj
@@ -9,9 +9,9 @@
             [source.jobs.handlers :as handlers]))
 
 (defn get
-  {:summary "get integration by id"
+  {:summary "Get metadata of integration by ID"
    :parameters {:path [:map [:id {:title "id"
-                                  :description "integration id"} :int]]}
+                                  :description "Integration ID"} :int]]}
    :responses {200 {:body [:map
                            [:id :int]
                            [:name :string]
@@ -36,9 +36,10 @@
     (res/response (assoc integration :content-types content-types))))
 
 (defn post
-  {:summary "update an integration by id"
+  {:summary "Update metadata of the given integration by ID"
+   :description "When the integration is updated, post selection is rerun based on the newly set desired categories and content types."
    :parameters {:path [:map [:id {:title "id"
-                                  :description "integration id"} :int]]
+                                  :description "Integration ID"} :int]]
                 :body [:map
                        [:name :string]
                        [:content-types [:vector
@@ -81,9 +82,10 @@
   (res/response {:message "successfully updated integration"}))
 
 (defn delete
-  {:summary "delete the integration with the given id"
+  {:summary "Delete the given integration by ID"
+   :description "Deletes the integration, bundle and kills the associated post selection job. This action cannot be undone."
    :parameters {:path [:map [:id {:title "id"
-                                  :description "integration id"} :int]]}
+                                  :description "Integration ID"} :int]]}
    :responses {200 {:body [:map [:message :string]]}
                403 {:body [:map [:message :string]]}}}
 
diff --git a/src/source/routes/integration_categories.clj b/src/source/routes/integration_categories.clj
index d5fd0812..d3e3cf0a 100644
--- a/src/source/routes/integration_categories.clj
+++ b/src/source/routes/integration_categories.clj
@@ -4,7 +4,8 @@
             [source.services.bundle-categories :as bundle-categories]))
 
 (defn get
-  {:summary "get all categories belonging to the integration with the given id"
+  {:summary "Get all categories belonging to the given integration by ID"
+   :description "This endpoint pulls all the categories on which post selection is based."
    :parameters {:path [:map [:id {:title "id"
                                   :description "integration id"} :int]]}
    :responses {200 {:body [:vector
@@ -16,7 +17,8 @@
   (res/response (bundles/categories-in-bundle ds (:id path-params))))
 
 (defn post
-  {:summary "update categories belonging to the integration with the given id"
+  {:summary "Update categories belonging to the given integration by ID"
+   :description "This endpoint updates the list of categories for the given integration on which post selection is based; however, post selection will not immediately be rerun. Post selection will be rerun 24 hours from the previous execution. If you want post selection to be rerun immediately upon update, please use `POST: /integrations{id}`."
    :parameters {:path [:map [:id {:title "id"
                                   :description "integration id"} :int]]
                 :body [:vector
diff --git a/src/source/routes/integration_filter_feed.clj b/src/source/routes/integration_filter_feed.clj
index ecffde6c..685ddb64 100644
--- a/src/source/routes/integration_filter_feed.clj
+++ b/src/source/routes/integration_filter_feed.clj
@@ -4,12 +4,12 @@
             [source.db.honey :as hon]))
 
 (defn get
-  {:summary "Returns true if the feed with the given id is filtered out by the integration with the given id"
+  {:summary "Returns true if the feed with the given id is filtered out by the given integration by ID"
    :parameters {:path [:map
                        [:id {:title "id"
-                             :description "integration id"} :int]
-                       [:feed-id {:title "feed-id"
-                                  :description "feed id"} :int]]}
+                             :description "Integration ID"} :int]
+                       [:feed-id {:title "FeedId"
+                                  :description "feed ID"} :int]]}
    :responses {200 {:body [:map [:filtered :boolean]]}
                401 {:body [:map [:message :string]]}
                403 {:body [:map [:message :string]]}}}
@@ -24,12 +24,13 @@
     (res/response {:filtered blocked})))
 
 (defn post
-  {:summary "filters out the feed with the given id from the bundle with the given bundle id"
+  {:summary "Filters out the feed with the given id from the given integration by ID"
+   :description "Filtering a feed out from the integration means that the given feed and all its posts will not appear when pulling content from the bundle."
    :parameters {:path [:map
                        [:id {:title "id"
-                             :description "bundle id"} :int]
-                       [:feed-id {:title "feed-id"
-                                  :description "feed id"} :int]]
+                             :description "Integration ID"} :int]
+                       [:feed-id {:title "feedId"
+                                  :description "Feed ID"} :int]]
                 :body [:map
                        [:filtered :boolean]]}
    :responses {:body {200 [:map [:message :string]]
diff --git a/src/source/routes/integration_filter_feeds.clj b/src/source/routes/integration_filter_feeds.clj
index e84ddfe2..2cce847e 100644
--- a/src/source/routes/integration_filter_feeds.clj
+++ b/src/source/routes/integration_filter_feeds.clj
@@ -3,7 +3,8 @@
             [source.db.honey :as hon]))
 
 (defn get
-  {:summary "gets all filtered feed ids by integration id"
+  {:summary "Gets all filtered feed IDs by integration ID"
+   :description "Returns a list of all feed IDs that have been filtered out from the given integration. If a feed appears in this list, it will not be returned when pulling content from the bundle."
    :parameters {:path [:map [:id {:title "id"
                                   :description "integration id"} :int]]}
    :responses {200 {:body [:vector
diff --git a/src/source/routes/integration_filter_post.clj b/src/source/routes/integration_filter_post.clj
index f124c980..c8891631 100644
--- a/src/source/routes/integration_filter_post.clj
+++ b/src/source/routes/integration_filter_post.clj
@@ -4,12 +4,12 @@
             [source.db.honey :as hon]))
 
 (defn get
-  {:summary "Returns true if the post with the given id is filtered out by the integration with the given id"
+  {:summary "Returns true if the post with the given ID is filtered out from the given integration by ID"
    :parameters {:path [:map
                        [:id {:title "id"
-                             :description "integration id"} :int]
-                       [:post-id {:title "post-id"
-                                  :description "post id"} :int]]}
+                             :description "Integration ID"} :int]
+                       [:post-id {:title "postId"
+                                  :description "Post ID"} :int]]}
    :responses {200 {:body [:map [:filtered :boolean]]}
                401 {:body [:map [:message :string]]}
                403 {:body [:map [:message :string]]}}}
@@ -24,7 +24,7 @@
     (res/response {:filtered blocked})))
 
 (defn post
-  {:summary "filters out the post with the given id from the bundle with the given bundle id"
+  {:summary "Filters out the post with the given id from the bundle with the given bundle ID"
    :parameters {:path [:map
                        [:id {:title "id"
                              :description "bundle id"} :int]
diff --git a/src/source/routes/integration_filter_posts.clj b/src/source/routes/integration_filter_posts.clj
index 65c4e3b7..0912839f 100644
--- a/src/source/routes/integration_filter_posts.clj
+++ b/src/source/routes/integration_filter_posts.clj
@@ -3,9 +3,10 @@
             [source.db.honey :as hon]))
 
 (defn get
-  {:summary "gets all filtered post ids by integration id"
+  {:summary "Gets all filtered post ids for the given integration by ID"
+   :description "If a post appears in this list, it will not be returned when pulling content from the bundle."
    :parameters {:path [:map [:id {:title "id"
-                                  :description "integration id"} :int]]}
+                                  :description "Integration ID"} :int]]}
    :responses {200 {:body [:vector
                            [:map
                             [:post-id :int]
diff --git a/src/source/routes/integrations.clj b/src/source/routes/integrations.clj
index ac05c7e5..1ccad6b6 100644
--- a/src/source/routes/integrations.clj
+++ b/src/source/routes/integrations.clj
@@ -8,7 +8,7 @@
             [congest.jobs :as congest]))
 
 (defn get
-  {:summary "get all integrations"
+  {:summary "Get metadata of all integrations on the user account"
    :responses {200 {:body [:vector
                            [:map
                             [:id :int]
@@ -28,7 +28,8 @@
   (res/response (bundles/bundles ds {:where [:= :user-id (:id user)]})))
 
 (defn post
-  {:summary "creates an integration and the associated bundle in which content is stored"
+  {:summary "Creates an integration and the associated bundle in which content is stored"
+   :description "When an integration is created, a job is scheduled to periodically run post selection every 24 hours. During post selection, the bundle is filled with relevant content according to desired categories, content types and analytics."
    :parameters {:body [:map
                        [:name :string]
                        [:ts-and-cs {:optional true} :int]

From c2e240f0cd614dff111d882195ed76d5ec6d2153 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Tue, 24 Feb 2026 11:44:31 +0200
Subject: [PATCH 302/391] optimised bundle categories endpoint and allowed for
 content type filtering

---
 src/source/routes/bundle_categories.clj | 10 +++++++---
 src/source/workers/bundles.clj          | 19 +++++++++----------
 2 files changed, 16 insertions(+), 13 deletions(-)

diff --git a/src/source/routes/bundle_categories.clj b/src/source/routes/bundle_categories.clj
index 3ab27364..5a0cd8ba 100644
--- a/src/source/routes/bundle_categories.clj
+++ b/src/source/routes/bundle_categories.clj
@@ -4,7 +4,10 @@
 
 (defn get
   {:summary "Get all categories for which content is present in the uuid-authorized bundle (RSS feeds / posts)."
-   :parameters {:query [:map [:uuid {:description "Bundle UUID"} :string]]}
+   :parameters {:query [:map
+                        [:uuid {:description "Bundle UUID"} :string]
+                        [:type {:optional true
+                                :description "Filters by content type ID"} :int]]}
    :responses {200 {:body [:vector
                            [:map
                             [:id :int]
@@ -12,5 +15,6 @@
                             [:display-picture {:optional true} [:maybe :string]]]]}
                404 {:body [:map [:message :string]]}}}
 
-  [{:keys [bundle-id ds] :as _request}]
-  (res/response (bundles/get-bundle-categories ds bundle-id)))
+  [{:keys [ds bundle-id query-params] :as _request}]
+  (res/response (bundles/get-bundle-categories ds {:bundle-id bundle-id
+                                                   :content-type-id (:type query-params)})))
diff --git a/src/source/workers/bundles.clj b/src/source/workers/bundles.clj
index 949423e7..a0ba017a 100644
--- a/src/source/workers/bundles.clj
+++ b/src/source/workers/bundles.clj
@@ -8,16 +8,13 @@
 
 (defn get-bundle-categories
   "Get all categories for feeds/posts in bundle"
-  [ds bundle-id]
-  (let [feed-ids (->> (hon/find ds (db.util/tname :outgoing-posts bundle-id))
-                      (mapv :feed-id))
-        category-ids (->> (hon/find ds {:tname :feed-categories
-                                        :where (when (seq feed-ids) [:in :feed-id feed-ids])
-                                        :ret :*})
-                          (mapv :category-id))]
-    (hon/find ds {:tname :categories
-                  :where (when (seq category-ids) [:in :id category-ids])
-                  :ret :*})))
+  [ds {:keys [bundle-id content-type-id]}]
+  (->> (-> (hsql/select-distinct :c.*)
+           (hsql/from [:categories :c])
+           (hsql/join [:feed-categories :fc] [:= :c.id :fc.category-id])
+           (hsql/join [(:tname (db.util/tname :outgoing-posts bundle-id)) :p] [:= :fc.feed-id :p.feed-id])
+           (hsql/where (when content-type-id [:= :p.content-type-id content-type-id])))
+       (hon/execute! ds)))
 
 (defn get-outgoing-feeds
   "Gets a filtered list of outgoing feeds for the associated bundle."
@@ -87,4 +84,6 @@
                                             :category-ids [50 52 54]
                                             :latest "false"}))
 
+  (time (get-bundle-categories (db.util/conn) {:bundle-id 14}))
+
   ())

From 48b92073935817b1d0d8507c5cf03e60aa8d81a3 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Tue, 24 Feb 2026 11:45:16 +0200
Subject: [PATCH 303/391] added bundle exists endpoint

---
 src/source/routes/bundle.clj | 22 +++++++++++++++++++++-
 src/source/routes/reitit.clj |  1 +
 2 files changed, 22 insertions(+), 1 deletion(-)

diff --git a/src/source/routes/bundle.clj b/src/source/routes/bundle.clj
index 491724f1..f08a8ef1 100644
--- a/src/source/routes/bundle.clj
+++ b/src/source/routes/bundle.clj
@@ -1,6 +1,9 @@
 (ns source.routes.bundle
   (:require [ring.util.response :as res]
-            [source.db.honey :as hon]))
+            [source.db.honey :as hon]
+            [source.db.util :as db.util]
+            [honey.sql.helpers :as hsql]
+            [pg.core :as pg]))
 
 (defn get
   {:summary "Get metadata for the associated uuid-authorized bundle."
@@ -21,3 +24,20 @@
   [{:keys [ds bundle-id] :as _request}]
   (res/response (hon/find-one ds {:tname :bundles
                                   :where [:= :id bundle-id]})))
+
+(defn exists
+  {:summary "Check for the existence of a bundle with the bundle UUID provided"
+   :parameters {:query [:map [:uuid {:description "Bundle UUID"} :string]]}
+   :responses {200 {:body [:map [:exists :boolean]]}
+               404 {:body [:map [:message :string]]}}}
+  [{:keys [ds query-params] :as _request}]
+  (res/response
+   (-> ds
+       (pg/execute "SELECT EXISTS(SELECT 1 FROM bundles WHERE uuid = $1) AS exists" {:params [(:uuid query-params)]})
+       (first))))
+
+(comment
+  (time (exists {:ds (db.util/conn)
+                 :query-params {:uuid "2bbeb46bbd70c82b"}}))
+  ())
+
diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index 0b869c0b..1836bb30 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -225,6 +225,7 @@
       ["/feeds/:id/posts/:post-id" (get bundle-feed-post/get)]
       ["/posts" (post bundle-posts/post)]
       ["/posts/:id" (get bundle-post/get)]]
+     ["/bundle/exists" (get bundle/exists)]
 
      ["/admin" {:middleware [[mw/apply-auth {:required-type :admin}]]
                 :tags #{"admin"}

From 4989faa255bd562df47228ee34d428654fd8e5ca Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Tue, 24 Feb 2026 11:45:29 +0200
Subject: [PATCH 304/391] updated postman collection

---
 Source.postman_collection.json | 28 +++++++++++++++++++++++++++-
 1 file changed, 27 insertions(+), 1 deletion(-)

diff --git a/Source.postman_collection.json b/Source.postman_collection.json
index 68ca22ef..623aa8c9 100644
--- a/Source.postman_collection.json
+++ b/Source.postman_collection.json
@@ -625,7 +625,7 @@
 			"response": []
 		},
 		{
-			"name": "Bundle Categories",
+			"name": "Bundle Exists",
 			"request": {
 				"auth": {
 					"type": "bearer",
@@ -661,6 +661,32 @@
 			},
 			"response": []
 		},
+		{
+			"name": "Bundle Categories",
+			"request": {
+				"method": "GET",
+				"header": [],
+				"url": {
+					"raw": "http://localhost:3000/bundle/categories?uuid=2bbeb46bbd70c82b",
+					"protocol": "http",
+					"host": [
+						"localhost"
+					],
+					"port": "3000",
+					"path": [
+						"bundle",
+						"categories"
+					],
+					"query": [
+						{
+							"key": "uuid",
+							"value": "2bbeb46bbd70c82b"
+						}
+					]
+				}
+			},
+			"response": []
+		},
 		{
 			"name": "Bundle Feeds",
 			"request": {

From af0dcb843b9de2b443c47706fea07829aa47c84e Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Tue, 24 Feb 2026 11:49:47 +0200
Subject: [PATCH 305/391] updated dependencies and response schema in exists
 endpoint

---
 src/source/routes/bundle.clj | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/src/source/routes/bundle.clj b/src/source/routes/bundle.clj
index f08a8ef1..56ddd569 100644
--- a/src/source/routes/bundle.clj
+++ b/src/source/routes/bundle.clj
@@ -2,7 +2,6 @@
   (:require [ring.util.response :as res]
             [source.db.honey :as hon]
             [source.db.util :as db.util]
-            [honey.sql.helpers :as hsql]
             [pg.core :as pg]))
 
 (defn get
@@ -28,8 +27,7 @@
 (defn exists
   {:summary "Check for the existence of a bundle with the bundle UUID provided"
    :parameters {:query [:map [:uuid {:description "Bundle UUID"} :string]]}
-   :responses {200 {:body [:map [:exists :boolean]]}
-               404 {:body [:map [:message :string]]}}}
+   :responses {200 {:body [:map [:exists :boolean]]}}}
   [{:keys [ds query-params] :as _request}]
   (res/response
    (-> ds

From b998cbc5220938adb3e671519b9338c9102adc56 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Tue, 24 Feb 2026 11:50:36 +0200
Subject: [PATCH 306/391] removed db.util as dependency from exists endpoint

---
 src/source/routes/bundle.clj | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/src/source/routes/bundle.clj b/src/source/routes/bundle.clj
index 56ddd569..c36c5388 100644
--- a/src/source/routes/bundle.clj
+++ b/src/source/routes/bundle.clj
@@ -1,7 +1,6 @@
 (ns source.routes.bundle
   (:require [ring.util.response :as res]
             [source.db.honey :as hon]
-            [source.db.util :as db.util]
             [pg.core :as pg]))
 
 (defn get
@@ -34,8 +33,3 @@
        (pg/execute "SELECT EXISTS(SELECT 1 FROM bundles WHERE uuid = $1) AS exists" {:params [(:uuid query-params)]})
        (first))))
 
-(comment
-  (time (exists {:ds (db.util/conn)
-                 :query-params {:uuid "2bbeb46bbd70c82b"}}))
-  ())
-

From ed999a22bcdfbcf016a11dbb624c6e0e14439f0b Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Tue, 24 Feb 2026 13:03:32 +0200
Subject: [PATCH 307/391] updated bundle category filtering to force checking
 intersection of all content types if none are specified

---
 src/source/workers/bundles.clj | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/source/workers/bundles.clj b/src/source/workers/bundles.clj
index a0ba017a..95405a61 100644
--- a/src/source/workers/bundles.clj
+++ b/src/source/workers/bundles.clj
@@ -13,7 +13,13 @@
            (hsql/from [:categories :c])
            (hsql/join [:feed-categories :fc] [:= :c.id :fc.category-id])
            (hsql/join [(:tname (db.util/tname :outgoing-posts bundle-id)) :p] [:= :fc.feed-id :p.feed-id])
-           (hsql/where (when content-type-id [:= :p.content-type-id content-type-id])))
+           (hsql/group-by :c.id :c.name :c.display-picture)
+           (hsql/where (when content-type-id [:= :p.content-type-id content-type-id]))
+           (hsql/having (when (nil? content-type-id)
+                          [:=
+                           [:count [:distinct :p.content-type-id]]
+                           (-> (hsql/select [[:count :id]])
+                               (hsql/from :content-types))])))
        (hon/execute! ds)))
 
 (defn get-outgoing-feeds

From 5c7e2faffb97c9a32419e149d4e8af7697cb3160 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Tue, 24 Feb 2026 13:15:02 +0200
Subject: [PATCH 308/391] improved pagination for bundle posts endpoint

---
 src/source/routes/bundle_posts.clj | 55 ++++++++++++++++--------------
 src/source/workers/bundles.clj     | 15 ++++++--
 2 files changed, 42 insertions(+), 28 deletions(-)

diff --git a/src/source/routes/bundle_posts.clj b/src/source/routes/bundle_posts.clj
index 65a6bd34..b89480a9 100644
--- a/src/source/routes/bundle_posts.clj
+++ b/src/source/routes/bundle_posts.clj
@@ -28,35 +28,40 @@
                           :description "Filters by most recently uploaded posts, not determined by analytics"}
                          [:enum "true" "false"]]
                         [:seed {:optional true} [:maybe :string]]]}
-   :responses {200 {:body [:vector
-                           [:map
-                            [:id :int]
-                            [:post-id :string]
-                            [:feed-id :int]
-                            [:creator-id :int]
-                            [:content-type-id :int]
-                            [:title :string]
-                            [:thumbnail [:maybe :string]]
-                            [:info [:maybe :string]]
-                            [:url [:maybe :string]]
-                            [:stream-url [:maybe :string]]
-                            [:season [:maybe :int]]
-                            [:episode [:maybe :int]]
-                            [:redacted {:optional true} [:maybe :int]]
-                            [:posted-at [:maybe :string]]]]}
+   :responses {200 {:body [:map
+                           [:page-size :int]
+                           [:total-size :int]
+                           [:current-index :int]
+                           [:next-index :int]
+                           [:data [:vector
+                                   [:map
+                                    [:id :int]
+                                    [:post-id :string]
+                                    [:feed-id :int]
+                                    [:creator-id :int]
+                                    [:content-type-id :int]
+                                    [:title :string]
+                                    [:thumbnail [:maybe :string]]
+                                    [:info [:maybe :string]]
+                                    [:url [:maybe :string]]
+                                    [:stream-url [:maybe :string]]
+                                    [:season [:maybe :int]]
+                                    [:episode [:maybe :int]]
+                                    [:redacted {:optional true} [:maybe :int]]
+                                    [:posted-at [:maybe :string]]]]]]}
                404 {:boy [:map [:message :string]]}}}
 
   [{:keys [ds bundle-id query-params body] :as _request}]
   (let [{:keys [limit start type latest seed]} (walk/keywordize-keys query-params)
-        posts (->> {:bundle-id bundle-id
-                    :limit limit
-                    :start start
-                    :type type
-                    :latest latest
-                    :seed seed
-                    :category-ids (:category-ids body)}
-                   (bundles/get-outgoing-posts ds))]
+        {:keys [data] :as posts} (->> {:bundle-id bundle-id
+                                       :limit limit
+                                       :start start
+                                       :type type
+                                       :latest latest
+                                       :seed seed
+                                       :category-ids (:category-ids body)}
+                                      (bundles/get-outgoing-posts ds))]
     (try
-      (analytics/insert-post-impressions! ds posts bundle-id)
+      (analytics/insert-post-impressions! ds data bundle-id)
       (catch Exception e (println (.getMessage e))))
     (res/response posts)))
diff --git a/src/source/workers/bundles.clj b/src/source/workers/bundles.clj
index a0ba017a..13e2d968 100644
--- a/src/source/workers/bundles.clj
+++ b/src/source/workers/bundles.clj
@@ -67,7 +67,9 @@
                               (sort-by #(get order-map (first %)))
                               (mapv last)))
 
-        valid-start? (and (some? start) (>= start 0) (< start (count shuffled-posts)))
+        total-size (count shuffled-posts)
+
+        valid-start? (and (some? start) (>= start 0) (< start total-size))
         started-posts (if valid-start?
                         (subvec shuffled-posts start)
                         shuffled-posts)
@@ -75,8 +77,15 @@
         valid-limit? (and (some? limit) (> (count started-posts) limit))
         limited-posts (if valid-limit?
                         (subvec started-posts 0 limit)
-                        started-posts)]
-    limited-posts))
+                        started-posts)
+
+        next-index (+ start limit)]
+
+    {:page-size (count limited-posts)
+     :total-size total-size
+     :current-index start
+     :next-index (when (< next-index total-size) next-index)
+     :data limited-posts}))
 
 (comment
 

From 5aefb4bc72d59f981c700d58de8ec092d8bdaf3e Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 25 Feb 2026 09:40:14 +0200
Subject: [PATCH 309/391] updated returned result to have a pagination field
 with the data and updated the schema such that pagination fields are optional

---
 src/source/routes/bundle_posts.clj | 12 +++++++-----
 src/source/workers/bundles.clj     |  8 ++++----
 2 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/src/source/routes/bundle_posts.clj b/src/source/routes/bundle_posts.clj
index b89480a9..f0722dab 100644
--- a/src/source/routes/bundle_posts.clj
+++ b/src/source/routes/bundle_posts.clj
@@ -2,7 +2,8 @@
   (:require [clojure.walk :as walk]
             [ring.util.response :as res]
             [source.services.analytics.interface :as analytics]
-            [source.workers.bundles :as bundles]))
+            [source.workers.bundles :as bundles]
+            [source.routes.openapi :as api]))
 
 (defn post
   {:summary "Get a list of posts in the uuid-authorized bundle, determined by analytics.
@@ -29,10 +30,11 @@
                          [:enum "true" "false"]]
                         [:seed {:optional true} [:maybe :string]]]}
    :responses {200 {:body [:map
-                           [:page-size :int]
-                           [:total-size :int]
-                           [:current-index :int]
-                           [:next-index :int]
+                           [:pagination [:map
+                                         (api/sometimes :page-size :int)
+                                         (api/sometimes :total-size :int)
+                                         (api/sometimes :current-index :int)
+                                         (api/sometimes :next-index :int)]]
                            [:data [:vector
                                    [:map
                                     [:id :int]
diff --git a/src/source/workers/bundles.clj b/src/source/workers/bundles.clj
index 13e2d968..d42537ad 100644
--- a/src/source/workers/bundles.clj
+++ b/src/source/workers/bundles.clj
@@ -81,10 +81,10 @@
 
         next-index (+ start limit)]
 
-    {:page-size (count limited-posts)
-     :total-size total-size
-     :current-index start
-     :next-index (when (< next-index total-size) next-index)
+    {:pagination {:page-size (count limited-posts)
+                  :total-size total-size
+                  :current-index start
+                  :next-index (when (< next-index total-size) next-index)}
      :data limited-posts}))
 
 (comment

From 6c31e6f7ef45d155f9a63f19b1a9b8ac84ecd372 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 25 Feb 2026 12:11:32 +0200
Subject: [PATCH 310/391] removed 404

---
 src/source/routes/bundle_posts.clj | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/src/source/routes/bundle_posts.clj b/src/source/routes/bundle_posts.clj
index f0722dab..2192ab41 100644
--- a/src/source/routes/bundle_posts.clj
+++ b/src/source/routes/bundle_posts.clj
@@ -31,9 +31,9 @@
                         [:seed {:optional true} [:maybe :string]]]}
    :responses {200 {:body [:map
                            [:pagination [:map
-                                         (api/sometimes :page-size :int)
-                                         (api/sometimes :total-size :int)
-                                         (api/sometimes :current-index :int)
+                                         [:page-size :int]
+                                         [:total-size :int]
+                                         [:current-index :int]
                                          (api/sometimes :next-index :int)]]
                            [:data [:vector
                                    [:map
@@ -50,8 +50,7 @@
                                     [:season [:maybe :int]]
                                     [:episode [:maybe :int]]
                                     [:redacted {:optional true} [:maybe :int]]
-                                    [:posted-at [:maybe :string]]]]]]}
-               404 {:boy [:map [:message :string]]}}}
+                                    [:posted-at [:maybe :string]]]]]]}}}
 
   [{:keys [ds bundle-id query-params body] :as _request}]
   (let [{:keys [limit start type latest seed]} (walk/keywordize-keys query-params)

From 2bd7c787318147111c64873b1f1f080d3cbf0272 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 25 Feb 2026 12:34:17 +0200
Subject: [PATCH 311/391] updated response on bundle_posts endpoint to use
 constructed responses from top level schemas

---
 src/source/routes/bundle_posts.clj | 26 +++---------------------
 src/source/workers/schemas.clj     | 32 +++++++++++++++++++++++++++++-
 2 files changed, 34 insertions(+), 24 deletions(-)

diff --git a/src/source/routes/bundle_posts.clj b/src/source/routes/bundle_posts.clj
index 2192ab41..8d326d73 100644
--- a/src/source/routes/bundle_posts.clj
+++ b/src/source/routes/bundle_posts.clj
@@ -3,7 +3,8 @@
             [ring.util.response :as res]
             [source.services.analytics.interface :as analytics]
             [source.workers.bundles :as bundles]
-            [source.routes.openapi :as api]))
+            [source.routes.openapi :as api]
+            [source.workers.schemas :as schemas]))
 
 (defn post
   {:summary "Get a list of posts in the uuid-authorized bundle, determined by analytics.
@@ -29,28 +30,7 @@
                           :description "Filters by most recently uploaded posts, not determined by analytics"}
                          [:enum "true" "false"]]
                         [:seed {:optional true} [:maybe :string]]]}
-   :responses {200 {:body [:map
-                           [:pagination [:map
-                                         [:page-size :int]
-                                         [:total-size :int]
-                                         [:current-index :int]
-                                         (api/sometimes :next-index :int)]]
-                           [:data [:vector
-                                   [:map
-                                    [:id :int]
-                                    [:post-id :string]
-                                    [:feed-id :int]
-                                    [:creator-id :int]
-                                    [:content-type-id :int]
-                                    [:title :string]
-                                    [:thumbnail [:maybe :string]]
-                                    [:info [:maybe :string]]
-                                    [:url [:maybe :string]]
-                                    [:stream-url [:maybe :string]]
-                                    [:season [:maybe :int]]
-                                    [:episode [:maybe :int]]
-                                    [:redacted {:optional true} [:maybe :int]]
-                                    [:posted-at [:maybe :string]]]]]]}}}
+   :responses (api/success (schemas/paginated schemas/Posts))}
 
   [{:keys [ds bundle-id query-params body] :as _request}]
   (let [{:keys [limit start type latest seed]} (walk/keywordize-keys query-params)
diff --git a/src/source/workers/schemas.clj b/src/source/workers/schemas.clj
index 7a3c94ba..afd9a6c6 100644
--- a/src/source/workers/schemas.clj
+++ b/src/source/workers/schemas.clj
@@ -1,5 +1,6 @@
 (ns source.workers.schemas
-  (:require [malli.util :as mu]))
+  (:require [malli.util :as mu]
+            [source.routes.openapi :as api]))
 
 (def Business
   [:map
@@ -109,6 +110,15 @@
       (mu/assoc :baseline Baseline)
       (mu/assoc :provider Provider)))
 
+(defn paginated [data-schema]
+  [:map
+   [:paginated [:map
+                [:page-size :int]
+                [:total-size :int]
+                [:current-index :int]
+                (api/sometimes :next-index :int)]]
+   [:data data-schema]])
+
 (def IncomingPostRecord
   [:map
    [:id :int]
@@ -142,6 +152,26 @@
    [:episode :int]
    [:posted-at :string]])
 
+(def Post
+  [:map
+   [:id :int]
+   [:post-id :string]
+   [:feed-id :int]
+   [:creator-id :int]
+   [:content-type-id :int]
+   [:title :string]
+   [:thumbnail (api/maybe :string)]
+   [:info (api/maybe :string)]
+   [:url (api/maybe :string)]
+   [:stream-url (api/maybe :string)]
+   [:season (api/maybe :int)]
+   [:episode (api/maybe :int)]
+   (api/sometimes :redacted :int)
+   [:posted-at (api/maybe :string)]])
+
+(def Posts
+  [:vector Post])
+
 (def JobStatus [:enum ["running" "stopped"]])
 
 (def Job

From dcca91b278e23a77a46852284881c6345f829335 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 25 Feb 2026 12:44:18 +0200
Subject: [PATCH 312/391] removed thread last in bundle posts endpoint

---
 src/source/routes/bundle_posts.clj | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/src/source/routes/bundle_posts.clj b/src/source/routes/bundle_posts.clj
index 8d326d73..13c30043 100644
--- a/src/source/routes/bundle_posts.clj
+++ b/src/source/routes/bundle_posts.clj
@@ -34,14 +34,15 @@
 
   [{:keys [ds bundle-id query-params body] :as _request}]
   (let [{:keys [limit start type latest seed]} (walk/keywordize-keys query-params)
-        {:keys [data] :as posts} (->> {:bundle-id bundle-id
-                                       :limit limit
-                                       :start start
-                                       :type type
-                                       :latest latest
-                                       :seed seed
-                                       :category-ids (:category-ids body)}
-                                      (bundles/get-outgoing-posts ds))]
+        {:keys [data] :as posts} (bundles/get-outgoing-posts
+                                  ds
+                                  {:bundle-id bundle-id
+                                   :limit limit
+                                   :start start
+                                   :type type
+                                   :latest latest
+                                   :seed seed
+                                   :category-ids (:category-ids body)})]
     (try
       (analytics/insert-post-impressions! ds data bundle-id)
       (catch Exception e (println (.getMessage e))))

From 2f1904b22382a6ed7d8a4b8ed8df2d6fcd756069 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 25 Feb 2026 13:01:25 +0200
Subject: [PATCH 313/391] put all analytics writing functions into transaction
 blocks

---
 src/source/services/analytics/core.clj | 128 +++++++++++++------------
 1 file changed, 67 insertions(+), 61 deletions(-)

diff --git a/src/source/services/analytics/core.clj b/src/source/services/analytics/core.clj
index 6cc96b5b..6a348dd4 100644
--- a/src/source/services/analytics/core.clj
+++ b/src/source/services/analytics/core.clj
@@ -5,7 +5,8 @@
             [source.util :as util]
             [source.services.feed-categories :as feed-categories]
             [honey.sql :as sql]
-            [clojure.walk :as walk]))
+            [clojure.walk :as walk]
+            [pg.core :as pg]))
 
 (defn convert-all-datetimes-to-string
   "Uses postwalk to convert all instances of java.time.LocalDate into string"
@@ -209,86 +210,91 @@
   "Given a list of feeds and a bundle id, inserts impression event reconds 
   for each given feed. Inserts event categories for each feed."
   [ds feeds bundle-id]
-  (let [bundle (bundles/bundle ds {:id bundle-id})
-        events (mapv (fn [{:keys [id content-type-id user-id]}]
-                       {:timestamp (util/get-utc-timestamp-string)
-                        :event "impression"
-                        :feed-id id
-                        :content-type-id content-type-id
-                        :creator-id user-id
-                        :bundle-id bundle-id
-                        :distributor-id (:user-id bundle)}) feeds)
-        events' (insert-event! ds {:data events
-                                   :ret :*})]
-    (insert-feed-event-categories! ds events' feeds)))
+  (pg/with-transaction [ds ds]
+    (let [bundle (bundles/bundle ds {:id bundle-id})
+          events (mapv (fn [{:keys [id content-type-id user-id]}]
+                         {:timestamp (util/get-utc-timestamp-string)
+                          :event "impression"
+                          :feed-id id
+                          :content-type-id content-type-id
+                          :creator-id user-id
+                          :bundle-id bundle-id
+                          :distributor-id (:user-id bundle)}) feeds)
+          events' (insert-event! ds {:data events
+                                     :ret :*})]
+      (insert-feed-event-categories! ds events' feeds))))
 
 (defn insert-post-impressions!
   "Given a list of posts and a bundle id, inserts impression event reconds 
   for each given post. Inserts event categories for each post."
   [ds posts bundle-id]
-  (let [bundle (bundles/bundle ds {:id bundle-id})
-        events (mapv (fn [{:keys [id feed-id content-type-id creator-id]}]
-                       {:timestamp (util/get-utc-timestamp-string)
-                        :event "impression"
-                        :feed-id feed-id
-                        :post-id id
-                        :content-type-id content-type-id
-                        :creator-id creator-id
-                        :bundle-id bundle-id
-                        :distributor-id (:user-id bundle)}) posts)
-        events' (when (seq posts) (insert-event! ds {:data events
-                                                     :ret :*}))]
-    (when (seq events') (insert-post-event-categories! ds events' posts))))
+  (pg/with-transaction [ds ds]
+    (let [bundle (bundles/bundle ds {:id bundle-id})
+          events (mapv (fn [{:keys [id feed-id content-type-id creator-id]}]
+                         {:timestamp (util/get-utc-timestamp-string)
+                          :event "impression"
+                          :feed-id feed-id
+                          :post-id id
+                          :content-type-id content-type-id
+                          :creator-id creator-id
+                          :bundle-id bundle-id
+                          :distributor-id (:user-id bundle)}) posts)
+          events' (when (seq posts) (insert-event! ds {:data events
+                                                       :ret :*}))]
+      (when (seq events') (insert-post-event-categories! ds events' posts)))))
 
 (defn insert-feed-click!
   "Given a feed and a bundle id, inserts a click event record 
   for the given feed"
   [ds {:keys [id content-type-id user-id] :as feed} bundle-id]
-  (let [bundle (bundles/bundle ds {:id bundle-id})
-        event {:timestamp (util/get-utc-timestamp-string)
-               :event "click"
-               :feed-id id
-               :content-type-id content-type-id
-               :creator-id user-id
-               :bundle-id bundle-id
-               :distributor-id (:user-id bundle)}
-        event' (insert-event! ds {:data event
-                                  :ret :*})]
-    (insert-feed-event-categories! ds event' feed)))
+  (pg/with-transaction [ds ds]
+    (let [bundle (bundles/bundle ds {:id bundle-id})
+          event {:timestamp (util/get-utc-timestamp-string)
+                 :event "click"
+                 :feed-id id
+                 :content-type-id content-type-id
+                 :creator-id user-id
+                 :bundle-id bundle-id
+                 :distributor-id (:user-id bundle)}
+          event' (insert-event! ds {:data event
+                                    :ret :*})]
+      (insert-feed-event-categories! ds event' feed))))
 
 (defn insert-post-click!
   "Given a post and a bundle id, inserts a click event record 
   for the given post"
   [ds {:keys [id feed-id content-type-id creator-id] :as post} bundle-id]
-  (let [bundle (bundles/bundle ds {:id bundle-id})
-        event {:timestamp (util/get-utc-timestamp-string)
-               :event "click"
-               :feed-id feed-id
-               :post-id id
-               :content-type-id content-type-id
-               :creator-id creator-id
-               :bundle-id bundle-id
-               :distributor-id (:user-id bundle)}
-        event' (insert-event! ds {:data event
-                                  :ret :*})]
-    (insert-post-event-categories! ds event' post)))
+  (pg/with-transaction [ds ds]
+    (let [bundle (bundles/bundle ds {:id bundle-id})
+          event {:timestamp (util/get-utc-timestamp-string)
+                 :event "click"
+                 :feed-id feed-id
+                 :post-id id
+                 :content-type-id content-type-id
+                 :creator-id creator-id
+                 :bundle-id bundle-id
+                 :distributor-id (:user-id bundle)}
+          event' (insert-event! ds {:data event
+                                    :ret :*})]
+      (insert-post-event-categories! ds event' post))))
 
 (defn insert-post-view!
   "Given a post and a bundle id, inserts a view event record 
   for the given post"
   [ds {:keys [id feed-id content-type-id creator-id] :as post} bundle-id]
-  (let [bundle (bundles/bundle ds {:id bundle-id})
-        event {:timestamp (util/get-utc-timestamp-string)
-               :event "view"
-               :feed-id feed-id
-               :post-id id
-               :content-type-id content-type-id
-               :creator-id creator-id
-               :bundle-id bundle-id
-               :distributor-id (:user-id bundle)}
-        event' (insert-event! ds {:data event
-                                  :ret :*})]
-    (insert-post-event-categories! ds event' post)))
+  (pg/with-transaction [ds ds]
+    (let [bundle (bundles/bundle ds {:id bundle-id})
+          event {:timestamp (util/get-utc-timestamp-string)
+                 :event "view"
+                 :feed-id feed-id
+                 :post-id id
+                 :content-type-id content-type-id
+                 :creator-id creator-id
+                 :bundle-id bundle-id
+                 :distributor-id (:user-id bundle)}
+          event' (insert-event! ds {:data event
+                                    :ret :*})]
+      (insert-post-event-categories! ds event' post))))
 
 (comment
   (require '[source.db.util :as db.util])

From 5da8fd7941ea863932d8999f918af7897565a69c Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 25 Feb 2026 14:15:19 +0200
Subject: [PATCH 314/391] fixed schema

---
 src/source/workers/schemas.clj | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/source/workers/schemas.clj b/src/source/workers/schemas.clj
index afd9a6c6..86e1d36f 100644
--- a/src/source/workers/schemas.clj
+++ b/src/source/workers/schemas.clj
@@ -112,7 +112,7 @@
 
 (defn paginated [data-schema]
   [:map
-   [:paginated [:map
+   [:pagination [:map
                 [:page-size :int]
                 [:total-size :int]
                 [:current-index :int]

From d29ecde9dad973ce637b83a0c91c3936cedb0dec Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 25 Feb 2026 14:49:26 +0200
Subject: [PATCH 315/391] added migration for descriptions on feeds

---
 .../migrations/012_feed_descriptions.clj       | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)
 create mode 100644 src/source/migrations/012_feed_descriptions.clj

diff --git a/src/source/migrations/012_feed_descriptions.clj b/src/source/migrations/012_feed_descriptions.clj
new file mode 100644
index 00000000..fd468f5f
--- /dev/null
+++ b/src/source/migrations/012_feed_descriptions.clj
@@ -0,0 +1,18 @@
+(ns source.migrations.012-feed-descriptions
+  (:require [source.db.master]
+            [source.db.honey :as hon]
+            [honey.sql.helpers :as hsql]))
+
+(defn run-up! [context]
+  (let [ds-master (:db-master context)]
+    (hon/execute!
+     ds-master
+     (-> (hsql/alter-table :feeds)
+         (hsql/add-column :description :text)))))
+
+(defn run-down! [context]
+  (let [ds-master (:db-master context)]
+    (hon/execute!
+     ds-master
+     (-> (hsql/alter-table :feeds)
+         (hsql/drop-column :description)))))

From 4fd242b1cab4e135eb2f4519168ca06d74d22520 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 25 Feb 2026 14:50:05 +0200
Subject: [PATCH 316/391] updated extraction to pull descriptions

---
 src/source/jobs/handlers.clj | 1 +
 src/source/workers/feeds.clj | 1 +
 2 files changed, 2 insertions(+)

diff --git a/src/source/jobs/handlers.clj b/src/source/jobs/handlers.clj
index 55c6bf4c..3bb7f81b 100644
--- a/src/source/jobs/handlers.clj
+++ b/src/source/jobs/handlers.clj
@@ -61,6 +61,7 @@
           (hon/update! ds {:tname :feeds
                            :where [:= :id feed-id]
                            :data {:title (get-in extracted [:feed :title])
+                                  :description (get-in extracted [:feed :description])
                                   :display-picture (if (and (:display-picture existing-feed)
                                                             (seq (:display-picture existing-feed)))
                                                      (:display-picture existing-feed)
diff --git a/src/source/workers/feeds.clj b/src/source/workers/feeds.clj
index e9988cc5..f5320317 100644
--- a/src/source/workers/feeds.clj
+++ b/src/source/workers/feeds.clj
@@ -24,6 +24,7 @@
                   {:tname :feeds
                    :data (merge feed-metadata {:title (get-in extracted [:feed :title])
                                                :display-picture (get-in extracted [:feed :display-picture])
+                                               :description (get-in extracted [:feed :description])
                                                :user-id user-id
                                                :created-at datetime
                                                :state "pending"})

From a243e2ac0cb267ca1df210160cfa212de143ed7b Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 25 Feb 2026 14:50:29 +0200
Subject: [PATCH 317/391] updated schemas to use top level schemas in requests
 and added description

---
 src/source/routes/bundle_feed.clj  | 22 +++---------
 src/source/routes/bundle_feeds.clj | 23 +++---------
 src/source/routes/feed.clj         | 36 +++++++------------
 src/source/routes/feeds.clj        | 56 ++++++++----------------------
 src/source/routes/openapi.clj      |  2 +-
 src/source/workers/schemas.clj     | 20 ++++++++---
 6 files changed, 54 insertions(+), 105 deletions(-)

diff --git a/src/source/routes/bundle_feed.clj b/src/source/routes/bundle_feed.clj
index d6d679ab..9457c677 100644
--- a/src/source/routes/bundle_feed.clj
+++ b/src/source/routes/bundle_feed.clj
@@ -1,7 +1,9 @@
 (ns source.routes.bundle-feed
   (:require [ring.util.response :as res]
             [source.db.honey :as hon]
-            [source.services.analytics.interface :as analytics]))
+            [source.services.analytics.interface :as analytics]
+            [source.routes.openapi :as api]
+            [source.workers.schemas :as schemas]))
 
 (defn get
   {:summary "Get a single RSS feed by id from RSS feeds within the uuid-authorized bundle. 
@@ -9,22 +11,8 @@
    :parameters {:query [:map [:uuid {:description "Bundle UUID"} :string]]
                 :path [:map [:id {:title "id"
                                   :description "Feed ID"} :int]]}
-   :responses  {200 {:body [:map
-                            [:id :int]
-                            [:title :string]
-                            [:display-picture [:maybe :string]]
-                            [:url [:maybe :string]]
-                            [:rss-url :string]
-                            [:user-id :int]
-                            [:provider-id [:maybe :int]]
-                            [:created-at :string]
-                            [:updated-at [:maybe :string]]
-                            [:content-type-id :int]
-                            [:cadence-id :int]
-                            [:baseline-id :int]
-                            [:ts-and-cs [:maybe :int]]
-                            [:state [:enum "live" "not live" "pending"]]]}
-                404 {:body [:map [:message :string]]}}}
+   :responses (-> (api/success schemas/FeedWithIDs)
+                  (api/not-found))}
 
   [{:keys [ds bundle-id path-params] :as _request}]
   (let [feed (hon/find-one ds {:tname :feeds
diff --git a/src/source/routes/bundle_feeds.clj b/src/source/routes/bundle_feeds.clj
index c6b8610e..6c0db769 100644
--- a/src/source/routes/bundle_feeds.clj
+++ b/src/source/routes/bundle_feeds.clj
@@ -2,7 +2,9 @@
   (:require [ring.util.response :as res]
             [source.db.util :as db.util]
             [source.workers.bundles :as bundles]
-            [source.services.analytics.interface :as analytics]))
+            [source.services.analytics.interface :as analytics]
+            [source.routes.openapi :as api]
+            [source.workers.schemas :as schemas]))
 
 (defn post
   {:summary "Get all RSS feeds present in the bundle authorised by uuid.
@@ -17,23 +19,8 @@
                         [:nonfiltered {:optional true
                                        :description "Marking this field as true will disable all filters"} :boolean]]
                 :body [:map [:category-ids [:vector :int]]]}
-   :responses {200 {:body [:vector
-                           [:map
-                            [:id :int]
-                            [:title :string]
-                            [:display-picture [:maybe :string]]
-                            [:url [:maybe :string]]
-                            [:rss-url :string]
-                            [:user-id :int]
-                            [:provider-id [:maybe :int]]
-                            [:created-at :string]
-                            [:updated-at [:maybe :string]]
-                            [:content-type-id :int]
-                            [:cadence-id :int]
-                            [:baseline-id :int]
-                            [:ts-and-cs [:maybe :int]]
-                            [:state [:enum "live" "not live" "pending"]]]]}
-               404 {:body [:map [:message :string]]}}}
+   :responses (-> (api/success schemas/FeedsWithIDs)
+                  (api/not-found))}
 
   [{:keys [ds bundle-id query-params body] :as _request}]
   (let [{:keys [type latest nonfiltered]} query-params
diff --git a/src/source/routes/feed.clj b/src/source/routes/feed.clj
index 4b4f3ac4..355b63fb 100644
--- a/src/source/routes/feed.clj
+++ b/src/source/routes/feed.clj
@@ -2,27 +2,16 @@
   (:require [ring.util.response :as res]
             [source.db.honey :as hon]
             [source.workers.feeds :as feeds]
-            [source.jobs.handlers :as handlers]))
+            [source.jobs.handlers :as handlers]
+            [source.routes.openapi :as api]
+            [source.workers.schemas :as schemas]
+            [malli.util :as mu]))
 
 (defn get
   {:summary "get feed by id"
    :parameters {:path [:map [:id {:title "id"
                                   :description "feed id"} :int]]}
-   :responses  {200 {:body [:map
-                            [:id :int]
-                            [:title :string]
-                            [:display-picture [:maybe :string]]
-                            [:url [:maybe :string]]
-                            [:rss-url :string]
-                            [:user-id :int]
-                            [:provider-id [:maybe :int]]
-                            [:created-at :string]
-                            [:updated-at [:maybe :string]]
-                            [:content-type-id :int]
-                            [:cadence-id :int]
-                            [:baseline-id :int]
-                            [:ts-and-cs [:maybe :int]]
-                            [:state [:enum "live" "not live" "pending"]]]}}}
+   :responses (api/success schemas/FeedWithIDs)}
 
   [{:keys [ds path-params] :as _request}]
   (-> (hon/find-one ds {:tname :feeds
@@ -33,13 +22,14 @@
   {:summary "update feed by id"
    :parameters {:path [:map [:id {:title "id"
                                   :description "feed id"} :int]]
-                :body [:map
-                       [:title :string]
-                       [:display-picture {:optional true} :string]
-                       [:url {:optional true} :string]
-                       [:ts-and-cs {:optional true} :int]
-                       [:cadence-id :int]
-                       [:baseline-id :int]]}
+                :body (-> (api/maybe-keys schemas/FeedWithIDs)
+                          (mu/dissoc :id)
+                          (mu/dissoc :user-id)
+                          (mu/dissoc :provider-id)
+                          (mu/dissoc :created-at)
+                          (mu/dissoc :updated-at)
+                          (mu/dissoc :state)
+                          (mu/dissoc :rss-url))}
    :responses  {200 {:body [:map [:message :string]]}
                 401 {:body [:map [:message :string]]}
                 403 {:body [:map [:message :string]]}}}
diff --git a/src/source/routes/feeds.clj b/src/source/routes/feeds.clj
index 64bb8c00..b65adfee 100644
--- a/src/source/routes/feeds.clj
+++ b/src/source/routes/feeds.clj
@@ -4,26 +4,14 @@
             [ring.util.response :as res]
             [source.db.honey :as hon]
             [source.util :as util]
-            [source.jobs.core :as jobs]))
+            [source.jobs.core :as jobs]
+            [source.routes.openapi :as api]
+            [source.workers.schemas :as schemas]
+            [malli.util :as mu]))
 
 (defn get
   {:summary "get all feeds"
-   :responses  {200 {:body [:vector
-                            [:map
-                             [:id :int]
-                             [:title :string]
-                             [:display-picture [:maybe :string]]
-                             [:url [:maybe :string]]
-                             [:rss-url :string]
-                             [:user-id :int]
-                             [:provider-id [:maybe :int]]
-                             [:created-at :string]
-                             [:updated-at [:maybe :string]]
-                             [:content-type-id :int]
-                             [:cadence-id :int]
-                             [:baseline-id :int]
-                             [:ts-and-cs [:maybe :int]]
-                             [:state [:enum "live" "not live" "pending"]]]]}}}
+   :responses (api/success schemas/FeedsWithIDs)}
 
   [{:keys [ds user] :as _request}]
   (-> (hon/find ds {:tname :feeds
@@ -32,30 +20,14 @@
 
 (defn post
   {:summary "adds a feed and extracts data from RSS feed URL to create incoming posts and schedules a job to keep them updated"
-   :parameters {:body [:map
-                       [:display-picture {:optional true} :string]
-                       [:url {:optional true} :string]
-                       [:rss-url :string]
-                       [:provider-id :int]
-                       [:content-type-id :int]
-                       [:cadence-id :int]
-                       [:baseline-id :int]
-                       [:ts-and-cs {:optional true} :int]]}
-   :responses {200 {:body [:map
-                           [:id :int]
-                           [:title :string]
-                           [:display-picture [:maybe :string]]
-                           [:url [:maybe :string]]
-                           [:rss-url :string]
-                           [:user-id :int]
-                           [:provider-id [:maybe :int]]
-                           [:created-at :string]
-                           [:updated-at [:maybe :string]]
-                           [:content-type-id :int]
-                           [:cadence-id :int]
-                           [:baseline-id :int]
-                           [:ts-and-cs [:maybe :int]]
-                           [:state [:enum "live" "not live" "pending"]]]}}}
+   :parameters {:body (-> schemas/FeedWithIDs
+                          (mu/dissoc :id)
+                          (mu/dissoc :title)
+                          (mu/dissoc :user-id)
+                          (mu/dissoc :created-at)
+                          (mu/dissoc :updated-at)
+                          (mu/dissoc :state))}
+   :responses (api/success schemas/FeedWithIDs)}
 
   [{:keys [js ds user body] :as _request}]
   (let [exists (hon/exists? ds {:tname :feeds
@@ -101,7 +73,7 @@
   (get {:ds (db.util/conn) :user {:id 3}})
   (post {:ds (db.util/conn)
          :js (congest/create-job-service [])
-         :user {:id 3}
+         :user {:id 5}
          :body {:rss-url "https://www.youtube.com/feeds/videos.xml?channel_id=UCUyeluBRhGPCW4rPe_UvBZQ"
                 :provider-id 1
                 :content-type-id 1
diff --git a/src/source/routes/openapi.clj b/src/source/routes/openapi.clj
index 065e8a0c..326050ac 100644
--- a/src/source/routes/openapi.clj
+++ b/src/source/routes/openapi.clj
@@ -91,7 +91,7 @@
     (merge parameters (reduce assoc-param {} (partition 2 opts)))))
 
 (defn- sometimes-entry [[k _ s]] [k {:optional true} [:maybe s]])
-(defn- maybe-keys [schema]
+(defn maybe-keys [schema]
   (mu/transform-entries
    schema
    #(mapv sometimes-entry %)))
diff --git a/src/source/workers/schemas.clj b/src/source/workers/schemas.clj
index 86e1d36f..9e8fa801 100644
--- a/src/source/workers/schemas.clj
+++ b/src/source/workers/schemas.clj
@@ -94,12 +94,13 @@
   [:map
    [:id :int]
    [:title :string]
-   [:display-picture :string]
-   [:url :string]
+   (api/sometimes :display-picture :string)
+   (api/sometimes :description :string)
+   (api/sometimes :url :string)
    [:rss-url :string]
    [:created-at :string]
-   [:updated-at :string]
-   [:ts-and-cs :int]
+   (api/sometimes :updated-at :string)
+   (api/sometimes :ts-and-cs :int)
    [:state FeedStatus]])
 
 (def Feed
@@ -110,6 +111,17 @@
       (mu/assoc :baseline Baseline)
       (mu/assoc :provider Provider)))
 
+(def FeedWithIDs
+  (-> FeedRecord
+      (mu/assoc :user-id :int)
+      (mu/assoc :content-type-id :int)
+      (mu/assoc :cadence-id :int)
+      (mu/assoc :baseline-id :int)
+      (mu/assoc :provider-id :int)))
+
+(def FeedsWithIDs
+  [:vector FeedWithIDs])
+
 (defn paginated [data-schema]
   [:map
    [:pagination [:map

From 2ddfd2291a42cb979f97637ac1e5129e4a0b5959 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 25 Feb 2026 17:53:42 +0200
Subject: [PATCH 318/391] fixed nil exception when pagination is not used

---
 src/source/workers/bundles.clj | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/source/workers/bundles.clj b/src/source/workers/bundles.clj
index aa787277..6148e878 100644
--- a/src/source/workers/bundles.clj
+++ b/src/source/workers/bundles.clj
@@ -85,12 +85,12 @@
                         (subvec started-posts 0 limit)
                         started-posts)
 
-        next-index (+ start limit)]
+        next-index (when (and start limit) (+ start limit))]
 
     {:pagination {:page-size (count limited-posts)
                   :total-size total-size
                   :current-index start
-                  :next-index (when (< next-index total-size) next-index)}
+                  :next-index (when (and next-index (< next-index total-size)) next-index)}
      :data limited-posts}))
 
 (comment

From 7f08ffad2934fd1bc2efa5592983fca93a156269 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Thu, 26 Feb 2026 12:27:08 +0200
Subject: [PATCH 319/391] added support for regular youtube link as rss feed
 and fetching youtube channel images

---
 deps.edn                     |  3 ++-
 src/source/rss/youtube.clj   | 17 ++++++++++++++++-
 src/source/workers/feeds.clj | 17 +++++++++++++++--
 3 files changed, 33 insertions(+), 4 deletions(-)

diff --git a/deps.edn b/deps.edn
index 1ce2e076..3e4aa744 100644
--- a/deps.edn
+++ b/deps.edn
@@ -40,4 +40,5 @@
         metosin/reitit {:mvn/version "0.9.1"}
         metosin/reitit-middleware {:mvn/version "0.9.1"}
         metosin/reitit-swagger {:mvn/version "0.9.1"}
-        metosin/reitit-swagger-ui {:mvn/version "0.9.1"}}}
+        metosin/reitit-swagger-ui {:mvn/version "0.9.1"}
+        enlive/enlive {:mvn/version "1.1.6"}}}
diff --git a/src/source/rss/youtube.clj b/src/source/rss/youtube.clj
index 965a7616..7d9c7b73 100644
--- a/src/source/rss/youtube.clj
+++ b/src/source/rss/youtube.clj
@@ -1,7 +1,8 @@
 (ns source.rss.youtube
   (:require [clojure.string :as s]
             [source.rss.squash :as squash]
-            [clojure.xml :as xml]))
+            [clojure.xml :as xml]
+            [net.cgrand.enlive-html :as html]))
 
 (defn find-channel-id
   "a rudimentary first implementation, we should probably consider using
@@ -23,5 +24,19 @@
    (xml/parse)
    (squash/squash)))
 
+(defn channel-image [channel-url]
+  (->> (-> channel-url
+           (java.net.URL.)
+           (html/html-resource)
+           (second)
+           (get-in [:content])
+           (first)
+           (get-in [:content]))
+       (filter (fn [{:keys [tag attrs]}]
+                 (and (= tag :link) (= (:rel attrs) "image_src"))))
+       (first)
+       (:attrs)
+       (:href)))
+
 (comment
   (find-channel-id "https://www.youtube.com/@CodingWithLewis"))
diff --git a/src/source/workers/feeds.clj b/src/source/workers/feeds.clj
index e9988cc5..552c82ca 100644
--- a/src/source/workers/feeds.clj
+++ b/src/source/workers/feeds.clj
@@ -2,13 +2,21 @@
   (:require [source.util :as utils]
             [source.workers.xml-schemas :as xml]
             [congest.jobs :as congest]
-            [source.db.honey :as hon]))
+            [source.db.honey :as hon]
+            [source.rss.youtube :as yt]
+            [clojure.string :as string]))
 
 (defn create-feed!
   "Creates feed with incoming posts pulled from RSS feed and starts associated job"
   [ds {:keys [user-id feed-metadata]}]
   (let [{:keys [provider-id rss-url content-type-id]} feed-metadata
         datetime (utils/get-utc-timestamp-string)
+        youtube? (= provider-id 1)
+        rss-url (if (and youtube? (not (string/includes? rss-url "feeds")))
+                  (->> (yt/find-channel-id rss-url)
+                       (str "https://www.youtube.com/feeds/videos.xml?channel_id="))
+                  rss-url)
+
         selection-schemas (->> [:= :provider-id provider-id]
                                (assoc {} :where)
                                (xml/selection-schemas ds))
@@ -19,11 +27,16 @@
         extracted (when-not (= latest-ss -1)
                     (xml/extract-data ds latest-ss rss-url))
         extracted-posts (get-in extracted [:feed :posts])
+
+        display-picture (if youtube?
+                          (yt/channel-image (get-in extracted [:feed :url]))
+                          (get-in extracted [:feed :display-picture]))
+
         new-feed (hon/insert!
                   ds
                   {:tname :feeds
                    :data (merge feed-metadata {:title (get-in extracted [:feed :title])
-                                               :display-picture (get-in extracted [:feed :display-picture])
+                                               :display-picture display-picture
                                                :user-id user-id
                                                :created-at datetime
                                                :state "pending"})

From ff7166b590ae3f66c51ff594d9e0196fbf684bac Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Thu, 26 Feb 2026 12:42:22 +0200
Subject: [PATCH 320/391] added feed title in posts endpoints

---
 src/source/routes/bundle_post.clj  | 31 ++++++++++++------------------
 src/source/routes/bundle_posts.clj |  6 ++++--
 src/source/workers/bundles.clj     |  3 ++-
 3 files changed, 18 insertions(+), 22 deletions(-)

diff --git a/src/source/routes/bundle_post.clj b/src/source/routes/bundle_post.clj
index 7f0feee3..fbdbffb2 100644
--- a/src/source/routes/bundle_post.clj
+++ b/src/source/routes/bundle_post.clj
@@ -3,7 +3,10 @@
             [source.services.analytics.interface :as analytics]
             [source.db.honey :as hon]
             [source.db.util :as db.util]
-            [honey.sql.helpers :as hsql]))
+            [honey.sql.helpers :as hsql]
+            [source.routes.openapi :as api]
+            [source.workers.schemas :as schemas]
+            [malli.util :as mu]))
 
 (defn get
   {:summary "Get a single post by post id in the uuid-authorized bundle.
@@ -13,26 +16,16 @@
    :parameters {:query [:map [:uuid {:description "Bundle UUID"} :string]]
                 :path [:map [:id {:title "id"
                                   :description "Post ID"} :int]]}
-   :responses {200 {:body [:map
-                           [:id :int]
-                           [:post-id :string]
-                           [:feed-id :int]
-                           [:creator-id :int]
-                           [:content-type-id :int]
-                           [:title :string]
-                           [:thumbnail [:maybe :string]]
-                           [:info [:maybe :string]]
-                           [:url [:maybe :string]]
-                           [:stream-url [:maybe :string]]
-                           [:season [:maybe :int]]
-                           [:episode [:maybe :int]]
-                           [:redacted {:optional true} [:maybe :int]]
-                           [:posted-at [:maybe :string]]]}
-               404 {:body [:map [:message :string]]}}}
+   :responses (-> (api/success (-> schemas/Post
+                                   (mu/assoc :feed-title :string)))
+                  (api/not-found))}
 
   [{:keys [ds bundle-id path-params] :as _request}]
-  (let [post (hon/find-one ds (-> (db.util/tname :outgoing-posts bundle-id)
-                                  (hsql/where [:= :id (:id path-params)])))]
+  (let [post (hon/execute! ds (-> (hsql/select-distinct :p.* [:f.title :feed-title])
+                                  (hsql/from [(:tname (db.util/tname :outgoing-posts bundle-id)) :p])
+                                  (hsql/join [:feeds :f] [:= :p.feed-id :f.id])
+                                  (hsql/where [:= :p.id (:id path-params)]))
+                           {:ret :1})]
     (try
       (analytics/insert-post-click! ds post bundle-id)
       (catch Exception e (println (.getMessage e))))
diff --git a/src/source/routes/bundle_posts.clj b/src/source/routes/bundle_posts.clj
index 13c30043..04d32e08 100644
--- a/src/source/routes/bundle_posts.clj
+++ b/src/source/routes/bundle_posts.clj
@@ -4,7 +4,8 @@
             [source.services.analytics.interface :as analytics]
             [source.workers.bundles :as bundles]
             [source.routes.openapi :as api]
-            [source.workers.schemas :as schemas]))
+            [source.workers.schemas :as schemas]
+            [malli.util :as mu]))
 
 (defn post
   {:summary "Get a list of posts in the uuid-authorized bundle, determined by analytics.
@@ -30,7 +31,8 @@
                           :description "Filters by most recently uploaded posts, not determined by analytics"}
                          [:enum "true" "false"]]
                         [:seed {:optional true} [:maybe :string]]]}
-   :responses (api/success (schemas/paginated schemas/Posts))}
+   :responses (api/success (schemas/paginated [:vector (-> schemas/Post
+                                                           (mu/assoc :feed-title :string))]))}
 
   [{:keys [ds bundle-id query-params body] :as _request}]
   (let [{:keys [limit start type latest seed]} (walk/keywordize-keys query-params)
diff --git a/src/source/workers/bundles.clj b/src/source/workers/bundles.clj
index 6148e878..a79d0ee1 100644
--- a/src/source/workers/bundles.clj
+++ b/src/source/workers/bundles.clj
@@ -45,8 +45,9 @@
   [ds {:keys [bundle-id limit start type latest category-ids seed]}]
   (let [filtered-posts (hon/execute!
                         ds
-                        (-> (hsql/select-distinct :p.*)
+                        (-> (hsql/select-distinct :p.* [:f.title :feed-title])
                             (hsql/from [(:tname (db.util/tname :outgoing-posts bundle-id)) :p])
+                            (hsql/join [:feeds :f] [:= :p.feed-id :f.id])
                             (hsql/join [:feed-categories :fc] [:= :p.feed-id :fc.feed-id])
                             (hsql/join [:categories :c] [:= :fc.category-id :c.id])
                             (hsql/where

From b277afb66d3f423e00621a64be7cbe2c0d8d2134 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Thu, 26 Feb 2026 14:55:52 +0200
Subject: [PATCH 321/391] updated feed schemas and added a malli dissoc helper
 that accepts multiple arguments

---
 src/source/routes/bundle_feed.clj  |  2 +-
 src/source/routes/bundle_feeds.clj |  2 +-
 src/source/routes/bundle_posts.clj |  4 ++--
 src/source/routes/feed.clj         | 15 ++++-----------
 src/source/routes/feeds.clj        | 16 +++++-----------
 src/source/routes/openapi.clj      | 15 +++++++++++++++
 src/source/workers/schemas.clj     | 20 ++++++--------------
 7 files changed, 34 insertions(+), 40 deletions(-)

diff --git a/src/source/routes/bundle_feed.clj b/src/source/routes/bundle_feed.clj
index 9457c677..65b9a3e9 100644
--- a/src/source/routes/bundle_feed.clj
+++ b/src/source/routes/bundle_feed.clj
@@ -11,7 +11,7 @@
    :parameters {:query [:map [:uuid {:description "Bundle UUID"} :string]]
                 :path [:map [:id {:title "id"
                                   :description "Feed ID"} :int]]}
-   :responses (-> (api/success schemas/FeedWithIDs)
+   :responses (-> (api/success schemas/Feed)
                   (api/not-found))}
 
   [{:keys [ds bundle-id path-params] :as _request}]
diff --git a/src/source/routes/bundle_feeds.clj b/src/source/routes/bundle_feeds.clj
index 6c0db769..983e8cab 100644
--- a/src/source/routes/bundle_feeds.clj
+++ b/src/source/routes/bundle_feeds.clj
@@ -19,7 +19,7 @@
                         [:nonfiltered {:optional true
                                        :description "Marking this field as true will disable all filters"} :boolean]]
                 :body [:map [:category-ids [:vector :int]]]}
-   :responses (-> (api/success schemas/FeedsWithIDs)
+   :responses (-> (api/success schemas/Feeds)
                   (api/not-found))}
 
   [{:keys [ds bundle-id query-params body] :as _request}]
diff --git a/src/source/routes/bundle_posts.clj b/src/source/routes/bundle_posts.clj
index 04d32e08..6466ff36 100644
--- a/src/source/routes/bundle_posts.clj
+++ b/src/source/routes/bundle_posts.clj
@@ -31,8 +31,8 @@
                           :description "Filters by most recently uploaded posts, not determined by analytics"}
                          [:enum "true" "false"]]
                         [:seed {:optional true} [:maybe :string]]]}
-   :responses (api/success (schemas/paginated [:vector (-> schemas/Post
-                                                           (mu/assoc :feed-title :string))]))}
+   :responses (api/success (api/paginated [:vector (-> schemas/Post
+                                                       (mu/assoc :feed-title :string))]))}
 
   [{:keys [ds bundle-id query-params body] :as _request}]
   (let [{:keys [limit start type latest seed]} (walk/keywordize-keys query-params)
diff --git a/src/source/routes/feed.clj b/src/source/routes/feed.clj
index 355b63fb..fdff0b37 100644
--- a/src/source/routes/feed.clj
+++ b/src/source/routes/feed.clj
@@ -4,14 +4,13 @@
             [source.workers.feeds :as feeds]
             [source.jobs.handlers :as handlers]
             [source.routes.openapi :as api]
-            [source.workers.schemas :as schemas]
-            [malli.util :as mu]))
+            [source.workers.schemas :as schemas]))
 
 (defn get
   {:summary "get feed by id"
    :parameters {:path [:map [:id {:title "id"
                                   :description "feed id"} :int]]}
-   :responses (api/success schemas/FeedWithIDs)}
+   :responses (api/success schemas/Feed)}
 
   [{:keys [ds path-params] :as _request}]
   (-> (hon/find-one ds {:tname :feeds
@@ -22,14 +21,8 @@
   {:summary "update feed by id"
    :parameters {:path [:map [:id {:title "id"
                                   :description "feed id"} :int]]
-                :body (-> (api/maybe-keys schemas/FeedWithIDs)
-                          (mu/dissoc :id)
-                          (mu/dissoc :user-id)
-                          (mu/dissoc :provider-id)
-                          (mu/dissoc :created-at)
-                          (mu/dissoc :updated-at)
-                          (mu/dissoc :state)
-                          (mu/dissoc :rss-url))}
+                :body (-> (api/maybe-keys schemas/Feed)
+                          (api/missoc :id :user-id :provider-id :created-at :updated-at :state :rss-url))}
    :responses  {200 {:body [:map [:message :string]]}
                 401 {:body [:map [:message :string]]}
                 403 {:body [:map [:message :string]]}}}
diff --git a/src/source/routes/feeds.clj b/src/source/routes/feeds.clj
index b65adfee..3768d258 100644
--- a/src/source/routes/feeds.clj
+++ b/src/source/routes/feeds.clj
@@ -6,12 +6,11 @@
             [source.util :as util]
             [source.jobs.core :as jobs]
             [source.routes.openapi :as api]
-            [source.workers.schemas :as schemas]
-            [malli.util :as mu]))
+            [source.workers.schemas :as schemas]))
 
 (defn get
   {:summary "get all feeds"
-   :responses (api/success schemas/FeedsWithIDs)}
+   :responses (api/success schemas/Feeds)}
 
   [{:keys [ds user] :as _request}]
   (-> (hon/find ds {:tname :feeds
@@ -20,14 +19,9 @@
 
 (defn post
   {:summary "adds a feed and extracts data from RSS feed URL to create incoming posts and schedules a job to keep them updated"
-   :parameters {:body (-> schemas/FeedWithIDs
-                          (mu/dissoc :id)
-                          (mu/dissoc :title)
-                          (mu/dissoc :user-id)
-                          (mu/dissoc :created-at)
-                          (mu/dissoc :updated-at)
-                          (mu/dissoc :state))}
-   :responses (api/success schemas/FeedWithIDs)}
+   :parameters {:body (-> schemas/Feed
+                          (api/missoc :id :title :user-id :created-at :updated-at :state))}
+   :responses (api/success schemas/Feed)}
 
   [{:keys [js ds user body] :as _request}]
   (let [exists (hon/exists? ds {:tname :feeds
diff --git a/src/source/routes/openapi.clj b/src/source/routes/openapi.clj
index 326050ac..1067dcd1 100644
--- a/src/source/routes/openapi.clj
+++ b/src/source/routes/openapi.clj
@@ -96,6 +96,12 @@
    schema
    #(mapv sometimes-entry %)))
 
+(defn missoc
+  "Executes mu/dissoc with multiple keys"
+  [schema & ks]
+  (reduce (fn [acc k]
+            (mu/dissoc acc k)) schema ks))
+
 ;; MALLI SCHEMAS
 
 (defn optional [key type]
@@ -107,6 +113,15 @@
 (defn sometimes [key type]
   (optional key (maybe type)))
 
+(defn paginated [data-schema]
+  [:map
+   [:pagination [:map
+                 [:page-size :int]
+                 [:total-size :int]
+                 [:current-index :int]
+                 (sometimes :next-index :int)]]
+   [:data data-schema]])
+
 (def RegisterParams
   [:map
    [:email :string]
diff --git a/src/source/workers/schemas.clj b/src/source/workers/schemas.clj
index 9e8fa801..5c7f2a7f 100644
--- a/src/source/workers/schemas.clj
+++ b/src/source/workers/schemas.clj
@@ -104,14 +104,6 @@
    [:state FeedStatus]])
 
 (def Feed
-  (-> FeedRecord
-      (mu/assoc :user User)
-      (mu/assoc :content-type ContentType)
-      (mu/assoc :cadence Cadence)
-      (mu/assoc :baseline Baseline)
-      (mu/assoc :provider Provider)))
-
-(def FeedWithIDs
   (-> FeedRecord
       (mu/assoc :user-id :int)
       (mu/assoc :content-type-id :int)
@@ -119,16 +111,16 @@
       (mu/assoc :baseline-id :int)
       (mu/assoc :provider-id :int)))
 
-(def FeedsWithIDs
-  [:vector FeedWithIDs])
+(def Feeds
+  [:vector Feed])
 
 (defn paginated [data-schema]
   [:map
    [:pagination [:map
-                [:page-size :int]
-                [:total-size :int]
-                [:current-index :int]
-                (api/sometimes :next-index :int)]]
+                 [:page-size :int]
+                 [:total-size :int]
+                 [:current-index :int]
+                 (api/sometimes :next-index :int)]]
    [:data data-schema]])
 
 (def IncomingPostRecord

From 6afa60882747b24c91a5de6c51e50c44d45b2b88 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Thu, 26 Feb 2026 15:02:03 +0200
Subject: [PATCH 322/391] fixed sql query

---
 src/source/workers/bundles.clj | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/source/workers/bundles.clj b/src/source/workers/bundles.clj
index a79d0ee1..e1aa5a5d 100644
--- a/src/source/workers/bundles.clj
+++ b/src/source/workers/bundles.clj
@@ -51,7 +51,7 @@
                             (hsql/join [:feed-categories :fc] [:= :p.feed-id :fc.feed-id])
                             (hsql/join [:categories :c] [:= :fc.category-id :c.id])
                             (hsql/where
-                             (when type [:= :content-type-id type])
+                             (when type [:= :p.content-type-id type])
                              [:not-in :p.id (-> (hsql/select :post-id)
                                                 (hsql/from :filtered-posts)
                                                 (hsql/where [:= :bundle-id bundle-id]))]

From 62af1c5ffb61646ffca90e93fbe8b105c03b4ab1 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Thu, 26 Feb 2026 15:29:56 +0200
Subject: [PATCH 323/391] added query parameter to allow truncation

---
 src/source/routes/bundle_posts.clj |  7 ++++++-
 src/source/workers/bundles.clj     | 21 +++++++++++++++++++--
 2 files changed, 25 insertions(+), 3 deletions(-)

diff --git a/src/source/routes/bundle_posts.clj b/src/source/routes/bundle_posts.clj
index 04d32e08..b519d29f 100644
--- a/src/source/routes/bundle_posts.clj
+++ b/src/source/routes/bundle_posts.clj
@@ -30,12 +30,16 @@
                          {:optional true
                           :description "Filters by most recently uploaded posts, not determined by analytics"}
                          [:enum "true" "false"]]
+                        [:truncate
+                         {:optional true
+                          :description "Truncates text in posts to a maximum of 100 characters. Defaults to true."}
+                         [:enum "true" "false"]]
                         [:seed {:optional true} [:maybe :string]]]}
    :responses (api/success (schemas/paginated [:vector (-> schemas/Post
                                                            (mu/assoc :feed-title :string))]))}
 
   [{:keys [ds bundle-id query-params body] :as _request}]
-  (let [{:keys [limit start type latest seed]} (walk/keywordize-keys query-params)
+  (let [{:keys [limit start type latest seed truncate]} (walk/keywordize-keys query-params)
         {:keys [data] :as posts} (bundles/get-outgoing-posts
                                   ds
                                   {:bundle-id bundle-id
@@ -44,6 +48,7 @@
                                    :type type
                                    :latest latest
                                    :seed seed
+                                   :truncate truncate
                                    :category-ids (:category-ids body)})]
     (try
       (analytics/insert-post-impressions! ds data bundle-id)
diff --git a/src/source/workers/bundles.clj b/src/source/workers/bundles.clj
index e1aa5a5d..0be883ec 100644
--- a/src/source/workers/bundles.clj
+++ b/src/source/workers/bundles.clj
@@ -40,12 +40,29 @@
                                           filtered-query))]
     filtered-feeds))
 
+(defn- select-outgoing-post [truncate]
+  (hsql/select-distinct
+   :p.id
+   :p.post-id
+   :p.feed-id
+   :p.creator-id
+   :p.content-type-id
+   :p.title
+   :p.thumbnail
+   (if (= truncate "false") :p.info [[:left :p.info 100] :info])
+   :p.url
+   :p.stream-url
+   :p.season
+   :p.episode
+   :p.posted-at
+   [:f.title :feed-title]))
+
 (defn get-outgoing-posts
   "Get outgoing posts based on short heuristics and update analytics impressions"
-  [ds {:keys [bundle-id limit start type latest category-ids seed]}]
+  [ds {:keys [bundle-id limit start type latest category-ids seed truncate]}]
   (let [filtered-posts (hon/execute!
                         ds
-                        (-> (hsql/select-distinct :p.* [:f.title :feed-title])
+                        (-> (select-outgoing-post truncate)
                             (hsql/from [(:tname (db.util/tname :outgoing-posts bundle-id)) :p])
                             (hsql/join [:feeds :f] [:= :p.feed-id :f.id])
                             (hsql/join [:feed-categories :fc] [:= :p.feed-id :fc.feed-id])

From 5c997fad6658c75070bf18ddf418fd0d87826437 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Fri, 27 Feb 2026 09:58:55 +0200
Subject: [PATCH 324/391] refactored to use hickory instead of enlive

---
 deps.edn                   |  3 +--
 src/source/rss/youtube.clj | 26 +++++++++++++++-----------
 2 files changed, 16 insertions(+), 13 deletions(-)

diff --git a/deps.edn b/deps.edn
index 3e4aa744..1ce2e076 100644
--- a/deps.edn
+++ b/deps.edn
@@ -40,5 +40,4 @@
         metosin/reitit {:mvn/version "0.9.1"}
         metosin/reitit-middleware {:mvn/version "0.9.1"}
         metosin/reitit-swagger {:mvn/version "0.9.1"}
-        metosin/reitit-swagger-ui {:mvn/version "0.9.1"}
-        enlive/enlive {:mvn/version "1.1.6"}}}
+        metosin/reitit-swagger-ui {:mvn/version "0.9.1"}}}
diff --git a/src/source/rss/youtube.clj b/src/source/rss/youtube.clj
index 7d9c7b73..3d3aecc7 100644
--- a/src/source/rss/youtube.clj
+++ b/src/source/rss/youtube.clj
@@ -2,7 +2,8 @@
   (:require [clojure.string :as s]
             [source.rss.squash :as squash]
             [clojure.xml :as xml]
-            [net.cgrand.enlive-html :as html]))
+            [org.httpkit.client :as http]
+            [hickory.core :as h]))
 
 (defn find-channel-id
   "a rudimentary first implementation, we should probably consider using
@@ -25,18 +26,21 @@
    (squash/squash)))
 
 (defn channel-image [channel-url]
-  (->> (-> channel-url
-           (java.net.URL.)
-           (html/html-resource)
-           (second)
-           (get-in [:content])
-           (first)
-           (get-in [:content]))
-       (filter (fn [{:keys [tag attrs]}]
-                 (and (= tag :link) (= (:rel attrs) "image_src"))))
+  (->> @(http/request {:url channel-url})
+       (:body)
+       (h/parse)
+       (h/as-hickory)
+       (:content)
+       (second)
+       (:content)
+       (first)
+       (:content)
+       (filterv (fn [{:keys [type attrs]}]
+                  (and (= type :element) (= (:rel attrs) "image_src"))))
        (first)
        (:attrs)
        (:href)))
 
 (comment
-  (find-channel-id "https://www.youtube.com/@CodingWithLewis"))
+  (find-channel-id "https://www.youtube.com/@Veritasium")
+  (channel-image "https://www.youtube.com/@Veritasium"))

From bdcc8a5115a4a373437298e5125d09aa26c14cff Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Fri, 27 Feb 2026 11:42:46 +0200
Subject: [PATCH 325/391] tightened rss feed checking

---
 src/source/workers/feeds.clj | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/source/workers/feeds.clj b/src/source/workers/feeds.clj
index 0c51ec63..a459573e 100644
--- a/src/source/workers/feeds.clj
+++ b/src/source/workers/feeds.clj
@@ -12,7 +12,7 @@
   (let [{:keys [provider-id rss-url content-type-id]} feed-metadata
         datetime (utils/get-utc-timestamp-string)
         youtube? (= provider-id 1)
-        rss-url (if (and youtube? (not (string/includes? rss-url "feeds")))
+        rss-url (if (and youtube? (not (string/includes? rss-url "/feeds/videos.xml?channel_id=")))
                   (->> (yt/find-channel-id rss-url)
                        (str "https://www.youtube.com/feeds/videos.xml?channel_id="))
                   rss-url)

From 24499f1bd4ee649ab0cda702d93fdb969f2f432a Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Fri, 27 Feb 2026 12:00:10 +0200
Subject: [PATCH 326/391] extracted query parameters in bundle posts

---
 src/source/routes/bundle_posts.clj | 39 ++++++++++++++----------------
 src/source/workers/schemas.clj     | 25 +++++++++++++++++++
 2 files changed, 43 insertions(+), 21 deletions(-)

diff --git a/src/source/routes/bundle_posts.clj b/src/source/routes/bundle_posts.clj
index f1e53a14..f4b3d7a9 100644
--- a/src/source/routes/bundle_posts.clj
+++ b/src/source/routes/bundle_posts.clj
@@ -7,34 +7,31 @@
             [source.workers.schemas :as schemas]
             [malli.util :as mu]))
 
+(def QueryTruncatePosts
+  [:truncate
+   {:optional true
+    :description "Truncates text in posts to a maximum of 100 characters. Defaults to true."}
+   [:enum "true" "false"]])
+
+(def QuerySeed
+  [:seed {:optional true} [:maybe :string]])
+
 (defn post
   {:summary "Get a list of posts in the uuid-authorized bundle, determined by analytics.
    This endpoint updates impression analytics for the returned posts."
    :description "This endpoint pulls a curated list of content (determined by analytics) of the posts that made it into the bundle during post selection. This can be filtered by content type ID, category IDs, or latest (most recently added posts). If results are filtered by latest, they will not be curated by analytics.
    
    Results can be paginated using the `start` and `limit` query parameters."
-   :parameters {:body [:map [:category-ids [:vector :int]]]
+   :parameters (api/params
+                :body [:map [:category-ids [:vector :int]]]
                 :query [:map
-                        [:uuid {:description "Bundle UUID"} :string]
-                        [:limit
-                         {:optional true
-                          :description "Used for pagination. Specifies a number of posts to be returned."}
-                         :int]
-                        [:start
-                         {:optional true
-                          :description "Used for pagination. Specifies the starting point for the returned posts, incremented by the limit."}
-                         :int]
-                        [:type {:optional true
-                                :description "Filters by content type ID"} :int]
-                        [:latest
-                         {:optional true
-                          :description "Filters by most recently uploaded posts, not determined by analytics"}
-                         [:enum "true" "false"]]
-                        [:truncate
-                         {:optional true
-                          :description "Truncates text in posts to a maximum of 100 characters. Defaults to true."}
-                         [:enum "true" "false"]]
-                        [:seed {:optional true} [:maybe :string]]]}
+                        schemas/QueryUUID
+                        schemas/QueryLimit
+                        schemas/QueryStart
+                        schemas/QueryContentType
+                        schemas/QueryLatest
+                        QueryTruncatePosts
+                        QuerySeed])
    :responses (api/success (api/paginated [:vector (-> schemas/Post
                                                        (mu/assoc :feed-title :string))]))}
 
diff --git a/src/source/workers/schemas.clj b/src/source/workers/schemas.clj
index 5c7f2a7f..e100d42b 100644
--- a/src/source/workers/schemas.clj
+++ b/src/source/workers/schemas.clj
@@ -176,6 +176,31 @@
 (def Posts
   [:vector Post])
 
+(def QueryUUID
+  [:uuid {:description "Bundle UUID"} :string])
+
+(def QueryStart
+  [:start
+   {:optional true
+    :description "Used for pagination. Specifies the starting point for the returned items, incremented by the limit."}
+   :int])
+
+(def QueryLimit
+  [:limit
+   {:optional true
+    :description "Used for pagination. Specifies a number of items to be returned."}
+   :int])
+
+(def QueryContentType
+  [:type {:optional true
+          :description "Filters by content type ID"} :int])
+
+(def QueryLatest
+  [:latest
+   {:optional true
+    :description "Filters by most recently published"}
+   [:enum "true" "false"]])
+
 (def JobStatus [:enum ["running" "stopped"]])
 
 (def Job

From 986499f26ab07784c835c586c80b0c995433cbac Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 2 Mar 2026 12:48:42 +0200
Subject: [PATCH 327/391] added better error handling in important worker
 functions

---
 src/source/workers/feeds.clj        | 17 +++++++++++++++--
 src/source/workers/integrations.clj | 10 +++++++++-
 2 files changed, 24 insertions(+), 3 deletions(-)

diff --git a/src/source/workers/feeds.clj b/src/source/workers/feeds.clj
index a459573e..c5b9e10e 100644
--- a/src/source/workers/feeds.clj
+++ b/src/source/workers/feeds.clj
@@ -24,8 +24,17 @@
                        (reduce (fn [acc {:keys [id]}]
                                  (conj acc id)) [])
                        (apply max -1))
-        extracted (when-not (= latest-ss -1)
-                    (xml/extract-data ds latest-ss rss-url))
+        extracted (try
+                    (when-not (= latest-ss -1)
+                      (xml/extract-data ds latest-ss rss-url))
+                    (catch Exception e
+                      (throw (ex-info (str "Data extraction failed for feed creation - RSS feed url: " rss-url " creator-id " user-id)
+                                      {:panic? "Not a huge deal, possibly just user error - but panic if it's not user error"
+                                       :possible-cause "RSS feed url might be incorrect or provider is unsupported"
+                                       :next-steps (str
+                                                    "Check if the RSS feed url is correct. If it is, test data extraction in the admin panel with provider-id "
+                                                    provider-id)
+                                       :raw-error (.getMessage e)}))))
         extracted-posts (get-in extracted [:feed :posts])
 
         display-picture (if youtube?
@@ -42,6 +51,10 @@
                                                :created-at datetime
                                                :state "pending"})
                    :ret :1})
+
+        _ (when (or (nil? display-picture) (= display-picture ""))
+            (println "Failed to pull display picture for feed [" (:id new-feed) (:title new-feed) "] provider-id" provider-id))
+
         extended-posts (mapv (fn [post]
                                (merge post
                                       {:feed-id (:id new-feed)
diff --git a/src/source/workers/integrations.clj b/src/source/workers/integrations.clj
index 760778ee..0fb7354f 100644
--- a/src/source/workers/integrations.clj
+++ b/src/source/workers/integrations.clj
@@ -11,7 +11,15 @@
 (defn create-integration! [ds {:keys [user-id bundle-metadata categories content-types]}]
   (let [new-bundle (bundles/create-bundle! ds {:user-id user-id
                                                :bundle-metadata bundle-metadata})]
-    (migrate/migrate-bundle (:id new-bundle) ["up"])
+
+    (try
+      (migrate/migrate-bundle (:id new-bundle) ["up"])
+      (catch Exception e
+        (throw (ex-info (str "Migration for new bundle with id " (:id new-bundle) " failed. This bundle belongs to the user with id " user-id)
+                        {:panic? "Yes, if this isn't working, it's likely no one will be able to create new integrations"
+                         :possible-cause "Something could be wrong in one of the bundle migrations"
+                         :next-steps "Go see what's going on in bundle migrations ASAP, you may be able to see the problem in the raw error message"
+                         :raw-error (.getMessage e)}))))
 
     (bundle-categories/insert-bundle-categories! ds {:bundle-id (:id new-bundle)
                                                      :categories categories})

From 2bf9e6f1c1a668c591f0c926bf1eba9b8aae697a Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 2 Mar 2026 12:49:49 +0200
Subject: [PATCH 328/391] added better error logs in job handlers

---
 src/source/jobs/handlers.clj | 29 +++++++++++++++++++++++++----
 1 file changed, 25 insertions(+), 4 deletions(-)

diff --git a/src/source/jobs/handlers.clj b/src/source/jobs/handlers.clj
index 3bb7f81b..8ce3f298 100644
--- a/src/source/jobs/handlers.clj
+++ b/src/source/jobs/handlers.clj
@@ -39,7 +39,15 @@
                              (reduce (fn [acc {:keys [id]}]
                                        (conj acc id)) [])
                              (apply max -1))
-              extracted (xml/extract-data ds latest-ss url)
+              extracted (try
+                          (xml/extract-data ds latest-ss url)
+                          (catch Exception e
+                            (throw (ex-info (str "Data extraction for feed job failed: feed-id " feed-id " creator-id " creator-id)
+                                            {:panic? "Yes, if data extraction fails here it will likely fail for others."
+                                             :possible-cause "Could possibly be an incorrect selection schema or output schema"
+                                             :next-steps (str "Check selection-schema-id " latest-ss " and feed-id " feed-id ". Test extraction manually.")
+                                             :raw-error (.getMessage e)}))))
+
               extracted-posts (get-in extracted [:feed :posts])
               extracted-display (get-in extracted [:feed :display-picture])
               extended-posts (mapv (fn [{:keys [posted-at thumbnail] :as post}]
@@ -123,7 +131,7 @@
             ; top 1000 post-heuristics records ordered by long heuristic in descending order
       (let [top-by-long-heuristics (services/top-posts-by-heuristic ds
                                                                     {:heuristic :long-heuristic
-                                                                     :limit 1000
+                                                                     :limit 2000
                                                                      :bundle-id bundle-id})
             ; convert into a vector of id numbers
             ids (mapv :post-id top-by-long-heuristics)
@@ -148,7 +156,20 @@
         (when (seq posts-in)
           (hon/delete! ds (db.util/tname :outgoing-posts bundle-id))
           (hon/insert! ds (-> (db.util/tname :outgoing-posts bundle-id)
-                              (assoc :data outgoing-posts))))
+                              (assoc :data outgoing-posts)))
+          (when (< (count outgoing-posts) 10)
+            (throw (ex-info (str
+                             "bundle job for bundle-id "
+                             bundle-id
+                             " pulled "
+                             (count outgoing-posts)
+                             ". Active creator id count: "
+                             (count active-creator-ids)
+                             ". Incoming posts pulled: "
+                             (count posts-in))
+                            {:panic? "Yes, the embed for this bundle will now be useless"
+                             :possible-cause "If no posts made it into the bundle, it's possible post heuristics failed or there's no incoming posts"
+                             :next-steps "Check for errors thrown in this job, ensure all tables for this bundle exist"}))))
 
         (println "bundle" bundle-id "job done")))))
 
@@ -162,4 +183,4 @@
     (try
       (let [{:keys [user-type user-id]} args]
         (users/hard-delete-user! ds (keyword user-type) user-id))
-      (catch Exception e (println "Failed to delete user: " e) :fail))))
+      (catch Exception e (println "Failed to delete user-id" (:user-id args) ":" e) :fail))))

From 6eac469f89fae9f01dc0b412c995df02b09687df Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 2 Mar 2026 12:50:20 +0200
Subject: [PATCH 329/391] added error logs in bundle authorisation and error
 handling middleware

---
 src/source/middleware/auth/core.clj | 8 +++++---
 src/source/middleware/core.clj      | 8 +++++---
 2 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/src/source/middleware/auth/core.clj b/src/source/middleware/auth/core.clj
index 751aa910..189eeb11 100644
--- a/src/source/middleware/auth/core.clj
+++ b/src/source/middleware/auth/core.clj
@@ -56,9 +56,11 @@
         (-> request
             (assoc :bundle-id id)
             (handler))
-        (->
-         (res/response {:message "The bundle you are looking for does not exist."})
-         (res/status 404))))))
+        (do
+          (println "Bundle authorization attempt failed with uuid:" bundle-uuid)
+          (->
+           (res/response {:message "The bundle you are looking for does not exist."})
+           (res/status 404)))))))
 
 (comment
   (let [authed-request {:headers {"Authorization"
diff --git a/src/source/middleware/core.clj b/src/source/middleware/core.clj
index e5a461bf..1c25cc67 100644
--- a/src/source/middleware/core.clj
+++ b/src/source/middleware/core.clj
@@ -43,7 +43,7 @@
     (try
       (handler req)
       (catch Exception e
-        (println "Unhandled Exception:\n" e)
+        (println "Unhandled Exception on endpoint URI " (:uri req) ": " e)
         (-> (res/response {:message "Internal Server Error"})
             (res/status 500))))))
 
@@ -74,8 +74,10 @@
           request (->> validations
                        (attach-validations request))]
       (if (seq errors)
-        (-> (res/response {:message (string/join "\n" errors)})
-            (res/status 400))
+        (do
+          (println "Schema validation failed on endpoint URI" (:uri request) ":" (string/join "\n" errors))
+          (-> (res/response {:message (string/join "\n" errors)})
+              (res/status 400)))
         (handler request)))))
 
 (defn process-body [{:keys [body] :as req} t-fn]

From 36ad591506e92d994321db0ff212dac0e6cff033 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 2 Mar 2026 14:34:59 +0200
Subject: [PATCH 330/391] wrapped various database writes in
 pg/with-transaction to transactionalise their work

---
 src/source/routes/register.clj      |   1 -
 src/source/services/auth.clj        |  22 +++--
 src/source/workers/feeds.clj        | 144 ++++++++++++++--------------
 src/source/workers/integrations.clj |  98 ++++++++++---------
 src/source/workers/users.clj        |  30 +++---
 5 files changed, 154 insertions(+), 141 deletions(-)

diff --git a/src/source/routes/register.clj b/src/source/routes/register.clj
index f616b0f9..a6d8d382 100644
--- a/src/source/routes/register.clj
+++ b/src/source/routes/register.clj
@@ -1,7 +1,6 @@
 (ns source.routes.register
   (:require [source.services.interface :as services]
             [ring.util.response :as res]
-            [source.util :as util]
             [source.db.honey :as hon]))
 
 (defn post
diff --git a/src/source/services/auth.clj b/src/source/services/auth.clj
index 642d9098..579b5e6c 100644
--- a/src/source/services/auth.clj
+++ b/src/source/services/auth.clj
@@ -1,7 +1,8 @@
 (ns source.services.auth
   (:require [source.password :as pw]
             [source.middleware.auth.core :as auth]
-            [source.db.honey :as hon]))
+            [source.db.honey :as hon]
+            [pg.core :as pg]))
 
 (defn login [ds {:keys [user] :as _login}]
   (merge
@@ -9,15 +10,16 @@
    (auth/create-session (select-keys user [:id :type]))))
 
 (defn register [ds {:keys [email password] :as user}]
-  (hon/insert! ds {:tname :users
-                   :data (-> user
-                             (dissoc :confirm-password)
-                             (assoc :password (pw/hash-password password)))})
-  (let [user (hon/find-one ds {:tname :users
-                               :where [:= :email email]})]
-    (merge
-     {:user (dissoc user :password)}
-     (auth/create-session (select-keys user [:id :type])))))
+  (pg/with-transaction [ds ds]
+    (hon/insert! ds {:tname :users
+                     :data (-> user
+                               (dissoc :confirm-password)
+                               (assoc :password (pw/hash-password password)))})
+    (let [user (hon/find-one ds {:tname :users
+                                 :where [:= :email email]})]
+      (merge
+       {:user (dissoc user :password)}
+       (auth/create-session (select-keys user [:id :type]))))))
 
 (comment
   (require '[source.db.interface :as db])
diff --git a/src/source/workers/feeds.clj b/src/source/workers/feeds.clj
index a459573e..c3e459b8 100644
--- a/src/source/workers/feeds.clj
+++ b/src/source/workers/feeds.clj
@@ -4,57 +4,59 @@
             [congest.jobs :as congest]
             [source.db.honey :as hon]
             [source.rss.youtube :as yt]
-            [clojure.string :as string]))
+            [clojure.string :as string]
+            [pg.core :as pg]))
 
 (defn create-feed!
   "Creates feed with incoming posts pulled from RSS feed and starts associated job"
   [ds {:keys [user-id feed-metadata]}]
-  (let [{:keys [provider-id rss-url content-type-id]} feed-metadata
-        datetime (utils/get-utc-timestamp-string)
-        youtube? (= provider-id 1)
-        rss-url (if (and youtube? (not (string/includes? rss-url "/feeds/videos.xml?channel_id=")))
-                  (->> (yt/find-channel-id rss-url)
-                       (str "https://www.youtube.com/feeds/videos.xml?channel_id="))
-                  rss-url)
+  (pg/with-transaction [ds ds]
+    (let [{:keys [provider-id rss-url content-type-id]} feed-metadata
+          datetime (utils/get-utc-timestamp-string)
+          youtube? (= provider-id 1)
+          rss-url (if (and youtube? (not (string/includes? rss-url "/feeds/videos.xml?channel_id=")))
+                    (->> (yt/find-channel-id rss-url)
+                         (str "https://www.youtube.com/feeds/videos.xml?channel_id="))
+                    rss-url)
 
-        selection-schemas (->> [:= :provider-id provider-id]
-                               (assoc {} :where)
-                               (xml/selection-schemas ds))
-        latest-ss (->> selection-schemas
-                       (reduce (fn [acc {:keys [id]}]
-                                 (conj acc id)) [])
-                       (apply max -1))
-        extracted (when-not (= latest-ss -1)
-                    (xml/extract-data ds latest-ss rss-url))
-        extracted-posts (get-in extracted [:feed :posts])
+          selection-schemas (->> [:= :provider-id provider-id]
+                                 (assoc {} :where)
+                                 (xml/selection-schemas ds))
+          latest-ss (->> selection-schemas
+                         (reduce (fn [acc {:keys [id]}]
+                                   (conj acc id)) [])
+                         (apply max -1))
+          extracted (when-not (= latest-ss -1)
+                      (xml/extract-data ds latest-ss rss-url))
+          extracted-posts (get-in extracted [:feed :posts])
 
-        display-picture (if youtube?
-                          (yt/channel-image (get-in extracted [:feed :url]))
-                          (get-in extracted [:feed :display-picture]))
+          display-picture (if youtube?
+                            (yt/channel-image (get-in extracted [:feed :url]))
+                            (get-in extracted [:feed :display-picture]))
 
-        new-feed (hon/insert!
-                  ds
-                  {:tname :feeds
-                   :data (merge feed-metadata {:title (get-in extracted [:feed :title])
-                                               :display-picture display-picture
-                                               :description (get-in extracted [:feed :description])
-                                               :user-id user-id
-                                               :created-at datetime
-                                               :state "pending"})
-                   :ret :1})
-        extended-posts (mapv (fn [post]
-                               (merge post
-                                      {:feed-id (:id new-feed)
-                                       :creator-id user-id
-                                       :content-type-id content-type-id
-                                       :thumbnail (or (:thumbnail post) (:display-picture new-feed))}))
-                             extracted-posts)]
-    (if (some? extracted-posts)
-      (do
-        (hon/insert! ds {:tname :incoming-posts
-                         :data extended-posts})
-        new-feed)
-      false)))
+          new-feed (hon/insert!
+                    ds
+                    {:tname :feeds
+                     :data (merge feed-metadata {:title (get-in extracted [:feed :title])
+                                                 :display-picture display-picture
+                                                 :description (get-in extracted [:feed :description])
+                                                 :user-id user-id
+                                                 :created-at datetime
+                                                 :state "pending"})
+                     :ret :1})
+          extended-posts (mapv (fn [post]
+                                 (merge post
+                                        {:feed-id (:id new-feed)
+                                         :creator-id user-id
+                                         :content-type-id content-type-id
+                                         :thumbnail (or (:thumbnail post) (:display-picture new-feed))}))
+                               extracted-posts)]
+      (if (some? extracted-posts)
+        (do
+          (hon/insert! ds {:tname :incoming-posts
+                           :data extended-posts})
+          new-feed)
+        false))))
 
 (defn update-feed! [ds {:keys [feed-id feed-metadata]}]
   (hon/update! ds {:tname :feeds
@@ -63,34 +65,36 @@
                    :ret :1}))
 
 (defn hard-delete-feed! [ds js job-id feed-id]
-  (let [post-ids (mapv :id (hon/find ds {:tname :incoming-posts
-                                         :where [:= :feed-id feed-id]}))
-        event-ids (:mapv :id (hon/find ds {:tname :events
-                                           :where [:= :feed-id feed-id]}))]
-    (hon/delete! ds {:tname :filtered-feeds
-                     :where [:= :feed-id feed-id]})
-    (hon/delete! ds {:tname :filtered-posts
-                     :where [:in :post-id post-ids]})
-    (hon/delete! ds {:tname :incoming-posts
-                     :where [:= :feed-id feed-id]})
-    (hon/delete! ds {:tname :feed-categories
-                     :where [:= :feed-id feed-id]})
-    (when (seq event-ids)
-      (hon/delete! ds {:tname :event-categories
-                       :where [:in :event-id event-ids]}))
-    (hon/delete! ds {:tname :events
-                     :where [:= :feed-id feed-id]})
-    (hon/delete! ds {:tname :feeds
-                     :where [:= :id feed-id]})
-    (congest/deregister! js job-id)))
+  (pg/with-transaction [ds ds]
+    (let [post-ids (mapv :id (hon/find ds {:tname :incoming-posts
+                                           :where [:= :feed-id feed-id]}))
+          event-ids (:mapv :id (hon/find ds {:tname :events
+                                             :where [:= :feed-id feed-id]}))]
+      (hon/delete! ds {:tname :filtered-feeds
+                       :where [:= :feed-id feed-id]})
+      (hon/delete! ds {:tname :filtered-posts
+                       :where [:in :post-id post-ids]})
+      (hon/delete! ds {:tname :incoming-posts
+                       :where [:= :feed-id feed-id]})
+      (hon/delete! ds {:tname :feed-categories
+                       :where [:= :feed-id feed-id]})
+      (when (seq event-ids)
+        (hon/delete! ds {:tname :event-categories
+                         :where [:in :event-id event-ids]}))
+      (hon/delete! ds {:tname :events
+                       :where [:= :feed-id feed-id]})
+      (hon/delete! ds {:tname :feeds
+                       :where [:= :id feed-id]})
+      (congest/deregister! js job-id))))
 
 (defn update-feed-categories! [ds {:keys [feed-id categories]}]
   (let [update-data (mapv (fn [{:keys [id]}]
                             {:feed-id feed-id
                              :category-id id}) categories)]
-    (when (seq update-data)
-      (hon/delete! ds {:tname :feed-categories
-                       :where [:= :feed-id feed-id]})
-      (hon/insert! ds {:tname :feed-categories
-                       :data update-data
-                       :ret :*}))))
+    (pg/with-transaction [ds ds]
+      (when (seq update-data)
+        (hon/delete! ds {:tname :feed-categories
+                         :where [:= :feed-id feed-id]})
+        (hon/insert! ds {:tname :feed-categories
+                         :data update-data
+                         :ret :*})))))
diff --git a/src/source/workers/integrations.clj b/src/source/workers/integrations.clj
index 760778ee..d2c45642 100644
--- a/src/source/workers/integrations.clj
+++ b/src/source/workers/integrations.clj
@@ -6,60 +6,66 @@
             [source.services.bundle-content-types :as bundle-content-types]
             [source.db.tables :as tables]
             [source.db.honey :as hon]
-            [congest.jobs :as congest]))
+            [congest.jobs :as congest]
+            [pg.core :as pg]))
 
 (defn create-integration! [ds {:keys [user-id bundle-metadata categories content-types]}]
-  (let [new-bundle (bundles/create-bundle! ds {:user-id user-id
-                                               :bundle-metadata bundle-metadata})]
-    (migrate/migrate-bundle (:id new-bundle) ["up"])
+  (pg/with-transaction [ds ds]
+    (let [new-bundle (bundles/create-bundle! ds {:user-id user-id
+                                                 :bundle-metadata bundle-metadata})]
+      (migrate/migrate-bundle (:id new-bundle) ["up"])
 
-    (bundle-categories/insert-bundle-categories! ds {:bundle-id (:id new-bundle)
-                                                     :categories categories})
-    (bundle-content-types/insert-bundle-content-types! ds {:bundle-id (:id new-bundle)
-                                                           :content-types content-types})
-    new-bundle))
+      (bundle-categories/insert-bundle-categories! ds {:bundle-id (:id new-bundle)
+                                                       :categories categories})
+      (bundle-content-types/insert-bundle-content-types! ds {:bundle-id (:id new-bundle)
+                                                             :content-types content-types})
+      new-bundle)))
 
 (defn update-integration! [ds {:keys [bundle-id bundle-metadata categories content-types]}]
-  (bundles/update-bundle! ds {:id bundle-id
-                              :data bundle-metadata})
-  (bundle-categories/update-bundle-categories! ds {:bundle-id bundle-id
-                                                   :categories categories})
-  (bundle-content-types/update-bundle-content-types! ds {:bundle-id bundle-id
-                                                         :content-types content-types}))
+  (pg/with-transaction [ds ds]
+    (bundles/update-bundle! ds {:id bundle-id
+                                :data bundle-metadata})
+    (bundle-categories/update-bundle-categories! ds {:bundle-id bundle-id
+                                                     :categories categories})
+    (bundle-content-types/update-bundle-content-types! ds {:bundle-id bundle-id
+                                                           :content-types content-types})))
 
 (defn hard-delete-bundle! [ds js job-id bundle-id]
-  (hon/delete! ds {:tname :filtered-feeds
-                   :where [:= :bundle-id bundle-id]})
-  (hon/delete! ds {:tname :filtered-posts
-                   :where [:= :bundle-id bundle-id]})
-  (hon/delete! ds {:tname :bundle-content-types
-                   :where [:= :bundle-id bundle-id]})
-  (hon/delete! ds {:tname :events
-                   :where [:= :bundle-id bundle-id]})
-  (tables/drop-tables! ds (db.util/tnames [:outgoing-posts
-                                           :bundle-categories
-                                           :post-heuristics]
-                                          bundle-id))
-  (hon/delete! ds {:tname :bundles
-                   :where [:= :id bundle-id]})
-  (congest/deregister! js job-id))
+  (pg/with-transaction [ds ds]
+    (hon/delete! ds {:tname :filtered-feeds
+                     :where [:= :bundle-id bundle-id]})
+    (hon/delete! ds {:tname :filtered-posts
+                     :where [:= :bundle-id bundle-id]})
+    (hon/delete! ds {:tname :bundle-content-types
+                     :where [:= :bundle-id bundle-id]})
+    (hon/delete! ds {:tname :events
+                     :where [:= :bundle-id bundle-id]})
+    (tables/drop-tables! ds (db.util/tnames [:outgoing-posts
+                                             :bundle-categories
+                                             :post-heuristics]
+                                            bundle-id))
+    (hon/delete! ds {:tname :bundles
+                     :where [:= :id bundle-id]})
+    (congest/deregister! js job-id)))
 
 (defn update-filtered-feeds! [ds {:keys [filtered bundle-id feed-id]}]
-  (if filtered
-    (hon/insert! ds {:tname :filtered-feeds
-                     :data {:feed-id feed-id
-                            :bundle-id bundle-id}})
-    (hon/delete! ds {:tname :filtered-feeds
-                     :where [:and
-                             [:= :feed-id feed-id]
-                             [:= :bundle-id bundle-id]]})))
+  (pg/with-transaction [ds ds]
+    (if filtered
+      (hon/insert! ds {:tname :filtered-feeds
+                       :data {:feed-id feed-id
+                              :bundle-id bundle-id}})
+      (hon/delete! ds {:tname :filtered-feeds
+                       :where [:and
+                               [:= :feed-id feed-id]
+                               [:= :bundle-id bundle-id]]}))))
 
 (defn update-filtered-posts! [ds {:keys [filtered bundle-id post-id]}]
-  (if filtered
-    (hon/insert! ds {:tname :filtered-posts
-                     :data {:post-id post-id
-                            :bundle-id bundle-id}})
-    (hon/delete! ds {:tname :filtered-posts
-                     :where [:and
-                             [:= :post-id post-id]
-                             [:= :bundle-id bundle-id]]})))
+  (pg/with-transaction [ds ds]
+    (if filtered
+      (hon/insert! ds {:tname :filtered-posts
+                       :data {:post-id post-id
+                              :bundle-id bundle-id}})
+      (hon/delete! ds {:tname :filtered-posts
+                       :where [:and
+                               [:= :post-id post-id]
+                               [:= :bundle-id bundle-id]]}))))
diff --git a/src/source/workers/users.clj b/src/source/workers/users.clj
index 206774aa..9a86dea7 100644
--- a/src/source/workers/users.clj
+++ b/src/source/workers/users.clj
@@ -1,7 +1,8 @@
 (ns source.workers.users
   (:require [source.workers.feeds :as feeds]
             [source.workers.integrations :as integrations]
-            [source.db.honey :as hon]))
+            [source.db.honey :as hon]
+            [pg.core :as pg]))
 
 (defn hard-delete-creator! [ds js user-id email]
   (let [feed-ids (mapv :id (hon/find ds {:tname :feeds
@@ -18,20 +19,21 @@
                      :where [:= :distributor-id user-id]})))
 
 (defn hard-delete-user! [ds js user-type user-id]
-  (let [{:keys [email business-id]} (hon/find-one ds {:tname :users
-                                                      :where [:= :id user-id]})]
-    (cond
-      (= user-type :creator)
-      (hard-delete-creator! ds js user-id email)
-      (= user-type :distributor)
-      (hard-delete-distributor! ds js user-id))
+  (pg/with-transaction [ds ds]
+    (let [{:keys [email business-id]} (hon/find-one ds {:tname :users
+                                                        :where [:= :id user-id]})]
+      (cond
+        (= user-type :creator)
+        (hard-delete-creator! ds js user-id email)
+        (= user-type :distributor)
+        (hard-delete-distributor! ds js user-id))
 
-    (hon/delete! ds {:tname :user-sectors
-                     :where [:= :user-id user-id]})
-    (when (some? business-id) (hon/delete! ds {:tname :businesses
-                                               :where [:= :id business-id]}))
-    (hon/delete! ds {:tname :users
-                     :where [:= :id user-id]})))
+      (hon/delete! ds {:tname :user-sectors
+                       :where [:= :user-id user-id]})
+      (when (some? business-id) (hon/delete! ds {:tname :businesses
+                                                 :where [:= :id business-id]}))
+      (hon/delete! ds {:tname :users
+                       :where [:= :id user-id]}))))
 
 (defn soft-delete-user! [ds user-id]
   (hon/update! ds {:tname :users

From 75f3ee79f588b2e9f023631e62b90ff6e47bfb00 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 2 Mar 2026 15:11:49 +0200
Subject: [PATCH 331/391] added logger utility and replaced println calls with
 logger calls

---
 src/source/admins.clj                   |  5 +++--
 src/source/jobs/handlers.clj            | 17 +++++++++--------
 src/source/logger.clj                   | 12 ++++++++++++
 src/source/middleware/auth/core.clj     |  5 +++--
 src/source/middleware/auth/util.clj     |  6 +++---
 src/source/middleware/core.clj          |  7 ++++---
 src/source/routes/bundle_feed.clj       |  5 +++--
 src/source/routes/bundle_feed_post.clj  |  5 +++--
 src/source/routes/bundle_feed_posts.clj |  5 +++--
 src/source/routes/bundle_feeds.clj      |  5 +++--
 src/source/routes/bundle_post.clj       |  5 +++--
 src/source/routes/bundle_posts.clj      |  5 +++--
 src/source/server.clj                   | 11 ++++++-----
 src/source/util.clj                     |  1 -
 src/source/workers/feeds.clj            |  5 +++--
 15 files changed, 61 insertions(+), 38 deletions(-)
 create mode 100644 src/source/logger.clj

diff --git a/src/source/admins.clj b/src/source/admins.clj
index 9fa2b6f0..2ca19de7 100644
--- a/src/source/admins.clj
+++ b/src/source/admins.clj
@@ -1,7 +1,8 @@
 (ns source.admins
   (:require [source.crypt-fs :as crypt]
             [source.config :as conf]
-            [clojure.data.json :as json]))
+            [clojure.data.json :as json]
+            [source.logger :as logger]))
 
 (defn encrypt! []
   (crypt/write-file-crypt! (conf/read-value :admins-encrypted-path)
@@ -16,5 +17,5 @@
         (crypt/read-file-crypt (conf/read-value :supersecretkey))
         (json/read-json))
     (catch Exception e
-      (println (str "Couldn't read the admins file: " (.getMessage e)))
+      (logger/log-error (str "Couldn't read the admins file: " (.getMessage e)))
       [])))
diff --git a/src/source/jobs/handlers.clj b/src/source/jobs/handlers.clj
index 8ce3f298..9d44f986 100644
--- a/src/source/jobs/handlers.clj
+++ b/src/source/jobs/handlers.clj
@@ -7,7 +7,8 @@
             [source.db.util :as db.util]
             [clojure.set :as set]
             [clojure.string :as string]
-            [source.db.honey :as hon]))
+            [source.db.honey :as hon]
+            [source.logger :as logger]))
 
 (defmulti handler
   (fn [opts]
@@ -31,7 +32,7 @@
     (try
       (when (users/removed? ds (:creator-id args))
         (let [{:keys [feed-id creator-id content-type-id provider-id url]} args
-              _ (println "feed" feed-id "job started")
+              _ (logger/log (str "feed " feed-id " job started."))
               selection-schemas (->> [:= :provider-id provider-id]
                                      (assoc {} :where)
                                      (xml/selection-schemas ds))
@@ -84,9 +85,9 @@
                (hon/insert! ds {:tname :incoming-posts
                                 :data post})))
            extended-posts)
-          (println "feed" feed-id "job finished")))
+          (logger/log (str "feed " feed-id " job finished."))))
 
-      (catch Exception e (println "feed job failed: " e) :fail))))
+      (catch Exception e (logger/log-error (str "feed job failed: " e)) :fail))))
 
 (defn update-bundle-job-id
   "returns the job id of an update-bundle job with the given bundle id"
@@ -115,7 +116,7 @@
 (defmethod handler :update-bundle [_]
   (fn [{:keys [args ds]}]
     (let [{:keys [bundle-id categories]} args
-          _ (println "starting bundle" bundle-id "job")
+          _ (logger/log (str "starting bundle " bundle-id " job."))
           incoming-posts (services/incoming-posts-with-feeds ds {:where [:= :feeds.state "live"]})
           posts-categories (incoming-posts/categories-by-posts ds {:where [:= :state "live"]})
           heuristics (mapv
@@ -125,7 +126,7 @@
       (try
         (services/upsert-post-heuristics! ds {:bundle-id bundle-id
                                               :data heuristics})
-        (catch Exception e (println "bundle" bundle-id "upserting post heuristics failed: " (.getMessage e))))
+        (catch Exception e (logger/log-error (str "bundle " bundle-id " upserting post heuristics failed: " (.getMessage e)))))
 
       ; pull highest scored posts by long heuristics into outgoing posts
             ; top 1000 post-heuristics records ordered by long heuristic in descending order
@@ -171,7 +172,7 @@
                              :possible-cause "If no posts made it into the bundle, it's possible post heuristics failed or there's no incoming posts"
                              :next-steps "Check for errors thrown in this job, ensure all tables for this bundle exist"}))))
 
-        (println "bundle" bundle-id "job done")))))
+        (logger/log (str "bundle " bundle-id " job done."))))))
 
 (defn user-deletion-job-id
   "returns the job id of a user deletion job with the given user id"
@@ -183,4 +184,4 @@
     (try
       (let [{:keys [user-type user-id]} args]
         (users/hard-delete-user! ds (keyword user-type) user-id))
-      (catch Exception e (println "Failed to delete user-id" (:user-id args) ":" e) :fail))))
+      (catch Exception e (logger/log-error (str "Failed to delete user-id " (:user-id args) ":" e)) :fail))))
diff --git a/src/source/logger.clj b/src/source/logger.clj
new file mode 100644
index 00000000..5b67e82c
--- /dev/null
+++ b/src/source/logger.clj
@@ -0,0 +1,12 @@
+(ns source.logger
+  (:require
+   [source.util :as util]))
+
+(defn log [message]
+  (println (str "SOURCE [" (util/get-utc-timestamp-string) "] LOG: " message)))
+
+(defn log-warning [message]
+  (println (str "SOURCE [" (util/get-utc-timestamp-string) "] WARNING: " message)))
+
+(defn log-error [message]
+  (println (str "SOURCE [" (util/get-utc-timestamp-string) "] ERROR: " message)))
diff --git a/src/source/middleware/auth/core.clj b/src/source/middleware/auth/core.clj
index 189eeb11..f506db45 100644
--- a/src/source/middleware/auth/core.clj
+++ b/src/source/middleware/auth/core.clj
@@ -3,7 +3,8 @@
             [source.db.util :as db.util]
             [ring.util.response :as res]
             [source.services.bundles :as bundles]
-            [source.db.honey :as db]))
+            [source.db.honey :as db]
+            [source.logger :as logger]))
 
 (defn create-session [user]
   (let [payload {:id (:id user)
@@ -57,7 +58,7 @@
             (assoc :bundle-id id)
             (handler))
         (do
-          (println "Bundle authorization attempt failed with uuid:" bundle-uuid)
+          (logger/log-warning (str "Bundle authorization attempt failed with uuid: " bundle-uuid))
           (->
            (res/response {:message "The bundle you are looking for does not exist."})
            (res/status 404)))))))
diff --git a/src/source/middleware/auth/util.clj b/src/source/middleware/auth/util.clj
index 058a5f3c..cacc75c8 100644
--- a/src/source/middleware/auth/util.clj
+++ b/src/source/middleware/auth/util.clj
@@ -1,7 +1,8 @@
 (ns source.middleware.auth.util
   (:require [buddy.sign.jwt :as jwt]
             [source.config :as conf]
-            [clojure.string :as str]))
+            [clojure.string :as str]
+            [source.logger :as logger]))
 
 (defn auth-header [request]
   (or (get-in request [:headers "Authorization"])
@@ -25,6 +26,5 @@
   (try
     (jwt/decrypt token (conf/read-value :supersecretkey))
     (catch Exception e
-      (println (.getMessage e))
+      (logger/log-warning (str "JWT Verification failed: " (.getMessage e)))
       false)))
-
diff --git a/src/source/middleware/core.clj b/src/source/middleware/core.clj
index 1c25cc67..920d49c7 100644
--- a/src/source/middleware/core.clj
+++ b/src/source/middleware/core.clj
@@ -12,7 +12,8 @@
             [ring.middleware.cookies :as cookies]
             [clojure.walk :as walk]
             [source.util :as util]
-            [clojure.string :as string]))
+            [clojure.string :as string]
+            [source.logger :as logger]))
 
 (defn wrap-ds [handler ds]
   (fn [request]
@@ -43,7 +44,7 @@
     (try
       (handler req)
       (catch Exception e
-        (println "Unhandled Exception on endpoint URI " (:uri req) ": " e)
+        (logger/log-error (str "Unhandled Exception on endpoint URI " (:uri req) ": " e))
         (-> (res/response {:message "Internal Server Error"})
             (res/status 500))))))
 
@@ -75,7 +76,7 @@
                        (attach-validations request))]
       (if (seq errors)
         (do
-          (println "Schema validation failed on endpoint URI" (:uri request) ":" (string/join "\n" errors))
+          (logger/log-warning (str "Schema validation failed on endpoint URI " (:uri request) ": " (string/join "\n" errors)))
           (-> (res/response {:message (string/join "\n" errors)})
               (res/status 400)))
         (handler request)))))
diff --git a/src/source/routes/bundle_feed.clj b/src/source/routes/bundle_feed.clj
index 65b9a3e9..6f25e79b 100644
--- a/src/source/routes/bundle_feed.clj
+++ b/src/source/routes/bundle_feed.clj
@@ -3,7 +3,8 @@
             [source.db.honey :as hon]
             [source.services.analytics.interface :as analytics]
             [source.routes.openapi :as api]
-            [source.workers.schemas :as schemas]))
+            [source.workers.schemas :as schemas]
+            [source.logger :as logger]))
 
 (defn get
   {:summary "Get a single RSS feed by id from RSS feeds within the uuid-authorized bundle. 
@@ -19,5 +20,5 @@
                                :where [:= :id (:id path-params)]})]
     (try
       (analytics/insert-feed-click! ds feed bundle-id)
-      (catch Exception e (println (.getMessage e))))
+      (catch Exception e (logger/log-error (str "Failed to insert feed click for bundle feed: " (.getMessage e)))))
     (res/response feed)))
diff --git a/src/source/routes/bundle_feed_post.clj b/src/source/routes/bundle_feed_post.clj
index 0b6f3ed8..a630cc7a 100644
--- a/src/source/routes/bundle_feed_post.clj
+++ b/src/source/routes/bundle_feed_post.clj
@@ -3,7 +3,8 @@
             [source.db.honey :as hon]
             [source.services.analytics.interface :as analytics]
             [source.db.util :as db.util]
-            [honey.sql.helpers :as hsql]))
+            [honey.sql.helpers :as hsql]
+            [source.logger :as logger]))
 
 (defn get
   {:summary "Get a single post by post id belonging to an RSS feed in the associated uuid-authorized bundle. 
@@ -35,5 +36,5 @@
                                   (hsql/where [:= :id (:post-id path-params)])))]
     (try
       (analytics/insert-post-click! ds post bundle-id)
-      (catch Exception e (println (.getMessage e))))
+      (catch Exception e (logger/log-error (str "Failed to insert post click on bundle feed post: " (.getMessage e)))))
     (res/response post)))
diff --git a/src/source/routes/bundle_feed_posts.clj b/src/source/routes/bundle_feed_posts.clj
index eac89c13..b1a744ee 100644
--- a/src/source/routes/bundle_feed_posts.clj
+++ b/src/source/routes/bundle_feed_posts.clj
@@ -1,7 +1,8 @@
 (ns source.routes.bundle-feed-posts
   (:require [ring.util.response :as res]
             [source.db.honey :as hon]
-            [source.services.analytics.interface :as analytics]))
+            [source.services.analytics.interface :as analytics]
+            [source.logger :as logger]))
 
 (defn get
   {:summary "Get all posts present within a given RSS feed by feed id, within the uuid-authorized bundle.
@@ -35,5 +36,5 @@
                             :ret :*})]
     (try
       (analytics/insert-post-impressions! ds posts bundle-id)
-      (catch Exception e (println (.getMessage e))))
+      (catch Exception e (logger/log-error (str "Failed to insert post impressions for bundle feed posts: " (.getMessage e)))))
     (res/response posts)))
diff --git a/src/source/routes/bundle_feeds.clj b/src/source/routes/bundle_feeds.clj
index 983e8cab..a93ec76a 100644
--- a/src/source/routes/bundle_feeds.clj
+++ b/src/source/routes/bundle_feeds.clj
@@ -4,7 +4,8 @@
             [source.workers.bundles :as bundles]
             [source.services.analytics.interface :as analytics]
             [source.routes.openapi :as api]
-            [source.workers.schemas :as schemas]))
+            [source.workers.schemas :as schemas]
+            [source.logger :as logger]))
 
 (defn post
   {:summary "Get all RSS feeds present in the bundle authorised by uuid.
@@ -32,7 +33,7 @@
                    (bundles/get-outgoing-feeds ds))]
     (try
       (analytics/insert-feed-impressions! ds feeds bundle-id)
-      (catch Exception e (println (.getMessage e))))
+      (catch Exception e (logger/log-error (str "Failed to insert feed impressions for bundle feeds: " (.getMessage e)))))
     (res/response feeds)))
 
 (comment
diff --git a/src/source/routes/bundle_post.clj b/src/source/routes/bundle_post.clj
index fbdbffb2..cf61c36f 100644
--- a/src/source/routes/bundle_post.clj
+++ b/src/source/routes/bundle_post.clj
@@ -6,7 +6,8 @@
             [honey.sql.helpers :as hsql]
             [source.routes.openapi :as api]
             [source.workers.schemas :as schemas]
-            [malli.util :as mu]))
+            [malli.util :as mu]
+            [source.logger :as logger]))
 
 (defn get
   {:summary "Get a single post by post id in the uuid-authorized bundle.
@@ -28,5 +29,5 @@
                            {:ret :1})]
     (try
       (analytics/insert-post-click! ds post bundle-id)
-      (catch Exception e (println (.getMessage e))))
+      (catch Exception e (logger/log-error (str "Failed to insert post click on bundle post: " (.getMessage e)))))
     (res/response post)))
diff --git a/src/source/routes/bundle_posts.clj b/src/source/routes/bundle_posts.clj
index f4b3d7a9..e099c5bc 100644
--- a/src/source/routes/bundle_posts.clj
+++ b/src/source/routes/bundle_posts.clj
@@ -5,7 +5,8 @@
             [source.workers.bundles :as bundles]
             [source.routes.openapi :as api]
             [source.workers.schemas :as schemas]
-            [malli.util :as mu]))
+            [malli.util :as mu]
+            [source.logger :as logger]))
 
 (def QueryTruncatePosts
   [:truncate
@@ -49,5 +50,5 @@
                                    :category-ids (:category-ids body)})]
     (try
       (analytics/insert-post-impressions! ds data bundle-id)
-      (catch Exception e (println (.getMessage e))))
+      (catch Exception e (logger/log-error (str "Failed to insert post impressions on bundle posts: " (.getMessage e)))))
     (res/response posts)))
diff --git a/src/source/server.clj b/src/source/server.clj
index 49184673..1056334b 100644
--- a/src/source/server.clj
+++ b/src/source/server.clj
@@ -4,7 +4,8 @@
             [congest.jobs :as congest]
             [source.jobs.core :as jobs]
             [source.routes.interface :as routes]
-            [source.util :as util]))
+            [source.util :as util]
+            [source.logger :as logger]))
 
 (defonce ^:private *components (atom nil))
 
@@ -32,7 +33,7 @@
   (try
     (when (deps-on? deps)
       (swap! *components assoc name (init-fn @*components)))
-    (catch Exception e (println (str "Failed to initialise " name ":") e))))
+    (catch Exception e (logger/log (str "Failed to initialise " name ": " e)))))
 
 (defn initialise-components! [components]
   (run! initialise! components))
@@ -43,7 +44,7 @@
 (defn start-server []
   (cond (not (some? (:server @*components)))
         (do
-          (println "Starting server on port 3000...")
+          (logger/log "Starting server on port 3000...")
           (initialise-components! [{:name :ds
                                     :init-fn (fn [_deps] (db/ds :master))
                                     :deps []}
@@ -54,10 +55,10 @@
                                     :deps [:ds :js]
                                     :init-fn initialise-server!}]))
         :else
-        (println "Server already running!")))
+        (logger/log "Server already running!")))
 
 (defn stop-server []
-  (println "Stopping server...")
+  (logger/log "Stopping server...")
   (when (some? (:js @*components))
     (congest/kill! (:js @*components)))
   (when (some? (:server @*components))
diff --git a/src/source/util.clj b/src/source/util.clj
index 68692274..73871d82 100644
--- a/src/source/util.clj
+++ b/src/source/util.clj
@@ -3,7 +3,6 @@
             [buddy.core.nonce :as nonce]
             [clojure.main :refer [demunge]]
             [malli.core :as m]
-            [malli.error :as me]
             [malli.transform :as mt]
             [clojure.string :as string])
   (:import (java.math BigInteger)
diff --git a/src/source/workers/feeds.clj b/src/source/workers/feeds.clj
index c5b9e10e..1cfc914b 100644
--- a/src/source/workers/feeds.clj
+++ b/src/source/workers/feeds.clj
@@ -4,7 +4,8 @@
             [congest.jobs :as congest]
             [source.db.honey :as hon]
             [source.rss.youtube :as yt]
-            [clojure.string :as string]))
+            [clojure.string :as string]
+            [source.logger :as logger]))
 
 (defn create-feed!
   "Creates feed with incoming posts pulled from RSS feed and starts associated job"
@@ -53,7 +54,7 @@
                    :ret :1})
 
         _ (when (or (nil? display-picture) (= display-picture ""))
-            (println "Failed to pull display picture for feed [" (:id new-feed) (:title new-feed) "] provider-id" provider-id))
+            (logger/log-error (str "Failed to pull display picture for feed [ " (:id new-feed) " " (:title new-feed) "] provider-id " provider-id)))
 
         extended-posts (mapv (fn [post]
                                (merge post

From db0c7764aef70eb3c870cf68058f4c9070d06f3b Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Tue, 3 Mar 2026 10:19:58 +0200
Subject: [PATCH 332/391] added schemas to job endpoints

---
 src/source/routes/job.clj            | 11 +++++++++--
 src/source/routes/job_deregister.clj |  8 ++++++--
 src/source/routes/job_start.clj      |  8 ++++++--
 src/source/routes/job_stop.clj       |  8 ++++++--
 src/source/routes/jobs.clj           | 29 ++++++++++++++++++++++++----
 src/source/routes/jobs_view.clj      | 22 ++++++++++++++++++---
 src/source/workers/schemas.clj       |  5 ++++-
 7 files changed, 75 insertions(+), 16 deletions(-)

diff --git a/src/source/routes/job.clj b/src/source/routes/job.clj
index ae8bec01..1255b73d 100644
--- a/src/source/routes/job.clj
+++ b/src/source/routes/job.clj
@@ -1,8 +1,15 @@
 (ns source.routes.job
   (:require [source.services.interface :as services]
-            [ring.util.response :as res]))
+            [ring.util.response :as res]
+            [source.routes.openapi :as api]
+            [source.workers.schemas :as schemas]))
 
-(defn get [{:keys [ds path-params] :as _req}]
+(defn get
+  {:summary "View a single job's raw metadata by id"
+   :parameters (api/params :path [:map [:id :int]])
+   :responses (api/success schemas/JobWithMetadata)}
+
+  [{:keys [ds path-params] :as _req}]
   (let [job (services/job ds path-params)
         metadata (services/job-metadata ds {:id (:job-metadata-id job)})]
     (if (some? job)
diff --git a/src/source/routes/job_deregister.clj b/src/source/routes/job_deregister.clj
index 8d39ce28..39fcd3b2 100644
--- a/src/source/routes/job_deregister.clj
+++ b/src/source/routes/job_deregister.clj
@@ -1,9 +1,13 @@
 (ns source.routes.job-deregister
   (:require [congest.jobs :as jobs]
             [ring.util.response :as res]
-            [source.services.interface :as services]))
+            [source.services.interface :as services]
+            [source.routes.openapi :as api]))
 
-(defn get [{:keys [js ds path-params] :as _req}]
+(defn get
+  {:summary "Deregister a job by id"
+   :params (api/params :path [:map [:id :int]])}
+  [{:keys [js ds path-params] :as _req}]
   (let [job (services/job ds path-params)]
     (jobs/deregister! js (:job-id job))
     (res/response {:message "successfully deregistered job"})))
diff --git a/src/source/routes/job_start.clj b/src/source/routes/job_start.clj
index 6a6fe08e..6a2144d1 100644
--- a/src/source/routes/job_start.clj
+++ b/src/source/routes/job_start.clj
@@ -1,9 +1,13 @@
 (ns source.routes.job-start
   (:require [source.jobs.core :as jobs]
             [ring.util.response :as res]
-            [source.services.interface :as services]))
+            [source.services.interface :as services]
+            [source.routes.openapi :as api]))
 
-(defn get [{:keys [js ds path-params]}]
+(defn get
+  {:summary "start a job by id"
+   :params (api/params :path [:map [:id :int]])}
+  [{:keys [js ds path-params]}]
   (let [job (services/job ds path-params)]
     (jobs/start! js ds (:job-id job))
     (res/response {:message "successfully started job"})))
diff --git a/src/source/routes/job_stop.clj b/src/source/routes/job_stop.clj
index 2a3624b1..f60c1671 100644
--- a/src/source/routes/job_stop.clj
+++ b/src/source/routes/job_stop.clj
@@ -1,9 +1,13 @@
 (ns source.routes.job-stop
   (:require [congest.jobs :as jobs]
             [ring.util.response :as res]
-            [source.services.interface :as services]))
+            [source.services.interface :as services]
+            [source.routes.openapi :as api]))
 
-(defn get [{:keys [js ds path-params]}]
+(defn get
+  {:summary "stop a job by id"
+   :params (api/params :path [:map [:id :int]])}
+  [{:keys [js ds path-params]}]
   (let [job (services/job ds path-params)]
     (jobs/stop! js (:job-id job) false)
     (res/response {:message "successfully stopped job"})))
diff --git a/src/source/routes/jobs.clj b/src/source/routes/jobs.clj
index 7aa0a9fd..72dc1e01 100644
--- a/src/source/routes/jobs.clj
+++ b/src/source/routes/jobs.clj
@@ -2,9 +2,15 @@
   (:require [source.services.interface :as services]
             [congest.jobs :as congest]
             [source.jobs.core :as jobs]
-            [ring.util.response :as res]))
+            [ring.util.response :as res]
+            [source.routes.openapi :as api]
+            [source.workers.schemas :as schemas]
+            [source.util :as util]))
 
-(defn get [{:keys [ds] :as _req}]
+(defn get
+  {:summary "get list of raw job metadata"
+   :responses (api/success schemas/JobsWithMetadata)}
+  [{:keys [ds] :as _req}]
   (->>
    (services/jobs ds)
    (mapv (fn [job]
@@ -13,7 +19,22 @@
                     {:metadata metadata}))))
    (res/response)))
 
-(defn post [{:keys [js ds body] :as _req}]
-  (let [{:keys [metadata]} body]
+(defn post
+  {:summary "Register a new job with metadata"
+   :params (api/params :body [:map
+                              [:metadata [:map
+                                          [:id :string]
+                                          [:initial-delay :int]
+                                          [:auto-start :boolean]
+                                          [:stop-after-fail :boolean]
+                                          [:interval :int]
+                                          [:recurring? :boolean]
+                                          [:args [:map-of :keyword :any]]
+                                          [:handler :keyword]
+                                          [:sleep :boolean]]]])
+   :responses (api/success (api/response-schema))}
+  [{:keys [js ds body] :as _req}]
+  (let [{:keys [metadata]} body
+        metadata (assoc metadata :created-at (util/get-utc-timestamp-string))]
     (congest/register! js (jobs/prepare-congest-metadata ds metadata))
     (res/response {:message "successfully registered job"})))
diff --git a/src/source/routes/jobs_view.clj b/src/source/routes/jobs_view.clj
index 30ee5d1e..5436f72a 100644
--- a/src/source/routes/jobs_view.clj
+++ b/src/source/routes/jobs_view.clj
@@ -1,7 +1,8 @@
 (ns source.routes.jobs-view
   (:require [ring.util.response :as res]
             [congest.jobs :as congest]
-            [clojure.walk :as walk]))
+            [clojure.walk :as walk]
+            [source.routes.openapi :as api]))
 
 (defn stringify-unknowns [x]
   (walk/postwalk
@@ -16,7 +17,22 @@
        v
        (str v))) x))
 
-(defn get [{:keys [js]}]
+(defn get
+  {:summary "gets a list of all jobs"
+   :responses (api/success [:vector [:map
+                                     [:args [:map-of :string :any]]
+                                     [:initial-delay :int]
+                                     (api/sometimes :kill-after :int)
+                                     (api/sometimes :auto-start :boolean)
+                                     [:created-at :int]
+                                     [:handler-name :string]
+                                     [:num-calls :int]
+                                     [:id :string]
+                                     [:stop-after-fail :boolean]
+                                     [:interval :int]
+                                     [:recurring? :boolean]
+                                     [:sleep :boolean]]])}
+  [{:keys [js]}]
   (let [raw-jobs (congest/view js)
-        formatted (stringify-unknowns raw-jobs)]
+        formatted (mapv (fn [[_ v]] v) (stringify-unknowns raw-jobs))]
     (res/response formatted)))
diff --git a/src/source/workers/schemas.clj b/src/source/workers/schemas.clj
index e100d42b..a89b8b0a 100644
--- a/src/source/workers/schemas.clj
+++ b/src/source/workers/schemas.clj
@@ -217,7 +217,7 @@
    [:initial-delay :int]
    [:auto-start :int]
    [:stop-after-fail :int]
-   [:kill-after :int]
+   (api/sometimes :kill-after :int)
    [:num-calls :int]
    [:interval :int]
    [:recurring :int]
@@ -228,6 +228,9 @@
   (-> Job
       (mu/assoc :job-metadata JobMetadata)))
 
+(def JobsWithMetadata
+  [:vector JobWithMetadata])
+
 (def Bundle
   [:map
    [:id :int]

From b1519f466d081c5fc00824b4be9d487078a0f290 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Tue, 3 Mar 2026 10:22:06 +0200
Subject: [PATCH 333/391] added missing response schemas

---
 src/source/routes/job_deregister.clj | 3 ++-
 src/source/routes/job_start.clj      | 3 ++-
 src/source/routes/job_stop.clj       | 3 ++-
 3 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/src/source/routes/job_deregister.clj b/src/source/routes/job_deregister.clj
index 39fcd3b2..7a5e1552 100644
--- a/src/source/routes/job_deregister.clj
+++ b/src/source/routes/job_deregister.clj
@@ -6,7 +6,8 @@
 
 (defn get
   {:summary "Deregister a job by id"
-   :params (api/params :path [:map [:id :int]])}
+   :params (api/params :path [:map [:id :int]])
+   :responses (api/success (api/response-schema))}
   [{:keys [js ds path-params] :as _req}]
   (let [job (services/job ds path-params)]
     (jobs/deregister! js (:job-id job))
diff --git a/src/source/routes/job_start.clj b/src/source/routes/job_start.clj
index 6a2144d1..b00f3120 100644
--- a/src/source/routes/job_start.clj
+++ b/src/source/routes/job_start.clj
@@ -6,7 +6,8 @@
 
 (defn get
   {:summary "start a job by id"
-   :params (api/params :path [:map [:id :int]])}
+   :params (api/params :path [:map [:id :int]])
+   :responses (api/success (api/response-schema))}
   [{:keys [js ds path-params]}]
   (let [job (services/job ds path-params)]
     (jobs/start! js ds (:job-id job))
diff --git a/src/source/routes/job_stop.clj b/src/source/routes/job_stop.clj
index f60c1671..5b01c050 100644
--- a/src/source/routes/job_stop.clj
+++ b/src/source/routes/job_stop.clj
@@ -6,7 +6,8 @@
 
 (defn get
   {:summary "stop a job by id"
-   :params (api/params :path [:map [:id :int]])}
+   :params (api/params :path [:map [:id :int]])
+   :responses (api/success (api/response-schema))}
   [{:keys [js ds path-params]}]
   (let [job (services/job ds path-params)]
     (jobs/stop! js (:job-id job) false)

From ef018aa2346ae08abc1d4ad7d68b55bbcf9770db Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Tue, 3 Mar 2026 13:18:08 +0200
Subject: [PATCH 334/391] use modified top level schemas instead of rewriting

---
 src/source/routes/jobs.clj      | 21 ++++++++++-----------
 src/source/routes/jobs_view.clj | 25 +++++++++++--------------
 src/source/workers/schemas.clj  |  5 +++--
 3 files changed, 24 insertions(+), 27 deletions(-)

diff --git a/src/source/routes/jobs.clj b/src/source/routes/jobs.clj
index 72dc1e01..fbaadd9f 100644
--- a/src/source/routes/jobs.clj
+++ b/src/source/routes/jobs.clj
@@ -5,7 +5,8 @@
             [ring.util.response :as res]
             [source.routes.openapi :as api]
             [source.workers.schemas :as schemas]
-            [source.util :as util]))
+            [source.util :as util]
+            [malli.util :as mu]))
 
 (defn get
   {:summary "get list of raw job metadata"
@@ -22,16 +23,14 @@
 (defn post
   {:summary "Register a new job with metadata"
    :params (api/params :body [:map
-                              [:metadata [:map
-                                          [:id :string]
-                                          [:initial-delay :int]
-                                          [:auto-start :boolean]
-                                          [:stop-after-fail :boolean]
-                                          [:interval :int]
-                                          [:recurring? :boolean]
-                                          [:args [:map-of :keyword :any]]
-                                          [:handler :keyword]
-                                          [:sleep :boolean]]]])
+                              [:metadata
+                               (-> schemas/JobMetadata
+                                   (api/missoc :recurring :num-calls)
+                                   (mu/assoc :auto-start :boolean)
+                                   (mu/assoc :stop-after-fail :boolean)
+                                   (mu/assoc :recurring? :boolean)
+                                   (mu/assoc :sleep :boolean)
+                                   (mu/assoc :handler :string))]])
    :responses (api/success (api/response-schema))}
   [{:keys [js ds body] :as _req}]
   (let [{:keys [metadata]} body
diff --git a/src/source/routes/jobs_view.clj b/src/source/routes/jobs_view.clj
index 5436f72a..a576dde6 100644
--- a/src/source/routes/jobs_view.clj
+++ b/src/source/routes/jobs_view.clj
@@ -2,7 +2,9 @@
   (:require [ring.util.response :as res]
             [congest.jobs :as congest]
             [clojure.walk :as walk]
-            [source.routes.openapi :as api]))
+            [source.routes.openapi :as api]
+            [source.workers.schemas :as schemas]
+            [malli.util :as mu]))
 
 (defn stringify-unknowns [x]
   (walk/postwalk
@@ -19,19 +21,14 @@
 
 (defn get
   {:summary "gets a list of all jobs"
-   :responses (api/success [:vector [:map
-                                     [:args [:map-of :string :any]]
-                                     [:initial-delay :int]
-                                     (api/sometimes :kill-after :int)
-                                     (api/sometimes :auto-start :boolean)
-                                     [:created-at :int]
-                                     [:handler-name :string]
-                                     [:num-calls :int]
-                                     [:id :string]
-                                     [:stop-after-fail :boolean]
-                                     [:interval :int]
-                                     [:recurring? :boolean]
-                                     [:sleep :boolean]]])}
+   :responses (api/success [:vector
+                            (-> schemas/JobMetadata
+                                (api/missoc :created-at :recurring)
+                                (mu/assoc :args [:map-of :string :any])
+                                (mu/assoc :auto-start :boolean)
+                                (mu/assoc :handler-name :string)
+                                (mu/assoc :recurring? :boolean)
+                                (mu/assoc :sleep :boolean))])}
   [{:keys [js]}]
   (let [raw-jobs (congest/view js)
         formatted (mapv (fn [[_ v]] v) (stringify-unknowns raw-jobs))]
diff --git a/src/source/workers/schemas.clj b/src/source/workers/schemas.clj
index a89b8b0a..1365e36b 100644
--- a/src/source/workers/schemas.clj
+++ b/src/source/workers/schemas.clj
@@ -214,14 +214,15 @@
 
 (def JobMetadata
   [:map
+   [:id :string]
    [:initial-delay :int]
-   [:auto-start :int]
    [:stop-after-fail :int]
+   (api/sometimes :auto-start :int)
    (api/sometimes :kill-after :int)
    [:num-calls :int]
    [:interval :int]
    [:recurring :int]
-   [:created-at :string]
+   (api/sometimes :created-at :string)
    [:sleep :int]])
 
 (def JobWithMetadata

From 9e9fc6f9501f0ddb71df93a44c505ee10f955f34 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Tue, 3 Mar 2026 14:51:02 +0200
Subject: [PATCH 335/391] added admin endpoints to manage categories

---
 src/source/routes/categories.clj  | 36 ++++++++++++++++++++++++++++++-
 src/source/routes/reitit.clj      |  3 +++
 src/source/workers/categories.clj | 19 +++++++++++++---
 3 files changed, 54 insertions(+), 4 deletions(-)

diff --git a/src/source/routes/categories.clj b/src/source/routes/categories.clj
index 2f83e7a4..c5060924 100644
--- a/src/source/routes/categories.clj
+++ b/src/source/routes/categories.clj
@@ -1,7 +1,10 @@
 (ns source.routes.categories
   (:require [ring.util.response :as res]
             [source.db.honey :as hon]
-            [source.workers.categories :as categories]))
+            [source.workers.categories :as categories]
+            [source.routes.openapi :as api]
+            [source.workers.schemas :as schemas]
+            [malli.util :as mu]))
 
 (defn get
   {:summary "get all categories"
@@ -23,3 +26,34 @@
                             [:display-picture {:optional true} [:maybe :string]]]]}}}
   [{:keys [ds] :as _request}]
   (res/response (categories/used-categories ds)))
+
+(defn add-category
+  {:summary "add a new category to the system"
+   :parameters (api/params :body (-> schemas/Category
+                                     (mu/dissoc :id)))
+   :responses (api/success (api/response-schema))}
+  [{:keys [ds body]}]
+  (hon/insert! ds {:tname :categories
+                   :data body})
+  (res/response {:message "successfully added new category"}))
+
+(defn update-category
+  {:summary "add a new category to the system"
+   :parameters (api/params
+                :path [:map [:id :int]]
+                :body (-> schemas/Category
+                          (mu/dissoc :id)))
+   :responses (api/success (api/response-schema))}
+  [{:keys [ds body path-params]}]
+  (hon/update! ds {:tname :categories
+                   :where [:= :id (:id path-params)]
+                   :data body})
+  (res/response {:message "successfully updated category"}))
+
+(defn delete-category
+  {:summary "delete a category from the system"
+   :parameters (api/params :path [:map [:id :int]])
+   :responses (api/success (api/response-schema))}
+  [{:keys [ds path-params]}]
+  (categories/delete-category! ds (:id path-params))
+  (res/response {:message "successfully deleted category"}))
diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index 1836bb30..91bb5a50 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -232,6 +232,9 @@
                 :swagger {:security [{"auth" []}]}
                 :openapi {:security [{:bearerAuth []}]}}
 
+      ["/categories" (post categories/add-category)]
+      ["/categories/:id" (-> (post categories/update-category)
+                             (delete categories/delete-category))]
       ["/business/types" (-> (post business-types/post)
                              (patch business-types/patch)
                              (delete business-types/delete))]
diff --git a/src/source/workers/categories.clj b/src/source/workers/categories.clj
index 0fb886f3..a032ad4b 100644
--- a/src/source/workers/categories.clj
+++ b/src/source/workers/categories.clj
@@ -1,5 +1,8 @@
 (ns source.workers.categories
-  (:require [source.db.honey :as hon]))
+  (:require [source.db.honey :as hon]
+            [pg.core :as pg]
+            [source.db.util :as db.util]
+            [honey.sql.helpers :as hsql]))
 
 (defn used-categories
   "returns all categories for which feeds exist."
@@ -11,8 +14,18 @@
                            [:in :id category-ids]
                            [:= :id -1])})))
 
-(comment
-  (require '[source.db.util :as db.util])
+(defn delete-category! [ds category-id]
+  (pg/with-transaction [ds ds]
+    (let [bundle-ids (mapv :id (hon/find ds {:tname :bundles}))]
+      (hon/delete! ds {:tname :feed-categories
+                       :where [:= :category-id category-id]})
+      (hon/delete! ds {:tname :event-categories
+                       :where [:= :category-id category-id]})
+      (run!
+       #(hon/delete! ds (-> (db.util/tname :bundle-categories %)
+                            (hsql/where [:= :category-id category-id])))
+       bundle-ids))))
 
+(comment
   (used-categories (db.util/conn))
   ())

From d275146524c9f210a63edc99eae535adb3b0a6d7 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Tue, 3 Mar 2026 15:19:54 +0200
Subject: [PATCH 336/391] actually delete the category itself

---
 src/source/workers/categories.clj | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/source/workers/categories.clj b/src/source/workers/categories.clj
index a032ad4b..82c787ee 100644
--- a/src/source/workers/categories.clj
+++ b/src/source/workers/categories.clj
@@ -24,7 +24,9 @@
       (run!
        #(hon/delete! ds (-> (db.util/tname :bundle-categories %)
                             (hsql/where [:= :category-id category-id])))
-       bundle-ids))))
+       bundle-ids)
+      (hon/delete! ds {:tname :categories
+                       :where [:= :id category-id]}))))
 
 (comment
   (used-categories (db.util/conn))

From e2565f729a40ecc3b66790b30494c19a47c40a64 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Tue, 3 Mar 2026 16:07:55 +0200
Subject: [PATCH 337/391] implemented email verification endpoints

---
 fly.dev.toml                             |   1 +
 resources/config.edn                     |   1 +
 src/source/config.clj                    |   1 +
 src/source/email/templates.clj           | 103 +++++++++++++++--------
 src/source/migrations/013_email_hash.clj |  18 ++++
 src/source/routes/me.clj                 |  15 +++-
 src/source/routes/register.clj           |   1 -
 src/source/routes/reitit.clj             |   4 +
 src/source/routes/user.clj               |  26 +++++-
 src/source/routes/users.clj              |  12 ++-
 src/source/services/auth.clj             |  11 ++-
 11 files changed, 150 insertions(+), 43 deletions(-)
 create mode 100644 src/source/migrations/013_email_hash.clj

diff --git a/fly.dev.toml b/fly.dev.toml
index 93667d56..87ffeb89 100644
--- a/fly.dev.toml
+++ b/fly.dev.toml
@@ -10,6 +10,7 @@ primary_region = 'fra'
 
 [env]
   CORS_ORIGIN = 'https://source-staging.fly.dev'
+  BASE_URL = 'https://source-be-staging.fly.dev'
   DATABASE_DIR = '/data'
   EMAIL_USERNAME = 'merveillevaneck@gmail.com'
   ENV = 'prod'
diff --git a/resources/config.edn b/resources/config.edn
index 0a354152..049697a2 100644
--- a/resources/config.edn
+++ b/resources/config.edn
@@ -2,6 +2,7 @@
  :admins-path "resources/admins.json"
  :admins-encrypted-path "resources/admins_encrypted.json"
  :cors-origin #or [#env CORS_ORIGIN "http://localhost:8080"]
+ :base-url #or [#env BASE_URL "https://localhost:3000"]
  :env #or [#env ENV "dev"]
  :database {:url #or [#env DATABASE_URL "postgres://localhost"]
             :type "postgresql"}
diff --git a/src/source/config.clj b/src/source/config.clj
index 45b6d597..2557bd93 100644
--- a/src/source/config.clj
+++ b/src/source/config.clj
@@ -22,6 +22,7 @@
    [:admins-path {:optional true} :string]
    [:admins-encrypted-path :string]
    [:cors-origin :string]
+   [:base-url :string]
    [:env :string]
    [:database [:map
                [:url :string]
diff --git a/src/source/email/templates.clj b/src/source/email/templates.clj
index 8806d079..ab184d25 100644
--- a/src/source/email/templates.clj
+++ b/src/source/email/templates.clj
@@ -68,6 +68,39 @@
           "The Source Team"]]
         (footer)]]]]]))
 
+(defn email-verification
+  "Returns the completed HTML for a feed rejection email"
+  [{:keys [email-hash]}]
+  (h/html5
+   {:lang "en"}
+   (head-metadata)
+   [:body {:style "font-family: 'Switzer', sans-serif"}
+    [:table {:width "100%" :border "0" :cellspacing "0" :cellpadding "0"}
+     [:tr
+      [:td {:align "center" :style "padding: 20px;"}
+       [:table {:class "content"
+                :width "600"
+                :border "0"
+                :cellspacing "0"
+                :cellpadding "0"
+                :style "border-collapse: collapse; border: 1px solid #cccccc;"}
+        (header)
+        [:tr
+         [:td {:class "body"
+               :style "padding: 40px; text-align: left; font-size: 16px; line-height: 1.6;"}
+          "Welcome to Source!"
+          [:br] [:br]
+          "Thanks for signing up. Please click the button below to verify your email."]]
+        (button {:text "Verify"
+                 :redirect (str (conf/read-value :base-url) "/email/verify/" email-hash)})
+        [:tr
+         [:td {:class "body"
+               :style "padding: 40px; text-align: left; font-size: 16px; line-height: 1.6;"}
+          "Regards,"
+          [:br]
+          "The Source Team"]]
+        (footer)]]]]]))
+
 (defn feed-approval
   "Returns the completed HTML for a feed approval email"
   [{:keys [creator-name feed-title feed-id]}]
@@ -110,38 +143,38 @@
                             (str (subs message 0 15) "...")
                             message)]
     (h/html5
-          {:lang "en"}
-          (head-metadata)
-          [:body {:style "font-family: 'Switzer', sans-serif"}
-           [:table {:width "100%" :border "0" :cellspacing "0" :cellpadding "0"}
-            [:tr
-             [:td {:align "center" :style "padding: 20px;"}
-              [:table {:class "content"
-                       :width "600"
-                       :border "0"
-                       :cellspacing "0"
-                       :cellpadding "0"
-                       :style "border-collapse: collapse; border: 1px solid #cccccc;"}
-               (header)
-               [:tr
-                [:td {:class "body"
-                      :style "padding: 40px; text-align: left; font-size: 16px; line-height: 1.6;"}
-                 "A user has reported a problem:"
-                 [:br] [:br]
-                 message
-                 [:br] [:br]
-                 (str "User ID: " user-id)
-                 [:br]
-                 (str "User email address: " user-email)
-                 [:br]
-                 (str "User type: " user-type)
-                 [:br]
-                 [:br]
-                 "Click on the link below to respond"]]
-               (button {:text "Respond"
-                        :redirect (str "mailto:" user-email "?subject=Source Team Re:" shortened-message)})
-               [:tr
-                [:td {:class "body"
-                      :style "padding: 40px; text-align: left; font-size: 11px; line-height: 1.6;"}
-                 "This is an automated message. Please do not reply directly to this email."]]
-               (footer)]]]]])))
+     {:lang "en"}
+     (head-metadata)
+     [:body {:style "font-family: 'Switzer', sans-serif"}
+      [:table {:width "100%" :border "0" :cellspacing "0" :cellpadding "0"}
+       [:tr
+        [:td {:align "center" :style "padding: 20px;"}
+         [:table {:class "content"
+                  :width "600"
+                  :border "0"
+                  :cellspacing "0"
+                  :cellpadding "0"
+                  :style "border-collapse: collapse; border: 1px solid #cccccc;"}
+          (header)
+          [:tr
+           [:td {:class "body"
+                 :style "padding: 40px; text-align: left; font-size: 16px; line-height: 1.6;"}
+            "A user has reported a problem:"
+            [:br] [:br]
+            message
+            [:br] [:br]
+            (str "User ID: " user-id)
+            [:br]
+            (str "User email address: " user-email)
+            [:br]
+            (str "User type: " user-type)
+            [:br]
+            [:br]
+            "Click on the link below to respond"]]
+          (button {:text "Respond"
+                   :redirect (str "mailto:" user-email "?subject=Source Team Re:" shortened-message)})
+          [:tr
+           [:td {:class "body"
+                 :style "padding: 40px; text-align: left; font-size: 11px; line-height: 1.6;"}
+            "This is an automated message. Please do not reply directly to this email."]]
+          (footer)]]]]])))
diff --git a/src/source/migrations/013_email_hash.clj b/src/source/migrations/013_email_hash.clj
new file mode 100644
index 00000000..e8730c86
--- /dev/null
+++ b/src/source/migrations/013_email_hash.clj
@@ -0,0 +1,18 @@
+(ns source.migrations.013-email-hash
+  (:require [source.db.master]
+            [source.db.honey :as hon]
+            [honey.sql.helpers :as hsql]))
+
+(defn run-up! [context]
+  (let [ds-master (:db-master context)]
+    (hon/execute!
+     ds-master
+     (-> (hsql/alter-table :users)
+         (hsql/add-column :email-hash :text)))))
+
+(defn run-down! [context]
+  (let [ds-master (:db-master context)]
+    (hon/execute!
+     ds-master
+     (-> (hsql/alter-table :users)
+         (hsql/drop-column :email-hash)))))
diff --git a/src/source/routes/me.clj b/src/source/routes/me.clj
index 55fa80c3..cb8a72b5 100644
--- a/src/source/routes/me.clj
+++ b/src/source/routes/me.clj
@@ -5,7 +5,9 @@
             [source.jobs.core :as jobs]
             [source.jobs.handlers :as handlers]
             [congest.jobs :as congest]
-            [source.workers.users :as users]))
+            [source.workers.users :as users]
+            [source.email.gmail :as gmail]
+            [source.email.templates :as templates]))
 
 (defn get
   {:summary "get logged in user by access token"
@@ -60,7 +62,6 @@
     ; TODO: service needed
     (->> (jobs/prepare-congest-metadata
           ds
-          js
           {:id job-id
            :initial-delay (* 1000 60 60 24 30)
            :auto-start true
@@ -86,3 +87,13 @@
     (users/cancel-soft-user-deletion! ds id)
     (congest/deregister! js job-id)
     (res/response {:message "successfully cancelled user deletion"})))
+
+(defn resend-email
+  {:summary "Resend verification email"}
+  [{:keys [ds user]}]
+  (let [{:keys [email email-hash]} (hon/find-one ds {:tname :users
+                                                     :where [:= :id (:id user)]})]
+    (gmail/send-email {:to email
+                       :subject "Source - Verify your email"
+                       :body (templates/email-verification email-hash)
+                       :type :text/html})))
diff --git a/src/source/routes/register.clj b/src/source/routes/register.clj
index f616b0f9..a6d8d382 100644
--- a/src/source/routes/register.clj
+++ b/src/source/routes/register.clj
@@ -1,7 +1,6 @@
 (ns source.routes.register
   (:require [source.services.interface :as services]
             [ring.util.response :as res]
-            [source.util :as util]
             [source.db.honey :as hon]))
 
 (defn post
diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index 1836bb30..00400012 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -91,6 +91,8 @@
       ["/:id" (-> (get user/get)
                   (patch user/patch))]]
 
+     ["/email/verify/:hash" (get user/verify-email)]
+
      ["/me" {:middleware [[mw/apply-auth]]
              :tags #{"me"}
              :swagger {:security [{"auth" []}]}
@@ -99,6 +101,7 @@
       ["" (-> (get me/get)
               (post me/post)
               (delete me/delete-user))]
+      ["/email/resend" (get me/resend-email)]
       ["/deletion/cancel" (get me/cancel-deletion)]
       ["/business" (-> (get me-business/get)
                        (post me-business/post))]
@@ -232,6 +235,7 @@
                 :swagger {:security [{"auth" []}]}
                 :openapi {:security [{:bearerAuth []}]}}
 
+      ["/users/:id/verify" (get users/verify-email)]
       ["/business/types" (-> (post business-types/post)
                              (patch business-types/patch)
                              (delete business-types/delete))]
diff --git a/src/source/routes/user.clj b/src/source/routes/user.clj
index b24f1b39..b2086b47 100644
--- a/src/source/routes/user.clj
+++ b/src/source/routes/user.clj
@@ -1,7 +1,8 @@
 (ns source.routes.user
   (:require [ring.util.response :as res]
-            [source.util :as util]
-            [source.db.honey :as hon]))
+            [source.db.honey :as hon]
+            [source.routes.openapi :as api]
+            [source.config :as conf]))
 
 (defn get
   {:summary "get user by id"
@@ -66,6 +67,27 @@
                    :data body})
   (res/response {:message "successfully updated user"}))
 
+(defn verify-email
+  {:summary "verify user email with email hash"
+   :parameters (api/params :path [:map [:hash :string]])
+   :responses {302 {:body [:map [:message :string]]}
+               403 {:body [:map [:message :string]]}}}
+  [{:keys [ds path-params]}]
+  (let [email-hash (:hash path-params)
+        user (hon/find-one ds {:tname :users
+                               :where [:= :email-hash email-hash]})]
+    (if (some? user)
+      (do
+        (hon/update! ds {:tname :users
+                         :where [:= :id (:id user)]
+                         :data {:email-verified true
+                                :email-hash ""}})
+        (-> (conf/read-value :cors-origin)
+            (str "/dashboard/onboarding")
+            (res/redirect)))
+      (-> (res/response {:message "unauthorized"})
+          (res/status 403)))))
+
 (comment
   (require '[source.db.interface :as db])
   (get {:ds (db/ds :master) :path-params {:id 5}})
diff --git a/src/source/routes/users.clj b/src/source/routes/users.clj
index e778a93b..b1d40182 100644
--- a/src/source/routes/users.clj
+++ b/src/source/routes/users.clj
@@ -1,6 +1,7 @@
 (ns source.routes.users
   (:require [ring.util.response :as res]
-            [source.db.honey :as hon]))
+            [source.db.honey :as hon]
+            [source.routes.openapi :as api]))
 
 (defn get
   {:summary "get all users"
@@ -25,6 +26,15 @@
   (res/response {:users (hon/find ds {:tname :users
                                       :ret :*})}))
 
+(defn verify-email
+  {:summary "verify user email with email hash"
+   :parameters (api/params :path [:map [:id :int]])
+   :responses (api/success (api/response-schema))}
+  [{:keys [ds path-params]}]
+  (hon/update! ds {:tname :users
+                   :where [:= :id (:id path-params)]
+                   :data {:email-verified true}}))
+
 (comment
   (require '[source.db.interface :as db])
   (get {:ds (db/ds :master)})
diff --git a/src/source/services/auth.clj b/src/source/services/auth.clj
index 642d9098..20846ca3 100644
--- a/src/source/services/auth.clj
+++ b/src/source/services/auth.clj
@@ -1,7 +1,9 @@
 (ns source.services.auth
   (:require [source.password :as pw]
             [source.middleware.auth.core :as auth]
-            [source.db.honey :as hon]))
+            [source.db.honey :as hon]
+            [source.email.gmail :as gmail]
+            [source.email.templates :as templates]))
 
 (defn login [ds {:keys [user] :as _login}]
   (merge
@@ -12,7 +14,12 @@
   (hon/insert! ds {:tname :users
                    :data (-> user
                              (dissoc :confirm-password)
-                             (assoc :password (pw/hash-password password)))})
+                             (assoc :password (pw/hash-password password)
+                                    :email-hash (pw/hash-password email)))})
+  (gmail/send-email {:to email
+                     :subject "Source - Verify your email"
+                     :body (templates/email-verification (pw/hash-password email))
+                     :type :text/html})
   (let [user (hon/find-one ds {:tname :users
                                :where [:= :email email]})]
     (merge

From 60515c48988e43aa07c4efa676d331279f1f0faa Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Tue, 3 Mar 2026 16:09:26 +0200
Subject: [PATCH 338/391] fixed feed enum schema

---
 src/source/workers/schemas.clj | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/source/workers/schemas.clj b/src/source/workers/schemas.clj
index 1365e36b..711e40af 100644
--- a/src/source/workers/schemas.clj
+++ b/src/source/workers/schemas.clj
@@ -88,7 +88,7 @@
    [:min :int]
    [:max :int]])
 
-(def FeedStatus [:enum ["live" "not live" "pending"]])
+(def FeedStatus [:enum "live" "not live" "pending"])
 
 (def FeedRecord
   [:map

From bf85dd28e3933655a5bf12a84490301837695aa7 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 4 Mar 2026 10:18:05 +0200
Subject: [PATCH 339/391] updated email hash migration to ensure all admins
 have verified emails

---
 src/source/migrations/013_email_hash.clj |  3 +++
 src/source/routes/me.clj                 | 13 ++++++++-----
 src/source/routes/user.clj               |  2 +-
 src/source/routes/users.clj              |  4 +++-
 src/source/services/auth.clj             |  6 +++---
 5 files changed, 18 insertions(+), 10 deletions(-)

diff --git a/src/source/migrations/013_email_hash.clj b/src/source/migrations/013_email_hash.clj
index e8730c86..1dadb1bc 100644
--- a/src/source/migrations/013_email_hash.clj
+++ b/src/source/migrations/013_email_hash.clj
@@ -5,6 +5,9 @@
 
 (defn run-up! [context]
   (let [ds-master (:db-master context)]
+    (hon/update! ds-master {:tname :users
+                            :where [:= :type "admin"]
+                            :data {:email-verified 1}})
     (hon/execute!
      ds-master
      (-> (hsql/alter-table :users)
diff --git a/src/source/routes/me.clj b/src/source/routes/me.clj
index cb8a72b5..4f96433f 100644
--- a/src/source/routes/me.clj
+++ b/src/source/routes/me.clj
@@ -7,7 +7,8 @@
             [congest.jobs :as congest]
             [source.workers.users :as users]
             [source.email.gmail :as gmail]
-            [source.email.templates :as templates]))
+            [source.email.templates :as templates]
+            [source.routes.openapi :as api]))
 
 (defn get
   {:summary "get logged in user by access token"
@@ -28,7 +29,7 @@
   [{:keys [ds user] :as _request}]
   (let [user (hon/find-one ds {:tname :users
                                :where [:= :id (:id user)]})]
-    (->> (dissoc user :password)
+    (->> (dissoc user :password :email-hash)
          (res/response))))
 
 (defn post
@@ -89,11 +90,13 @@
     (res/response {:message "successfully cancelled user deletion"})))
 
 (defn resend-email
-  {:summary "Resend verification email"}
+  {:summary "Resend verification email"
+   :responses (api/success (api/response-schema))}
   [{:keys [ds user]}]
   (let [{:keys [email email-hash]} (hon/find-one ds {:tname :users
                                                      :where [:= :id (:id user)]})]
     (gmail/send-email {:to email
                        :subject "Source - Verify your email"
-                       :body (templates/email-verification email-hash)
-                       :type :text/html})))
+                       :body (templates/email-verification {:email-hash email-hash})
+                       :type :text/html})
+    (res/response {:message "successfully resent email-verification email"})))
diff --git a/src/source/routes/user.clj b/src/source/routes/user.clj
index b2086b47..bfb28bb1 100644
--- a/src/source/routes/user.clj
+++ b/src/source/routes/user.clj
@@ -80,7 +80,7 @@
       (do
         (hon/update! ds {:tname :users
                          :where [:= :id (:id user)]
-                         :data {:email-verified true
+                         :data {:email-verified 1
                                 :email-hash ""}})
         (-> (conf/read-value :cors-origin)
             (str "/dashboard/onboarding")
diff --git a/src/source/routes/users.clj b/src/source/routes/users.clj
index b1d40182..841fed03 100644
--- a/src/source/routes/users.clj
+++ b/src/source/routes/users.clj
@@ -33,7 +33,9 @@
   [{:keys [ds path-params]}]
   (hon/update! ds {:tname :users
                    :where [:= :id (:id path-params)]
-                   :data {:email-verified true}}))
+                   :data {:email-verified 1
+                          :email-hash ""}})
+  (res/response {:message "successfully verified user"}))
 
 (comment
   (require '[source.db.interface :as db])
diff --git a/src/source/services/auth.clj b/src/source/services/auth.clj
index 20846ca3..ecb5b6a9 100644
--- a/src/source/services/auth.clj
+++ b/src/source/services/auth.clj
@@ -7,7 +7,7 @@
 
 (defn login [ds {:keys [user] :as _login}]
   (merge
-   {:user (dissoc user :password)}
+   {:user (dissoc user :password :email-hash)}
    (auth/create-session (select-keys user [:id :type]))))
 
 (defn register [ds {:keys [email password] :as user}]
@@ -18,12 +18,12 @@
                                     :email-hash (pw/hash-password email)))})
   (gmail/send-email {:to email
                      :subject "Source - Verify your email"
-                     :body (templates/email-verification (pw/hash-password email))
+                     :body (templates/email-verification {:email-hash (pw/hash-password email)})
                      :type :text/html})
   (let [user (hon/find-one ds {:tname :users
                                :where [:= :email email]})]
     (merge
-     {:user (dissoc user :password)}
+     {:user (dissoc user :password :email-hash)}
      (auth/create-session (select-keys user [:id :type])))))
 
 (comment

From f3260e0ee0953c43441a7d52e005b4833009d758 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 4 Mar 2026 10:29:49 +0200
Subject: [PATCH 340/391] updated swagger schemas in user route

---
 src/source/routes/user.clj | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/source/routes/user.clj b/src/source/routes/user.clj
index bfb28bb1..8eaa1fc1 100644
--- a/src/source/routes/user.clj
+++ b/src/source/routes/user.clj
@@ -70,8 +70,8 @@
 (defn verify-email
   {:summary "verify user email with email hash"
    :parameters (api/params :path [:map [:hash :string]])
-   :responses {302 {:body [:map [:message :string]]}
-               403 {:body [:map [:message :string]]}}}
+   :responses {302 (api/response-schema)
+               403 (api/response-schema)}}
   [{:keys [ds path-params]}]
   (let [email-hash (:hash path-params)
         user (hon/find-one ds {:tname :users

From 3ec856bdb07461fdc1ea7b6927237833118bed7b Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 4 Mar 2026 15:13:22 +0200
Subject: [PATCH 341/391] fixed schema in email verify endpoint

---
 src/source/routes/user.clj | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/source/routes/user.clj b/src/source/routes/user.clj
index 8eaa1fc1..173f48c7 100644
--- a/src/source/routes/user.clj
+++ b/src/source/routes/user.clj
@@ -70,8 +70,8 @@
 (defn verify-email
   {:summary "verify user email with email hash"
    :parameters (api/params :path [:map [:hash :string]])
-   :responses {302 (api/response-schema)
-               403 (api/response-schema)}}
+   :responses {302 {:body (api/response-schema)}
+               403 {:body (api/response-schema)}}}
   [{:keys [ds path-params]}]
   (let [email-hash (:hash path-params)
         user (hon/find-one ds {:tname :users

From 8159added238dd328adaeecd0f33efb00767204e Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Fri, 6 Mar 2026 10:48:16 +0200
Subject: [PATCH 342/391] updated tnames to return only the table names,
 excluding the maps

---
 src/source/bundle_migrations/002_outgoing_posts.clj | 4 ++--
 src/source/db/util.clj                              | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/source/bundle_migrations/002_outgoing_posts.clj b/src/source/bundle_migrations/002_outgoing_posts.clj
index 77535736..180348d3 100644
--- a/src/source/bundle_migrations/002_outgoing_posts.clj
+++ b/src/source/bundle_migrations/002_outgoing_posts.clj
@@ -16,5 +16,5 @@
   (let [{:keys [ds-master bundle-id]} context]
     (tables/drop-tables!
      ds-master
-     (mapv :tname (-> [:outgoing-posts]
-                      (db.util/tnames bundle-id))))))
+     (-> [:outgoing-posts]
+         (db.util/tnames bundle-id)))))
diff --git a/src/source/db/util.clj b/src/source/db/util.clj
index 942903d9..e66042f2 100644
--- a/src/source/db/util.clj
+++ b/src/source/db/util.clj
@@ -37,7 +37,7 @@
         (assoc data-map :tname))))
 
 (defn tnames [tnames id]
-  (mapv #(tname % id) tnames))
+  (mapv #(:tname (tname % id)) tnames))
 
 (comment
   (def q "SELECT * FROM events")

From 9359906d468a0ccc48a560925ab3f3067662c666 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Fri, 6 Mar 2026 10:49:14 +0200
Subject: [PATCH 343/391] fixed job starter to have 0 as a base value for
 delays instead of nil

---
 src/source/jobs/core.clj | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/source/jobs/core.clj b/src/source/jobs/core.clj
index c00cbd15..7a745967 100644
--- a/src/source/jobs/core.clj
+++ b/src/source/jobs/core.clj
@@ -48,8 +48,12 @@
                                        :args args
                                        :handler handler))
                     metadata (assoc m
-                                    :initial-delay (+ initial-delay (* 1000 5 i))
-                                    :interval (+ interval (* 1000 5 i)))]
+                                    :initial-delay (if (some? initial-delay)
+                                                     (+ initial-delay (* 1000 5 i))
+                                                     0)
+                                    :interval (if (some? interval)
+                                                (+ interval (* 1000 5 i))
+                                                0))]
                 (prepare-congest-metadata ds metadata))))
           jobs
           (-> jobs count inc range))))

From a1245132b4dc723ec7b89a9b58c4aa90b6519114 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Fri, 6 Mar 2026 10:50:26 +0200
Subject: [PATCH 344/391] fixed bug when deleting integrations

---
 src/source/workers/integrations.clj | 35 ++++++++++++++++-------------
 1 file changed, 20 insertions(+), 15 deletions(-)

diff --git a/src/source/workers/integrations.clj b/src/source/workers/integrations.clj
index 760778ee..c31d58f4 100644
--- a/src/source/workers/integrations.clj
+++ b/src/source/workers/integrations.clj
@@ -28,21 +28,26 @@
                                                          :content-types content-types}))
 
 (defn hard-delete-bundle! [ds js job-id bundle-id]
-  (hon/delete! ds {:tname :filtered-feeds
-                   :where [:= :bundle-id bundle-id]})
-  (hon/delete! ds {:tname :filtered-posts
-                   :where [:= :bundle-id bundle-id]})
-  (hon/delete! ds {:tname :bundle-content-types
-                   :where [:= :bundle-id bundle-id]})
-  (hon/delete! ds {:tname :events
-                   :where [:= :bundle-id bundle-id]})
-  (tables/drop-tables! ds (db.util/tnames [:outgoing-posts
-                                           :bundle-categories
-                                           :post-heuristics]
-                                          bundle-id))
-  (hon/delete! ds {:tname :bundles
-                   :where [:= :id bundle-id]})
-  (congest/deregister! js job-id))
+  (let [event-ids (mapv :id (hon/find ds {:tname :events
+                                          :where [:= :bundle-id bundle-id]}))]
+    (hon/delete! ds {:tname :filtered-feeds
+                     :where [:= :bundle-id bundle-id]})
+    (hon/delete! ds {:tname :filtered-posts
+                     :where [:= :bundle-id bundle-id]})
+    (hon/delete! ds {:tname :bundle-content-types
+                     :where [:= :bundle-id bundle-id]})
+    (when (seq event-ids)
+      (hon/delete! ds {:tname :event-categories
+                       :where [:in :event-id event-ids]}))
+    (hon/delete! ds {:tname :events
+                     :where [:= :bundle-id bundle-id]})
+    (tables/drop-tables! ds (db.util/tnames [:outgoing-posts
+                                             :bundle-categories
+                                             :post-heuristics]
+                                            bundle-id))
+    (hon/delete! ds {:tname :bundles
+                     :where [:= :id bundle-id]})
+    (congest/deregister! js job-id)))
 
 (defn update-filtered-feeds! [ds {:keys [filtered bundle-id feed-id]}]
   (if filtered

From 294451b017a1ee3847cd9bf23b588e74a2b3705f Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Fri, 6 Mar 2026 10:51:26 +0200
Subject: [PATCH 345/391] fixed type issue in user deletion and function call
 in feed deletion

---
 src/source/workers/feeds.clj | 4 ++--
 src/source/workers/users.clj | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/source/workers/feeds.clj b/src/source/workers/feeds.clj
index a459573e..7d0eab4e 100644
--- a/src/source/workers/feeds.clj
+++ b/src/source/workers/feeds.clj
@@ -65,8 +65,8 @@
 (defn hard-delete-feed! [ds js job-id feed-id]
   (let [post-ids (mapv :id (hon/find ds {:tname :incoming-posts
                                          :where [:= :feed-id feed-id]}))
-        event-ids (:mapv :id (hon/find ds {:tname :events
-                                           :where [:= :feed-id feed-id]}))]
+        event-ids (mapv :id (hon/find ds {:tname :events
+                                          :where [:= :feed-id feed-id]}))]
     (hon/delete! ds {:tname :filtered-feeds
                      :where [:= :feed-id feed-id]})
     (hon/delete! ds {:tname :filtered-posts
diff --git a/src/source/workers/users.clj b/src/source/workers/users.clj
index 206774aa..f78fa6b0 100644
--- a/src/source/workers/users.clj
+++ b/src/source/workers/users.clj
@@ -36,12 +36,12 @@
 (defn soft-delete-user! [ds user-id]
   (hon/update! ds {:tname :users
                    :where [:= :id user-id]
-                   :data {:removed true}}))
+                   :data {:removed 1}}))
 
 (defn cancel-soft-user-deletion! [ds user-id]
   (hon/update! ds {:tname :users
                    :where [:= :id user-id]
-                   :data {:removed false}}))
+                   :data {:removed 0}}))
 
 (defn removed? [ds user-id]
   (let [removed? (-> (hon/find ds {:tname :users

From dc9a411f4a35da5d2fbda96139b55eb1841a5034 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Fri, 6 Mar 2026 14:10:02 +0200
Subject: [PATCH 346/391] made migration to insert business types

---
 src/source/migrations/014_business_types.clj | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)
 create mode 100644 src/source/migrations/014_business_types.clj

diff --git a/src/source/migrations/014_business_types.clj b/src/source/migrations/014_business_types.clj
new file mode 100644
index 00000000..c8a33273
--- /dev/null
+++ b/src/source/migrations/014_business_types.clj
@@ -0,0 +1,18 @@
+(ns source.migrations.014-business-types
+  (:require [source.db.master]
+            [source.db.honey :as hon]))
+
+(defn run-up! [context]
+  (let [ds-master (:db-master context)]
+    (hon/insert! ds-master {:tname :business-types
+                            :data [{:name "For-Profit"}
+                                   {:name "Non-Profit"}
+                                   {:name "Creator"}]})))
+
+(defn run-down! [context]
+  (let [ds-master (:db-master context)]
+    (hon/delete! ds-master {:tname :business-types
+                            :where [:or
+                                    [:= :name "For-Profit"]
+                                    [:= :name "Non-Profit"]
+                                    [:= :name "Creator"]]})))

From af4ba2410916e54f5f157bfab7ad737e31d0d376 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Fri, 6 Mar 2026 15:29:42 +0200
Subject: [PATCH 347/391] added migration to add and seed integration types,
 and added integration type id field to integrations

---
 .../migrations/015_integration_types.clj      | 32 +++++++++++++++++++
 1 file changed, 32 insertions(+)
 create mode 100644 src/source/migrations/015_integration_types.clj

diff --git a/src/source/migrations/015_integration_types.clj b/src/source/migrations/015_integration_types.clj
new file mode 100644
index 00000000..f25f55d9
--- /dev/null
+++ b/src/source/migrations/015_integration_types.clj
@@ -0,0 +1,32 @@
+(ns source.migrations.015-integration-types
+  (:require [source.db.master]
+            [source.db.honey :as hon]
+            [honey.sql.helpers :as hsql]
+            [source.db.tables :as tables]))
+
+(defn run-up! [context]
+  (let [ds-master (:db-master context)]
+    (tables/create-table!
+     ds-master
+     :source.db.master
+     :integration-types)
+
+    (hon/insert! ds-master {:tname :integration-types
+                            :data [{:name "Website"}
+                                   {:name "App"}
+                                   {:name "Plugin"}
+                                   {:name "Community"}]})
+
+    (hon/execute!
+     ds-master
+     (-> (hsql/alter-table :bundles)
+         (hsql/add-column :integration-id :int)))))
+
+(defn run-down! [context]
+  (let [ds-master (:db-master context)]
+    (hon/execute!
+     ds-master
+     (-> (hsql/alter-table :bundles)
+         (hsql/drop-column :integration-id)))
+
+    (tables/drop-table! ds-master :integration-types)))

From 6f86395c7b725e3e2fd26204fa89275fd82f5ecc Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Fri, 6 Mar 2026 15:30:02 +0200
Subject: [PATCH 348/391] updated db schema

---
 src/source/db/master.clj | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/src/source/db/master.clj b/src/source/db/master.clj
index d32e95aa..8d746662 100644
--- a/src/source/db/master.clj
+++ b/src/source/db/master.clj
@@ -260,6 +260,12 @@
    (tables/table-id)
    [:name :text :not nil]))
 
+(def integration-types
+  (tables/create-table-sql
+   :business-types
+   (tables/table-id)
+   [:name :text :not nil]))
+
 (comment
   (require '[honey.sql :as sql])
 
@@ -284,6 +290,7 @@
   (sql/format filtered-feeds)
   (sql/format filtered-posts)
   (sql/format business-types)
+  (sql/format integration-types)
 
   (sql/format events)
   (sql/format event-categories)

From 869331377f76fd3f8ae9671b0a20ae9090e77b9f Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Fri, 6 Mar 2026 15:30:32 +0200
Subject: [PATCH 349/391] added and updated endpoints to work with integration
 types and integration type ids

---
 src/source/routes/integration.clj  | 41 ++++++++---------------
 src/source/routes/integrations.clj | 52 +++++++++++-------------------
 src/source/routes/reitit.clj       |  6 +++-
 src/source/workers/schemas.clj     | 16 +++++++--
 4 files changed, 50 insertions(+), 65 deletions(-)

diff --git a/src/source/routes/integration.clj b/src/source/routes/integration.clj
index 956ce7e8..df13c634 100644
--- a/src/source/routes/integration.clj
+++ b/src/source/routes/integration.clj
@@ -6,27 +6,17 @@
             [congest.jobs :as congest]
             [source.jobs.core :as jobs]
             [source.util :as util]
-            [source.jobs.handlers :as handlers]))
+            [source.jobs.handlers :as handlers]
+            [source.workers.schemas :as schemas]
+            [malli.util :as mu]
+            [source.routes.openapi :as api]))
 
 (defn get
   {:summary "Get metadata of integration by ID"
    :parameters {:path [:map [:id {:title "id"
                                   :description "Integration ID"} :int]]}
-   :responses {200 {:body [:map
-                           [:id :int]
-                           [:name :string]
-                           [:uuid :string]
-                           [:user-id :int]
-                           [:video :int]
-                           [:podcast :int]
-                           [:blog :int]
-                           [:hash [:maybe :string]]
-                           [:content-type-id :int]
-                           [:ts-and-cs [:maybe :int]]
-                           [:content-types [:vector
-                                            [:map
-                                             [:id :int]
-                                             [:name :string]]]]]}
+   :responses {200 {:body (-> schemas/Bundle
+                              (mu/assoc :content-types schemas/ContentTypes))}
                401 {:body [:map [:message :string]]}
                403 {:body [:map [:message :string]]}}}
 
@@ -40,16 +30,11 @@
    :description "When the integration is updated, post selection is rerun based on the newly set desired categories and content types."
    :parameters {:path [:map [:id {:title "id"
                                   :description "Integration ID"} :int]]
-                :body [:map
-                       [:name :string]
-                       [:content-types [:vector
-                                        [:map
-                                         [:id :int]
-                                         [:name :string]]]]
-                       [:categories [:vector
-                                     [:map
-                                      [:id :int]
-                                      [:name :string]]]]]}
+                :body (-> [:map
+                           [:name :string]
+                           [:integration-type-id :int]]
+                          (mu/assoc :content-types [:vector schemas/ConstantSchema])
+                          (mu/assoc :categories [:vector schemas/ConstantSchema]))}
    :responses {200 {:body [:map [:message :string]]}
                401 {:body [:map [:message :string]]}
                403 {:body [:map [:message :string]]}}}
@@ -86,8 +71,8 @@
    :description "Deletes the integration, bundle and kills the associated post selection job. This action cannot be undone."
    :parameters {:path [:map [:id {:title "id"
                                   :description "Integration ID"} :int]]}
-   :responses {200 {:body [:map [:message :string]]}
-               403 {:body [:map [:message :string]]}}}
+   :responses {200 {:body (api/response-schema)}
+               403 {:body (api/response-schema)}}}
 
   [{:keys [ds js user path-params] :as _request}]
   (let [bundle-id (:id path-params)
diff --git a/src/source/routes/integrations.clj b/src/source/routes/integrations.clj
index 1ccad6b6..129e59ad 100644
--- a/src/source/routes/integrations.clj
+++ b/src/source/routes/integrations.clj
@@ -5,22 +5,15 @@
             [source.util :as util]
             [source.jobs.core :as jobs]
             [source.jobs.handlers :as handlers]
-            [congest.jobs :as congest]))
+            [congest.jobs :as congest]
+            [source.routes.openapi :as api]
+            [source.workers.schemas :as schemas]
+            [source.db.honey :as hon]
+            [malli.util :as mu]))
 
 (defn get
   {:summary "Get metadata of all integrations on the user account"
-   :responses {200 {:body [:vector
-                           [:map
-                            [:id :int]
-                            [:name :string]
-                            [:uuid :string]
-                            [:user-id :int]
-                            [:video :int]
-                            [:podcast :int]
-                            [:blog :int]
-                            [:hash [:maybe :string]]
-                            [:content-type-id :int]
-                            [:ts-and-cs [:maybe :int]]]]}
+   :responses {200 {:body schemas/Bundles}
                401 {:body [:map [:message :string]]}
                403 {:body [:map [:message :string]]}}}
 
@@ -30,28 +23,13 @@
 (defn post
   {:summary "Creates an integration and the associated bundle in which content is stored"
    :description "When an integration is created, a job is scheduled to periodically run post selection every 24 hours. During post selection, the bundle is filled with relevant content according to desired categories, content types and analytics."
-   :parameters {:body [:map
-                       [:name :string]
-                       [:ts-and-cs {:optional true} :int]
-                       [:content-types [:vector
-                                        [:map
-                                         [:id :int]
-                                         [:name :string]]]]
-                       [:categories [:vector
-                                     [:map
-                                      [:id :int]
-                                      [:name :string]]]]]}
-   :responses {201 {:body [:map
-                           [:id :int]
+   :parameters {:body (-> [:map
                            [:name :string]
-                           [:uuid :string]
-                           [:user-id :int]
-                           [:video :int]
-                           [:podcast :int]
-                           [:blog :int]
-                           [:hash {:optional true} [:maybe :string]]
-                           [:content-type-id :int]
-                           [:ts-and-cs {:optional true} :int]]}
+                           [:ts-and-cs {:optional true} :int]
+                           [:integration-type-id :int]]
+                          (mu/assoc :content-types [:vector schemas/ConstantSchema])
+                          (mu/assoc :categories [:vector schemas/ConstantSchema]))}
+   :responses {201 {:body schemas/Bundle}
                401 {:body [:map [:message :string]]}
                403 {:body [:map [:message :string]]}}}
 
@@ -78,3 +56,9 @@
            :sleep false})
          (congest/register! js))
     (res/response new-bundle)))
+
+(defn get-integration-types
+  {:summary "get all integration types"
+   :responses (api/success schemas/IntegrationTypes)}
+  [{:keys [ds]}]
+  (res/response (hon/find ds {:tname :integration-types})))
diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index 33c44868..426d06b3 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -187,6 +187,9 @@
       ["/:id/filter/posts/:post-id" (-> (get integration-filter-post/get)
                                         (post integration-filter-post/post))]]
 
+     ["/integration/types" {:tags #{"integrations"}}
+      ["" (get integrations/get-integration-types)]]
+
      ["/feeds" {:middleware [[mw/apply-auth]]
                 :tags #{"feeds"}}
 
@@ -228,7 +231,8 @@
       ["/feeds/:id/posts/:post-id" (get bundle-feed-post/get)]
       ["/posts" (post bundle-posts/post)]
       ["/posts/:id" (get bundle-post/get)]]
-     ["/bundle/exists" (get bundle/exists)]
+     ["/bundle/exists" {:tags #{"bundles"}}
+      ["" (get bundle/exists)]]
 
      ["/admin" {:middleware [[mw/apply-auth {:required-type :admin}]]
                 :tags #{"admin"}
diff --git a/src/source/workers/schemas.clj b/src/source/workers/schemas.clj
index 711e40af..d03f46d1 100644
--- a/src/source/workers/schemas.clj
+++ b/src/source/workers/schemas.clj
@@ -232,16 +232,28 @@
 (def JobsWithMetadata
   [:vector JobWithMetadata])
 
+(def IntegrationType
+  ConstantSchema)
+
+(def IntegrationTypes
+  [:vector IntegrationType])
+
 (def Bundle
   [:map
    [:id :int]
    [:name :string]
    [:uuid :string]
+   [:user-id :int]
    [:video :int]
    [:podcast :int]
    [:blog :int]
-   [:hash :string]
-   [:ts-and-cs :int]])
+   (api/sometimes :hash :string)
+   [:content-type-id :int]
+   (api/sometimes :integration-type-id :int)
+   [:ts-and-cs {:optional true} :int]])
+
+(def Bundles
+  [:vector Bundle])
 
 (def BundleWithUser
   (-> Bundle

From 525a2ad8dbbc764d72862a9ad948b0402dc693d9 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Tue, 10 Mar 2026 10:26:49 +0200
Subject: [PATCH 350/391] replaced logger with a new dependency called telemere

---
 deps.edn                                |  3 +-
 src/source/admins.clj                   |  5 +-
 src/source/jobs/handlers.clj            | 61 ++++++++++++++-----------
 src/source/logger.clj                   | 12 -----
 src/source/middleware/auth/core.clj     |  5 +-
 src/source/middleware/auth/util.clj     |  5 +-
 src/source/middleware/core.clj          |  8 ++--
 src/source/routes/bundle_feed.clj       |  5 +-
 src/source/routes/bundle_feed_post.clj  |  5 +-
 src/source/routes/bundle_feed_posts.clj |  5 +-
 src/source/routes/bundle_feeds.clj      |  5 +-
 src/source/routes/bundle_post.clj       |  5 +-
 src/source/routes/bundle_posts.clj      |  5 +-
 src/source/routes/util.clj              | 12 +++--
 src/source/server.clj                   | 11 +++--
 src/source/workers/feeds.clj            | 22 +++++----
 src/source/workers/integrations.clj     | 16 ++++---
 17 files changed, 107 insertions(+), 83 deletions(-)
 delete mode 100644 src/source/logger.clj

diff --git a/deps.edn b/deps.edn
index 1ce2e076..37af2dc7 100644
--- a/deps.edn
+++ b/deps.edn
@@ -40,4 +40,5 @@
         metosin/reitit {:mvn/version "0.9.1"}
         metosin/reitit-middleware {:mvn/version "0.9.1"}
         metosin/reitit-swagger {:mvn/version "0.9.1"}
-        metosin/reitit-swagger-ui {:mvn/version "0.9.1"}}}
+        metosin/reitit-swagger-ui {:mvn/version "0.9.1"}
+        com.taoensso/telemere {:mvn/version "1.2.1"}}}
diff --git a/src/source/admins.clj b/src/source/admins.clj
index 2ca19de7..a795bf5c 100644
--- a/src/source/admins.clj
+++ b/src/source/admins.clj
@@ -2,7 +2,7 @@
   (:require [source.crypt-fs :as crypt]
             [source.config :as conf]
             [clojure.data.json :as json]
-            [source.logger :as logger]))
+            [taoensso.telemere :as t]))
 
 (defn encrypt! []
   (crypt/write-file-crypt! (conf/read-value :admins-encrypted-path)
@@ -17,5 +17,6 @@
         (crypt/read-file-crypt (conf/read-value :supersecretkey))
         (json/read-json))
     (catch Exception e
-      (logger/log-error (str "Couldn't read the admins file: " (.getMessage e)))
+      (t/log! {:level :error
+               :msg (str "Couldn't read the admins file: " (.getMessage e))})
       [])))
diff --git a/src/source/jobs/handlers.clj b/src/source/jobs/handlers.clj
index 9d44f986..91bf85a8 100644
--- a/src/source/jobs/handlers.clj
+++ b/src/source/jobs/handlers.clj
@@ -8,7 +8,7 @@
             [clojure.set :as set]
             [clojure.string :as string]
             [source.db.honey :as hon]
-            [source.logger :as logger]))
+            [taoensso.telemere :as t]))
 
 (defmulti handler
   (fn [opts]
@@ -32,7 +32,7 @@
     (try
       (when (users/removed? ds (:creator-id args))
         (let [{:keys [feed-id creator-id content-type-id provider-id url]} args
-              _ (logger/log (str "feed " feed-id " job started."))
+              _ (t/log! (str "feed " feed-id " job started."))
               selection-schemas (->> [:= :provider-id provider-id]
                                      (assoc {} :where)
                                      (xml/selection-schemas ds))
@@ -43,11 +43,14 @@
               extracted (try
                           (xml/extract-data ds latest-ss url)
                           (catch Exception e
-                            (throw (ex-info (str "Data extraction for feed job failed: feed-id " feed-id " creator-id " creator-id)
-                                            {:panic? "Yes, if data extraction fails here it will likely fail for others."
-                                             :possible-cause "Could possibly be an incorrect selection schema or output schema"
-                                             :next-steps (str "Check selection-schema-id " latest-ss " and feed-id " feed-id ". Test extraction manually.")
-                                             :raw-error (.getMessage e)}))))
+                            (throw
+                             (t/error!
+                              ::data-extraction
+                              (ex-info (str "Data extraction for feed job failed: feed-id " feed-id " creator-id " creator-id)
+                                       {:panic? "Yes, if data extraction fails here it will likely fail for others."
+                                        :possible-cause "Could possibly be an incorrect selection schema or output schema"
+                                        :next-steps (str "Check selection-schema-id " latest-ss " and feed-id " feed-id ". Test extraction manually.")
+                                        :raw-error (.getMessage e)})))))
 
               extracted-posts (get-in extracted [:feed :posts])
               extracted-display (get-in extracted [:feed :display-picture])
@@ -85,9 +88,10 @@
                (hon/insert! ds {:tname :incoming-posts
                                 :data post})))
            extended-posts)
-          (logger/log (str "feed " feed-id " job finished."))))
+          (t/log! (str "feed " feed-id " job finished."))))
 
-      (catch Exception e (logger/log-error (str "feed job failed: " e)) :fail))))
+      (catch Exception e (t/log! {:level :error
+                                  :msg (str "feed job failed: " e)}) :fail))))
 
 (defn update-bundle-job-id
   "returns the job id of an update-bundle job with the given bundle id"
@@ -116,7 +120,7 @@
 (defmethod handler :update-bundle [_]
   (fn [{:keys [args ds]}]
     (let [{:keys [bundle-id categories]} args
-          _ (logger/log (str "starting bundle " bundle-id " job."))
+          _ (t/log! (str "starting bundle " bundle-id " job."))
           incoming-posts (services/incoming-posts-with-feeds ds {:where [:= :feeds.state "live"]})
           posts-categories (incoming-posts/categories-by-posts ds {:where [:= :state "live"]})
           heuristics (mapv
@@ -126,7 +130,8 @@
       (try
         (services/upsert-post-heuristics! ds {:bundle-id bundle-id
                                               :data heuristics})
-        (catch Exception e (logger/log-error (str "bundle " bundle-id " upserting post heuristics failed: " (.getMessage e)))))
+        (catch Exception e (t/log! {:level :error
+                                    :msg (str "bundle " bundle-id " upserting post heuristics failed: " (.getMessage e))})))
 
       ; pull highest scored posts by long heuristics into outgoing posts
             ; top 1000 post-heuristics records ordered by long heuristic in descending order
@@ -159,20 +164,23 @@
           (hon/insert! ds (-> (db.util/tname :outgoing-posts bundle-id)
                               (assoc :data outgoing-posts)))
           (when (< (count outgoing-posts) 10)
-            (throw (ex-info (str
-                             "bundle job for bundle-id "
-                             bundle-id
-                             " pulled "
-                             (count outgoing-posts)
-                             ". Active creator id count: "
-                             (count active-creator-ids)
-                             ". Incoming posts pulled: "
-                             (count posts-in))
-                            {:panic? "Yes, the embed for this bundle will now be useless"
-                             :possible-cause "If no posts made it into the bundle, it's possible post heuristics failed or there's no incoming posts"
-                             :next-steps "Check for errors thrown in this job, ensure all tables for this bundle exist"}))))
-
-        (logger/log (str "bundle " bundle-id " job done."))))))
+            (throw
+             (t/error!
+              ::update-bundle-job
+              (ex-info (str
+                        "bundle job for bundle-id "
+                        bundle-id
+                        " pulled "
+                        (count outgoing-posts)
+                        ". Active creator id count: "
+                        (count active-creator-ids)
+                        ". Incoming posts pulled: "
+                        (count posts-in))
+                       {:panic? "Yes, the embed for this bundle will now be useless"
+                        :possible-cause "If no posts made it into the bundle, it's possible post heuristics failed or there's no incoming posts"
+                        :next-steps "Check for errors thrown in this job, ensure all tables for this bundle exist"})))))
+
+        (t/log! (str "bundle " bundle-id " job done."))))))
 
 (defn user-deletion-job-id
   "returns the job id of a user deletion job with the given user id"
@@ -184,4 +192,5 @@
     (try
       (let [{:keys [user-type user-id]} args]
         (users/hard-delete-user! ds (keyword user-type) user-id))
-      (catch Exception e (logger/log-error (str "Failed to delete user-id " (:user-id args) ":" e)) :fail))))
+      (catch Exception e (t/log! {:level :error
+                                  :msg (str "Failed to delete user-id " (:user-id args) ":" e)}) :fail))))
diff --git a/src/source/logger.clj b/src/source/logger.clj
deleted file mode 100644
index 5b67e82c..00000000
--- a/src/source/logger.clj
+++ /dev/null
@@ -1,12 +0,0 @@
-(ns source.logger
-  (:require
-   [source.util :as util]))
-
-(defn log [message]
-  (println (str "SOURCE [" (util/get-utc-timestamp-string) "] LOG: " message)))
-
-(defn log-warning [message]
-  (println (str "SOURCE [" (util/get-utc-timestamp-string) "] WARNING: " message)))
-
-(defn log-error [message]
-  (println (str "SOURCE [" (util/get-utc-timestamp-string) "] ERROR: " message)))
diff --git a/src/source/middleware/auth/core.clj b/src/source/middleware/auth/core.clj
index f506db45..33fb4f05 100644
--- a/src/source/middleware/auth/core.clj
+++ b/src/source/middleware/auth/core.clj
@@ -4,7 +4,7 @@
             [ring.util.response :as res]
             [source.services.bundles :as bundles]
             [source.db.honey :as db]
-            [source.logger :as logger]))
+            [taoensso.telemere :as t]))
 
 (defn create-session [user]
   (let [payload {:id (:id user)
@@ -58,7 +58,8 @@
             (assoc :bundle-id id)
             (handler))
         (do
-          (logger/log-warning (str "Bundle authorization attempt failed with uuid: " bundle-uuid))
+          (t/log! {:level :warn
+                   :msg (str "Bundle authorization attempt failed with uuid: " bundle-uuid)})
           (->
            (res/response {:message "The bundle you are looking for does not exist."})
            (res/status 404)))))))
diff --git a/src/source/middleware/auth/util.clj b/src/source/middleware/auth/util.clj
index cacc75c8..29b3faf1 100644
--- a/src/source/middleware/auth/util.clj
+++ b/src/source/middleware/auth/util.clj
@@ -2,7 +2,7 @@
   (:require [buddy.sign.jwt :as jwt]
             [source.config :as conf]
             [clojure.string :as str]
-            [source.logger :as logger]))
+            [taoensso.telemere :as t]))
 
 (defn auth-header [request]
   (or (get-in request [:headers "Authorization"])
@@ -26,5 +26,6 @@
   (try
     (jwt/decrypt token (conf/read-value :supersecretkey))
     (catch Exception e
-      (logger/log-warning (str "JWT Verification failed: " (.getMessage e)))
+      (t/log! {:level :warn
+               :msg (str "JWT Verification failed: " (.getMessage e))})
       false)))
diff --git a/src/source/middleware/core.clj b/src/source/middleware/core.clj
index 920d49c7..472d3743 100644
--- a/src/source/middleware/core.clj
+++ b/src/source/middleware/core.clj
@@ -13,7 +13,7 @@
             [clojure.walk :as walk]
             [source.util :as util]
             [clojure.string :as string]
-            [source.logger :as logger]))
+            [taoensso.telemere :as t]))
 
 (defn wrap-ds [handler ds]
   (fn [request]
@@ -44,7 +44,8 @@
     (try
       (handler req)
       (catch Exception e
-        (logger/log-error (str "Unhandled Exception on endpoint URI " (:uri req) ": " e))
+        (t/log! {:level :error
+                 :msg (str "Unhandled Exception on endpoint URI " (:uri req) ": " e)})
         (-> (res/response {:message "Internal Server Error"})
             (res/status 500))))))
 
@@ -76,7 +77,8 @@
                        (attach-validations request))]
       (if (seq errors)
         (do
-          (logger/log-warning (str "Schema validation failed on endpoint URI " (:uri request) ": " (string/join "\n" errors)))
+          (t/log! {:level :warn
+                   :msg (str "Schema validation failed on endpoint URI " (:uri request) ": " (string/join "\n" errors))})
           (-> (res/response {:message (string/join "\n" errors)})
               (res/status 400)))
         (handler request)))))
diff --git a/src/source/routes/bundle_feed.clj b/src/source/routes/bundle_feed.clj
index 6f25e79b..f4471a8b 100644
--- a/src/source/routes/bundle_feed.clj
+++ b/src/source/routes/bundle_feed.clj
@@ -4,7 +4,7 @@
             [source.services.analytics.interface :as analytics]
             [source.routes.openapi :as api]
             [source.workers.schemas :as schemas]
-            [source.logger :as logger]))
+            [taoensso.telemere :as t]))
 
 (defn get
   {:summary "Get a single RSS feed by id from RSS feeds within the uuid-authorized bundle. 
@@ -20,5 +20,6 @@
                                :where [:= :id (:id path-params)]})]
     (try
       (analytics/insert-feed-click! ds feed bundle-id)
-      (catch Exception e (logger/log-error (str "Failed to insert feed click for bundle feed: " (.getMessage e)))))
+      (catch Exception e (t/log! {:level :error
+                                  :msg (str "Failed to insert feed click for bundle feed: " (.getMessage e))})))
     (res/response feed)))
diff --git a/src/source/routes/bundle_feed_post.clj b/src/source/routes/bundle_feed_post.clj
index a630cc7a..c6d885f5 100644
--- a/src/source/routes/bundle_feed_post.clj
+++ b/src/source/routes/bundle_feed_post.clj
@@ -4,7 +4,7 @@
             [source.services.analytics.interface :as analytics]
             [source.db.util :as db.util]
             [honey.sql.helpers :as hsql]
-            [source.logger :as logger]))
+            [taoensso.telemere :as t]))
 
 (defn get
   {:summary "Get a single post by post id belonging to an RSS feed in the associated uuid-authorized bundle. 
@@ -36,5 +36,6 @@
                                   (hsql/where [:= :id (:post-id path-params)])))]
     (try
       (analytics/insert-post-click! ds post bundle-id)
-      (catch Exception e (logger/log-error (str "Failed to insert post click on bundle feed post: " (.getMessage e)))))
+      (catch Exception e (t/log! {:level :error
+                                  :msg (str "Failed to insert post click on bundle feed post: " (.getMessage e))})))
     (res/response post)))
diff --git a/src/source/routes/bundle_feed_posts.clj b/src/source/routes/bundle_feed_posts.clj
index b1a744ee..ecd20762 100644
--- a/src/source/routes/bundle_feed_posts.clj
+++ b/src/source/routes/bundle_feed_posts.clj
@@ -2,7 +2,7 @@
   (:require [ring.util.response :as res]
             [source.db.honey :as hon]
             [source.services.analytics.interface :as analytics]
-            [source.logger :as logger]))
+            [taoensso.telemere :as t]))
 
 (defn get
   {:summary "Get all posts present within a given RSS feed by feed id, within the uuid-authorized bundle.
@@ -36,5 +36,6 @@
                             :ret :*})]
     (try
       (analytics/insert-post-impressions! ds posts bundle-id)
-      (catch Exception e (logger/log-error (str "Failed to insert post impressions for bundle feed posts: " (.getMessage e)))))
+      (catch Exception e (t/log! {:level :error
+                                  :msg (str "Failed to insert post impressions for bundle feed posts: " (.getMessage e))})))
     (res/response posts)))
diff --git a/src/source/routes/bundle_feeds.clj b/src/source/routes/bundle_feeds.clj
index a93ec76a..6c18a607 100644
--- a/src/source/routes/bundle_feeds.clj
+++ b/src/source/routes/bundle_feeds.clj
@@ -5,7 +5,7 @@
             [source.services.analytics.interface :as analytics]
             [source.routes.openapi :as api]
             [source.workers.schemas :as schemas]
-            [source.logger :as logger]))
+            [taoensso.telemere :as t]))
 
 (defn post
   {:summary "Get all RSS feeds present in the bundle authorised by uuid.
@@ -33,7 +33,8 @@
                    (bundles/get-outgoing-feeds ds))]
     (try
       (analytics/insert-feed-impressions! ds feeds bundle-id)
-      (catch Exception e (logger/log-error (str "Failed to insert feed impressions for bundle feeds: " (.getMessage e)))))
+      (catch Exception e (t/log! {:level :error
+                                  :msg (str "Failed to insert feed impressions for bundle feeds: " (.getMessage e))})))
     (res/response feeds)))
 
 (comment
diff --git a/src/source/routes/bundle_post.clj b/src/source/routes/bundle_post.clj
index cf61c36f..9acbfc27 100644
--- a/src/source/routes/bundle_post.clj
+++ b/src/source/routes/bundle_post.clj
@@ -7,7 +7,7 @@
             [source.routes.openapi :as api]
             [source.workers.schemas :as schemas]
             [malli.util :as mu]
-            [source.logger :as logger]))
+            [taoensso.telemere :as t]))
 
 (defn get
   {:summary "Get a single post by post id in the uuid-authorized bundle.
@@ -29,5 +29,6 @@
                            {:ret :1})]
     (try
       (analytics/insert-post-click! ds post bundle-id)
-      (catch Exception e (logger/log-error (str "Failed to insert post click on bundle post: " (.getMessage e)))))
+      (catch Exception e (t/log! {:level :error
+                                  :msg (str "Failed to insert post click on bundle post: " (.getMessage e))})))
     (res/response post)))
diff --git a/src/source/routes/bundle_posts.clj b/src/source/routes/bundle_posts.clj
index e099c5bc..d714149b 100644
--- a/src/source/routes/bundle_posts.clj
+++ b/src/source/routes/bundle_posts.clj
@@ -6,7 +6,7 @@
             [source.routes.openapi :as api]
             [source.workers.schemas :as schemas]
             [malli.util :as mu]
-            [source.logger :as logger]))
+            [taoensso.telemere :as t]))
 
 (def QueryTruncatePosts
   [:truncate
@@ -50,5 +50,6 @@
                                    :category-ids (:category-ids body)})]
     (try
       (analytics/insert-post-impressions! ds data bundle-id)
-      (catch Exception e (logger/log-error (str "Failed to insert post impressions on bundle posts: " (.getMessage e)))))
+      (catch Exception e (t/log! {:level :error
+                                  :msg (str "Failed to insert post impressions on bundle posts: " (.getMessage e))})))
     (res/response posts)))
diff --git a/src/source/routes/util.clj b/src/source/routes/util.clj
index 34914ed5..03569cb9 100644
--- a/src/source/routes/util.clj
+++ b/src/source/routes/util.clj
@@ -6,7 +6,8 @@
             [reitit.coercion.malli :as coercion]
             [source.middleware.interface :as mw]
             [malli.util :as mu]
-            [source.routes.openapi :as api]))
+            [source.routes.openapi :as api]
+            [taoensso.telemere :as t]))
 
 (defn- extract-openapi-meta [handler]
   (-> (util/metadata handler)
@@ -44,8 +45,13 @@
     ([handler] (route {} method handler))
     ([route-map handler]
      (when (not (map? route-map))
-       (throw (ex-info "Invalid argument for resolve-route-map: route-map must be a map"
-                       {:panic? "not really"})))
+       (throw
+        (t/error!
+         ::route-map-resolution
+         (ex-info "Invalid argument for resolve-route-map: route-map must be a map"
+                  {:panic? "not really, but you should never get this in prod"
+                   :possible-cause "something is wrong in one of the endpoints' swagger docs, we're expecting a map here"
+                   :next-steps (str "check the swagger docs for something looking like this: " route-map)}))))
      (route route-map method handler))))
 
 (defn get [& opts]
diff --git a/src/source/server.clj b/src/source/server.clj
index 1056334b..d8d35826 100644
--- a/src/source/server.clj
+++ b/src/source/server.clj
@@ -5,7 +5,7 @@
             [source.jobs.core :as jobs]
             [source.routes.interface :as routes]
             [source.util :as util]
-            [source.logger :as logger]))
+            [taoensso.telemere :as t]))
 
 (defonce ^:private *components (atom nil))
 
@@ -33,7 +33,8 @@
   (try
     (when (deps-on? deps)
       (swap! *components assoc name (init-fn @*components)))
-    (catch Exception e (logger/log (str "Failed to initialise " name ": " e)))))
+    (catch Exception e (t/log! {:level :error
+                                :msg (str "Failed to initialise " name ": " e)}))))
 
 (defn initialise-components! [components]
   (run! initialise! components))
@@ -44,7 +45,7 @@
 (defn start-server []
   (cond (not (some? (:server @*components)))
         (do
-          (logger/log "Starting server on port 3000...")
+          (t/log! "Starting server on port 3000...")
           (initialise-components! [{:name :ds
                                     :init-fn (fn [_deps] (db/ds :master))
                                     :deps []}
@@ -55,10 +56,10 @@
                                     :deps [:ds :js]
                                     :init-fn initialise-server!}]))
         :else
-        (logger/log "Server already running!")))
+        (t/log! "Server already running!")))
 
 (defn stop-server []
-  (logger/log "Stopping server...")
+  (t/log! "Stopping server...")
   (when (some? (:js @*components))
     (congest/kill! (:js @*components)))
   (when (some? (:server @*components))
diff --git a/src/source/workers/feeds.clj b/src/source/workers/feeds.clj
index 1cfc914b..3c45fdab 100644
--- a/src/source/workers/feeds.clj
+++ b/src/source/workers/feeds.clj
@@ -5,7 +5,7 @@
             [source.db.honey :as hon]
             [source.rss.youtube :as yt]
             [clojure.string :as string]
-            [source.logger :as logger]))
+            [taoensso.telemere :as t]))
 
 (defn create-feed!
   "Creates feed with incoming posts pulled from RSS feed and starts associated job"
@@ -29,13 +29,16 @@
                     (when-not (= latest-ss -1)
                       (xml/extract-data ds latest-ss rss-url))
                     (catch Exception e
-                      (throw (ex-info (str "Data extraction failed for feed creation - RSS feed url: " rss-url " creator-id " user-id)
-                                      {:panic? "Not a huge deal, possibly just user error - but panic if it's not user error"
-                                       :possible-cause "RSS feed url might be incorrect or provider is unsupported"
-                                       :next-steps (str
-                                                    "Check if the RSS feed url is correct. If it is, test data extraction in the admin panel with provider-id "
-                                                    provider-id)
-                                       :raw-error (.getMessage e)}))))
+                      (throw
+                       (t/error!
+                        ::data-extraction
+                        (ex-info (str "Data extraction failed for feed creation - RSS feed url: " rss-url " creator-id " user-id)
+                                 {:panic? "Not a huge deal, possibly just user error - but panic if it's not user error"
+                                  :possible-cause "RSS feed url might be incorrect or provider is unsupported"
+                                  :next-steps (str
+                                               "Check if the RSS feed url is correct. If it is, test data extraction in the admin panel with provider-id "
+                                               provider-id)
+                                  :raw-error (.getMessage e)})))))
         extracted-posts (get-in extracted [:feed :posts])
 
         display-picture (if youtube?
@@ -54,7 +57,8 @@
                    :ret :1})
 
         _ (when (or (nil? display-picture) (= display-picture ""))
-            (logger/log-error (str "Failed to pull display picture for feed [ " (:id new-feed) " " (:title new-feed) "] provider-id " provider-id)))
+            (t/log! {:level :error
+                     :msg (str "Failed to pull display picture for feed [ " (:id new-feed) " " (:title new-feed) "] provider-id " provider-id)}))
 
         extended-posts (mapv (fn [post]
                                (merge post
diff --git a/src/source/workers/integrations.clj b/src/source/workers/integrations.clj
index 0fb7354f..bdece006 100644
--- a/src/source/workers/integrations.clj
+++ b/src/source/workers/integrations.clj
@@ -6,7 +6,8 @@
             [source.services.bundle-content-types :as bundle-content-types]
             [source.db.tables :as tables]
             [source.db.honey :as hon]
-            [congest.jobs :as congest]))
+            [congest.jobs :as congest]
+            [taoensso.telemere :as t]))
 
 (defn create-integration! [ds {:keys [user-id bundle-metadata categories content-types]}]
   (let [new-bundle (bundles/create-bundle! ds {:user-id user-id
@@ -15,11 +16,14 @@
     (try
       (migrate/migrate-bundle (:id new-bundle) ["up"])
       (catch Exception e
-        (throw (ex-info (str "Migration for new bundle with id " (:id new-bundle) " failed. This bundle belongs to the user with id " user-id)
-                        {:panic? "Yes, if this isn't working, it's likely no one will be able to create new integrations"
-                         :possible-cause "Something could be wrong in one of the bundle migrations"
-                         :next-steps "Go see what's going on in bundle migrations ASAP, you may be able to see the problem in the raw error message"
-                         :raw-error (.getMessage e)}))))
+        (throw
+         (t/error!
+          ::create-bundle
+          (ex-info (str "Migration for new bundle with id " (:id new-bundle) " failed. This bundle belongs to the user with id " user-id)
+                   {:panic? "Yes, if this isn't working, it's likely no one will be able to create new integrations"
+                    :possible-cause "Something could be wrong in one of the bundle migrations"
+                    :next-steps "Go see what's going on in bundle migrations ASAP, you may be able to see the problem in the raw error message"
+                    :raw-error (.getMessage e)})))))
 
     (bundle-categories/insert-bundle-categories! ds {:bundle-id (:id new-bundle)
                                                      :categories categories})

From 61368bc6b856f28b564b370bdc2d1d7ca1fc6a07 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Tue, 10 Mar 2026 14:26:58 +0200
Subject: [PATCH 351/391] wrapped more writes in transactions

---
 src/source/services/bundle_categories.clj    | 18 ++++++++++--------
 src/source/services/bundle_content_types.clj | 16 +++++++++-------
 src/source/services/user_sectors.clj         | 14 ++++++++------
 3 files changed, 27 insertions(+), 21 deletions(-)

diff --git a/src/source/services/bundle_categories.clj b/src/source/services/bundle_categories.clj
index 293c4baa..88e605ae 100644
--- a/src/source/services/bundle_categories.clj
+++ b/src/source/services/bundle_categories.clj
@@ -2,7 +2,8 @@
   (:require [source.db.interface :as db]
             [source.db.bundle :as bundle]
             [source.db.util :as db.util]
-            [honey.sql.helpers :as hsql]))
+            [honey.sql.helpers :as hsql]
+            [pg.core :as pg]))
 
 (defn category-id [ds {:keys [bundle-id where] :as opts}]
   (->> {:tname :bundle-categories
@@ -21,10 +22,11 @@
                        (assoc :data bundle-categories)))))
 
 (defn update-bundle-categories! [ds {:keys [bundle-id categories]}]
-  (let [bundle-categories (mapv (fn [{:keys [id]}]
-                                  {:bundle-id bundle-id
-                                   :category-id id}) categories)]
-    (db/delete! ds (-> (db.util/tname :bundle-categories bundle-id)
-                       (hsql/where [:= :bundle-id bundle-id])))
-    (db/insert! ds (-> (db.util/tname :bundle-categories bundle-id)
-                       (assoc :data bundle-categories)))))
+  (pg/with-transaction [ds ds]
+    (let [bundle-categories (mapv (fn [{:keys [id]}]
+                                    {:bundle-id bundle-id
+                                     :category-id id}) categories)]
+      (db/delete! ds (-> (db.util/tname :bundle-categories bundle-id)
+                         (hsql/where [:= :bundle-id bundle-id])))
+      (db/insert! ds (-> (db.util/tname :bundle-categories bundle-id)
+                         (assoc :data bundle-categories))))))
diff --git a/src/source/services/bundle_content_types.clj b/src/source/services/bundle_content_types.clj
index 5d26a177..4c795370 100644
--- a/src/source/services/bundle_content_types.clj
+++ b/src/source/services/bundle_content_types.clj
@@ -1,6 +1,7 @@
 (ns source.services.bundle-content-types
   (:require [source.db.interface :as db]
-            [source.db.honey :as hon]))
+            [source.db.honey :as hon]
+            [pg.core :as pg]))
 
 ;;NEW
 (defn insert-bundle-content-types! [ds {:keys [bundle-id content-types]}]
@@ -33,9 +34,10 @@
 
 ;;NEW
 (defn update-bundle-content-types! [ds {:keys [bundle-id content-types]}]
-  (let [content-types (mapv (fn [{:keys [id]}]
-                              {:bundle-id bundle-id
-                               :content-type-id id}) content-types)]
-    (delete-bundle-content-types! ds {:where [:= :bundle-id bundle-id]})
-    (hon/insert! ds {:tname :bundle-content-types
-                     :data content-types})))
+  (pg/with-transaction [ds ds]
+    (let [content-types (mapv (fn [{:keys [id]}]
+                                {:bundle-id bundle-id
+                                 :content-type-id id}) content-types)]
+      (delete-bundle-content-types! ds {:where [:= :bundle-id bundle-id]})
+      (hon/insert! ds {:tname :bundle-content-types
+                       :data content-types}))))
diff --git a/src/source/services/user_sectors.clj b/src/source/services/user_sectors.clj
index 99e387f5..8fce6216 100644
--- a/src/source/services/user_sectors.clj
+++ b/src/source/services/user_sectors.clj
@@ -1,6 +1,7 @@
 (ns source.services.user-sectors
   (:require [source.db.interface :as db]
-            [source.db.honey :as hon]))
+            [source.db.honey :as hon]
+            [pg.core :as pg]))
 
 (defn insert-user-sector! [ds {:keys [_data _ret] :as opts}]
   (->> {:tname :user-sectors}
@@ -28,8 +29,9 @@
 
 ;;NEW
 (defn update-user-sectors! [ds {:keys [user-id sectors]}]
-  (let [update-data (reduce (fn [acc {:keys [id]}]
-                              (conj acc {:user-id user-id
-                                         :sector-id id})) [] sectors)]
-    (delete-user-sector! ds {:where [:= :user-id user-id]})
-    (insert-user-sector! ds {:data update-data})))
+  (pg/with-transaction [ds ds]
+    (let [update-data (reduce (fn [acc {:keys [id]}]
+                                (conj acc {:user-id user-id
+                                           :sector-id id})) [] sectors)]
+      (delete-user-sector! ds {:where [:= :user-id user-id]})
+      (insert-user-sector! ds {:data update-data}))))

From 8b2e22e5fc7c03236c4b8803f085a92a7890d5a2 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Tue, 10 Mar 2026 14:31:03 +0200
Subject: [PATCH 352/391] fixed formatting

---
 src/source/workers/feeds.clj | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/source/workers/feeds.clj b/src/source/workers/feeds.clj
index 288081d2..17f4713d 100644
--- a/src/source/workers/feeds.clj
+++ b/src/source/workers/feeds.clj
@@ -19,7 +19,6 @@
                     (->> (yt/find-channel-id rss-url)
                          (str "https://www.youtube.com/feeds/videos.xml?channel_id="))
                     rss-url)
-
           selection-schemas (->> [:= :provider-id provider-id]
                                  (assoc {} :where)
                                  (xml/selection-schemas ds))

From d9bfc968d83371ce48b248f2ba454572179acbc9 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 11 Mar 2026 10:51:27 +0200
Subject: [PATCH 353/391] updated server to start job service first before
 initiating jobs

---
 src/source/jobs/core.clj | 26 ++++++++++++++------------
 src/source/server.clj    |  7 +++++--
 2 files changed, 19 insertions(+), 14 deletions(-)

diff --git a/src/source/jobs/core.clj b/src/source/jobs/core.clj
index 7a745967..1e039f20 100644
--- a/src/source/jobs/core.clj
+++ b/src/source/jobs/core.clj
@@ -3,11 +3,12 @@
             [clojure.data.json :as json]
             [source.services.interface :as services]
             [source.jobs.oplog :as oplog]
-            [source.jobs.handlers :as handlers]))
+            [source.jobs.handlers :as handlers]
+            [source.db.honey :as hon]))
 
 (defn prepare-congest-metadata
   "given raw job metadata, returns extended metadata necessary for use with congest"
-  [ds metadata]
+  [ds js metadata]
   (let [i->b (fn [i] (if (integer? i)
                        (if (= i 1) true false)
                        i))
@@ -21,7 +22,8 @@
                :handler-name (:handler metadata)
                :handler (handlers/handler metadata)
                :logger oplog/operation-logger
-               :ds ds)
+               :ds ds
+               :js js)
         (dissoc :recurring))))
 
 (defn start!
@@ -33,11 +35,11 @@
                             :args args
                             :handler handler))]
     (congest/deregister! js job-id)
-    (congest/register! js (prepare-congest-metadata ds metadata))))
+    (congest/register! js (prepare-congest-metadata ds js metadata))))
 
 (defn interrupted-jobs
-  "Get congest-ready metadata of all jobs marked as running"
-  [ds]
+  "Get vec of congest-ready metadata of all jobs marked as running"
+  [ds js]
   (let [jobs (services/jobs ds)]
     (mapv (fn [{:keys [job-id job-metadata-id args handler status]} i]
             (when (= status "running")
@@ -49,12 +51,12 @@
                                        :handler handler))
                     metadata (assoc m
                                     :initial-delay (if (some? initial-delay)
-                                                     (+ initial-delay (* 1000 5 i))
+                                                     (+' initial-delay (*' 1000 5 i))
                                                      0)
                                     :interval (if (some? interval)
-                                                (+ interval (* 1000 5 i))
+                                                (+' interval (*' 1000 5 i))
                                                 0))]
-                (prepare-congest-metadata ds metadata))))
+                (prepare-congest-metadata ds js metadata))))
           jobs
           (-> jobs count inc range))))
 
@@ -81,10 +83,10 @@
   (services/delete-job-metadata! ds {})
 
   (def js (congest/create-job-service []))
-  (congest/register! js (prepare-congest-metadata ds testjob))
-  (congest/deregister! js "test")
+  (congest/register! js (prepare-congest-metadata ds js testjob))
+  (congest/deregister! js "delete_creator_31")
   (congest/stop! js "test" false)
   (start! js ds "test")
-  (interrupted-jobs ds)
+  (interrupted-jobs ds js)
 
   ())
diff --git a/src/source/server.clj b/src/source/server.clj
index d8d35826..984a5ee7 100644
--- a/src/source/server.clj
+++ b/src/source/server.clj
@@ -16,8 +16,11 @@
    {:port 3000}))
 
 (defn initialise-job-service! [{:keys [ds] :as _deps}]
-  (->> (jobs/interrupted-jobs ds)
-       (congest/create-job-service)))
+  (let [js (congest/create-job-service [])]
+    (run!
+     #(congest/register! js %)
+     (jobs/interrupted-jobs ds js))
+    js))
 
 (defn component-on? [component]
   (if (some? (get @*components component))

From 2cca2037bca6dcdfac912d13fa881ea9c722324c Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 11 Mar 2026 10:52:25 +0200
Subject: [PATCH 354/391] updated prepare-congest-metadata everywhere to pass
 in the job service and therefore pass the js to the user deletion job

---
 src/source/jobs/handlers.clj       | 4 ++--
 src/source/routes/feeds.clj        | 1 +
 src/source/routes/integration.clj  | 1 +
 src/source/routes/integrations.clj | 1 +
 src/source/routes/jobs.clj         | 2 +-
 src/source/routes/me.clj           | 5 +++--
 6 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/src/source/jobs/handlers.clj b/src/source/jobs/handlers.clj
index 91bf85a8..e05acf66 100644
--- a/src/source/jobs/handlers.clj
+++ b/src/source/jobs/handlers.clj
@@ -188,9 +188,9 @@
   (str "delete_" user-type "_" user-id))
 
 (defmethod handler :delete-user [_]
-  (fn [{:keys [args ds]}]
+  (fn [{:keys [args ds js]}]
     (try
       (let [{:keys [user-type user-id]} args]
-        (users/hard-delete-user! ds (keyword user-type) user-id))
+        (users/hard-delete-user! ds js (keyword user-type) user-id))
       (catch Exception e (t/log! {:level :error
                                   :msg (str "Failed to delete user-id " (:user-id args) ":" e)}) :fail))))
diff --git a/src/source/routes/feeds.clj b/src/source/routes/feeds.clj
index 3768d258..a5ecce35 100644
--- a/src/source/routes/feeds.clj
+++ b/src/source/routes/feeds.clj
@@ -41,6 +41,7 @@
             ;TODO: service needed
             (->> (jobs/prepare-congest-metadata
                   ds
+                  js
                   {:id (str email "-" (:id new-feed))
                    :initial-delay (* 1000 60 60 24)
                    :auto-start true
diff --git a/src/source/routes/integration.clj b/src/source/routes/integration.clj
index df13c634..d4762121 100644
--- a/src/source/routes/integration.clj
+++ b/src/source/routes/integration.clj
@@ -51,6 +51,7 @@
     (congest/deregister! js job-id)
     (->> (jobs/prepare-congest-metadata
           ds
+          js
           {:id job-id
            :initial-delay (* 1000 60 60 24)
            :auto-start true
diff --git a/src/source/routes/integrations.clj b/src/source/routes/integrations.clj
index 129e59ad..0f7278ad 100644
--- a/src/source/routes/integrations.clj
+++ b/src/source/routes/integrations.clj
@@ -43,6 +43,7 @@
     ;TODO: service needed
     (->> (jobs/prepare-congest-metadata
           ds
+          js
           {:id (handlers/update-bundle-job-id (:id new-bundle))
            :initial-delay 0
            :auto-start true
diff --git a/src/source/routes/jobs.clj b/src/source/routes/jobs.clj
index fbaadd9f..c667569d 100644
--- a/src/source/routes/jobs.clj
+++ b/src/source/routes/jobs.clj
@@ -35,5 +35,5 @@
   [{:keys [js ds body] :as _req}]
   (let [{:keys [metadata]} body
         metadata (assoc metadata :created-at (util/get-utc-timestamp-string))]
-    (congest/register! js (jobs/prepare-congest-metadata ds metadata))
+    (congest/register! js (jobs/prepare-congest-metadata ds js metadata))
     (res/response {:message "successfully registered job"})))
diff --git a/src/source/routes/me.clj b/src/source/routes/me.clj
index 4f96433f..62d8d19b 100644
--- a/src/source/routes/me.clj
+++ b/src/source/routes/me.clj
@@ -63,11 +63,12 @@
     ; TODO: service needed
     (->> (jobs/prepare-congest-metadata
           ds
+          js
           {:id job-id
-           :initial-delay (* 1000 60 60 24 30)
+           :initial-delay (* 1000 60 60 24 24)
            :auto-start true
            :stop-after-fail false,
-           :interval (* 1000 60 60 24 30)
+           :interval (* 1000 60 60 24 24)
            :recurring? false
            :kill-after 1
            :args {:user-type type

From 45bdc8c028fd530eda0645c38ec0e3139307f628 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 11 Mar 2026 15:30:12 +0200
Subject: [PATCH 355/391] prevent soft-deleted users from using non-GET
 endpoints

---
 src/source/middleware/auth/core.clj | 22 ++++++++++++++--------
 1 file changed, 14 insertions(+), 8 deletions(-)

diff --git a/src/source/middleware/auth/core.clj b/src/source/middleware/auth/core.clj
index 33fb4f05..aff45ea5 100644
--- a/src/source/middleware/auth/core.clj
+++ b/src/source/middleware/auth/core.clj
@@ -17,15 +17,21 @@
       (util/auth-token)
       (util/verify-jwt)))
 
-(defn wrap-auth [handler]
+(defn wrap-auth
+  "Returns unauthenticated if the user JWT validation failed, or if a soft-deleted user tries to call a non-GET endpoint"
+  [handler]
   (fn [request]
-    (if-let [user (validate-request request)]
-      (-> request
-          (assoc :user user)
-          (handler))
-      (->
-       (res/response {:message "Unauthorized"})
-       (res/status 401)))))
+    (let [ds (db.util/conn :master)
+          {:keys [id] :as user} (validate-request request)
+          {:keys [removed]} (db/find-one ds {:tname :users
+                                             :where [:= :id id]})]
+      (if (and user (if (= (:request-method request) :get) true (= removed 0)))
+        (-> request
+            (assoc :user user)
+            (handler))
+        (->
+         (res/response {:message "Unauthorized"})
+         (res/status 401))))))
 
 (defn wrap-auth-user-type
   "returns an unauthorized response if the user's type is not the required user type (creator | distributor | admin)"

From fa4a55375edf0ea9beeea715700881ac5d5b80fe Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 11 Mar 2026 15:52:54 +0200
Subject: [PATCH 356/391] removed auth from /mail/report endpoint

---
 src/source/routes/reitit.clj | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj
index 426d06b3..878d1a19 100644
--- a/src/source/routes/reitit.clj
+++ b/src/source/routes/reitit.clj
@@ -108,11 +108,7 @@
       ["/sectors" (-> (get me-sectors/get)
                       (post me-sectors/post))]]
 
-     ["/mail" {:middleware [[mw/apply-auth]]
-               :tags #{"mail"}
-               :swagger {:security [{"auth" []}]}
-               :openapi {:security [{:bearerAuth []}]}}
-
+     ["/mail" {:tags #{"mail"}}
       ["/report" (post report/post)]]
 
      ["/businesses" {:middleware [[mw/apply-auth {:required-type :admin}]]

From e4af5625c59d4711816ddbeae699c515b410688f Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Thu, 12 Mar 2026 14:07:37 +0200
Subject: [PATCH 357/391] updated job deregistration to be done in soft
 deletion instead, avoiding the problem entirely

---
 src/source/jobs/core.clj            | 18 ++++++------
 src/source/jobs/handlers.clj        |  4 +--
 src/source/routes/feed.clj          |  3 +-
 src/source/routes/feeds.clj         |  1 -
 src/source/routes/integration.clj   |  4 +--
 src/source/routes/integrations.clj  |  1 -
 src/source/routes/jobs.clj          |  2 +-
 src/source/routes/me.clj            |  3 +-
 src/source/server.clj               |  7 ++---
 src/source/workers/feeds.clj        | 12 ++++----
 src/source/workers/integrations.clj | 12 ++++----
 src/source/workers/users.clj        | 44 ++++++++++++++++++++---------
 12 files changed, 63 insertions(+), 48 deletions(-)

diff --git a/src/source/jobs/core.clj b/src/source/jobs/core.clj
index 1e039f20..209ba8bc 100644
--- a/src/source/jobs/core.clj
+++ b/src/source/jobs/core.clj
@@ -3,12 +3,11 @@
             [clojure.data.json :as json]
             [source.services.interface :as services]
             [source.jobs.oplog :as oplog]
-            [source.jobs.handlers :as handlers]
-            [source.db.honey :as hon]))
+            [source.jobs.handlers :as handlers]))
 
 (defn prepare-congest-metadata
   "given raw job metadata, returns extended metadata necessary for use with congest"
-  [ds js metadata]
+  [ds metadata]
   (let [i->b (fn [i] (if (integer? i)
                        (if (= i 1) true false)
                        i))
@@ -22,8 +21,7 @@
                :handler-name (:handler metadata)
                :handler (handlers/handler metadata)
                :logger oplog/operation-logger
-               :ds ds
-               :js js)
+               :ds ds)
         (dissoc :recurring))))
 
 (defn start!
@@ -35,11 +33,11 @@
                             :args args
                             :handler handler))]
     (congest/deregister! js job-id)
-    (congest/register! js (prepare-congest-metadata ds js metadata))))
+    (congest/register! js (prepare-congest-metadata ds metadata))))
 
 (defn interrupted-jobs
   "Get vec of congest-ready metadata of all jobs marked as running"
-  [ds js]
+  [ds]
   (let [jobs (services/jobs ds)]
     (mapv (fn [{:keys [job-id job-metadata-id args handler status]} i]
             (when (= status "running")
@@ -56,7 +54,7 @@
                                     :interval (if (some? interval)
                                                 (+' interval (*' 1000 5 i))
                                                 0))]
-                (prepare-congest-metadata ds js metadata))))
+                (prepare-congest-metadata ds metadata))))
           jobs
           (-> jobs count inc range))))
 
@@ -83,10 +81,10 @@
   (services/delete-job-metadata! ds {})
 
   (def js (congest/create-job-service []))
-  (congest/register! js (prepare-congest-metadata ds js testjob))
+  (congest/register! js (prepare-congest-metadata ds testjob))
   (congest/deregister! js "delete_creator_31")
   (congest/stop! js "test" false)
   (start! js ds "test")
-  (interrupted-jobs ds js)
+  (interrupted-jobs ds)
 
   ())
diff --git a/src/source/jobs/handlers.clj b/src/source/jobs/handlers.clj
index e05acf66..91bf85a8 100644
--- a/src/source/jobs/handlers.clj
+++ b/src/source/jobs/handlers.clj
@@ -188,9 +188,9 @@
   (str "delete_" user-type "_" user-id))
 
 (defmethod handler :delete-user [_]
-  (fn [{:keys [args ds js]}]
+  (fn [{:keys [args ds]}]
     (try
       (let [{:keys [user-type user-id]} args]
-        (users/hard-delete-user! ds js (keyword user-type) user-id))
+        (users/hard-delete-user! ds (keyword user-type) user-id))
       (catch Exception e (t/log! {:level :error
                                   :msg (str "Failed to delete user-id " (:user-id args) ":" e)}) :fail))))
diff --git a/src/source/routes/feed.clj b/src/source/routes/feed.clj
index fdff0b37..2421cab9 100644
--- a/src/source/routes/feed.clj
+++ b/src/source/routes/feed.clj
@@ -51,7 +51,8 @@
         job-id (handlers/update-feed-posts-job-id email id)]
     (if (some? feed)
       (do
-        (feeds/hard-delete-feed! ds js job-id id)
+        (feeds/hard-delete-feed! ds id)
+        (feeds/deregister-feed-job! js job-id)
         (res/response {:message "successfully deleted feed"}))
       (-> (res/response {:message "unauthorized"})
           (res/status 403)))))
diff --git a/src/source/routes/feeds.clj b/src/source/routes/feeds.clj
index a5ecce35..3768d258 100644
--- a/src/source/routes/feeds.clj
+++ b/src/source/routes/feeds.clj
@@ -41,7 +41,6 @@
             ;TODO: service needed
             (->> (jobs/prepare-congest-metadata
                   ds
-                  js
                   {:id (str email "-" (:id new-feed))
                    :initial-delay (* 1000 60 60 24)
                    :auto-start true
diff --git a/src/source/routes/integration.clj b/src/source/routes/integration.clj
index d4762121..41692373 100644
--- a/src/source/routes/integration.clj
+++ b/src/source/routes/integration.clj
@@ -51,7 +51,6 @@
     (congest/deregister! js job-id)
     (->> (jobs/prepare-congest-metadata
           ds
-          js
           {:id job-id
            :initial-delay (* 1000 60 60 24)
            :auto-start true
@@ -83,7 +82,8 @@
         job-id (handlers/update-bundle-job-id bundle-id)]
     (if (some? bundle)
       (do
-        (integrations/hard-delete-bundle! ds js job-id bundle-id)
+        (integrations/hard-delete-bundle! ds bundle-id)
+        (integrations/deregister-bundle-job! js job-id)
         (res/response {:message "successfully deleted integration"}))
       (-> (res/response {:message "unauthorized"})
           (res/status 403)))))
diff --git a/src/source/routes/integrations.clj b/src/source/routes/integrations.clj
index 0f7278ad..129e59ad 100644
--- a/src/source/routes/integrations.clj
+++ b/src/source/routes/integrations.clj
@@ -43,7 +43,6 @@
     ;TODO: service needed
     (->> (jobs/prepare-congest-metadata
           ds
-          js
           {:id (handlers/update-bundle-job-id (:id new-bundle))
            :initial-delay 0
            :auto-start true
diff --git a/src/source/routes/jobs.clj b/src/source/routes/jobs.clj
index c667569d..fbaadd9f 100644
--- a/src/source/routes/jobs.clj
+++ b/src/source/routes/jobs.clj
@@ -35,5 +35,5 @@
   [{:keys [js ds body] :as _req}]
   (let [{:keys [metadata]} body
         metadata (assoc metadata :created-at (util/get-utc-timestamp-string))]
-    (congest/register! js (jobs/prepare-congest-metadata ds js metadata))
+    (congest/register! js (jobs/prepare-congest-metadata ds metadata))
     (res/response {:message "successfully registered job"})))
diff --git a/src/source/routes/me.clj b/src/source/routes/me.clj
index 62d8d19b..e1721c13 100644
--- a/src/source/routes/me.clj
+++ b/src/source/routes/me.clj
@@ -58,12 +58,11 @@
   [{:keys [ds js user] :as _request}]
   (let [{:keys [id type]} user
         job-id (handlers/user-deletion-job-id type id)]
-    (users/soft-delete-user! ds id)
+    (users/soft-delete-user! ds js id)
 
     ; TODO: service needed
     (->> (jobs/prepare-congest-metadata
           ds
-          js
           {:id job-id
            :initial-delay (* 1000 60 60 24 24)
            :auto-start true
diff --git a/src/source/server.clj b/src/source/server.clj
index 984a5ee7..d8d35826 100644
--- a/src/source/server.clj
+++ b/src/source/server.clj
@@ -16,11 +16,8 @@
    {:port 3000}))
 
 (defn initialise-job-service! [{:keys [ds] :as _deps}]
-  (let [js (congest/create-job-service [])]
-    (run!
-     #(congest/register! js %)
-     (jobs/interrupted-jobs ds js))
-    js))
+  (->> (jobs/interrupted-jobs ds)
+       (congest/create-job-service)))
 
 (defn component-on? [component]
   (if (some? (get @*components component))
diff --git a/src/source/workers/feeds.clj b/src/source/workers/feeds.clj
index 17f4713d..d4edd547 100644
--- a/src/source/workers/feeds.clj
+++ b/src/source/workers/feeds.clj
@@ -1,12 +1,12 @@
 (ns source.workers.feeds
   (:require [source.util :as utils]
             [source.workers.xml-schemas :as xml]
-            [congest.jobs :as congest]
             [source.db.honey :as hon]
             [source.rss.youtube :as yt]
             [clojure.string :as string]
             [taoensso.telemere :as t]
-            [pg.core :as pg]))
+            [pg.core :as pg]
+            [congest.jobs :as congest]))
 
 (defn create-feed!
   "Creates feed with incoming posts pulled from RSS feed and starts associated job"
@@ -79,7 +79,7 @@
                    :data feed-metadata
                    :ret :1}))
 
-(defn hard-delete-feed! [ds js job-id feed-id]
+(defn hard-delete-feed! [ds feed-id]
   (pg/with-transaction [ds ds]
     (let [post-ids (mapv :id (hon/find ds {:tname :incoming-posts
                                            :where [:= :feed-id feed-id]}))
@@ -99,8 +99,10 @@
       (hon/delete! ds {:tname :events
                        :where [:= :feed-id feed-id]})
       (hon/delete! ds {:tname :feeds
-                       :where [:= :id feed-id]})
-      (congest/deregister! js job-id))))
+                       :where [:= :id feed-id]}))))
+
+(defn deregister-feed-job! [js job-id]
+  (congest/deregister! js job-id))
 
 (defn update-feed-categories! [ds {:keys [feed-id categories]}]
   (let [update-data (mapv (fn [{:keys [id]}]
diff --git a/src/source/workers/integrations.clj b/src/source/workers/integrations.clj
index 268c1f94..9f058eed 100644
--- a/src/source/workers/integrations.clj
+++ b/src/source/workers/integrations.clj
@@ -6,9 +6,9 @@
             [source.services.bundle-content-types :as bundle-content-types]
             [source.db.tables :as tables]
             [source.db.honey :as hon]
-            [congest.jobs :as congest]
             [taoensso.telemere :as t]
-            [pg.core :as pg]))
+            [pg.core :as pg]
+            [congest.jobs :as congest]))
 
 (defn create-integration! [ds {:keys [user-id bundle-metadata categories content-types]}]
   (pg/with-transaction [ds ds]
@@ -42,7 +42,7 @@
     (bundle-content-types/update-bundle-content-types! ds {:bundle-id bundle-id
                                                            :content-types content-types})))
 
-(defn hard-delete-bundle! [ds js job-id bundle-id]
+(defn hard-delete-bundle! [ds bundle-id]
   (pg/with-transaction [ds ds]
     (let [event-ids (mapv :id (hon/find ds {:tname :events
                                             :where [:= :bundle-id bundle-id]}))]
@@ -62,8 +62,10 @@
                                                :post-heuristics]
                                               bundle-id))
       (hon/delete! ds {:tname :bundles
-                       :where [:= :id bundle-id]})
-      (congest/deregister! js job-id))))
+                       :where [:= :id bundle-id]}))))
+
+(defn deregister-bundle-job! [js job-id]
+  (congest/deregister! js job-id))
 
 (defn update-filtered-feeds! [ds {:keys [filtered bundle-id feed-id]}]
   (pg/with-transaction [ds ds]
diff --git a/src/source/workers/users.clj b/src/source/workers/users.clj
index daf7532d..5abeb636 100644
--- a/src/source/workers/users.clj
+++ b/src/source/workers/users.clj
@@ -4,29 +4,39 @@
             [source.db.honey :as hon]
             [pg.core :as pg]))
 
-(defn hard-delete-creator! [ds js user-id email]
+(defn hard-delete-creator! [ds user-id]
   (let [feed-ids (mapv :id (hon/find ds {:tname :feeds
                                          :where [:= :user-id user-id]}))]
-    (run! #(feeds/hard-delete-feed! ds js (str email "-" %) %) feed-ids)
+    (run! #(feeds/hard-delete-feed! ds %) feed-ids)
     (hon/delete! ds {:tname :events
                      :where [:= :creator-id user-id]})))
 
-(defn hard-delete-distributor! [ds js user-id]
+(defn deregister-creator-jobs! [ds js user-id email]
+  (let [feed-ids (mapv :id (hon/find ds {:tname :feeds
+                                         :where [:= :user-id user-id]}))]
+    (run! #(feeds/deregister-feed-job! js (str email "-" %)) feed-ids)))
+
+(defn hard-delete-distributor! [ds user-id]
   (let [bundle-ids (mapv :id (hon/find ds {:tname :bundles
                                            :where [:= :user-id user-id]}))]
-    (run! #(integrations/hard-delete-bundle! ds js (str "bundle_" %) %) bundle-ids)
+    (run! #(integrations/hard-delete-bundle! ds %) bundle-ids)
     (hon/delete! ds {:tname :events
                      :where [:= :distributor-id user-id]})))
 
-(defn hard-delete-user! [ds js user-type user-id]
+(defn deregister-distributor-jobs! [ds js user-id]
+  (let [bundle-ids (mapv :id (hon/find ds {:tname :bundles
+                                           :where [:= :user-id user-id]}))]
+    (run! #(integrations/hard-delete-bundle! js (str "bundle_" %)) bundle-ids)))
+
+(defn hard-delete-user! [ds user-type user-id]
   (pg/with-transaction [ds ds]
-    (let [{:keys [email business-id]} (hon/find-one ds {:tname :users
-                                                        :where [:= :id user-id]})]
+    (let [{:keys [business-id]} (hon/find-one ds {:tname :users
+                                                  :where [:= :id user-id]})]
       (cond
         (= user-type :creator)
-        (hard-delete-creator! ds js user-id email)
+        (hard-delete-creator! ds user-id)
         (= user-type :distributor)
-        (hard-delete-distributor! ds js user-id))
+        (hard-delete-distributor! ds user-id))
 
       (hon/delete! ds {:tname :user-sectors
                        :where [:= :user-id user-id]})
@@ -35,10 +45,18 @@
       (hon/delete! ds {:tname :users
                        :where [:= :id user-id]}))))
 
-(defn soft-delete-user! [ds user-id]
-  (hon/update! ds {:tname :users
-                   :where [:= :id user-id]
-                   :data {:removed 1}}))
+(defn soft-delete-user! [ds js user-id]
+  (let [{:keys [email type]} (hon/find-one ds {:tname :users
+                                               :where [:= :id user-id]})]
+    (cond
+      (= (keyword type) :creator)
+      (deregister-creator-jobs! ds js user-id email)
+      (= (keyword type) :distributor)
+      (deregister-distributor-jobs! ds js user-id))
+
+    (hon/update! ds {:tname :users
+                     :where [:= :id user-id]
+                     :data {:removed 1}})))
 
 (defn cancel-soft-user-deletion! [ds user-id]
   (hon/update! ds {:tname :users

From 3ed560c4c1d7e134456b8320e9fd575e6b3b88c0 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Fri, 13 Mar 2026 10:38:50 +0200
Subject: [PATCH 358/391] migration 15 was incorrect adding a column called
 integration_id instead of integration_type_id

---
 .../migrations/016_integration_type_ids.clj   | 26 +++++++++++++++++++
 1 file changed, 26 insertions(+)
 create mode 100644 src/source/migrations/016_integration_type_ids.clj

diff --git a/src/source/migrations/016_integration_type_ids.clj b/src/source/migrations/016_integration_type_ids.clj
new file mode 100644
index 00000000..5d3335ba
--- /dev/null
+++ b/src/source/migrations/016_integration_type_ids.clj
@@ -0,0 +1,26 @@
+(ns source.migrations.016-integration-type-ids
+  (:require [source.db.master]
+            [source.db.honey :as hon]
+            [honey.sql.helpers :as hsql]))
+
+(defn run-up! [context]
+  (let [ds-master (:db-master context)]
+    (hon/execute!
+     ds-master
+     (-> (hsql/alter-table :bundles)
+         (hsql/drop-column :integration-id)))
+    (hon/execute!
+     ds-master
+     (-> (hsql/alter-table :bundles)
+         (hsql/add-column :integration-type-id :int)))))
+
+(defn run-down! [context]
+  (let [ds-master (:db-master context)]
+    (hon/execute!
+     ds-master
+     (-> (hsql/alter-table :bundles)
+         (hsql/add-column :integration-id)))
+    (hon/execute!
+     ds-master
+     (-> (hsql/alter-table :bundles)
+         (hsql/drop-column :integration-type-id)))))

From 369ef120ebf5d4ee378f04b8d94edcde743fb867 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Fri, 13 Mar 2026 10:55:39 +0200
Subject: [PATCH 359/391] updated auth middleware such that it displays a
 different error message with a 403 code if the user has been archived

---
 src/source/middleware/auth/core.clj | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/src/source/middleware/auth/core.clj b/src/source/middleware/auth/core.clj
index aff45ea5..19a2c08c 100644
--- a/src/source/middleware/auth/core.clj
+++ b/src/source/middleware/auth/core.clj
@@ -20,15 +20,18 @@
 (defn wrap-auth
   "Returns unauthenticated if the user JWT validation failed, or if a soft-deleted user tries to call a non-GET endpoint"
   [handler]
-  (fn [request]
-    (let [ds (db.util/conn :master)
-          {:keys [id] :as user} (validate-request request)
+  (fn [{:keys [ds] :as request}]
+    (let [{:keys [id] :as user} (validate-request request)
           {:keys [removed]} (db/find-one ds {:tname :users
                                              :where [:= :id id]})]
-      (if (and user (if (= (:request-method request) :get) true (= removed 0)))
-        (-> request
-            (assoc :user user)
-            (handler))
+      (if user
+        (if (or (= (:request-method request) :get) (= removed 0))
+          (-> request
+              (assoc :user user)
+              (handler))
+          (->
+           (res/response {:message "The account of the user attempting to use this endpoint has been archived."})
+           (res/status 403)))
         (->
          (res/response {:message "Unauthorized"})
          (res/status 401))))))

From b4a25eb18b96d0bc4a9e4563ece443f9820db5ad Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Fri, 13 Mar 2026 11:01:26 +0200
Subject: [PATCH 360/391] updated auth middleware to return 403 if user has
 been archived, always 401 if regular auth issue

---
 src/source/middleware/auth/core.clj | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/source/middleware/auth/core.clj b/src/source/middleware/auth/core.clj
index 19a2c08c..bb5032dc 100644
--- a/src/source/middleware/auth/core.clj
+++ b/src/source/middleware/auth/core.clj
@@ -51,7 +51,7 @@
         (and (= user-type (name required-type)) (= user-type expected-type)) (handler request)
         :else (->
                (res/response {:message "Unauthorized"})
-               (res/status 403))))))
+               (res/status 401))))))
 
 (defn wrap-bundle-id
   "validates the bundle uuid in the query parameters of the request for 

From fa7d6ab22a61ff9224531ebe8ec40a3cff23bc78 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Tue, 17 Mar 2026 14:37:31 +0200
Subject: [PATCH 361/391] added config to allow changing the port number

---
 resources/config.edn  | 1 +
 src/source/config.clj | 1 +
 src/source/server.clj | 7 ++++---
 3 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/resources/config.edn b/resources/config.edn
index 049697a2..fa899698 100644
--- a/resources/config.edn
+++ b/resources/config.edn
@@ -3,6 +3,7 @@
  :admins-encrypted-path "resources/admins_encrypted.json"
  :cors-origin #or [#env CORS_ORIGIN "http://localhost:8080"]
  :base-url #or [#env BASE_URL "https://localhost:3000"]
+ :port #or [#env PORT 3000]
  :env #or [#env ENV "dev"]
  :database {:url #or [#env DATABASE_URL "postgres://localhost"]
             :type "postgresql"}
diff --git a/src/source/config.clj b/src/source/config.clj
index 2557bd93..a1b4cb98 100644
--- a/src/source/config.clj
+++ b/src/source/config.clj
@@ -23,6 +23,7 @@
    [:admins-encrypted-path :string]
    [:cors-origin :string]
    [:base-url :string]
+   [:port :int]
    [:env :string]
    [:database [:map
                [:url :string]
diff --git a/src/source/server.clj b/src/source/server.clj
index d8d35826..d1bfe0e6 100644
--- a/src/source/server.clj
+++ b/src/source/server.clj
@@ -5,7 +5,8 @@
             [source.jobs.core :as jobs]
             [source.routes.interface :as routes]
             [source.util :as util]
-            [taoensso.telemere :as t]))
+            [taoensso.telemere :as t]
+            [source.config :as conf]))
 
 (defonce ^:private *components (atom nil))
 
@@ -13,7 +14,7 @@
   (http/run-server
    (routes/create-app {:ds ds
                        :js js})
-   {:port 3000}))
+   {:port (conf/read-value :port)}))
 
 (defn initialise-job-service! [{:keys [ds] :as _deps}]
   (->> (jobs/interrupted-jobs ds)
@@ -45,7 +46,7 @@
 (defn start-server []
   (cond (not (some? (:server @*components)))
         (do
-          (t/log! "Starting server on port 3000...")
+          (t/log! (str "Starting server on port " (conf/read-value :port) "..."))
           (initialise-components! [{:name :ds
                                     :init-fn (fn [_deps] (db/ds :master))
                                     :deps []}

From 4c723623ff3d640711173d7e96f951fc888206c8 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 18 Mar 2026 13:42:31 +0200
Subject: [PATCH 362/391] updated deployment process

---
 README.md                    | 42 +++++++++++++++++++++++++++++++++---
 build.sh                     |  6 ++++++
 deploy.sh                    | 13 -----------
 merv_start.sh                |  7 ------
 resources/config.edn         |  2 +-
 resources/staging_config.edn | 20 +++++++++++++++++
 server_startup.sh            |  8 -------
 src/source/config.clj        |  9 ++++++--
 start.sh                     |  7 ++++--
 9 files changed, 78 insertions(+), 36 deletions(-)
 create mode 100755 build.sh
 delete mode 100755 deploy.sh
 delete mode 100755 merv_start.sh
 create mode 100644 resources/staging_config.edn
 delete mode 100755 server_startup.sh

diff --git a/README.md b/README.md
index f0bbef6f..4fd94489 100644
--- a/README.md
+++ b/README.md
@@ -13,8 +13,11 @@ This is the backend for the Source platform. You can find documentation on setup
 
 ```.env
 SUPER_SECRET_KEY [string, minimum 32 characters]
+EMAIL_USERNAME [string, email username used for sending emails]
+EMAIL_PASSWORD [string, email password used for sending emails]
 GOOGLE_CLIENT_ID [string, from Google Console]
 GOOGLE_CLIENT_SECRET [string, from Google Console]
+DATABASE_URL [string, connection URL to Postgres instance, excluding database name and trailing forward slash]
 ```
 
 - Run the provided shell script with `./nrepl.sh` to start your nrepl server. For this, you will need the required nrepl alias in `.clojure/deps.edn`.
@@ -23,9 +26,9 @@ When you have made changes, you can restart the server by evaluating `server/res
 
 ## Database Migration
 
-Database migration is handled using the provided migrate.sh shell script.
-You should have a db directory and ensure your configuration is pointing to it. The default is `.db/` in the root, 
-this can be configured with the `DATABASE_DIR` variable in the environment.
+Database migration is handled using the provided `migrate.sh` shell script.
+You need to have a Postgresql connection string in order to run migrations,
+this can be configured with the `DATABASE_URL` variable in the environment.
 
 The migration system makes use of the kepler/mallard library, take a look at their [docs](https://github.com/kepler16/mallard) to find out more on how it works.
 The datasource you will be operating on is passed in as context to the run-up and run-down functions.
@@ -60,3 +63,36 @@ We are making use of cognitect-labs/test-runner for running our tests. Run the p
 Unit tests are written and placed in clj files in `tests/source-be/`. These files should have the `_test` postfix (e.g. google_auth_test.clj) and tests should be 
 written per namespace and cover all important functions.
 
+## Production Setup
+
+- Pull the project from GitHub.
+- Ensure you have Java installed, minimum openjdk version 11.
+- Create a .env file containing the following required information:
+
+```.env
+# These values are secrets, ensure they are not exposed anywhere else on your system
+SUPER_SECRET_KEY [string, minimum 32 characters]
+EMAIL_USERNAME [string, email username used for sending emails]
+EMAIL_PASSWORD [string, email password used for sending emails]
+GOOGLE_CLIENT_ID [string, from Google Console]
+GOOGLE_CLIENT_SECRET [string, from Google Console]
+DATABASE_URL [string, connection URL to Postgres instance, excluding database name and trailing forward slash]
+
+# These values are required for config and compiling the project
+ENV [string, "dev" | "staging" | "prod"] # This should match your config file, e.g. if ENV is "staging", your config file should be called staging_config.edn
+JAVA_CMD [string, path pointing to your Java executable]
+```
+
+- If there isn't already a `config.edn` file for your target environment in the `resources/` directory, create a config file using the given `resources/config.edn`
+as a template as shown below:
+```bash
+cp resources/config.edn resources/{env_name}_config.edn
+```
+where `{env_name}` is the value of the `ENV` variable set in your `.env`.
+You can configure everything for your machine inside your config.edn file, alternatively, you can configure everything marked with `#env` from `config.edn` 
+within your `.env` file.
+
+- If everything before this point is set up correctly, you are ready to begin the next steps.
+
+- Run `./build.sh` to the compile the project.
+- Run `./start.sh` to start the server. The server will use the config according to the specified environment in `.env` and will run migrations before starting.
diff --git a/build.sh b/build.sh
new file mode 100755
index 00000000..6c7e3b91
--- /dev/null
+++ b/build.sh
@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+export $(grep '.*' .env | xargs)
+
+echo "Starting compilation..."
+clojure -T:build uber
diff --git a/deploy.sh b/deploy.sh
deleted file mode 100755
index 83065e1f..00000000
--- a/deploy.sh
+++ /dev/null
@@ -1,13 +0,0 @@
-#!/usr/bin/env bash
-
-systemctl stop source-be.service
-
-export $(grep '.*' .env | xargs)
-
-echo "Pulling latest changes..."
-git pull
-echo "Starting compilation..."
-export JAVA_CMD="/home/merv/.jenv/shims/java"
-clojure -T:build uber
-
-systemctl start source-be.service
diff --git a/merv_start.sh b/merv_start.sh
deleted file mode 100755
index 2a0d33ba..00000000
--- a/merv_start.sh
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/usr/bin/env bash
-export $(grep '.*' .env | xargs)
-
-export JAVA_CMD="/home/merv/.jenv/shims/java"
-clojure -M:migrate
-
-/home/merv/.jenv/shims/java -jar target/source-be-standalone.jar
diff --git a/resources/config.edn b/resources/config.edn
index fa899698..e96fe5b3 100644
--- a/resources/config.edn
+++ b/resources/config.edn
@@ -1,7 +1,7 @@
 {:supersecretkey #env SUPER_SECRET_KEY
  :admins-path "resources/admins.json"
  :admins-encrypted-path "resources/admins_encrypted.json"
- :cors-origin #or [#env CORS_ORIGIN "http://localhost:8080"]
+ :cors-origin #or [#env CORS_ORIGIN "http://localhost:3001"]
  :base-url #or [#env BASE_URL "https://localhost:3000"]
  :port #or [#env PORT 3000]
  :env #or [#env ENV "dev"]
diff --git a/resources/staging_config.edn b/resources/staging_config.edn
new file mode 100644
index 00000000..3df88ecb
--- /dev/null
+++ b/resources/staging_config.edn
@@ -0,0 +1,20 @@
+{:supersecretkey #env SUPER_SECRET_KEY
+ :admins-path "resources/admins.json"
+ :admins-encrypted-path "resources/admins_encrypted.json"
+ :cors-origin #or [#env CORS_ORIGIN "https://mervstation.tail4f070.ts.net"]
+ :base-url #or [#env BASE_URL "https://mervstation.tail4f070.ts.net/sourcebe"]
+ :port #or [#env PORT 3998]
+ :env #or [#env ENV "staging"]
+ :database {:url #or [#env DATABASE_URL "postgres://localhost"]
+            :type "postgresql"}
+ :email {:address #or [#env SUPPORT_ADDRESS "lee@wearesource.earth"]
+         :username #env EMAIL_USERNAME
+         :password #env EMAIL_PASSWORD}
+ :oauth2 {:google {:authorization-uri   "https://accounts.google.com/o/oauth2/auth"
+                   :access-token-uri    "https://oauth2.googleapis.com/token"
+                   :redirect-uri        #or [#env GOOGLE_REDIRECT_URI "https://mervstation.tail4f070.ts.net/sourcebe/oauth2/google/callback"]
+                   :client-id           #env GOOGLE_CLIENT_ID
+                   :client-secret       #env GOOGLE_CLIENT_SECRET
+                   :access-query-param  :access_token
+                   :scope               ["https://www.googleapis.com/auth/userinfo.email"]
+                   :grant-type          "authorization_code"}}}
diff --git a/server_startup.sh b/server_startup.sh
deleted file mode 100755
index 9d4d8a19..00000000
--- a/server_startup.sh
+++ /dev/null
@@ -1,8 +0,0 @@
-#!/bin/bash
-
-echo "Starting tailscale funnel..."
-tailscale funnel 3000 &
-cd /home/merv/Developer/source-be
-
-echo "Starting server..."
-./merv_start.sh
diff --git a/src/source/config.clj b/src/source/config.clj
index a1b4cb98..67cb69fe 100644
--- a/src/source/config.clj
+++ b/src/source/config.clj
@@ -30,8 +30,14 @@
                [:type :string]]]
    [:oauth2 [:map-of keyword? oauth2-provider-schema]]])
 
+(defn- get-env-config []
+  (let [{:keys [env] :as config} (aero/read-config (io/resource "config.edn"))]
+    (if (= env "dev")
+      config
+      (aero/read-config (io/resource (str env "_config.edn"))))))
+
 (defn- load-config []
-  (let [config (aero/read-config (io/resource "config.edn"))
+  (let [config (get-env-config)
         decoded (m/decode schema config mt/string-transformer)]
     (when-not (m/validate schema decoded)
       (println (->> decoded
@@ -52,4 +58,3 @@
   (read-value :cors-origin)
   (read-value :admins-path)
   (load-config))
-
diff --git a/start.sh b/start.sh
index d16c0e08..91693940 100755
--- a/start.sh
+++ b/start.sh
@@ -1,5 +1,8 @@
-#!/usr/bin/env bash
+#!/bin/bash
+export $(grep '.*' .env | xargs)
 
+echo "Running migrations before startup..."
 clojure -M:migrate
 
-java -jar target/source-be-standalone.jar
+echo "Starting server..."
+$JAVA_CMD -jar target/source-be-standalone.jar

From b3b66d41aea18c15eb5d7bf3ca39c6ce3b936ebc Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 18 Mar 2026 14:00:33 +0200
Subject: [PATCH 363/391] updated readme

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 4fd94489..cc9b7610 100644
--- a/README.md
+++ b/README.md
@@ -65,7 +65,7 @@ written per namespace and cover all important functions.
 
 ## Production Setup
 
-- Pull the project from GitHub.
+- Pull the project from GitHub and navigate to its directory.
 - Ensure you have Java installed, minimum openjdk version 11.
 - Create a .env file containing the following required information:
 

From 0d23b14c97db54dc57b42e0af0de69cde50539fd Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 18 Mar 2026 14:05:56 +0200
Subject: [PATCH 364/391] added logging info to readme

---
 README.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/README.md b/README.md
index cc9b7610..6447db91 100644
--- a/README.md
+++ b/README.md
@@ -96,3 +96,5 @@ within your `.env` file.
 
 - Run `./build.sh` to the compile the project.
 - Run `./start.sh` to start the server. The server will use the config according to the specified environment in `.env` and will run migrations before starting.
+
+The logs will be displayed when the server is run. If you are running the server via a systemd service, you can find them by running `journalctl -u {servicename}.service`.

From 2354f855b864682c54c498bf09de23fe9828b7b8 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 18 Mar 2026 14:51:47 +0200
Subject: [PATCH 365/391] updated readme to change minimum java version to 16

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 6447db91..a9c91f43 100644
--- a/README.md
+++ b/README.md
@@ -4,7 +4,7 @@ This is the backend for the Source platform. You can find documentation on setup
 
 ## Dependencies
 
-- >= openjdk version 11.
+- >= openjdk version 16.
 
 ## Development setup
 
@@ -66,7 +66,7 @@ written per namespace and cover all important functions.
 ## Production Setup
 
 - Pull the project from GitHub and navigate to its directory.
-- Ensure you have Java installed, minimum openjdk version 11.
+- Ensure you have Java installed, minimum openjdk version 16.
 - Create a .env file containing the following required information:
 
 ```.env

From 8acc6c0b46b92ebb48f64cd4332dcc6fb27d8a7f Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Thu, 19 Mar 2026 11:13:27 +0200
Subject: [PATCH 366/391] updated according to requested changes, used
 System/getenv to get env and renamed config.edn to dev_config.edn

---
 README.md                                |  4 ++--
 resources/{config.edn => dev_config.edn} |  0
 src/source/config.clj                    | 10 +++-------
 3 files changed, 5 insertions(+), 9 deletions(-)
 rename resources/{config.edn => dev_config.edn} (100%)

diff --git a/README.md b/README.md
index a9c91f43..3045530c 100644
--- a/README.md
+++ b/README.md
@@ -83,13 +83,13 @@ ENV [string, "dev" | "staging" | "prod"] # This should match your config file, e
 JAVA_CMD [string, path pointing to your Java executable]
 ```
 
-- If there isn't already a `config.edn` file for your target environment in the `resources/` directory, create a config file using the given `resources/config.edn`
+- If there isn't already a `config.edn` file for your target environment in the `resources/` directory, create a config file using the given `resources/dev_config.edn`
 as a template as shown below:
 ```bash
 cp resources/config.edn resources/{env_name}_config.edn
 ```
 where `{env_name}` is the value of the `ENV` variable set in your `.env`.
-You can configure everything for your machine inside your config.edn file, alternatively, you can configure everything marked with `#env` from `config.edn` 
+You can configure everything for your machine inside your config.edn file, alternatively, you can configure everything marked with `#env` from `{env_name}_config.edn` 
 within your `.env` file.
 
 - If everything before this point is set up correctly, you are ready to begin the next steps.
diff --git a/resources/config.edn b/resources/dev_config.edn
similarity index 100%
rename from resources/config.edn
rename to resources/dev_config.edn
diff --git a/src/source/config.clj b/src/source/config.clj
index 67cb69fe..63c33154 100644
--- a/src/source/config.clj
+++ b/src/source/config.clj
@@ -30,14 +30,10 @@
                [:type :string]]]
    [:oauth2 [:map-of keyword? oauth2-provider-schema]]])
 
-(defn- get-env-config []
-  (let [{:keys [env] :as config} (aero/read-config (io/resource "config.edn"))]
-    (if (= env "dev")
-      config
-      (aero/read-config (io/resource (str env "_config.edn"))))))
-
 (defn- load-config []
-  (let [config (get-env-config)
+  (let [environment (get (System/getenv) "ENV")
+        environment (if (nil? environment) "dev" environment)
+        config (aero/read-config (io/resource (str environment "_config.edn")))
         decoded (m/decode schema config mt/string-transformer)]
     (when-not (m/validate schema decoded)
       (println (->> decoded

From 2fc060fde63b44d7b9d597c6bb7eff9904c6ca0b Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Thu, 19 Mar 2026 11:48:47 +0200
Subject: [PATCH 367/391] updated check for rss feed url for youtube to allow
 for playlists as well

---
 src/source/workers/feeds.clj | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/source/workers/feeds.clj b/src/source/workers/feeds.clj
index d4edd547..f4f8fb74 100644
--- a/src/source/workers/feeds.clj
+++ b/src/source/workers/feeds.clj
@@ -15,7 +15,7 @@
     (let [{:keys [provider-id rss-url content-type-id]} feed-metadata
           datetime (utils/get-utc-timestamp-string)
           youtube? (= provider-id 1)
-          rss-url (if (and youtube? (not (string/includes? rss-url "/feeds/videos.xml?channel_id=")))
+          rss-url (if (and youtube? (not (string/includes? rss-url "/feeds/videos.xml?")))
                     (->> (yt/find-channel-id rss-url)
                          (str "https://www.youtube.com/feeds/videos.xml?channel_id="))
                     rss-url)

From 74f25ac65f1e8587b007b6fb5343748c7a8db676 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Thu, 19 Mar 2026 13:40:19 +0200
Subject: [PATCH 368/391] added macro for connecting to database depending on
 environment (dev / staging / prod)

---
 resources/dev_config.edn |  5 ++++-
 src/source/db/util.clj   | 14 ++++++++++++++
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/resources/dev_config.edn b/resources/dev_config.edn
index e96fe5b3..81ef3583 100644
--- a/resources/dev_config.edn
+++ b/resources/dev_config.edn
@@ -6,7 +6,10 @@
  :port #or [#env PORT 3000]
  :env #or [#env ENV "dev"]
  :database {:url #or [#env DATABASE_URL "postgres://localhost"]
-            :type "postgresql"}
+            :type "postgresql"
+            :dev #or [#env DEV_DATABASE_URL "postgres://localhost:5432"]
+            :staging #or [#env STAGING_DATABASE_URL "postgres://fly-user:password@localhost:16380"]
+            :prod #or [#env PROD_DATABASE_URL "postgres://fly-user:password@localhost:16380"]}
  :email {:address #or [#env SUPPORT_ADDRESS "lee@wearesource.earth"]
          :username #env EMAIL_USERNAME
          :password #env EMAIL_PASSWORD}
diff --git a/src/source/db/util.clj b/src/source/db/util.clj
index e66042f2..1651b2e8 100644
--- a/src/source/db/util.clj
+++ b/src/source/db/util.clj
@@ -39,9 +39,23 @@
 (defn tnames [tnames id]
   (mapv #(:tname (tname % id)) tnames))
 
+(defn conn-env [env]
+  {:connection-uri (str (conf/read-value :database env) "/master")})
+
+(defmacro with-env [args & body]
+  `(let [~(first args) (conn-env ~(last args))]
+     ~(cons 'do body)))
+
 (comment
   (def q "SELECT * FROM events")
 
+  (macroexpand '(with-env [ds :staging] (println ds)))
+
+  (with-env [ds :staging]
+    (time (pg/with-conn [conn ds]
+            (pg/query conn q)))
+    #_(hon/find ds {:tname :users}))
+
   (time (pg/with-conn [conn {:connection-uri "postgresql://postgres:postgres@localhost:5432/master?ssl=false"}]
           (pg/query conn q)))
 

From f37dc7327a65cf50d5750ab6d0ef4a405d75f7dd Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Fri, 20 Mar 2026 11:00:47 +0200
Subject: [PATCH 369/391] changed the default conn strings to be more generic

---
 resources/dev_config.edn | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/resources/dev_config.edn b/resources/dev_config.edn
index 81ef3583..90a137b1 100644
--- a/resources/dev_config.edn
+++ b/resources/dev_config.edn
@@ -8,8 +8,8 @@
  :database {:url #or [#env DATABASE_URL "postgres://localhost"]
             :type "postgresql"
             :dev #or [#env DEV_DATABASE_URL "postgres://localhost:5432"]
-            :staging #or [#env STAGING_DATABASE_URL "postgres://fly-user:password@localhost:16380"]
-            :prod #or [#env PROD_DATABASE_URL "postgres://fly-user:password@localhost:16380"]}
+            :staging #or [#env STAGING_DATABASE_URL "postgres://username:password@localhost:16380"]
+            :prod #or [#env PROD_DATABASE_URL "postgres://username:password@localhost:16380"]}
  :email {:address #or [#env SUPPORT_ADDRESS "lee@wearesource.earth"]
          :username #env EMAIL_USERNAME
          :password #env EMAIL_PASSWORD}

From f5917bbdb8951db325bdb17aafa465864d8cf4a8 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Fri, 20 Mar 2026 11:07:11 +0200
Subject: [PATCH 370/391] added docstrings for util functions

---
 resources/dev_config.edn | 1 -
 src/source/db/util.clj   | 9 +++++++--
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/resources/dev_config.edn b/resources/dev_config.edn
index 90a137b1..5f2aed99 100644
--- a/resources/dev_config.edn
+++ b/resources/dev_config.edn
@@ -7,7 +7,6 @@
  :env #or [#env ENV "dev"]
  :database {:url #or [#env DATABASE_URL "postgres://localhost"]
             :type "postgresql"
-            :dev #or [#env DEV_DATABASE_URL "postgres://localhost:5432"]
             :staging #or [#env STAGING_DATABASE_URL "postgres://username:password@localhost:16380"]
             :prod #or [#env PROD_DATABASE_URL "postgres://username:password@localhost:16380"]}
  :email {:address #or [#env SUPPORT_ADDRESS "lee@wearesource.earth"]
diff --git a/src/source/db/util.clj b/src/source/db/util.clj
index 1651b2e8..2317ae29 100644
--- a/src/source/db/util.clj
+++ b/src/source/db/util.clj
@@ -39,10 +39,15 @@
 (defn tnames [tnames id]
   (mapv #(:tname (tname % id)) tnames))
 
-(defn conn-env [env]
+(defn conn-env 
+  "Creates a connection to the master database for the given environment. There must be a connection string in config for the given environment."
+  [env]
   {:connection-uri (str (conf/read-value :database env) "/master")})
 
-(defmacro with-env [args & body]
+(defmacro with-env 
+  "This macro creates a let binding structure associating a custom binding with a database connection based with the given environment as a keyword.
+  e.g. (with-env [ds :staging] (hon/find ds {:tname :users}))"
+  [args & body]
   `(let [~(first args) (conn-env ~(last args))]
      ~(cons 'do body)))
 

From 87569ddad1a2bbf69f89f32852978a8dfb289910 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Tue, 24 Mar 2026 10:30:22 +0200
Subject: [PATCH 371/391] added migration for prod categories seeding

---
 .../migrations/017_category_seeding.clj       | 52 +++++++++++++++++++
 1 file changed, 52 insertions(+)
 create mode 100644 src/source/migrations/017_category_seeding.clj

diff --git a/src/source/migrations/017_category_seeding.clj b/src/source/migrations/017_category_seeding.clj
new file mode 100644
index 00000000..5ad9ded3
--- /dev/null
+++ b/src/source/migrations/017_category_seeding.clj
@@ -0,0 +1,52 @@
+(ns source.migrations.017-category-seeding
+  (:require [source.db.master]
+            [source.db.honey :as hon]
+            [source.workers.categories :as categories]))
+
+(defn run-up! [context]
+  (let [ds-master (:db-master context)]
+    (run!
+     #(categories/delete-category! ds-master (:id %))
+     (hon/find ds-master {:tname :categories}))
+
+    (hon/insert! ds-master {:tname :categories
+                            :data [{:name "Regenerative finance and investment"}
+                                   {:name "Regenerative business"}
+                                   {:name "Sustainable fashion and textiles"}
+                                   {:name "Regenerative agriculture and food"}
+                                   {:name "Energy and clean technology"}
+                                   {:name "Circularity and waste"}
+                                   {:name "Built environment"}
+                                   {:name "Transport and mobility"}
+                                   {:name "Travel and regenerative tourism"}
+                                   {:name "Technology and AI for good"}
+                                   {:name "Wellbeing and mental health"}
+                                   {:name "Social justice and equity"}
+                                   {:name "Indigenous wisdom and land rights"}
+                                   {:name "Leadership and inner development"}
+                                   {:name "Policy and systems change"}
+                                   {:name "Water and freshwater systems"}
+                                   {:name "Biodiversity and conservation"}
+                                   {:name "Ocean and marine"}
+                                   {:name "Bioregionalism and local economies"}
+                                   {:name "Community and culture"}
+                                   {:name "Arts and artivism"}
+                                   {:name "Education and climate literacy"}
+                                   {:name "Media and communications"}
+                                   {:name "Nonprofits and philanthropy"}
+                                   {:name "Consumer goods and conscious living"}
+                                   {:name "Spirituality and consciousness"}
+                                   {:name "Gender and women's leadership"}
+                                   {:name "Health and planetary health"}]})))
+
+(defn run-down! [context]
+  (let [ds-master (:db-master context)]
+    (run!
+     #(categories/delete-category! ds-master (:id %))
+     (hon/find ds-master {:tname :categories}))
+
+    (hon/insert! ds-master {:tname :categories
+                            :data [{:name "programming"}
+                                   {:name "game development"}
+                                   {:name "languages"}
+                                   {:name "technology"}]})))

From be36d2457b8a551acfd44adb17244a3c3da8d263 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Thu, 26 Mar 2026 14:35:46 +0200
Subject: [PATCH 372/391] added migration to replace sectors list with those
 from categories

---
 src/source/migrations/018_sector_seeding.clj | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)
 create mode 100644 src/source/migrations/018_sector_seeding.clj

diff --git a/src/source/migrations/018_sector_seeding.clj b/src/source/migrations/018_sector_seeding.clj
new file mode 100644
index 00000000..a27a6579
--- /dev/null
+++ b/src/source/migrations/018_sector_seeding.clj
@@ -0,0 +1,19 @@
+(ns source.migrations.018-sector-seeding
+  (:require [source.db.master]
+            [source.db.honey :as hon]))
+
+(defn run-up! [context]
+  (let [ds-master (:db-master context)
+        sectors (->> (hon/find ds-master {:tname :categories})
+                     (mapv #(dissoc % :display-picture)))]
+    (hon/delete! ds-master {:tname :sectors})
+    (hon/insert! ds-master {:tname :sectors
+                            :data sectors})))
+
+(defn run-down! [context]
+  (let [ds-master (:db-master context)]
+    (hon/delete! ds-master {:tname :sectors})
+    (hon/insert! ds-master {:tname :sectors
+                            :data [{:name "renewable energy"}
+                                   {:name "conservation ecology"}
+                                   {:name "recycling"}]})))

From 215c5b00e5686781d345d60c8412c72b9ce8e103 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Thu, 26 Mar 2026 15:28:42 +0200
Subject: [PATCH 373/391] added env to fly

---
 fly.dev.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fly.dev.toml b/fly.dev.toml
index 87ffeb89..d1d3daf1 100644
--- a/fly.dev.toml
+++ b/fly.dev.toml
@@ -13,7 +13,7 @@ primary_region = 'fra'
   BASE_URL = 'https://source-be-staging.fly.dev'
   DATABASE_DIR = '/data'
   EMAIL_USERNAME = 'merveillevaneck@gmail.com'
-  ENV = 'prod'
+  ENV = 'staging'
   GOOGLE_CLIENT_ID = '449412212863-h1rre1gdmfgq96jt160fdd30bfatjiqb.apps.googleusercontent.com'
   GOOGLE_REDIRECT_URI = 'https://source-be-staging.fly.dev/oauth2/google/callback'
   SUPPORT_ADDRESS = 'keaganncollins@gmail.com'

From 9ca4f1120cc2bc28a0e1f69327e1371d0c1a9e59 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Thu, 26 Mar 2026 15:37:34 +0200
Subject: [PATCH 374/391] fixed fly java path

---
 fly.dev.toml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/fly.dev.toml b/fly.dev.toml
index d1d3daf1..2ef46432 100644
--- a/fly.dev.toml
+++ b/fly.dev.toml
@@ -14,6 +14,7 @@ primary_region = 'fra'
   DATABASE_DIR = '/data'
   EMAIL_USERNAME = 'merveillevaneck@gmail.com'
   ENV = 'staging'
+  JAVA_CMD = 'java'
   GOOGLE_CLIENT_ID = '449412212863-h1rre1gdmfgq96jt160fdd30bfatjiqb.apps.googleusercontent.com'
   GOOGLE_REDIRECT_URI = 'https://source-be-staging.fly.dev/oauth2/google/callback'
   SUPPORT_ADDRESS = 'keaganncollins@gmail.com'

From 38b278446ef1ff843b7fa25270bd4ab4d3f1546c Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Fri, 27 Mar 2026 13:36:00 +0200
Subject: [PATCH 375/391] fixed endpoint schema for updating provider

---
 src/source/routes/provider.clj | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/source/routes/provider.clj b/src/source/routes/provider.clj
index c228bc02..47894287 100644
--- a/src/source/routes/provider.clj
+++ b/src/source/routes/provider.clj
@@ -27,7 +27,7 @@
                 :body [:map
                        [:name :string]
                        [:domain {:optional true} [:maybe :string]]
-                       [:content-type-id :int]
+                       [:content-type-id {:optional true} [:maybe :int]]
                        [:instructions {:optional true} [:maybe :string]]
                        [:placeholder-url {:optional true} [:maybe :string]]]}
    :responses {200 {:body [:map [:message :string]]}}}

From 1ee22ed750a5f2b5d174255b13e432725ddf54a6 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Fri, 27 Mar 2026 13:51:10 +0200
Subject: [PATCH 376/391] delete user sectors before deleting sectors

---
 src/source/migrations/018_sector_seeding.clj | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/source/migrations/018_sector_seeding.clj b/src/source/migrations/018_sector_seeding.clj
index a27a6579..29e27d46 100644
--- a/src/source/migrations/018_sector_seeding.clj
+++ b/src/source/migrations/018_sector_seeding.clj
@@ -6,12 +6,14 @@
   (let [ds-master (:db-master context)
         sectors (->> (hon/find ds-master {:tname :categories})
                      (mapv #(dissoc % :display-picture)))]
+    (hon/delete! ds-master {:tname :user-sectors})
     (hon/delete! ds-master {:tname :sectors})
     (hon/insert! ds-master {:tname :sectors
                             :data sectors})))
 
 (defn run-down! [context]
   (let [ds-master (:db-master context)]
+    (hon/delete! ds-master {:tname :user-sectors})
     (hon/delete! ds-master {:tname :sectors})
     (hon/insert! ds-master {:tname :sectors
                             :data [{:name "renewable energy"}

From 2cff5f93c921d9d375899eb64aba5e08cf0e8afa Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Fri, 27 Mar 2026 14:55:51 +0200
Subject: [PATCH 377/391] added github actions workflow for deploying to prod
 on merge to main

---
 .github/workflows/fly-deploy.yml  | 18 ------------------
 .github/workflows/prod-deploy.yml | 23 +++++++++++++++++++++++
 2 files changed, 23 insertions(+), 18 deletions(-)
 delete mode 100644 .github/workflows/fly-deploy.yml
 create mode 100644 .github/workflows/prod-deploy.yml

diff --git a/.github/workflows/fly-deploy.yml b/.github/workflows/fly-deploy.yml
deleted file mode 100644
index b0c246ed..00000000
--- a/.github/workflows/fly-deploy.yml
+++ /dev/null
@@ -1,18 +0,0 @@
-# See https://fly.io/docs/app-guides/continuous-deployment-with-github-actions/
-
-name: Fly Deploy
-on:
-  push:
-    branches:
-      - main
-jobs:
-  deploy:
-    name: Deploy app
-    runs-on: ubuntu-latest
-    concurrency: deploy-group    # optional: ensure only one action runs at a time
-    steps:
-      - uses: actions/checkout@v4
-      - uses: superfly/flyctl-actions/setup-flyctl@master
-      - run: flyctl deploy --remote-only
-        env:
-          FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
diff --git a/.github/workflows/prod-deploy.yml b/.github/workflows/prod-deploy.yml
new file mode 100644
index 00000000..88a1af92
--- /dev/null
+++ b/.github/workflows/prod-deploy.yml
@@ -0,0 +1,23 @@
+name: Prod Deploy
+on:
+  push:
+    branches:
+      - main
+jobs:
+  deploy:
+    name: Deploy to Prod
+    runs-on: ubuntu-latest
+    concurrency: deploy-group    # optional: ensure only one action runs at a time
+    steps:
+      - name: Execute remote SSH commands using password
+        uses: appleboy/ssh-action@v1
+        with:
+          host: ${{ secrets.HOST }}
+          username: ${{ secrets.USERNAME }}
+          password: ${{ secrets.PASSWORD }}
+          script: |
+            cd /home/keagan/source-be
+            git checkout main
+            git pull
+            ./build.sh
+            sudo /bin/systemctl restart source-be

From fae1dea461fd96ae093ed89fb208061b5e21af6b Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 30 Mar 2026 09:55:08 +0200
Subject: [PATCH 378/391] updated path in workflow

---
 .github/workflows/prod-deploy.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/prod-deploy.yml b/.github/workflows/prod-deploy.yml
index 88a1af92..f282b89b 100644
--- a/.github/workflows/prod-deploy.yml
+++ b/.github/workflows/prod-deploy.yml
@@ -16,7 +16,7 @@ jobs:
           username: ${{ secrets.USERNAME }}
           password: ${{ secrets.PASSWORD }}
           script: |
-            cd /home/keagan/source-be
+            cd /home/deploy/source-be
             git checkout main
             git pull
             ./build.sh

From e957a13f49f2fc96d784fe9bb9882b151358b4ec Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 30 Mar 2026 13:31:17 +0200
Subject: [PATCH 379/391] added github workflow for deploying to tailscale
 staging server

---
 .github/workflows/staging-deploy.yml | 26 ++++++++++++++++++++++++++
 .gitignore                           |  1 +
 2 files changed, 27 insertions(+)
 create mode 100644 .github/workflows/staging-deploy.yml

diff --git a/.github/workflows/staging-deploy.yml b/.github/workflows/staging-deploy.yml
new file mode 100644
index 00000000..26e76e43
--- /dev/null
+++ b/.github/workflows/staging-deploy.yml
@@ -0,0 +1,26 @@
+name: Staging Deply
+on:
+  push:
+    branches:
+      - dev
+jobs:
+  deploy:
+    name: Deploy to Staging
+    runs-on: ubuntu-latest
+    concurrency: deploy-group    # optional: ensure only one action runs at a time
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Connect to Tailscale
+        uses: tailscale/github-action@v4
+        with:
+          oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID }}
+          oauth-secret: ${{ secrets.TS_OAUTH_SECRET }}
+          tags: tag:ci
+
+      - name: Deploy via SSH
+        run: |
+          ssh -o StrictHostKeyChecking=no merv@mervstation.tail4f070.ts.net
+          cd /home/merv/Developer/source-be
+          git pull
+          systemctl restart source-be
diff --git a/.gitignore b/.gitignore
index 576bdd4f..11809f10 100644
--- a/.gitignore
+++ b/.gitignore
@@ -8,6 +8,7 @@ source-users
 dev/dev.clj
 .nrepl-port
 target/
+resources/admins_encrypted.json
 
 .db
 admins.json

From c7db4bed7c9080b873e68e8bff161e8c8ba371d5 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 30 Mar 2026 13:34:28 +0200
Subject: [PATCH 380/391] updated systemctl command to use sudo

---
 .github/workflows/staging-deploy.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/staging-deploy.yml b/.github/workflows/staging-deploy.yml
index 26e76e43..e554e80e 100644
--- a/.github/workflows/staging-deploy.yml
+++ b/.github/workflows/staging-deploy.yml
@@ -23,4 +23,4 @@ jobs:
           ssh -o StrictHostKeyChecking=no merv@mervstation.tail4f070.ts.net
           cd /home/merv/Developer/source-be
           git pull
-          systemctl restart source-be
+          sudo /bin/systemctl restart source-be

From 49e44f19ffc4e797acd6cd4ceb0da4fdbb214dd5 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Mon, 30 Mar 2026 14:53:47 +0200
Subject: [PATCH 381/391] fixed typo

---
 .github/workflows/staging-deploy.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/staging-deploy.yml b/.github/workflows/staging-deploy.yml
index e554e80e..eee1ce02 100644
--- a/.github/workflows/staging-deploy.yml
+++ b/.github/workflows/staging-deploy.yml
@@ -1,4 +1,4 @@
-name: Staging Deply
+name: Staging Deploy
 on:
   push:
     branches:

From 5e4554dda3e0f77baaa850c83bdad30fd576a359 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Tue, 31 Mar 2026 12:44:44 +0200
Subject: [PATCH 382/391] updated tailscale workflow to use authkey instead of
 oauth2

---
 .github/workflows/staging-deploy.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/staging-deploy.yml b/.github/workflows/staging-deploy.yml
index eee1ce02..1170aa1e 100644
--- a/.github/workflows/staging-deploy.yml
+++ b/.github/workflows/staging-deploy.yml
@@ -14,8 +14,7 @@ jobs:
       - name: Connect to Tailscale
         uses: tailscale/github-action@v4
         with:
-          oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID }}
-          oauth-secret: ${{ secrets.TS_OAUTH_SECRET }}
+          authKey: ${{ secrets.TS_AUTH_KEY }}
           tags: tag:ci
 
       - name: Deploy via SSH

From 1a2bb9f3a58c53fb2ff47e3cec6a4c007621186b Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Tue, 31 Mar 2026 13:00:09 +0200
Subject: [PATCH 383/391] updated tailscale workflow to specify statedir
 according to docs

---
 .github/workflows/fly.staging.yml    | 20 --------------------
 .github/workflows/staging-deploy.yml |  2 +-
 2 files changed, 1 insertion(+), 21 deletions(-)
 delete mode 100644 .github/workflows/fly.staging.yml

diff --git a/.github/workflows/fly.staging.yml b/.github/workflows/fly.staging.yml
deleted file mode 100644
index ef72aea5..00000000
--- a/.github/workflows/fly.staging.yml
+++ /dev/null
@@ -1,20 +0,0 @@
-# See https://fly.io/docs/app-guides/continuous-deployment-with-github-actions/
-
-name: Fly Staging Deploy
-on:
-  push:
-    branches:
-      - dev
-jobs:
-  deploy:
-    name: Deploy app
-    runs-on: ubuntu-latest
-    concurrency: deploy-group    # optional: ensure only one action runs at a time
-    steps:
-      - uses: actions/checkout@v4
-      - uses: superfly/flyctl-actions/setup-flyctl@master
-      - run: flyctl deploy --remote-only --config fly.dev.toml
-        env:
-          FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
-          SUPER_SECRET_KEY: ${{ secrets.SUPER_SECRET_KEY }}
-          GOOGLE_CLIENT_SECRET: ${{ secrets.GOOGLE_CLIENT_SECRET }}
diff --git a/.github/workflows/staging-deploy.yml b/.github/workflows/staging-deploy.yml
index 1170aa1e..e9696274 100644
--- a/.github/workflows/staging-deploy.yml
+++ b/.github/workflows/staging-deploy.yml
@@ -15,7 +15,7 @@ jobs:
         uses: tailscale/github-action@v4
         with:
           authKey: ${{ secrets.TS_AUTH_KEY }}
-          tags: tag:ci
+          statedir: /tmp/tailscale-state/
 
       - name: Deploy via SSH
         run: |

From 34fc5d99907de8819fca668a834d42e0390a6765 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Tue, 31 Mar 2026 13:21:13 +0200
Subject: [PATCH 384/391] updated staging workflow with tailscale

---
 .github/workflows/staging-deploy.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/staging-deploy.yml b/.github/workflows/staging-deploy.yml
index e9696274..2024bbfe 100644
--- a/.github/workflows/staging-deploy.yml
+++ b/.github/workflows/staging-deploy.yml
@@ -14,12 +14,12 @@ jobs:
       - name: Connect to Tailscale
         uses: tailscale/github-action@v4
         with:
-          authKey: ${{ secrets.TS_AUTH_KEY }}
-          statedir: /tmp/tailscale-state/
+          authkey: ${{ secrets.TS_AUTH_KEY }}
+          ping: mervstation.tail4f070.ts.net
 
       - name: Deploy via SSH
         run: |
-          ssh -o StrictHostKeyChecking=no merv@mervstation.tail4f070.ts.net
+          tailscale ssh merv@mervstation.tail4f070.ts.net
           cd /home/merv/Developer/source-be
           git pull
           sudo /bin/systemctl restart source-be

From 83595c525524abc7f58135074df5d0c64684e0c2 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 1 Apr 2026 09:22:49 +0200
Subject: [PATCH 385/391] attempt 3 using oauth to fix staging workflow

---
 .github/workflows/staging-deploy.yml | 23 ++++++++++++++---------
 1 file changed, 14 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/staging-deploy.yml b/.github/workflows/staging-deploy.yml
index e9696274..b2f5625e 100644
--- a/.github/workflows/staging-deploy.yml
+++ b/.github/workflows/staging-deploy.yml
@@ -7,19 +7,24 @@ jobs:
   deploy:
     name: Deploy to Staging
     runs-on: ubuntu-latest
-    concurrency: deploy-group    # optional: ensure only one action runs at a time
+    concurrency: deploy-group
     steps:
       - uses: actions/checkout@v4
 
       - name: Connect to Tailscale
         uses: tailscale/github-action@v4
         with:
-          authKey: ${{ secrets.TS_AUTH_KEY }}
-          statedir: /tmp/tailscale-state/
+          oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID }}
+          oauth-secret: ${{ secrets.TS_OAUTH_SECRET }}
+          tags: tag:ci
+          ping: mervstation.tail4f070.ts.net
 
-      - name: Deploy via SSH
-        run: |
-          ssh -o StrictHostKeyChecking=no merv@mervstation.tail4f070.ts.net
-          cd /home/merv/Developer/source-be
-          git pull
-          sudo /bin/systemctl restart source-be
+      - name: Execute remote SSH commands
+        uses: appleboy/ssh-action@v1
+        with:
+          host: mervstation.tail4f070.ts.net
+          username: merv
+          script: |
+            cd /home/merv/Developer/source-be
+            git pull
+            sudo /bin/systemctl restart source-be

From 81e4069e4c9dd20f6ac7ae2ef8a797c21f5bdf59 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 1 Apr 2026 09:32:09 +0200
Subject: [PATCH 386/391] attempt 4 fixing staging workflow by providing
 password

---
 .github/workflows/staging-deploy.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/staging-deploy.yml b/.github/workflows/staging-deploy.yml
index b2f5625e..a8d15368 100644
--- a/.github/workflows/staging-deploy.yml
+++ b/.github/workflows/staging-deploy.yml
@@ -24,6 +24,7 @@ jobs:
         with:
           host: mervstation.tail4f070.ts.net
           username: merv
+          password: tailscale
           script: |
             cd /home/merv/Developer/source-be
             git pull

From ef9a718d9a61ddb3b4f4ee266381dd3946bb29e5 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 1 Apr 2026 09:56:30 +0200
Subject: [PATCH 387/391] remove appleboy run ssh manually instead

---
 .github/workflows/staging-deploy.yml | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/staging-deploy.yml b/.github/workflows/staging-deploy.yml
index a8d15368..8d44c82d 100644
--- a/.github/workflows/staging-deploy.yml
+++ b/.github/workflows/staging-deploy.yml
@@ -20,12 +20,9 @@ jobs:
           ping: mervstation.tail4f070.ts.net
 
       - name: Execute remote SSH commands
-        uses: appleboy/ssh-action@v1
-        with:
-          host: mervstation.tail4f070.ts.net
-          username: merv
-          password: tailscale
-          script: |
+        run: |
+          ssh -o StrictHostKeyChecking=no merv@mervstation.tail4f070.ts.net '
             cd /home/merv/Developer/source-be
             git pull
             sudo /bin/systemctl restart source-be
+          '

From 67fc17fcb931926fd31bb2c26143e55149fb35b7 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Tue, 7 Apr 2026 13:11:58 +0200
Subject: [PATCH 388/391] fixed category seeding to keep old categories

---
 .../migrations/017_category_seeding.clj       | 40 ++++++++++++++-----
 1 file changed, 29 insertions(+), 11 deletions(-)

diff --git a/src/source/migrations/017_category_seeding.clj b/src/source/migrations/017_category_seeding.clj
index 5ad9ded3..539b453c 100644
--- a/src/source/migrations/017_category_seeding.clj
+++ b/src/source/migrations/017_category_seeding.clj
@@ -5,10 +5,6 @@
 
 (defn run-up! [context]
   (let [ds-master (:db-master context)]
-    (run!
-     #(categories/delete-category! ds-master (:id %))
-     (hon/find ds-master {:tname :categories}))
-
     (hon/insert! ds-master {:tname :categories
                             :data [{:name "Regenerative finance and investment"}
                                    {:name "Regenerative business"}
@@ -43,10 +39,32 @@
   (let [ds-master (:db-master context)]
     (run!
      #(categories/delete-category! ds-master (:id %))
-     (hon/find ds-master {:tname :categories}))
-
-    (hon/insert! ds-master {:tname :categories
-                            :data [{:name "programming"}
-                                   {:name "game development"}
-                                   {:name "languages"}
-                                   {:name "technology"}]})))
+     (hon/find ds-master {:tname :categories
+                          :where [:in :name ["Regenerative finance and investment"
+                                             "Regenerative business"
+                                             "Sustainable fashion and textiles"
+                                             "Regenerative agriculture and food"
+                                             "Energy and clean technology"
+                                             "Circularity and waste"
+                                             "Built environment"
+                                             "Transport and mobility"
+                                             "Travel and regenerative tourism"
+                                             "Technology and AI for good"
+                                             "Wellbeing and mental health"
+                                             "Social justice and equity"
+                                             "Indigenous wisdom and land rights"
+                                             "Leadership and inner development"
+                                             "Policy and systems change"
+                                             "Water and freshwater systems"
+                                             "Biodiversity and conservation"
+                                             "Ocean and marine"
+                                             "Bioregionalism and local economies"
+                                             "Community and culture"
+                                             "Arts and artivism"
+                                             "Education and climate literacy"
+                                             "Media and communications"
+                                             "Nonprofits and philanthropy"
+                                             "Consumer goods and conscious living"
+                                             "Spirituality and consciousness"
+                                             "Gender and women's leadership"
+                                             "Health and planetary health"]]}))))

From 46e523a2acad4123d949641e1ae4f2979466b3a2 Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Fri, 10 Apr 2026 11:46:25 +0200
Subject: [PATCH 389/391] concatenated ellipsis to truncated descriptions

---
 src/source/workers/bundles.clj | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/source/workers/bundles.clj b/src/source/workers/bundles.clj
index 0be883ec..9ec696cf 100644
--- a/src/source/workers/bundles.clj
+++ b/src/source/workers/bundles.clj
@@ -49,7 +49,7 @@
    :p.content-type-id
    :p.title
    :p.thumbnail
-   (if (= truncate "false") :p.info [[:left :p.info 100] :info])
+   (if (= truncate "false") :p.info [[:|| [:left :p.info 100] "..."] :info])
    :p.url
    :p.stream-url
    :p.season

From 9f12742a53114795491199ea959a1e3aba69fae4 Mon Sep 17 00:00:00 2001
From: Keagan <51060273+toasted226@users.noreply.github.com>
Date: Fri, 10 Apr 2026 12:10:06 +0200
Subject: [PATCH 390/391] Execute build script in staging deploy workflow

Add build script execution before restarting service.
---
 .github/workflows/staging-deploy.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/staging-deploy.yml b/.github/workflows/staging-deploy.yml
index 8d44c82d..8804c3d1 100644
--- a/.github/workflows/staging-deploy.yml
+++ b/.github/workflows/staging-deploy.yml
@@ -24,5 +24,6 @@ jobs:
           ssh -o StrictHostKeyChecking=no merv@mervstation.tail4f070.ts.net '
             cd /home/merv/Developer/source-be
             git pull
+            ./build.sh
             sudo /bin/systemctl restart source-be
           '

From 211e92dc8360011cfa478a06bb8c1e06bb6c8bfd Mon Sep 17 00:00:00 2001
From: Keagan <keaganncollins@gmail.com>
Date: Wed, 15 Apr 2026 09:58:40 +0200
Subject: [PATCH 391/391] send verification email after google registration

---
 src/source/routes/google_user.clj | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/src/source/routes/google_user.clj b/src/source/routes/google_user.clj
index 79e25506..e24c3112 100644
--- a/src/source/routes/google_user.clj
+++ b/src/source/routes/google_user.clj
@@ -2,7 +2,10 @@
   (:require [source.oauth2.google.interface :as google]
             [source.middleware.auth.core :as auth]
             [ring.util.response :as res]
-            [source.db.honey :as hon]))
+            [source.db.honey :as hon]
+            [source.password :as pw]
+            [source.email.templates :as templates]
+            [source.email.gmail :as gmail]))
 
 (defn get
   {:summary "completes the google oauth2 flow and returns the authenticated user"
@@ -44,4 +47,8 @@
                                          :where [:= :email email]})
               payload (dissoc new-user :password)
               session (auth/create-session payload)]
+          (gmail/send-email {:to email
+                             :subject "Source - Verify your email"
+                             :body (templates/email-verification {:email-hash (pw/hash-password email)})
+                             :type :text/html})
           (res/response (merge {:user payload} session)))))))