diff --git a/.github/workflows/fly-deploy.yml b/.github/workflows/fly-deploy.yml deleted file mode 100644 index b0c246ed..00000000 --- a/.github/workflows/fly-deploy.yml +++ /dev/null @@ -1,18 +0,0 @@ -# See https://fly.io/docs/app-guides/continuous-deployment-with-github-actions/ - -name: Fly Deploy -on: - push: - branches: - - main -jobs: - deploy: - name: Deploy app - runs-on: ubuntu-latest - concurrency: deploy-group # optional: ensure only one action runs at a time - steps: - - uses: actions/checkout@v4 - - uses: superfly/flyctl-actions/setup-flyctl@master - - run: flyctl deploy --remote-only - env: - FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }} diff --git a/.github/workflows/fly.staging.yml b/.github/workflows/fly.staging.yml deleted file mode 100644 index ef72aea5..00000000 --- a/.github/workflows/fly.staging.yml +++ /dev/null @@ -1,20 +0,0 @@ -# See https://fly.io/docs/app-guides/continuous-deployment-with-github-actions/ - -name: Fly Staging Deploy -on: - push: - branches: - - dev -jobs: - deploy: - name: Deploy app - runs-on: ubuntu-latest - concurrency: deploy-group # optional: ensure only one action runs at a time - steps: - - uses: actions/checkout@v4 - - uses: superfly/flyctl-actions/setup-flyctl@master - - run: flyctl deploy --remote-only --config fly.dev.toml - env: - FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }} - SUPER_SECRET_KEY: ${{ secrets.SUPER_SECRET_KEY }} - GOOGLE_CLIENT_SECRET: ${{ secrets.GOOGLE_CLIENT_SECRET }} diff --git a/.github/workflows/prod-deploy.yml b/.github/workflows/prod-deploy.yml new file mode 100644 index 00000000..f282b89b --- /dev/null +++ b/.github/workflows/prod-deploy.yml @@ -0,0 +1,23 @@ +name: Prod Deploy +on: + push: + branches: + - main +jobs: + deploy: + name: Deploy to Prod + runs-on: ubuntu-latest + concurrency: deploy-group # optional: ensure only one action runs at a time + steps: + - name: Execute remote SSH commands using password + uses: appleboy/ssh-action@v1 + with: + host: ${{ secrets.HOST }} + username: ${{ secrets.USERNAME }} + password: ${{ secrets.PASSWORD }} + script: | + cd /home/deploy/source-be + git checkout main + git pull + ./build.sh + sudo /bin/systemctl restart source-be diff --git a/.github/workflows/staging-deploy.yml b/.github/workflows/staging-deploy.yml new file mode 100644 index 00000000..8804c3d1 --- /dev/null +++ b/.github/workflows/staging-deploy.yml @@ -0,0 +1,29 @@ +name: Staging Deploy +on: + push: + branches: + - dev +jobs: + deploy: + name: Deploy to Staging + runs-on: ubuntu-latest + concurrency: deploy-group + steps: + - uses: actions/checkout@v4 + + - name: Connect to Tailscale + uses: tailscale/github-action@v4 + with: + oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID }} + oauth-secret: ${{ secrets.TS_OAUTH_SECRET }} + tags: tag:ci + ping: mervstation.tail4f070.ts.net + + - name: Execute remote SSH commands + run: | + ssh -o StrictHostKeyChecking=no merv@mervstation.tail4f070.ts.net ' + cd /home/merv/Developer/source-be + git pull + ./build.sh + sudo /bin/systemctl restart source-be + ' diff --git a/.gitignore b/.gitignore index 576bdd4f..11809f10 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ source-users dev/dev.clj .nrepl-port target/ +resources/admins_encrypted.json .db admins.json diff --git a/.java-version b/.java-version new file mode 100644 index 00000000..aabe6ec3 --- /dev/null +++ b/.java-version @@ -0,0 +1 @@ +21 diff --git a/README.md b/README.md index 6c1f189d..3045530c 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,10 @@ This is the backend for the Source platform. You can find documentation on setup for development below. +## Dependencies + +- >= openjdk version 16. + ## Development setup - Pull the project from GitHub. @@ -9,8 +13,11 @@ This is the backend for the Source platform. You can find documentation on setup ```.env SUPER_SECRET_KEY [string, minimum 32 characters] +EMAIL_USERNAME [string, email username used for sending emails] +EMAIL_PASSWORD [string, email password used for sending emails] GOOGLE_CLIENT_ID [string, from Google Console] GOOGLE_CLIENT_SECRET [string, from Google Console] +DATABASE_URL [string, connection URL to Postgres instance, excluding database name and trailing forward slash] ``` - Run the provided shell script with `./nrepl.sh` to start your nrepl server. For this, you will need the required nrepl alias in `.clojure/deps.edn`. @@ -19,9 +26,9 @@ When you have made changes, you can restart the server by evaluating `server/res ## Database Migration -Database migration is handled using the provided migrate.sh shell script. -You should have a db directory and ensure your configuration is pointing to it. The default is `.db/` in the root, -this can be configured with the `DATABASE_DIR` variable in the environment. +Database migration is handled using the provided `migrate.sh` shell script. +You need to have a Postgresql connection string in order to run migrations, +this can be configured with the `DATABASE_URL` variable in the environment. The migration system makes use of the kepler/mallard library, take a look at their [docs](https://github.com/kepler16/mallard) to find out more on how it works. The datasource you will be operating on is passed in as context to the run-up and run-down functions. @@ -56,3 +63,38 @@ We are making use of cognitect-labs/test-runner for running our tests. Run the p Unit tests are written and placed in clj files in `tests/source-be/`. These files should have the `_test` postfix (e.g. google_auth_test.clj) and tests should be written per namespace and cover all important functions. +## Production Setup + +- Pull the project from GitHub and navigate to its directory. +- Ensure you have Java installed, minimum openjdk version 16. +- Create a .env file containing the following required information: + +```.env +# These values are secrets, ensure they are not exposed anywhere else on your system +SUPER_SECRET_KEY [string, minimum 32 characters] +EMAIL_USERNAME [string, email username used for sending emails] +EMAIL_PASSWORD [string, email password used for sending emails] +GOOGLE_CLIENT_ID [string, from Google Console] +GOOGLE_CLIENT_SECRET [string, from Google Console] +DATABASE_URL [string, connection URL to Postgres instance, excluding database name and trailing forward slash] + +# These values are required for config and compiling the project +ENV [string, "dev" | "staging" | "prod"] # This should match your config file, e.g. if ENV is "staging", your config file should be called staging_config.edn +JAVA_CMD [string, path pointing to your Java executable] +``` + +- If there isn't already a `config.edn` file for your target environment in the `resources/` directory, create a config file using the given `resources/dev_config.edn` +as a template as shown below: +```bash +cp resources/config.edn resources/{env_name}_config.edn +``` +where `{env_name}` is the value of the `ENV` variable set in your `.env`. +You can configure everything for your machine inside your config.edn file, alternatively, you can configure everything marked with `#env` from `{env_name}_config.edn` +within your `.env` file. + +- If everything before this point is set up correctly, you are ready to begin the next steps. + +- Run `./build.sh` to the compile the project. +- Run `./start.sh` to start the server. The server will use the config according to the specified environment in `.env` and will run migrations before starting. + +The logs will be displayed when the server is run. If you are running the server via a systemd service, you can find them by running `journalctl -u {servicename}.service`. diff --git a/Source.postman_collection.json b/Source.postman_collection.json new file mode 100644 index 00000000..623aa8c9 --- /dev/null +++ b/Source.postman_collection.json @@ -0,0 +1,1381 @@ +{ + "info": { + "_postman_id": "5858dc0c-cdd1-4c48-a7b8-5e7bb0519be5", + "name": "Source", + "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json", + "_exporter_id": "23875094" + }, + "item": [ + { + "name": "Home", + "request": { + "auth": { + "type": "noauth" + }, + "method": "GET", + "header": [], + "url": { + "raw": "http://localhost:3000/", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "" + ] + } + }, + "response": [] + }, + { + "name": "Get Users", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "http://localhost:3000/users", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "users" + ] + } + }, + "response": [] + }, + { + "name": "Register", + "request": { + "auth": { + "type": "noauth" + }, + "method": "POST", + "header": [ + { + "key": "", + "value": "yeet", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"email\": \"toast@toast.com\",\n \"password\": \"Pa55w.rd\",\n \"confirm-password\": \"Pa55w.rd\",\n \"firstname\": \"toast\",\n \"lastname\": \"toast\",\n \"type\": \"creator\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "http://localhost:3000/register", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "register" + ] + } + }, + "response": [] + }, + { + "name": "Login", + "request": { + "auth": { + "type": "noauth" + }, + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"email\": \"keaganncollins@gmail.com\",\n \"password\": \"Pa55w.rd\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "http://localhost:3000/login", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "login" + ] + } + }, + "response": [] + }, + { + "name": "Me", + "request": { + "method": "GET", + "header": [] + }, + "response": [] + }, + { + "name": "Update User", + "request": { + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"email\": \"toast@toast.com\",\n \"type\": \"creator\",\n \"address\": \"Cape Town\",\n \"firstname\": \"Keagan\",\n \"lastname\": \"Collins\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "http://localhost:3000/users/3", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "users", + "3" + ] + } + }, + "response": [] + }, + { + "name": "Add Admin", + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"email\": \"toast@toast.com\",\n \"password\": \"Pa55w.rd\",\n \"confirm-password\": \"Pa55w.rd\",\n \"firstname\": \"toast\",\n \"lastname\": \"toast\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "http://localhost:3000/add-admin", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "add-admin" + ] + } + }, + "response": [] + }, + { + "name": "Get Businesses", + "protocolProfileBehavior": { + "disableBodyPruning": true + }, + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "eyJhbGciOiJkaXIiLCJlbmMiOiJBMTI4Q0JDLUhTMjU2In0..VeiUSfpzZQEfRiCUnTWfqw.seHlV7WJPNHxdwbzxGQ-NZzTp2RYYxmhDI2mPx4jsd0.TCN3XRVFi6jiwa6a6a4ttA", + "type": "string" + } + ] + }, + "method": "GET", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"name\": \"modulr\",\n \"url\": \"https://modulr.com\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "http://localhost:3000/businesses", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "businesses" + ] + } + }, + "response": [] + }, + { + "name": "Add Business", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "eyJhbGciOiJkaXIiLCJlbmMiOiJBMTI4Q0JDLUhTMjU2In0..VeiUSfpzZQEfRiCUnTWfqw.seHlV7WJPNHxdwbzxGQ-NZzTp2RYYxmhDI2mPx4jsd0.TCN3XRVFi6jiwa6a6a4ttA", + "type": "string" + } + ] + }, + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"name\": \"yeet\",\n \"url\": \"https://yeet.com\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "http://localhost:3000/businesses", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "businesses" + ] + } + }, + "response": [] + }, + { + "name": "Get Selection Schemas", + "protocolProfileBehavior": { + "disableBodyPruning": true + }, + "request": { + "method": "GET", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"record\": {\n \"providerId\": 1,\n \"outputSchemaId\": 1\n },\n \"schema\": {\n \"title\": {\n \"path\": [\"tag/body\", \"tag/feed\", \"tag/title\", \"content/0\"]\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "http://localhost:3000/admin/selection-schemas", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "admin", + "selection-schemas" + ] + } + }, + "response": [] + }, + { + "name": "Get Selection Schemas By Provider", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "http://localhost:3000/admin/selection-schemas/providers/1", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "admin", + "selection-schemas", + "providers", + "1" + ] + } + }, + "response": [] + }, + { + "name": "Add Selection Schema", + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"record\": {\n \"providerId\": 1,\n \"outputSchemaId\": 1\n },\n \"schema\": {\n \"title\": {\n \"path\": [\"tag/body\", \"tag/feed\", \"tag/title\", \"content/0\"]\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "http://localhost:3000/admin/selection-schemas", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "admin", + "selection-schemas" + ] + } + }, + "response": [] + }, + { + "name": "Get Output Schemas", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "https://source-be-staging.fly.dev/admin/output-schemas", + "protocol": "https", + "host": [ + "source-be-staging", + "fly", + "dev" + ], + "path": [ + "admin", + "output-schemas" + ] + } + }, + "response": [] + }, + { + "name": "Add Output Schema", + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"schema\": {\n \"feed\": {\n \"type\": \"map\",\n \"schema\": {\n \"title\": {\n \"type\": \"string\"\n },\n \"description\": {\n \"type\": \"string\"\n },\n \"display-picture\": {\n \"type\": \"string\"\n },\n \"posts\": {\n \"type\": \"vector\",\n \"schema\": {\n \"post-id\": {\n \"type\": \"string\"\n },\n \"title\": {\n \"type\": \"string\"\n },\n \"thumbnail\": {\n \"type\": \"string\"\n },\n \"info\": {\n \"type\": \"string\"\n },\n \"posted-at\": {\n \"type\": \"string\"\n },\n \"stream-url\": {\n \"type\": \"string\"\n }\n }\n }\n }\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "https://source-be-staging.fly.dev/admin/output-schemas", + "protocol": "https", + "host": [ + "source-be-staging", + "fly", + "dev" + ], + "path": [ + "admin", + "output-schemas" + ] + } + }, + "response": [] + }, + { + "name": "Extract Data", + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"schemaId\": 3,\n \"url\": \"https://www.youtube.com/feeds/videos.xml?channel_id=UCUyeluBRhGPCW4rPe_UvBZQ\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "http://localhost:3000/admin/extract-data", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "admin", + "extract-data" + ] + } + }, + "response": [] + }, + { + "name": "Get Providers", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "eyJhbGciOiJkaXIiLCJlbmMiOiJBMTI4Q0JDLUhTMjU2In0..1n1VNQ-I1Z7VhgW8IWfiPg.YkGk31r1DG073Q5uyW6I0rGbm9YNKB3r_8D_yWUfG60.suZYrgS9-JXGbRF05iRA0g", + "type": "string" + } + ] + }, + "method": "GET", + "header": [], + "url": { + "raw": "http://localhost:3000/providers", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "providers" + ] + } + }, + "response": [] + }, + { + "name": "Add Provider", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "eyJhbGciOiJkaXIiLCJlbmMiOiJBMTI4Q0JDLUhTMjU2In0..1n1VNQ-I1Z7VhgW8IWfiPg.YkGk31r1DG073Q5uyW6I0rGbm9YNKB3r_8D_yWUfG60.suZYrgS9-JXGbRF05iRA0g", + "type": "string" + } + ] + }, + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"name\": \"Spotify\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "http://localhost:3000/admin/providers", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "admin", + "providers" + ] + } + }, + "response": [] + }, + { + "name": "Get Jobs", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "eyJhbGciOiJkaXIiLCJlbmMiOiJBMTI4Q0JDLUhTMjU2In0..1n1VNQ-I1Z7VhgW8IWfiPg.YkGk31r1DG073Q5uyW6I0rGbm9YNKB3r_8D_yWUfG60.suZYrgS9-JXGbRF05iRA0g", + "type": "string" + } + ] + }, + "method": "GET", + "header": [], + "url": { + "raw": "http://localhost:3000/admin/jobs", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "admin", + "jobs" + ] + } + }, + "response": [] + }, + { + "name": "Get Job", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "eyJhbGciOiJkaXIiLCJlbmMiOiJBMTI4Q0JDLUhTMjU2In0..1n1VNQ-I1Z7VhgW8IWfiPg.YkGk31r1DG073Q5uyW6I0rGbm9YNKB3r_8D_yWUfG60.suZYrgS9-JXGbRF05iRA0g", + "type": "string" + } + ] + }, + "method": "GET", + "header": [], + "url": { + "raw": "http://localhost:3000/admin/jobs/1", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "admin", + "jobs", + "1" + ] + } + }, + "response": [] + }, + { + "name": "Get Feeds", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "eyJhbGciOiJkaXIiLCJlbmMiOiJBMTI4Q0JDLUhTMjU2In0..WnRJPqICwpp9h10Sr3w2cg.2luYqAPIVBDcUZSM_661Iqza0TCkP3lhg_-HbACQzdE.DG0evgm8d9aMn4ZcV8oHnQ", + "type": "string" + } + ] + }, + "method": "GET", + "header": [], + "url": { + "raw": "http://localhost:3000/feeds", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "feeds" + ] + } + }, + "response": [] + }, + { + "name": "Add Feed", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "eyJhbGciOiJkaXIiLCJlbmMiOiJBMTI4Q0JDLUhTMjU2In0..WnRJPqICwpp9h10Sr3w2cg.2luYqAPIVBDcUZSM_661Iqza0TCkP3lhg_-HbACQzdE.DG0evgm8d9aMn4ZcV8oHnQ", + "type": "string" + } + ] + }, + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"rssUrl\": \"https://www.youtube.com/feeds/videos.xml?channel_id=UCUyeluBRhGPCW4rPe_UvBZQ\",\n \"providerId\": 1,\n \"contentTypeId\": 1,\n \"cadenceId\": 1,\n \"baselineId\": 1\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "http://localhost:3000/feeds", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "feeds" + ] + } + }, + "response": [] + }, + { + "name": "Bundle Exists", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "4ba928b9604e00be952642e9ce7d210648dd713c9330df26ecfadbe900c72fac", + "type": "string" + } + ] + }, + "method": "GET", + "header": [], + "url": { + "raw": "http://localhost:3000/api/bundle/categories?uuid=1c1953c7cbeeefd2", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "api", + "bundle", + "categories" + ], + "query": [ + { + "key": "uuid", + "value": "1c1953c7cbeeefd2" + } + ] + } + }, + "response": [] + }, + { + "name": "Bundle Categories", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "http://localhost:3000/bundle/categories?uuid=2bbeb46bbd70c82b", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "bundle", + "categories" + ], + "query": [ + { + "key": "uuid", + "value": "2bbeb46bbd70c82b" + } + ] + } + }, + "response": [] + }, + { + "name": "Bundle Feeds", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "4ba928b9604e00be952642e9ce7d210648dd713c9330df26ecfadbe900c72fac", + "type": "string" + } + ] + }, + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"categoryIds\": [14, 15, 16]\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "http://localhost:3000/api/bundle/feeds?uuid=1c1953c7cbeeefd2&type=2", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "api", + "bundle", + "feeds" + ], + "query": [ + { + "key": "uuid", + "value": "1c1953c7cbeeefd2" + }, + { + "key": "type", + "value": "2" + } + ] + } + }, + "response": [] + }, + { + "name": "Bundle Feed", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "4ba928b9604e00be952642e9ce7d210648dd713c9330df26ecfadbe900c72fac", + "type": "string" + } + ] + }, + "method": "GET", + "header": [], + "url": { + "raw": "http://localhost:3000/bundle/feeds/15?uuid=d5a87ac8f4e90682", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "bundle", + "feeds", + "15" + ], + "query": [ + { + "key": "uuid", + "value": "d5a87ac8f4e90682" + } + ] + } + }, + "response": [] + }, + { + "name": "Bundle Feed Posts", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "4ba928b9604e00be952642e9ce7d210648dd713c9330df26ecfadbe900c72fac", + "type": "string" + } + ] + }, + "method": "GET", + "header": [], + "url": { + "raw": "http://localhost:3000/bundle/feeds/15/posts?uuid=d5a87ac8f4e90682", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "bundle", + "feeds", + "15", + "posts" + ], + "query": [ + { + "key": "uuid", + "value": "d5a87ac8f4e90682" + } + ] + } + }, + "response": [] + }, + { + "name": "Bundle Feed Post", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "4ba928b9604e00be952642e9ce7d210648dd713c9330df26ecfadbe900c72fac", + "type": "string" + } + ] + }, + "method": "GET", + "header": [], + "url": { + "raw": "http://localhost:3000/api/bundle/feeds/57/posts/4073?uuid=d5a87ac8f4e90682", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "api", + "bundle", + "feeds", + "57", + "posts", + "4073" + ], + "query": [ + { + "key": "uuid", + "value": "d5a87ac8f4e90682" + } + ] + } + }, + "response": [] + }, + { + "name": "Bundle Posts", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "4ba928b9604e00be952642e9ce7d210648dd713c9330df26ecfadbe900c72fac", + "type": "string" + } + ] + }, + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"categoryIds\": [14]\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "http://localhost:3000/api/bundle/posts?uuid=1c1953c7cbeeefd2&limit=6&start=0&type=2", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "api", + "bundle", + "posts" + ], + "query": [ + { + "key": "uuid", + "value": "1c1953c7cbeeefd2" + }, + { + "key": "limit", + "value": "6" + }, + { + "key": "start", + "value": "0" + }, + { + "key": "type", + "value": "2" + } + ] + } + }, + "response": [] + }, + { + "name": "Bundle Post", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "4ba928b9604e00be952642e9ce7d210648dd713c9330df26ecfadbe900c72fac", + "type": "string" + } + ] + }, + "method": "GET", + "header": [], + "url": { + "raw": "http://localhost:3000/bundle/posts/190?uuid=d5a87ac8f4e90682", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "bundle", + "posts", + "190" + ], + "query": [ + { + "key": "uuid", + "value": "d5a87ac8f4e90682" + } + ] + } + }, + "response": [] + }, + { + "name": "Categories", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "35d38319c1fb1a1f10d1cb83090ae296112a0c606862261b79b8a4f39a5941ab", + "type": "string" + } + ] + }, + "method": "GET", + "header": [], + "url": { + "raw": "http://localhost:3000/api/categories", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "api", + "categories" + ] + } + }, + "response": [] + }, + { + "name": "Category", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "4ba928b9604e00be952642e9ce7d210648dd713c9330df26ecfadbe900c72fac", + "type": "string" + } + ] + }, + "method": "GET", + "header": [], + "url": { + "raw": "http://localhost:3000/api/categories/11", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "api", + "categories", + "11" + ] + } + }, + "response": [] + }, + { + "name": "Create Integration", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "eyJhbGciOiJkaXIiLCJlbmMiOiJBMTI4Q0JDLUhTMjU2In0..avx8YJcoRzOQjg3PRUS6Rg.rTtvfTfHZCExbyV31miHRrHidmNUWbp79hkyWJdyOBA.0HOnXs8Ang1bofzTF5B68A", + "type": "string" + } + ] + }, + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"name\": \"poop\",\n \"tsAndCs\": 1,\n \"contentTypes\": [{\"id\": 1}],\n \"categories\": [{\"id\": 9}, {\"id\": 10}]\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "https://source-be-staging.fly.dev/integrations", + "protocol": "https", + "host": [ + "source-be-staging", + "fly", + "dev" + ], + "path": [ + "integrations" + ] + } + }, + "response": [] + }, + { + "name": "Delete Feed", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "eyJhbGciOiJkaXIiLCJlbmMiOiJBMTI4Q0JDLUhTMjU2In0..qHwUsdiT1O9yEQ7PvmDGRg.VMc4H5xe4T_6_pItihZJUi8UJQpnX4glob4fgmzKeBc.5EQrSl_mPb_2WXhsG3dH1g", + "type": "string" + } + ] + }, + "method": "DELETE", + "header": [], + "url": { + "raw": "http://localhost:3000/feeds/18", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "feeds", + "18" + ] + } + }, + "response": [] + }, + { + "name": "Delete Integration", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "eyJhbGciOiJkaXIiLCJlbmMiOiJBMTI4Q0JDLUhTMjU2In0..qHwUsdiT1O9yEQ7PvmDGRg.VMc4H5xe4T_6_pItihZJUi8UJQpnX4glob4fgmzKeBc.5EQrSl_mPb_2WXhsG3dH1g", + "type": "string" + } + ] + }, + "method": "DELETE", + "header": [], + "url": { + "raw": "http://localhost:3000/integrations/2", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "integrations", + "2" + ] + } + }, + "response": [] + }, + { + "name": "Delete User", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "eyJhbGciOiJkaXIiLCJlbmMiOiJBMTI4Q0JDLUhTMjU2In0..x0B7HOqbRp8kU2WhSt_vFg.J316r0lA44pCWJYtR7D51_yOhuAIhDZFe--rK6UMA34.wCp-1HYrbfg5OJ3QKaDJkQ", + "type": "string" + } + ] + }, + "method": "DELETE", + "header": [], + "url": { + "raw": "http://localhost:3000/me", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "me" + ] + } + }, + "response": [] + }, + { + "name": "Cancel Delete User", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "eyJhbGciOiJkaXIiLCJlbmMiOiJBMTI4Q0JDLUhTMjU2In0..qHwUsdiT1O9yEQ7PvmDGRg.VMc4H5xe4T_6_pItihZJUi8UJQpnX4glob4fgmzKeBc.5EQrSl_mPb_2WXhsG3dH1g", + "type": "string" + } + ] + }, + "method": "DELETE", + "header": [], + "url": { + "raw": "http://localhost:3000/integrations/2", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "integrations", + "2" + ] + } + }, + "response": [] + }, + { + "name": "Admin All Feeds", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "", + "type": "string" + } + ] + }, + "method": "GET", + "header": [], + "url": { + "raw": "http://localhost:3000/admin/feeds", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "admin", + "feeds" + ] + } + }, + "response": [] + }, + { + "name": "Get All Categories", + "request": { + "method": "GET", + "header": [] + }, + "response": [] + }, + { + "name": "Analytics Creator General", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "", + "type": "string" + } + ] + }, + "method": "GET", + "header": [], + "url": { + "raw": "http://localhost:3000/analytics/creator/general?mindate=2026-01-01&maxdate=2026-02-24", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "analytics", + "creator", + "general" + ], + "query": [ + { + "key": "mindate", + "value": "2026-01-01" + }, + { + "key": "maxdate", + "value": "2026-02-24" + } + ] + } + }, + "response": [] + }, + { + "name": "Analytics Creator Deltas", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "", + "type": "string" + } + ] + }, + "method": "GET", + "header": [], + "url": { + "raw": "http://localhost:3000/analytics/creator/deltas?mindate=2026-01-01&maxdate=2026-02-24", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "analytics", + "creator", + "deltas" + ], + "query": [ + { + "key": "mindate", + "value": "2026-01-01" + }, + { + "key": "maxdate", + "value": "2026-02-24" + } + ] + } + }, + "response": [] + }, + { + "name": "Analytics Creator Top", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "", + "type": "string" + } + ] + }, + "method": "GET", + "header": [], + "url": { + "raw": "http://localhost:3000/analytics/creator/top?mindate=2026-01-01&maxdate=2026-02-24&n=10&top=post", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "analytics", + "creator", + "top" + ], + "query": [ + { + "key": "mindate", + "value": "2026-01-01" + }, + { + "key": "maxdate", + "value": "2026-02-24" + }, + { + "key": "n", + "value": "10" + }, + { + "key": "top", + "value": "post" + } + ] + } + }, + "response": [] + }, + { + "name": "Analytics Creator Top Average", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "", + "type": "string" + } + ] + }, + "method": "GET", + "header": [], + "url": { + "raw": "http://localhost:3000/analytics/creator/top/average?mindate=2026-01-01&maxdate=2026-02-24", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "analytics", + "creator", + "top", + "average" + ], + "query": [ + { + "key": "mindate", + "value": "2026-01-01" + }, + { + "key": "maxdate", + "value": "2026-02-24" + } + ] + } + }, + "response": [] + } + ] +} \ No newline at end of file diff --git a/build.sh b/build.sh new file mode 100755 index 00000000..6c7e3b91 --- /dev/null +++ b/build.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +export $(grep '.*' .env | xargs) + +echo "Starting compilation..." +clojure -T:build uber diff --git a/deps.edn b/deps.edn index 5cd12548..37af2dc7 100644 --- a/deps.edn +++ b/deps.edn @@ -14,9 +14,12 @@ ring/ring-json {:mvn/version "0.5.1"} ring/ring-defaults {:mvn/version "0.6.0"} ring-cors/ring-cors {:mvn/version "0.1.13"} - metosin/jsonista {:mvn/version "0.3.13"} - com.github.seancorfield/next.jdbc {:mvn/version "1.3.994"} - org.xerial/sqlite-jdbc {:mvn/version "3.49.1.0"} + io.github.modulr-software/congest {:mvn/version "0.1.7"} + com.github.seancorfield/next.jdbc {:mvn/version "1.3.1070"} + org.postgresql/postgresql {:mvn/version "42.2.10"} + com.github.igrishaev/pg2-core {:mvn/version "0.1.44"} + com.github.igrishaev/pg2-honey {:mvn/version "0.1.44"} + org.xerial/sqlite-jdbc {:mvn/version "3.51.0.0"} buddy/buddy-core {:mvn/version "1.12.0-430"} buddy/buddy-sign {:mvn/version "3.5.351"} aero/aero {:mvn/version "1.1.6"} @@ -27,5 +30,15 @@ com.gfredericks/test.chuck {:mvn/version "0.2.14"} com.kepler16/mallard {:mvn/version "3.2.1"} com.kepler16/mallard-sqlite-store {:mvn/version "3.2.1"} + com.kepler16/mallard-postgres-store {:mvn/version "3.2.5"} com.github.seancorfield/honeysql {:mvn/version "2.7.1310"} - metosin/reitit {:mvn/version "0.9.1"}}} + hickory/hickory {:mvn/version "0.7.1"} + com.draines/postal {:mvn/version "2.0.5"} + hiccup/hiccup {:mvn/version "2.0.0"} + clojure.java-time/clojure.java-time {:mvn/version "1.4.3"} + metosin/jsonista {:mvn/version "0.3.13"} + metosin/reitit {:mvn/version "0.9.1"} + metosin/reitit-middleware {:mvn/version "0.9.1"} + metosin/reitit-swagger {:mvn/version "0.9.1"} + metosin/reitit-swagger-ui {:mvn/version "0.9.1"} + com.taoensso/telemere {:mvn/version "1.2.1"}}} diff --git a/fly.dev.toml b/fly.dev.toml index 3fb8325e..2ef46432 100644 --- a/fly.dev.toml +++ b/fly.dev.toml @@ -1,33 +1,38 @@ -# fly.toml app configuration file generated for source-be-staging on 2025-07-08T14:36:46+02:00 +# fly.toml app configuration file generated for source-be-staging on 2026-02-10T10:41:37+02:00 # # See https://fly.io/docs/reference/configuration/ for information about how to use this file. # app = 'source-be-staging' -primary_region = 'jnb' +primary_region = 'fra' [build] [env] - CORS_ORIGIN = 'http://localhost:3001' - ENV = 'prod' - GOOGLE_CLIENT_ID = '449412212863-h1rre1gdmfgq96jt160fdd30bfatjiqb.apps.googleusercontent.com' + CORS_ORIGIN = 'https://source-staging.fly.dev' + BASE_URL = 'https://source-be-staging.fly.dev' DATABASE_DIR = '/data' + EMAIL_USERNAME = 'merveillevaneck@gmail.com' + ENV = 'staging' + JAVA_CMD = 'java' + GOOGLE_CLIENT_ID = '449412212863-h1rre1gdmfgq96jt160fdd30bfatjiqb.apps.googleusercontent.com' + GOOGLE_REDIRECT_URI = 'https://source-be-staging.fly.dev/oauth2/google/callback' + SUPPORT_ADDRESS = 'keaganncollins@gmail.com' + +[[mounts]] + source = 'source_storage_staging' + destination = '/data' + initial_size = '50gb' + processes = ['app'] [http_service] internal_port = 3000 force_https = true - auto_stop_machines = 'stop' + auto_stop_machines = 'off' min_machines_running = 0 processes = ['app'] [[vm]] - memory = '1gb' + memory = '2gb' cpu_kind = 'shared' - cpus = 1 - -[mounts] - source = 'source_storage_staging' - destination = '/data' - processes = ['app'] - initial_size = '50gb' + cpus = 2 diff --git a/fly.prod.toml b/fly.prod.toml index 269b438a..ac35715c 100644 --- a/fly.prod.toml +++ b/fly.prod.toml @@ -22,6 +22,7 @@ primary_region = 'jnb' cpus = 1 [env] + GOOGLE_REDIRECT_URI = "https://source-be.fly.dev" GOOGLE_CLIENT_ID = "449412212863-h1rre1gdmfgq96jt160fdd30bfatjiqb.apps.googleusercontent.com" - CORS_ORIGIN = "http://localhost:3001" + CORS_ORIGIN = "https://source.fly.dev" ENV = "prod" diff --git a/resources/admins_encrypted.json b/resources/admins_encrypted.json index 50d35965..1418c113 100644 --- a/resources/admins_encrypted.json +++ b/resources/admins_encrypted.json @@ -1 +1 @@ -205f937d56f65fb4f74c29508e2dca11bd4747257dceb2840a4a9610fa086aa55996313547f4fafa6c319b463049fc6d11d781a8f1d97ecc03353b6c3c3c80e2ab99e78cab2728cc9a208bf7d922a6d9a2fd9cc0d0d19d1dbe43f1e0e7de06ed8a7f3132e53d72939b99c7e524ed791872fcf019c200eb5cf81d963462235c6970dcf425351e4d491f997885302513066ad671891a4ea95e4436d5f3f10fdbb861cb26c107d6ad103ec774f86b332b56d135a8b53fb4b4dc95599638ab07753cf4b4675e689609edfcbbc2c6694d1945d539c9e00b4e21667d48586c3c08277dab000afae8c76352164b4771bf169166bf71338e1cb38dc6af02a01d2a715cecec93f7b66f8f4e0387947918e2c3f56f2406443936bf5147491b77de91d2267de864a7316e8ef59971cfde209d1a86b1a022a8af0f653d1d0f380dfb6ca85f610aa6ed74a4d59041cd481d498d0e8bf6bf2d674f07b750a798ee8c4c7720b165eb2710170aab171c93a73967ac037ce00442793e4b11c74d32415b895ab101126af9db912befc603bf2bbe2d7aa651cbb9adb044b54737d6163986d460156840ead2d5e2b9ab80eddd047da9f319865a \ No newline at end of file +1f849fdbcaaa4d5cf82b56107541fa49174399ba9cc32f67fd7eceb7f0847b8050f85973837676f7d7f8d92fa4d6cacf9e175a8fd6fff154212d1187c2888ec39e5d813d2a247e571991dc61e3ab567e796a92e0fbb7a4ee4ff62567a2c405d3008ce6fa89e6b85f7515aff0420cf59ed7e6112e232dec6952b2165f46a167d875b46ec1d1fa4f2a035112fe739607040da8f90bc8abd8f40b798020b6046535a4415e373bc1be463b447cb3218d88419b05d47d99cd053e16f985610cbe9565db3192db3c2810958176f2ebcbd8808a55e2c7e970f2f12b201ce39babdfd1ffe1cf22910395aeb8ff16a5af98896765046a0514c23b1f4fee34bc1a7ad293406151deba329de10c71b919038c165f0c848b60de52a4737ed0f27793c4bc85e7a3c1373e8982280fd53edddad4fca12eeaf212633a55c63ef457185887477ab6068957502a76932f9a267afb03f18ccacf53e52faea2fc56527fedc89570261403b9ade0a6e43e2244223a998338ca623e291ed714b133913e870f1fc07dff801e4450a0a67ac328b6b22f18ffd3ac4704a8e8d6e63dd1940a1c02b6f9c4183358585dc600857fdc0b8a72309e4edb41d1cd2fa2f450b4c6ff4dca70a2a09b9fcafac195493dc43d4d25062bee0b7b29e4fb89c592c7d0eee06724b8fd60c501a5ed49cde6a25679f86c207527c6cbc52c6c7f4215a2d1c857a4de92d14d016b \ No newline at end of file diff --git a/resources/config.edn b/resources/dev_config.edn similarity index 57% rename from resources/config.edn rename to resources/dev_config.edn index 993553d1..5f2aed99 100644 --- a/resources/config.edn +++ b/resources/dev_config.edn @@ -1,10 +1,17 @@ {:supersecretkey #env SUPER_SECRET_KEY :admins-path "resources/admins.json" :admins-encrypted-path "resources/admins_encrypted.json" - :cors-origin #or [#env CORS_ORIGIN "http://localhost:8080"] + :cors-origin #or [#env CORS_ORIGIN "http://localhost:3001"] + :base-url #or [#env BASE_URL "https://localhost:3000"] + :port #or [#env PORT 3000] :env #or [#env ENV "dev"] - :database {:dir #or [#env DATABASE_DIR ".db/"] - :type "sqlite"} + :database {:url #or [#env DATABASE_URL "postgres://localhost"] + :type "postgresql" + :staging #or [#env STAGING_DATABASE_URL "postgres://username:password@localhost:16380"] + :prod #or [#env PROD_DATABASE_URL "postgres://username:password@localhost:16380"]} + :email {:address #or [#env SUPPORT_ADDRESS "lee@wearesource.earth"] + :username #env EMAIL_USERNAME + :password #env EMAIL_PASSWORD} :oauth2 {:google {:authorization-uri "https://accounts.google.com/o/oauth2/auth" :access-token-uri "https://oauth2.googleapis.com/token" :redirect-uri #or [#env GOOGLE_REDIRECT_URI "http://localhost:3000/oauth2/google/callback"] @@ -13,4 +20,3 @@ :access-query-param :access_token :scope ["https://www.googleapis.com/auth/userinfo.email"] :grant-type "authorization_code"}}} - diff --git a/resources/staging_config.edn b/resources/staging_config.edn new file mode 100644 index 00000000..3df88ecb --- /dev/null +++ b/resources/staging_config.edn @@ -0,0 +1,20 @@ +{:supersecretkey #env SUPER_SECRET_KEY + :admins-path "resources/admins.json" + :admins-encrypted-path "resources/admins_encrypted.json" + :cors-origin #or [#env CORS_ORIGIN "https://mervstation.tail4f070.ts.net"] + :base-url #or [#env BASE_URL "https://mervstation.tail4f070.ts.net/sourcebe"] + :port #or [#env PORT 3998] + :env #or [#env ENV "staging"] + :database {:url #or [#env DATABASE_URL "postgres://localhost"] + :type "postgresql"} + :email {:address #or [#env SUPPORT_ADDRESS "lee@wearesource.earth"] + :username #env EMAIL_USERNAME + :password #env EMAIL_PASSWORD} + :oauth2 {:google {:authorization-uri "https://accounts.google.com/o/oauth2/auth" + :access-token-uri "https://oauth2.googleapis.com/token" + :redirect-uri #or [#env GOOGLE_REDIRECT_URI "https://mervstation.tail4f070.ts.net/sourcebe/oauth2/google/callback"] + :client-id #env GOOGLE_CLIENT_ID + :client-secret #env GOOGLE_CLIENT_SECRET + :access-query-param :access_token + :scope ["https://www.googleapis.com/auth/userinfo.email"] + :grant-type "authorization_code"}}} diff --git a/src/source/admins.clj b/src/source/admins.clj index 9fa2b6f0..a795bf5c 100644 --- a/src/source/admins.clj +++ b/src/source/admins.clj @@ -1,7 +1,8 @@ (ns source.admins (:require [source.crypt-fs :as crypt] [source.config :as conf] - [clojure.data.json :as json])) + [clojure.data.json :as json] + [taoensso.telemere :as t])) (defn encrypt! [] (crypt/write-file-crypt! (conf/read-value :admins-encrypted-path) @@ -16,5 +17,6 @@ (crypt/read-file-crypt (conf/read-value :supersecretkey)) (json/read-json)) (catch Exception e - (println (str "Couldn't read the admins file: " (.getMessage e))) + (t/log! {:level :error + :msg (str "Couldn't read the admins file: " (.getMessage e))}) []))) diff --git a/src/source/bundle_migrations/001_init_bundle_db.clj b/src/source/bundle_migrations/001_init_bundle_db.clj new file mode 100644 index 00000000..80377dbd --- /dev/null +++ b/src/source/bundle_migrations/001_init_bundle_db.clj @@ -0,0 +1,24 @@ +(ns source.bundle-migrations.001-init-bundle-db + (:require [source.db.bundle] + [source.db.tables :as tables] + [source.db.util :as db.util])) + +(defn run-up! [context] + (let [{:keys [ds-master bundle-id]} context + tables [:outgoing-posts + :bundle-categories + :post-heuristics]] + (tables/create-bundle-tables! + ds-master + :source.db.bundle + tables + bundle-id))) + +(defn run-down! [context] + (let [{:keys [ds-master bundle-id]} context] + (tables/drop-tables! + ds-master + (-> [:outgoing-posts + :bundle-categories + :post-heuristics] + (db.util/tnames bundle-id))))) diff --git a/src/source/bundle_migrations/002_outgoing_posts.clj b/src/source/bundle_migrations/002_outgoing_posts.clj new file mode 100644 index 00000000..180348d3 --- /dev/null +++ b/src/source/bundle_migrations/002_outgoing_posts.clj @@ -0,0 +1,20 @@ +(ns source.bundle-migrations.002-outgoing-posts + (:require [source.db.bundle] + [source.db.tables :as tables] + [source.db.util :as db.util])) + +(defn run-up! [context] + (let [{:keys [ds-master bundle-id]} context + tables [:outgoing-posts]] + (tables/create-bundle-tables! + ds-master + :source.db.bundle + tables + bundle-id))) + +(defn run-down! [context] + (let [{:keys [ds-master bundle-id]} context] + (tables/drop-tables! + ds-master + (-> [:outgoing-posts] + (db.util/tnames bundle-id))))) diff --git a/src/source/bundle_migrations/003_bundle_indices.clj b/src/source/bundle_migrations/003_bundle_indices.clj new file mode 100644 index 00000000..003b5ac8 --- /dev/null +++ b/src/source/bundle_migrations/003_bundle_indices.clj @@ -0,0 +1,29 @@ +(ns source.bundle-migrations.003-bundle-indices + (:require [source.db.bundle] + [pg.core :as pg] + [source.db.util :as db.util] + [camel-snake-kebab.core :as csk])) + +(defn run-up! [context] + (let [{:keys [ds-master bundle-id]} context + op (->> bundle-id + (db.util/tname :outgoing-posts) + (:tname) + (csk/->snake_case_string))] + + (pg/with-connection [ds-master ds-master] + (pg/execute ds-master (str "CREATE INDEX idx_" op "_feed_id ON " op " (feed_id);")) + (pg/execute ds-master (str "CREATE INDEX idx_" op "_creator_id ON " op " (creator_id);")) + (pg/execute ds-master (str "CREATE INDEX idx_" op "_content_type_id ON " op " (content_type_id);"))))) + +(defn run-down! [context] + (let [{:keys [ds-master bundle-id]} context + op (->> bundle-id + (db.util/tname :outgoing-posts) + (:tname) + (csk/->snake_case_string))] + + (pg/with-connection [ds-master ds-master] + (pg/execute ds-master (str "DROP INDEX IF EXISTS idx_" op "_content_type_id")) + (pg/execute ds-master (str "DROP INDEX IF EXISTS idx_" op "_creator_id;")) + (pg/execute ds-master (str "DROP INDEX IF EXISTS idx_" op "_feed_id;"))))) diff --git a/src/source/config.clj b/src/source/config.clj index f6397c59..63c33154 100644 --- a/src/source/config.clj +++ b/src/source/config.clj @@ -22,14 +22,18 @@ [:admins-path {:optional true} :string] [:admins-encrypted-path :string] [:cors-origin :string] + [:base-url :string] + [:port :int] [:env :string] [:database [:map - [:dir :string] + [:url :string] [:type :string]]] [:oauth2 [:map-of keyword? oauth2-provider-schema]]]) (defn- load-config [] - (let [config (aero/read-config (io/resource "config.edn")) + (let [environment (get (System/getenv) "ENV") + environment (if (nil? environment) "dev" environment) + config (aero/read-config (io/resource (str environment "_config.edn"))) decoded (m/decode schema config mt/string-transformer)] (when-not (m/validate schema decoded) (println (->> decoded @@ -46,9 +50,7 @@ (comment (read-value :supersecretkey) - (read-value :database :dir) (read-value :oauth2 :google) (read-value :cors-origin) (read-value :admins-path) (load-config)) - diff --git a/src/source/db/bundle.clj b/src/source/db/bundle.clj index 620415ea..53a1d6b6 100644 --- a/src/source/db/bundle.clj +++ b/src/source/db/bundle.clj @@ -1,39 +1,67 @@ -(ns source.db.bundle +(ns source.db.bundle (:require [source.db.tables :as tables] [honey.sql :as sql])) (def event-categories (tables/create-table-sql - :event-categories - (tables/table-id) - [:event-id :integer :not nil] - [:category-id :text :not nil] - (tables/foreign-key :event-id :analytics :id))) + :event-categories + (tables/table-id) + [:event-id :integer :not nil] + [:category-id :text :not nil])) -(def outgoing-posts +(def outgoing-posts (tables/create-table-sql - :outgoing-posts - (tables/table-id) - [:title :text] - [:subtitle :text] - [:stream-url :text [:default nil]] - [:content-type :text [:default nil]] - [:feed-id :integer] - [:creator-id :integer] - (tables/foreign-key :creator-id :users :id) - (tables/foreign-key :feed-id :feeds :id))) - -(def analytics + :outgoing-posts + (tables/table-id) + [:post-id :text :not nil] + [:feed-id :integer :not nil] + [:title :text :not nil] + [:thumbnail :text] + [:info :text] + [:url :text] + [:stream-url :text] + [:creator-id :integer :not nil] + [:season :integer] + [:episode :integer] + [:content-type-id :integer :not nil] + [:posted-at :datetime])) + +(def bundle-categories + (tables/create-table-sql + :bundle-categories + (tables/table-id) + [:bundle-id :int :not nil] + [:category-id :int :not nil])) + +(def post-heuristics (tables/create-table-sql - :analytics - (tables/table-id) - [:post-id :integer :not nil] - [:event-type :text :not nil] - [:timestamp :text :not nil])) + :post-heuristics + (tables/table-id) + [:post-id :integer :not nil] + [:long-heuristic :integer] + [:short-heuristic :integer] + [[:unique [:composite :post-id]]])) + +(def analytics + (tables/create-table-sql + :analytics + (tables/table-id) + [:post-id :integer :not nil] + [:event-type :text :not nil] + [:timestamp :text :not nil])) + +(defn tname [tname id] + (-> (str (name tname) "-" id) + (keyword))) + +(defn tnames [tnames id] + (mapv #(tname % id) tnames)) (comment (sql/format event-categories) (sql/format outgoing-posts) + (sql/format bundle-categories) + (sql/format post-heuristics) (sql/format analytics) ()) diff --git a/src/source/db/event.clj b/src/source/db/event.clj index 87919369..80594636 100644 --- a/src/source/db/event.clj +++ b/src/source/db/event.clj @@ -1,40 +1,40 @@ (ns source.db.event - (:require [source.services.event-categories :as ec] - [source.services.analytics :as analytics] - [source.services.outgoing-posts :as outgoing-posts] + (:require [source.services.analytics.interface :as analytics] [source.services.feed-categories :as feed-categories] [source.db.util :as db.util] [source.util :as util] - [source.db.honey :as db])) + [source.db.honey :as db] + [source.db.bundle :as bundle] + [honey.sql.helpers :as hsql])) -(defn get-post-categories [bundle-ds ds post-id] - (let [feed-id (-> (outgoing-posts/outgoing-post bundle-ds {:id post-id}) +(defn get-post-categories [ds post-id bundle-id] + (let [feed-id (-> (db/find-one ds (-> (db.util/tname :outgoing-posts bundle-id) + (hsql/where [:= :id post-id]))) (:feed-id))] (feed-categories/category-id ds {:feed-id feed-id}))) (defn log! [{:keys [post-id bundle-id type]}] (let [ds (db.util/conn :master) timestamp (util/get-utc-timestamp-string) - bundle-ds (->> bundle-id - (db.util/db-name :bundle) - (db.util/conn)) creator-ds (->> {:post-id post-id} (db/find ds) (:creator-id) (db.util/db-name :creator) (db.util/conn)) - categories (get-post-categories bundle-ds ds post-id)] - (let [event-id (-> (analytics/insert-event! bundle-ds {:data {:post_id post-id - :event_type type - :timestamp timestamp} - :ret :*}) + categories (get-post-categories ds post-id bundle-id)] + (let [event-id (-> (analytics/insert-event! ds {:data {:post_id post-id + :event_type type + :timestamp timestamp} + :ret :*}) (first))] - (ec/insert-event-category! bundle-ds {:data {:event-id event-id - :category-id (:category-id categories)}})) + (db/insert! ds (-> (db.util/tname :event-categories bundle-id) + (assoc :data {:event-id event-id + :category-id (:category-id categories)})))) (let [event-id (-> (analytics/insert-event! creator-ds {:data {:post_id post-id :event_type type :timestamp timestamp} :ret :*}) (first))] - (ec/insert-event-category! creator-ds {:data {:event-id event-id - :category-id (:category-id categories)}})))) + (db/insert! creator-ds {:tname :event-categories + :data {:event-id event-id + :category-id (:category-id categories)}})))) diff --git a/src/source/db/honey.clj b/src/source/db/honey.clj index cb626077..029377ae 100644 --- a/src/source/db/honey.clj +++ b/src/source/db/honey.clj @@ -1,43 +1,39 @@ (ns source.db.honey - (:require [honey.sql :as sql] - [honey.sql.helpers :as hsql] - [source.db.util :as db.util] - [camel-snake-kebab.core :as csk] + (:require [camel-snake-kebab.core :as csk] [camel-snake-kebab.extras :as cske] - [next.jdbc :as jdbc] - [next.jdbc.result-set :as rs])) + [honey.sql.helpers :as hsql] + [pg.core :as pg] + [pg.honey :as pgh])) (defn execute! "computes a prepared statement for an sql map and executes select one or select all. returns results as unqualified lower maps by default." - [ds sqlmap & {:keys [ret exec-opts]}] + [ds sqlmap & {:keys [ret]}] (assert (and (some? ds) (some? sqlmap) (or (some? ret) (nil? ret)))) - (let [ps (sql/format sqlmap) - exec-opts' (merge - {:builder-fn rs/as-unqualified-lower-maps} - exec-opts) - result (cske/transform-keys + (let [result (cske/transform-keys csk/->kebab-case-keyword - (jdbc/execute! ds ps exec-opts'))] + (pg/with-conn [conn ds] + (pgh/execute conn sqlmap)))] (cond (= ret :1) (first result) - (= ret :*) result - :else nil))) + :else result))) (defn find "does find one or find all for a given table name and where clause. The where clause follows the same data DSL as honeysql. Automatically transforms kebab case keys into snake case for sql. e.g. :provider-id becomes \"provider_id\" when honey sql prepares the statement in execute!" - [ds {:keys [tname where ret]}] + [ds {:keys [tname where order-by limit ret]}] (execute! ds (-> (hsql/select :*) (hsql/from (csk/->snake_case_keyword tname)) + (merge (if (some? order-by) {:order-by order-by} {})) + (merge (if (some? limit) (hsql/limit limit) {})) (hsql/where (or (cske/transform-keys csk/->snake_case_keyword where) []))) - :ret ret)) + :ret (or ret :*))) (defn find-one [ds opts] (->> {:ret :1} @@ -56,7 +52,7 @@ (-> (hsql/insert-into (csk/->snake_case_keyword tname)) (hsql/values vals) (hsql/returning :*)) - :ret ret))) + :ret (or ret nil)))) (defn delete! "deletes a record or set of records that match a predicate where clause. the where @@ -95,24 +91,22 @@ (comment (hsql/where :or [:= :id 1] [:= :id 2]) - - (def ds (db.util/conn :master)) - - (find ds {:tname :users - :ret :1}) + (def ds {}) (insert! ds {:tname :sectors :values {:name "something"} :ret :*}) (delete! ds - {:tname :sectors - :where [:> :id 3] + {:tname :feeds + :where [:= :id 6] :ret :*}) (update! ds - {:tname :sectors - :where [:= :id 7] - :values {:name "something else"}}) + {:tname :users + :where [:= :id 3] + :values {:type "creator"}}) + + (find ds {:tname :bundles}) ()) diff --git a/src/source/db/interface.clj b/src/source/db/interface.clj index a1622be2..a36c93f7 100644 --- a/src/source/db/interface.clj +++ b/src/source/db/interface.clj @@ -11,7 +11,7 @@ (defn execute! [ds opts] (hon/execute! ds opts)) -(defn find [ds opts] +(defn find [ds {:keys [_tname _where _ret] :as opts}] (hon/find ds opts)) (defn find-one [ds opts] diff --git a/src/source/db/master.clj b/src/source/db/master.clj index a7cb951b..8d746662 100644 --- a/src/source/db/master.clj +++ b/src/source/db/master.clj @@ -14,7 +14,9 @@ [:onboarded :integer [:default 0]] [:address :text] [:mobile :text] - [:profile-image :text])) + [:profile-image :text] + [:business-id :int] + (tables/foreign-key :business-id :businesses :id))) (def sectors (tables/create-table-sql @@ -53,14 +55,45 @@ (tables/create-table-sql :bundles (tables/table-id) + [:name :text :not nil] [:uuid :text :not nil :unique] [:user-id :integer] [:video :integer :not nil [:default 0]] [:podcast :integer :not nil [:default 0]] [:blog :integer :not nil [:default 0]] [:hash :text] + [:content-type-id :integer :not nil] + [:ts-and-cs :integer] + (tables/foreign-key :content-type-id :content-types :id) (tables/foreign-key :user-id :users :id))) +(def bundle-content-types + (tables/create-table-sql + :bundle-content-types + (tables/table-id) + [:bundle-id :integer :not nil] + [:content-type-id :integer :not nil] + (tables/foreign-key :bundle-id :bundles :id) + (tables/foreign-key :content-type-id :content-types :id))) + +(def filtered-feeds + (tables/create-table-sql + :filtered-feeds + (tables/table-id) + [:feed-id :integer :not nil] + [:bundle-id :integer :not nil] + (tables/foreign-key :feed-id :feeds :id) + (tables/foreign-key :bundle-id :bundles :id))) + +(def filtered-posts + (tables/create-table-sql + :filtered-posts + (tables/table-id) + [:post-id :integer :not nil] + [:bundle-id :integer :not nil] + (tables/foreign-key :post-id :incoming-posts :id) + (tables/foreign-key :bundle-id :bundles :id))) + (def feeds (tables/create-table-sql :feeds @@ -69,15 +102,15 @@ [:display-picture :text] [:url :text] [:rss-url :text :not nil] - [:user-id :integer] + [:user-id :integer :not nil] [:provider-id :integer] [:created-at :datetime :not nil] [:updated-at :datetime] [:content-type-id :integer :not nil] [:cadence-id :integer :not nil] [:baseline-id :integer :not nil] - [:ts-and-cs :text] - [:state :text] + [:ts-and-cs :integer] + [:state :text [:check [:in :state ["live" "not live" "pending"]]]] (tables/foreign-key :user-id :users :id) (tables/foreign-key :provider-id :providers :id) (tables/foreign-key :cadence-id :cadences :id) @@ -91,7 +124,8 @@ [:feed-id :integer :not nil] [:category-id :integer :not nil] (tables/foreign-key :feed-id :feeds :id) - (tables/foreign-key :category-id :categories :id))) + (tables/foreign-key :category-id :categories :id) + [[:unique [:composite :feed-id :category-id]]])) (def providers (tables/create-table-sql @@ -107,6 +141,7 @@ :businesses (tables/table-id) [:name :text] + [:address :text [:default nil]] [:url :text [:default nil]] [:linkedin :text [:default nil]] [:twitter :text [:default nil]])) @@ -129,6 +164,108 @@ (tables/foreign-key :feed-id :feeds :id) (tables/foreign-key :sector-id :sectors :id))) +(def selection-schemas + (tables/create-table-sql + :selection-schemas + (tables/table-id) + [:output-schema-id :integer :not nil] + [:provider-id :integer :not nil] + (tables/foreign-key :provider-id :providers :id))) + +(def output-schemas + (tables/create-table-sql + :output-schemas + (tables/table-id) + [:schema :text])) + +(def incoming-posts + (tables/create-table-sql + :incoming-posts + (tables/table-id) + [:post-id :text :not nil] + [:feed-id :integer :not nil] + [:title :text :not nil] + [:thumbnail :text] + [:info :text] + [:url :text] + [:stream-url :text] + [:creator-id :integer :not nil] + [:season :integer] + [:episode :integer] + [:content-type-id :integer :not nil] + [:redacted :integer] + [:posted-at :datetime] + (tables/foreign-key :feed-id :feeds :id) + (tables/foreign-key :creator-id :users :id) + (tables/foreign-key :content-type-id :content-types :id))) + +(def jobs + (tables/create-table-sql + :jobs + (tables/table-id) + [:job-id :text :not nil] + [:status :text [:check [:in :status ["running" "stopped"]]]] + [:args :text] + [:handler :text :not nil] + [:last-heartbeat :datetime] + [:job-metadata-id :integer :not nil] + (tables/foreign-key :job-metadata-id :job-metadata :id) + [[:foreign-key :job-metadata-id] [:references :job-metadata :id] :on-delete :cascade])) + +(def job-metadata + (tables/create-table-sql + :job-metadata + (tables/table-id) + [:initial-delay :integer] + [:auto-start :integer] + [:stop-after-fail :integer] + [:kill-after :integer] + [:num-calls :integer] + [:interval :integer] + [:recurring :integer] + [:created-at :datetime] + [:sleep :integer])) + +(def events + (tables/create-table-sql + :events + (tables/table-id) + [:timestamp :datetime] + [:event :text [:check [:in :event ["impression" "click" "view"]]]] + [:feed-id :integer :not nil] + [:post-id :integer] + [:content-type-id :integer :not nil] + [:creator-id :integer :not nil] + [:bundle-id :integer :not nil] + [:distributor-id :integer :not nil] + (tables/foreign-key :feed-id :feeds :id) + (tables/foreign-key :post-id :incoming-posts :id) + (tables/foreign-key :content-type-id :content-types :id) + (tables/foreign-key :creator-id :users :id) + (tables/foreign-key :bundle-id :bundles :id) + (tables/foreign-key :distributor-id :users :id))) + +(def event-categories + (tables/create-table-sql + :event-categories + (tables/table-id) + [:event-id :integer :not nil] + [:category-id :integer :not nil] + (tables/foreign-key :event-id :events :id) + (tables/foreign-key :category-id :categories :id))) + +(def business-types + (tables/create-table-sql + :business-types + (tables/table-id) + [:name :text :not nil])) + +(def integration-types + (tables/create-table-sql + :business-types + (tables/table-id) + [:name :text :not nil])) + (comment (require '[honey.sql :as sql]) @@ -139,10 +276,22 @@ (sql/format categories) (sql/format baselines) (sql/format bundles) + (sql/format bundle-content-types) (sql/format feeds) (sql/format feed-categories) (sql/format providers) (sql/format businesses) (sql/format user-sectors) (sql/format feed-sectors) + (sql/format selection-schemas) + (sql/format incoming-posts) + (sql/format jobs) + (sql/format job-metadata) + (sql/format filtered-feeds) + (sql/format filtered-posts) + (sql/format business-types) + (sql/format integration-types) + + (sql/format events) + (sql/format event-categories) ()) diff --git a/src/source/db/tables.clj b/src/source/db/tables.clj index 88b0f6e6..a3863f17 100644 --- a/src/source/db/tables.clj +++ b/src/source/db/tables.clj @@ -1,6 +1,8 @@ (ns source.db.tables (:require [source.db.honey :as hon] - [honey.sql.helpers :as hsql])) + [honey.sql.helpers :as hsql] + [source.db.util :as db.util] + [honey.sql :as sql])) (defn create-table-sql "returns a honey data DSL structure for creating a table tname @@ -30,32 +32,44 @@ "Given keywords ns and table, parses the keywords into a resolvable keyword, resolves the symbol to retrieve defined sql create table honey statements, prepares jdbc statements from them, - and executes with next.jdbc, returning the result of the execution." + and executes with next.jdbc, returning the result of the execution. + tname can either be a resolvable var keyword, or a vector containing + a resolvable var keyword and a table name to be assigned" [ds ns tname] - (->> (resolve-sql-def ns tname) - (hon/execute! ds))) + (let [multi? (vector? tname) + tname' (if multi? (first tname) tname) + new-tname (if multi? (last tname) tname) + table-stmt (-> (resolve-sql-def ns tname') + (assoc :create-table [new-tname :if-not-exists]))] + (hon/execute! ds table-stmt))) (defn create-tables! "Like create-table! but accepts a vector of keywords for table names and runs create-table with ns on every table name keyword in the vector." [ds ns tables] - (mapv #(create-table! ds ns %) - tables)) + (mapv #(create-table! ds ns %) tables)) + +(defn create-bundle-tables! + [ds ns tables bundle-id] + (->> (mapv (fn [t] [t (-> (db.util/tname t bundle-id) + :tname)]) tables) + (mapv #(create-table! ds ns %)))) (defn tables - "returns all current tables in a sqlite datasource" + "returns all current tables in a postgres datasource" [ds] - (->> {:tname :sqlite-master - :where [:and [:= :type "table"] [:<> :name "sqlite_sequence"]]} + (->> {:tname :information_schema.tables + :where [:and [:= :table-schema "public"] [:= :table-type "BASE TABLE"]] + :ret :*} (hon/find ds))) (defn table-name "return the name of a table record" [table] - (:name table)) + (:table-name table)) (defn table-names - "retrieves and returns all table names for an sqlite datasource" + "retrieves and returns all table names for an postgres datasource" [ds] (->> (tables ds) (mapv table-name))) @@ -65,12 +79,12 @@ with defaults: integer primary key autoincrement. Can be used with (create-table-sql) to simplify creating table ids." [] - [:id :integer [:primary-key] :autoincrement]) + [:id :integer :generated :by :default :as :identity :primary :key]) (defn drop-table-sql "returns a honey data DSL structure for dropping a table tname" [tname] - (hsql/drop-table tname)) + {:drop-table [tname [:cascade]]}) (defn drop-table! "given a table tname, this function drops the table tname from @@ -110,11 +124,13 @@ :source.db.master [:users :cadences + :sectors :businesses]) (drop-tables! (db.util/conn :master) [:users + :sectors :cadences]) (drop-all-tables! (db.util/conn :master)) diff --git a/src/source/db/util.clj b/src/source/db/util.clj index 7161b18f..2317ae29 100644 --- a/src/source/db/util.clj +++ b/src/source/db/util.clj @@ -1,15 +1,8 @@ (ns source.db.util (:require [source.config :as conf] [next.jdbc :as jdbc] - [next.jdbc.result-set :as rs])) - -(defn db-path [dbname] - (let [db-dir (conf/read-value :database :dir)] - (str - db-dir - (when (not (= (last db-dir) \/)) - "/") - dbname))) + [next.jdbc.result-set :as rs] + [pg.core :as pg])) (defn db-name ([type] @@ -17,19 +10,61 @@ ([type id] (str (name type) "_" id))) +(defn get-connection [ds] + (let [conn (-> ds + (jdbc/get-connection))] + (try (jdbc/execute! conn ["CREATE DOMAIN DATETIME TEXT"]) (catch Exception _)) + (jdbc/with-options conn {:builder-fn rs/as-unqualified-lower-maps}) + conn)) + (defn- -conn [dbname] - (-> {:dbtype (conf/read-value :database :type)} - (merge {:dbname (db-path dbname)}) - (jdbc/get-connection) - (jdbc/with-options {:builder-fn rs/as-unqualified-lower-maps}))) + {:connection-uri (str (conf/read-value :database :url) ":5432/" dbname)}) (defn conn ([] (conn :master)) ([db-type] - (assert (= db-type :master)) - (-conn (db-name db-type))) - ([db-type id] - (assert (or (= db-type :bundle) (= db-type :creator))) - (-conn (db-name db-type id)))) + (assert (or (= db-type :master) (= db-type :migrate))) + (-conn (db-name db-type)))) + +(defn tname + ([tname id] + {:tname (->> (str (name tname) "-" id) + (keyword))}) + ([data-map tname id] + (->> (str (name tname) "-" id) + (keyword) + (assoc data-map :tname)))) + +(defn tnames [tnames id] + (mapv #(:tname (tname % id)) tnames)) + +(defn conn-env + "Creates a connection to the master database for the given environment. There must be a connection string in config for the given environment." + [env] + {:connection-uri (str (conf/read-value :database env) "/master")}) + +(defmacro with-env + "This macro creates a let binding structure associating a custom binding with a database connection based with the given environment as a keyword. + e.g. (with-env [ds :staging] (hon/find ds {:tname :users}))" + [args & body] + `(let [~(first args) (conn-env ~(last args))] + ~(cons 'do body))) + +(comment + (def q "SELECT * FROM events") + + (macroexpand '(with-env [ds :staging] (println ds))) + + (with-env [ds :staging] + (time (pg/with-conn [conn ds] + (pg/query conn q))) + #_(hon/find ds {:tname :users})) + + (time (pg/with-conn [conn {:connection-uri "postgresql://postgres:postgres@localhost:5432/master?ssl=false"}] + (pg/query conn q))) + + (time (jdbc/execute! {:dbtype "postgresql" + :jdbcUrl (str "jdbc:" (conf/read-value :database :url) ":5432/master")} [q])) + ()) diff --git a/src/source/email/gmail.clj b/src/source/email/gmail.clj new file mode 100644 index 00000000..770caea2 --- /dev/null +++ b/src/source/email/gmail.clj @@ -0,0 +1,54 @@ +(ns source.email.gmail + (:require [postal.core :as postal] + [source.config :as conf])) + +(defn postal-config [] + (let [email-username (conf/read-value :email :username) + gmail-password (conf/read-value :email :password)] + {:host "smtp.gmail.com" + :user email-username + :pass gmail-password + :port 587 + :tls true})) + +(defn send-email [{:keys [to subject body type] :as _opts}] + (let [email-username (conf/read-value :email :username)] + (-> (postal-config) + (postal/send-message + {:from email-username + :to to + :subject subject + :body [{:type (str (namespace type) "/" (name type)) + :content body}]})))) + +(comment + (require '[source.email.templates :as templates]) + + (send-email {:to "keaganncollins@gmail.com" + :subject "test email" + :body "hi this is a test coming from source-be" + :type :text/plain}) + + (send-email {:to "keaganncollins@gmail.com" + :subject "feed rejection template" + :body (templates/feed-rejection {:creator-name "Keagan" + :feed-title "Keagan's Mukbang Channel" + :reason "too cringe frfr"}) + :type :text/html}) + + (send-email {:to "keaganncollins@gmail.com" + :subject "feed approval template" + :body (templates/feed-approval {:creator-name "Keagan" + :feed-title "Keagan's Mukbang Channel" + :feed-id 2}) + :type :text/html}) + + (send-email {:to "keaganncollins@gmail.com" + :subject "admin reported problem" + :type :text/html + :body (templates/admin-reported-problem {:user-id 3 + :user-email "keaganncollins@gmail.com" + :user-type "creator" + :message "no burger king foot lettuce :("})}) + + ()) diff --git a/src/source/email/templates.clj b/src/source/email/templates.clj new file mode 100644 index 00000000..ab184d25 --- /dev/null +++ b/src/source/email/templates.clj @@ -0,0 +1,180 @@ +(ns source.email.templates + (:require [hiccup.page :as h] + [source.config :as conf])) + +(defn head-metadata [] + [:head + [:meta {:charset "UTF-8"}] + [:meta {:name "viewport" :content "width=device-width, initial-scale=1.0"}]]) + +(defn header [] + [:tr + [:td {:class "header" + :style "background-color: #0F172A; padding: 40px; text-align: center; color: white; font-size: 36px;"} + "Source"]]) + +(defn button [{:keys [text redirect]}] + [:tr + [:td {:style "padding: 0px 40px 0px 40px; text-align: center;"} + [:table {:cellspacing "0" :cellpadding "0" :style "margin: auto;"} + [:tr + [:td {:align "center" + :style "background-color: #0F172A; padding: 10px 20px; border-radius: 5px;"} + [:a {:href redirect + :target "_blank" + :style "color: #ffffff; text-decoration: none; font-weight: bold;"} + text]]]]]]) + +(defn footer [] + [:tr + [:td {:class "footer" + :style "background-color: #0F172A; padding: 20px; text-align: center; color: white; font-size: 14px;"} + "Copyright © 2025 | Wearesource"]]) + +(defn feed-rejection + "Returns the completed HTML for a feed rejection email" + [{:keys [creator-name feed-title reason]}] + (h/html5 + {:lang "en"} + (head-metadata) + [:body {:style "font-family: 'Switzer', sans-serif"} + [:table {:width "100%" :border "0" :cellspacing "0" :cellpadding "0"} + [:tr + [:td {:align "center" :style "padding: 20px;"} + [:table {:class "content" + :width "600" + :border "0" + :cellspacing "0" + :cellpadding "0" + :style "border-collapse: collapse; border: 1px solid #cccccc;"} + (header) + [:tr + [:td {:class "body" + :style "padding: 40px; text-align: left; font-size: 16px; line-height: 1.6;"} + (str "Hi " creator-name) + [:br] + (str "Unfortunately, the feed \"" feed-title "\" that you recently added was rejected.") + [:br] [:br] + (str reason) + [:br] [:br] + "If you believe this was in error, you can reply to this email or click on the link below to leave us a message."]] + (button {:text "Leave us a message" + :redirect (str (conf/read-value :cors-origin) "/report-a-problem")}) + [:tr + [:td {:class "body" + :style "padding: 40px; text-align: left; font-size: 16px; line-height: 1.6;"} + "Regards," + [:br] + "The Source Team"]] + (footer)]]]]])) + +(defn email-verification + "Returns the completed HTML for a feed rejection email" + [{:keys [email-hash]}] + (h/html5 + {:lang "en"} + (head-metadata) + [:body {:style "font-family: 'Switzer', sans-serif"} + [:table {:width "100%" :border "0" :cellspacing "0" :cellpadding "0"} + [:tr + [:td {:align "center" :style "padding: 20px;"} + [:table {:class "content" + :width "600" + :border "0" + :cellspacing "0" + :cellpadding "0" + :style "border-collapse: collapse; border: 1px solid #cccccc;"} + (header) + [:tr + [:td {:class "body" + :style "padding: 40px; text-align: left; font-size: 16px; line-height: 1.6;"} + "Welcome to Source!" + [:br] [:br] + "Thanks for signing up. Please click the button below to verify your email."]] + (button {:text "Verify" + :redirect (str (conf/read-value :base-url) "/email/verify/" email-hash)}) + [:tr + [:td {:class "body" + :style "padding: 40px; text-align: left; font-size: 16px; line-height: 1.6;"} + "Regards," + [:br] + "The Source Team"]] + (footer)]]]]])) + +(defn feed-approval + "Returns the completed HTML for a feed approval email" + [{:keys [creator-name feed-title feed-id]}] + (h/html5 + {:lang "en"} + (head-metadata) + [:body {:style "font-family: 'Switzer', sans-serif"} + [:table {:width "100%" :border "0" :cellspacing "0" :cellpadding "0"} + [:tr + [:td {:align "center" :style "padding: 20px;"} + [:table {:class "content" + :width "600" + :border "0" + :cellspacing "0" + :cellpadding "0" + :style "border-collapse: collapse; border: 1px solid #cccccc;"} + (header) + [:tr + [:td {:class "body" + :style "padding: 40px; text-align: left; font-size: 16px; line-height: 1.6;"} + (str "Hi " creator-name) + [:br] + (str "Good news! The feed \"" feed-title "\" that you recently added was approved and is now live on the platform.") + [:br] [:br] + "Click on the link below to go to your dashboard and view your feed."]] + (button {:text "View your feed" + :redirect (str (conf/read-value :cors-origin) "/dashboard/feed/" feed-id)}) + [:tr + [:td {:class "body" + :style "padding: 40px; text-align: left; font-size: 16px; line-height: 1.6;"} + "Regards," + [:br] + "The Source Team"]] + (footer)]]]]])) + +(defn admin-reported-problem + "Returns the completed HTML for an admin problem report email" + [{:keys [user-id user-type user-email message]}] + (let [shortened-message (if (> (count message) 15) + (str (subs message 0 15) "...") + message)] + (h/html5 + {:lang "en"} + (head-metadata) + [:body {:style "font-family: 'Switzer', sans-serif"} + [:table {:width "100%" :border "0" :cellspacing "0" :cellpadding "0"} + [:tr + [:td {:align "center" :style "padding: 20px;"} + [:table {:class "content" + :width "600" + :border "0" + :cellspacing "0" + :cellpadding "0" + :style "border-collapse: collapse; border: 1px solid #cccccc;"} + (header) + [:tr + [:td {:class "body" + :style "padding: 40px; text-align: left; font-size: 16px; line-height: 1.6;"} + "A user has reported a problem:" + [:br] [:br] + message + [:br] [:br] + (str "User ID: " user-id) + [:br] + (str "User email address: " user-email) + [:br] + (str "User type: " user-type) + [:br] + [:br] + "Click on the link below to respond"]] + (button {:text "Respond" + :redirect (str "mailto:" user-email "?subject=Source Team Re:" shortened-message)}) + [:tr + [:td {:class "body" + :style "padding: 40px; text-align: left; font-size: 11px; line-height: 1.6;"} + "This is an automated message. Please do not reply directly to this email."]] + (footer)]]]]]))) diff --git a/src/source/jobs/core.clj b/src/source/jobs/core.clj new file mode 100644 index 00000000..209ba8bc --- /dev/null +++ b/src/source/jobs/core.clj @@ -0,0 +1,90 @@ +(ns source.jobs.core + (:require [congest.jobs :as congest] + [clojure.data.json :as json] + [source.services.interface :as services] + [source.jobs.oplog :as oplog] + [source.jobs.handlers :as handlers])) + +(defn prepare-congest-metadata + "given raw job metadata, returns extended metadata necessary for use with congest" + [ds metadata] + (let [i->b (fn [i] (if (integer? i) + (if (= i 1) true false) + i)) + args (:args metadata)] + (-> metadata + (assoc :auto-start (i->b (:auto-start metadata)) + :stop-after-fail (i->b (:stop-after-fail metadata)) + :recurring? (i->b (or (:recurring metadata) (:recurring? metadata))) + :sleep (i->b (:sleep metadata)) + :args (if (string? args) (json/read-str (:args metadata) {:key-fn keyword}) args) + :handler-name (:handler metadata) + :handler (handlers/handler metadata) + :logger oplog/operation-logger + :ds ds) + (dissoc :recurring)))) + +(defn start! + "given a job-id, re-registers an existing job from the database" + [js ds job-id] + (let [{:keys [job-metadata-id args handler]} (services/job ds {:where [:= :job-id job-id]}) + metadata (-> (services/job-metadata ds {:id job-metadata-id}) + (assoc :id job-id + :args args + :handler handler))] + (congest/deregister! js job-id) + (congest/register! js (prepare-congest-metadata ds metadata)))) + +(defn interrupted-jobs + "Get vec of congest-ready metadata of all jobs marked as running" + [ds] + (let [jobs (services/jobs ds)] + (mapv (fn [{:keys [job-id job-metadata-id args handler status]} i] + (when (= status "running") + (let [{:keys [initial-delay + interval] + :as m} (-> (services/job-metadata ds {:id job-metadata-id}) + (assoc :id job-id + :args args + :handler handler)) + metadata (assoc m + :initial-delay (if (some? initial-delay) + (+' initial-delay (*' 1000 5 i)) + 0) + :interval (if (some? interval) + (+' interval (*' 1000 5 i)) + 0))] + (prepare-congest-metadata ds metadata)))) + jobs + (-> jobs count inc range)))) + +(comment + (require '[source.db.util :as db.util]) + + (def ds (db.util/conn)) + + (def testjob {:id "test" + :initial-delay 10 + :auto-start true + :stop-after-fail false, + :interval 3000 + :recurring? true + :args {:name "congest"} + :handler :test + :created-at nil + :sleep false}) + + (services/jobs ds) + (services/job-metadata ds {:id 5}) + + (services/delete-job! ds {}) + (services/delete-job-metadata! ds {}) + + (def js (congest/create-job-service [])) + (congest/register! js (prepare-congest-metadata ds testjob)) + (congest/deregister! js "delete_creator_31") + (congest/stop! js "test" false) + (start! js ds "test") + (interrupted-jobs ds) + + ()) diff --git a/src/source/jobs/handlers.clj b/src/source/jobs/handlers.clj new file mode 100644 index 00000000..91bf85a8 --- /dev/null +++ b/src/source/jobs/handlers.clj @@ -0,0 +1,196 @@ +(ns source.jobs.handlers + (:require [source.services.interface :as services] + [source.workers.xml-schemas :as xml] + [source.workers.users :as users] + [source.util :as util] + [source.services.incoming-posts :as incoming-posts] + [source.db.util :as db.util] + [clojure.set :as set] + [clojure.string :as string] + [source.db.honey :as hon] + [taoensso.telemere :as t])) + +(defmulti handler + (fn [opts] + (keyword (:handler opts)))) + +(defmethod handler :default [opts] + (throw (IllegalArgumentException. + (str "Handler with name " (:handler opts) " does not exist.")))) + +(defmethod handler :test [_] + (fn [{:keys [args]}] + (println "hello" (get args :name) args))) + +(defn update-feed-posts-job-id + "returns the job id of an update-feed-posts job with the given email and feed-id" + [email feed-id] + (str email "-" feed-id)) + +(defmethod handler :update-feed-posts [_] + (fn [{:keys [args ds]}] + (try + (when (users/removed? ds (:creator-id args)) + (let [{:keys [feed-id creator-id content-type-id provider-id url]} args + _ (t/log! (str "feed " feed-id " job started.")) + selection-schemas (->> [:= :provider-id provider-id] + (assoc {} :where) + (xml/selection-schemas ds)) + latest-ss (->> selection-schemas + (reduce (fn [acc {:keys [id]}] + (conj acc id)) []) + (apply max -1)) + extracted (try + (xml/extract-data ds latest-ss url) + (catch Exception e + (throw + (t/error! + ::data-extraction + (ex-info (str "Data extraction for feed job failed: feed-id " feed-id " creator-id " creator-id) + {:panic? "Yes, if data extraction fails here it will likely fail for others." + :possible-cause "Could possibly be an incorrect selection schema or output schema" + :next-steps (str "Check selection-schema-id " latest-ss " and feed-id " feed-id ". Test extraction manually.") + :raw-error (.getMessage e)}))))) + + extracted-posts (get-in extracted [:feed :posts]) + extracted-display (get-in extracted [:feed :display-picture]) + extended-posts (mapv (fn [{:keys [posted-at thumbnail] :as post}] + (merge post + {:feed-id feed-id + :creator-id creator-id + :content-type-id content-type-id + :posted-at (util/format-rss-date posted-at) + :thumbnail (if (and thumbnail + (seq thumbnail) + (not (string/includes? thumbnail ".mp3"))) + thumbnail + extracted-display)})) + extracted-posts) + existing-posts (hon/find ds {:tname :incoming-posts + :where [:= :creator-id creator-id]}) + existing-feed (hon/find-one ds {:tname :feeds + :where [:= :id feed-id]})] + (hon/update! ds {:tname :feeds + :where [:= :id feed-id] + :data {:title (get-in extracted [:feed :title]) + :description (get-in extracted [:feed :description]) + :display-picture (if (and (:display-picture existing-feed) + (seq (:display-picture existing-feed))) + (:display-picture existing-feed) + extracted-display) + :updated-at (util/get-utc-timestamp-string)}}) + (run! + (fn [post] + (if (some #(= (:post-id post) (:post-id %)) existing-posts) + (hon/update! ds {:tname :incoming-posts + :where [:= :post-id (:post-id post)] + :data post}) + (hon/insert! ds {:tname :incoming-posts + :data post}))) + extended-posts) + (t/log! (str "feed " feed-id " job finished.")))) + + (catch Exception e (t/log! {:level :error + :msg (str "feed job failed: " e)}) :fail)))) + +(defn update-bundle-job-id + "returns the job id of an update-bundle job with the given bundle id" + [bundle-id] + (str "bundle_" bundle-id)) + +(defn determine-post-score [post posts-categories categories] + ; calculate score for post + ; determine number of categories matched + ; get vector of category ids in the given post, e.g. [1 3] + (let [post-categories-vec (->> posts-categories + (mapv (fn [{:keys [post-id id]}] + (when (= post-id (:id post)) id))) + (filterv identity)) + ; get vector of category ids in categories to match, e.g. [1 2 3 4] + match-categories-vec (reduce (fn [acc {:keys [id]}] + (conj acc id)) [] categories) + ; get number of matches between the 2 vectors, e.g. #{1 3} intersect #{1 2 3 4} -> (count #{1 3}) -> 2 + matches (count (set/intersection (set post-categories-vec) + (set match-categories-vec)))] + ; use matches as a score to upsert long-heuristic for this post + {:post-id (:id post) + :long-heuristic matches})) + +; run long heuristics and pull the highest scoring incoming posts into the bundle's outgoing posts +(defmethod handler :update-bundle [_] + (fn [{:keys [args ds]}] + (let [{:keys [bundle-id categories]} args + _ (t/log! (str "starting bundle " bundle-id " job.")) + incoming-posts (services/incoming-posts-with-feeds ds {:where [:= :feeds.state "live"]}) + posts-categories (incoming-posts/categories-by-posts ds {:where [:= :state "live"]}) + heuristics (mapv + #(determine-post-score % posts-categories categories) + incoming-posts)] + ; use precalculated heuristics and insert this data to the database + (try + (services/upsert-post-heuristics! ds {:bundle-id bundle-id + :data heuristics}) + (catch Exception e (t/log! {:level :error + :msg (str "bundle " bundle-id " upserting post heuristics failed: " (.getMessage e))}))) + + ; pull highest scored posts by long heuristics into outgoing posts + ; top 1000 post-heuristics records ordered by long heuristic in descending order + (let [top-by-long-heuristics (services/top-posts-by-heuristic ds + {:heuristic :long-heuristic + :limit 2000 + :bundle-id bundle-id}) + ; convert into a vector of id numbers + ids (mapv :post-id top-by-long-heuristics) + + ; get all incoming posts with the above id numbers + posts-in (hon/find ds {:tname :incoming-posts + :where [:in :id ids]}) + + creator-ids (mapv :creator-id posts-in) + active-creator-ids (->> (hon/find ds {:tname :users + :where [:in :id creator-ids]}) + (filterv #(or (nil? (:removed %)) (= (:removed %) 0))) + (mapv :id)) + + ; remove redacted posts + outgoing-posts (reduce (fn [acc {:keys [redacted creator-id] :as post}] + (if (and (or (nil? redacted) (= redacted 0)) + (some #{creator-id} active-creator-ids)) + (conj acc (dissoc post :redacted)) + acc)) + [] posts-in)] + (when (seq posts-in) + (hon/delete! ds (db.util/tname :outgoing-posts bundle-id)) + (hon/insert! ds (-> (db.util/tname :outgoing-posts bundle-id) + (assoc :data outgoing-posts))) + (when (< (count outgoing-posts) 10) + (throw + (t/error! + ::update-bundle-job + (ex-info (str + "bundle job for bundle-id " + bundle-id + " pulled " + (count outgoing-posts) + ". Active creator id count: " + (count active-creator-ids) + ". Incoming posts pulled: " + (count posts-in)) + {:panic? "Yes, the embed for this bundle will now be useless" + :possible-cause "If no posts made it into the bundle, it's possible post heuristics failed or there's no incoming posts" + :next-steps "Check for errors thrown in this job, ensure all tables for this bundle exist"}))))) + + (t/log! (str "bundle " bundle-id " job done.")))))) + +(defn user-deletion-job-id + "returns the job id of a user deletion job with the given user id" + [user-type user-id] + (str "delete_" user-type "_" user-id)) + +(defmethod handler :delete-user [_] + (fn [{:keys [args ds]}] + (try + (let [{:keys [user-type user-id]} args] + (users/hard-delete-user! ds (keyword user-type) user-id)) + (catch Exception e (t/log! {:level :error + :msg (str "Failed to delete user-id " (:user-id args) ":" e)}) :fail)))) diff --git a/src/source/jobs/oplog.clj b/src/source/jobs/oplog.clj new file mode 100644 index 00000000..694b25f7 --- /dev/null +++ b/src/source/jobs/oplog.clj @@ -0,0 +1,75 @@ +(ns source.jobs.oplog + (:require [source.services.interface :as services] + [source.util :as util] + [clojure.data.json :as json])) + +(defn delete-job! + "Delete job record and metadata associated therewith" + [{:keys [ds id]}] + (let [{:keys [job-metadata-id id]} (services/job ds {:where [:= :job-id id]})] + (services/delete-job! ds {:id id}) + (services/delete-job-metadata! ds {:id job-metadata-id}))) + +(defn update-job! + "Update job status and last heartbeat" + [{:keys [ds id]} status] + (services/update-job! ds {:where [:= :job-id id] + :data {:status status + :last-heartbeat (util/get-utc-timestamp-string)}})) + +(defn update-job-metadata! + "Update the number of calls in job metadata" + [{:keys [ds id num-calls]}] + (let [metadata-id (:job-metadata-id (services/job ds {:where [:= :job-id id]}))] + (services/update-job-metadata! ds {:id metadata-id + :data {:num-calls num-calls}}))) + +(defn create-job! + "Inserts a new job record and its associated metadata if the job doesn't already exist in the database." + [{:keys [ds id args handler-name] :as job-metadata}] + (let [b->i (fn [b] (if b 1 0)) + metadata {:initial-delay (:initial-delay job-metadata) + :auto-start (b->i (:auto-start job-metadata)) + :stop-after-fail (b->i (:stop-after-fail job-metadata)) + :kill-after (:kill-after job-metadata) + :num-calls (:num-calls job-metadata) + :interval (:interval job-metadata) + :recurring (b->i (:recurring? job-metadata)) + :created-at (:created-at job-metadata) + :sleep (b->i (:sleep job-metadata))} + job (services/job ds {:where [:= :job-id id]}) + metadata-id (when-not (some? job) + (:id (services/insert-job-metadata! ds {:data metadata + :ret :1})))] + (if (some? job) + (services/update-job! ds {:where [:= :job-id id] + :data {:status "running" + :last-heartbeat (util/get-utc-timestamp-string)}}) + (services/insert-job! ds {:data {:job-id id + :status "running" + :args (json/json-str args) + :handler (name handler-name) + :last-heartbeat (util/get-utc-timestamp-string) + :job-metadata-id metadata-id} + :ret :1})))) + +(defn operation-logger + "This function serves as the operation logger called from within congest to handle the persistence layer" + [{:keys [action] :as metadata}] + (cond + (= action "register") + (create-job! metadata) + + (= action "deregister") + (delete-job! metadata) + + (= action "start") + (update-job! metadata "running") + + (or (= action "stop") (= action "kill")) + (update-job! metadata "stopped") + + (= action "run") + (do + (update-job! metadata "running") + (update-job-metadata! metadata)))) diff --git a/src/source/middleware/auth/core.clj b/src/source/middleware/auth/core.clj index 32ddd064..bb5032dc 100644 --- a/src/source/middleware/auth/core.clj +++ b/src/source/middleware/auth/core.clj @@ -2,9 +2,9 @@ (:require [source.middleware.auth.util :as util] [source.db.util :as db.util] [ring.util.response :as res] - [source.services.users :as users] [source.services.bundles :as bundles] - [source.db.honey :as db])) + [source.db.honey :as db] + [taoensso.telemere :as t])) (defn create-session [user] (let [payload {:id (:id user) @@ -17,31 +17,41 @@ (util/auth-token) (util/verify-jwt))) -(defn wrap-auth [handler] - (fn [request] - (if-let [user (validate-request request)] - (-> request - (assoc :user user) - (handler)) - (-> - (res/response {:message "Unauthorized"}) - (res/status 401))))) +(defn wrap-auth + "Returns unauthenticated if the user JWT validation failed, or if a soft-deleted user tries to call a non-GET endpoint" + [handler] + (fn [{:keys [ds] :as request}] + (let [{:keys [id] :as user} (validate-request request) + {:keys [removed]} (db/find-one ds {:tname :users + :where [:= :id id]})] + (if user + (if (or (= (:request-method request) :get) (= removed 0)) + (-> request + (assoc :user user) + (handler)) + (-> + (res/response {:message "The account of the user attempting to use this endpoint has been archived."}) + (res/status 403))) + (-> + (res/response {:message "Unauthorized"}) + (res/status 401)))))) (defn wrap-auth-user-type - "returns an unauthorized response if the user's type is not the required user type (provider | distributor | admin)" + "returns an unauthorized response if the user's type is not the required user type (creator | distributor | admin)" [handler & {:keys [required-type]}] (fn [request] (let [ds (db.util/conn :master) user-type (get-in request [:user :type]) - expected-type (->> {:id (get-in request [:user :id])} - (users/user ds) + expected-type (->> {:tname :users + :where [:= :id (get-in request [:user :id])]} + (db/find-one ds) (:type))] (cond (not (some? required-type)) (handler request) - (and (= user-type (name :admin)) (= user-type expected-type)) (handler request) + (and (= user-type (name required-type)) (= user-type expected-type)) (handler request) :else (-> (res/response {:message "Unauthorized"}) - (res/status 403)))))) + (res/status 401)))))) (defn wrap-bundle-id "validates the bundle uuid in the query parameters of the request for @@ -49,15 +59,19 @@ [handler] (fn [request] (let [ds (db.util/conn :master) - bundle-uuid (get-in request [:query-params "uuid"])] - - (if (db/exists? ds {:tname :bundles - :where [:= :uuid bundle-uuid]}) - (handler request) - - (-> - (res/response {:message "Unauthorized"}) - (res/status 403)))))) + bundle-uuid (get-in request [:query-params :uuid]) + {:keys [id]} (db/find-one ds {:tname :bundles + :where [:= :uuid bundle-uuid]})] + (if (some? id) + (-> request + (assoc :bundle-id id) + (handler)) + (do + (t/log! {:level :warn + :msg (str "Bundle authorization attempt failed with uuid: " bundle-uuid)}) + (-> + (res/response {:message "The bundle you are looking for does not exist."}) + (res/status 404))))))) (comment (let [authed-request {:headers {"Authorization" @@ -88,19 +102,21 @@ (println "Test passed")) (require '[source.util :as utils]) - (let [garbage-request {:query-params {"uuid" "garbage"}} + (let [garbage-request {:query-params {:uuid "garbage"}} + ds (db.util/conn) uuid (utils/uuid) - bundle-request {:query-params {"uuid" uuid}} + bundle-request {:query-params {:uuid uuid}} test-handler (-> (fn [request] request) (wrap-bundle-id))] - - (bundles/insert-bundle! (db.util/conn :master) {:data {:uuid uuid + (bundles/insert-bundle! (db.util/conn :master) {:data {:name (str "test-bundle-" uuid) + :uuid uuid + :content-type-id 1 :video 0 :podcast 0 :blog 0}}) (assert (= - 403 + 404 (-> garbage-request (test-handler) (:status)))) @@ -110,6 +126,10 @@ (-> bundle-request (test-handler) (:bundle-id)))) - (println "tests passed")) - ()) + (println "tests passed") + (db/delete! ds + {:tname :bundles + :where [:like :name "test-bundle-%"] + :ret :*})) + ()) diff --git a/src/source/middleware/auth/util.clj b/src/source/middleware/auth/util.clj index 058a5f3c..29b3faf1 100644 --- a/src/source/middleware/auth/util.clj +++ b/src/source/middleware/auth/util.clj @@ -1,7 +1,8 @@ (ns source.middleware.auth.util (:require [buddy.sign.jwt :as jwt] [source.config :as conf] - [clojure.string :as str])) + [clojure.string :as str] + [taoensso.telemere :as t])) (defn auth-header [request] (or (get-in request [:headers "Authorization"]) @@ -25,6 +26,6 @@ (try (jwt/decrypt token (conf/read-value :supersecretkey)) (catch Exception e - (println (.getMessage e)) + (t/log! {:level :warn + :msg (str "JWT Verification failed: " (.getMessage e))}) false))) - diff --git a/src/source/middleware/core.clj b/src/source/middleware/core.clj index 88c9986c..472d3743 100644 --- a/src/source/middleware/core.clj +++ b/src/source/middleware/core.clj @@ -2,11 +2,18 @@ (:require [source.middleware.auth.core :as auth] [source.middleware.content-type :as content-type] [source.config :as conf] + [camel-snake-kebab.core :as csk] + [camel-snake-kebab.extras :as cske] + [ring.util.response :as res] [ring.middleware.cors :refer [wrap-cors]] [ring.middleware.params :refer [wrap-params]] [ring.middleware.defaults :refer [wrap-defaults site-defaults]] [ring.middleware.json :as ring] - [ring.middleware.cookies :as cookies])) + [ring.middleware.cookies :as cookies] + [clojure.walk :as walk] + [source.util :as util] + [clojure.string :as string] + [taoensso.telemere :as t])) (defn wrap-ds [handler ds] (fn [request] @@ -14,13 +21,99 @@ (assoc :ds ds) (handler)))) +(defn wrap-js + "attaches the provided job service to the handler's request" + [handler js] + (fn [request] + (-> request + (assoc :js js) + (handler)))) + (defn apply-ds [app ds] (-> app (wrap-ds ds))) -(defn apply-generic [app & {:keys [ds]}] +(defn apply-js + "middleware for attaching the job service to the request" + [app js] (-> app + (wrap-js js))) + +(defn wrap-exception-logger [handler] + (fn [req] + (try + (handler req) + (catch Exception e + (t/log! {:level :error + :msg (str "Unhandled Exception on endpoint URI " (:uri req) ": " e)}) + (-> (res/response {:message "Internal Server Error"}) + (res/status 500)))))) + +(defn- validate-param [request [param-type schema]] + (let [{:keys [error] :as validated} (-> (cond + (= param-type :body) (:body request) + (= param-type :path) (:path-params request) + (= param-type :query) (:query-params request)) + (util/validate schema))] + (->> (when error + (str "In " (name param-type) ":\n" error)) + (assoc validated :param-type param-type :error)))) + +(defn- attach-validations [request validations] + (reduce (fn [acc {:keys [data param-type]}] + (cond + (= param-type :body) (assoc acc :body data) + (= param-type :path) (assoc acc :path-params data) + (= param-type :query) (assoc acc :query-params data) + :else acc)) request validations)) + +(defn wrap-input-validation [handler openapi-meta] + (fn [request] + (let [validations (->> (mapv (partial validate-param request) (:parameters openapi-meta))) + errors (->> validations + (filter #(:error %)) + (mapv :error)) + request (->> validations + (attach-validations request))] + (if (seq errors) + (do + (t/log! {:level :warn + :msg (str "Schema validation failed on endpoint URI " (:uri request) ": " (string/join "\n" errors))}) + (-> (res/response {:message (string/join "\n" errors)}) + (res/status 400))) + (handler request))))) + +(defn process-body [{:keys [body] :as req} t-fn] + (assoc req + :body + (if + (or (map? body) (vector? body) (seq? body)) + (cske/transform-keys (fn [k] + (if (or (keyword? k) (string? k)) + (t-fn k) + k)) body) + body))) + +(defn wrap-case-conversion [handler] + (fn [request] + (-> request + (process-body csk/->kebab-case-keyword) + (handler) + (process-body csk/->camelCaseKeyword)))) + +(defn wrap-query [handler] + (fn [{:keys [query-params] :as request}] + (-> request + (assoc :query-params (walk/keywordize-keys query-params)) + (handler)))) + +(defn apply-generic [app & {:keys [ds js]}] + (-> app + (wrap-exception-logger) (apply-ds ds) + (apply-js js) + (wrap-case-conversion) + (wrap-query) (content-type/wrap-content-type) (wrap-cors :access-control-allow-origin [(re-pattern (conf/read-value :cors-origin))] :access-control-allow-methods [:get :put :post :delete]) @@ -38,4 +131,3 @@ (defn apply-bundle [app] (-> app (auth/wrap-bundle-id))) - diff --git a/src/source/middleware/interface.clj b/src/source/middleware/interface.clj index ffb97174..d8959b84 100644 --- a/src/source/middleware/interface.clj +++ b/src/source/middleware/interface.clj @@ -1,8 +1,8 @@ (ns source.middleware.interface (:require [source.middleware.core :as mw])) -(defn apply-generic [app & {:keys [ds]}] - (mw/apply-generic app :ds ds)) +(defn apply-generic [app & {:keys [ds js]}] + (mw/apply-generic app :ds ds :js js)) (defn apply-auth "accepts required-type as an optional parameter to authorize the route only for the specified user type" @@ -12,3 +12,5 @@ (defn apply-bundle [app] (mw/apply-bundle app)) +(defn apply-validation [app openapi-meta] + (mw/wrap-input-validation app openapi-meta)) diff --git a/src/source/migrate.clj b/src/source/migrate.clj index e6e3e4f1..019b23c3 100644 --- a/src/source/migrate.clj +++ b/src/source/migrate.clj @@ -1,10 +1,13 @@ (ns source.migrate - (:require - [k16.mallard :as mallard] - [k16.mallard.store.sqlite :as store] - [k16.mallard.loader.fs :as loader.fs] - [next.jdbc :as jdbc] - [source.db.util :as db.util])) + (:require [k16.mallard :as mallard] + [k16.mallard.store.postgres :as store] + [k16.mallard.loader.fs :as loader.fs] + [next.jdbc :as jdbc] + [pg.core :as pg] + [source.db.util :as db.util] + [source.db.honey :as db] + [source.config :as conf] + [source.db.tables :as tables])) ;; This is our interface for running migrations. ;; @@ -17,17 +20,42 @@ (def ^:private migrations (loader.fs/load! "src/source/migrations")) +(def ^:private bundle-migrations + (loader.fs/load! "src/source/bundle_migrations")) + (defn run-migrations [args] - (let [context {:db-master (jdbc/get-datasource {:dbname (db.util/db-path "master") :dbtype "sqlite"})} - db-migrate (jdbc/get-datasource {:dbname (db.util/db-path "migrate") :dbtype "sqlite"}) + (let [context {:db-master (db.util/conn)} + db-migrate (jdbc/get-datasource {:dbtype "postgresql" + :jdbcUrl (str "jdbc:" (conf/read-value :database :url) ":5432/migrate")}) datastore (store/create-datastore - {:db db-migrate + {:ds db-migrate :table-name "migrations"})] + #_(try (jdbc/execute! (:db-master context) ["CREATE DOMAIN DATETIME TEXT"]) (catch Exception _)) (mallard/run {:context context :store datastore :operations migrations} args))) +(defn migrate-bundle [bundle-id args] + (let [context {:ds-master (db.util/conn) + :bundle-id bundle-id} + datastore (store/create-datastore + {:ds (jdbc/get-datasource {:dbtype "postgresql" + :jdbcUrl (str "jdbc:" (conf/read-value :database :url) ":5432/master")}) + :table-name (str "migrations_" bundle-id)})] + (mallard/run {:context context + :store datastore + :operations bundle-migrations} + args))) + +(defn run-bundle-migrations [args] + (let [ds-master (db.util/conn :master) + bundles (if (some #(= % "bundles") (tables/table-names ds-master)) + (db/find ds-master {:tname :bundles + :ret :*}) + [])] + (run! #(migrate-bundle (:id %) args) bundles))) + (defn -main [& args] + (run-bundle-migrations args) (run-migrations args)) - diff --git a/src/source/migrations/001_init_master_db.clj b/src/source/migrations/001_init_master_db.clj index 4b6cae24..2fd17daf 100644 --- a/src/source/migrations/001_init_master_db.clj +++ b/src/source/migrations/001_init_master_db.clj @@ -49,6 +49,13 @@ :domain "www.medium.com" :content-type-id 3}]}) +(def categories-seed + {:tname :categories + :data [{:name "programming"} + {:name "game development"} + {:name "languages"} + {:name "technology"}]}) + (def sectors-seed {:tname :sectors :data [{:name "renewable energy"} @@ -61,24 +68,29 @@ (tables/create-tables! ds-master :source.db.master - [:users + [:businesses + :users + :content-types + :providers :sectors :categories - :content-types :cadences :baselines :bundles :feeds :feed-categories - :providers - :businesses :user-sectors - :feed-sectors]) + :feed-sectors + :selection-schemas + :incoming-posts + :job-metadata + :jobs]) (db/insert! ds-master baselines-seed) (db/insert! ds-master cadences-seed) (db/insert! ds-master content-types-seed) (db/insert! ds-master providers-seed) + (db/insert! ds-master categories-seed) (db/insert! ds-master sectors-seed) (when (= (conf/read-value :env) "dev") @@ -95,6 +107,7 @@ (defn run-down! [context] (let [ds-master (:db-master context)] + (tables/drop-all-tables! ds-master))) (comment diff --git a/src/source/migrations/002_incoming_posts.clj b/src/source/migrations/002_incoming_posts.clj new file mode 100644 index 00000000..5f7787bc --- /dev/null +++ b/src/source/migrations/002_incoming_posts.clj @@ -0,0 +1,14 @@ +(ns source.migrations.002-incoming-posts + (:require [source.db.master] + [source.db.tables :as tables])) + +(defn run-up! [context] + (let [ds-master (:db-master context)] + (tables/create-tables! + ds-master + :source.db.master + [:incoming-posts]))) + +(defn run-down! [context] + (let [ds-master (:db-master context)] + (tables/drop-table! ds-master :incoming-posts))) diff --git a/src/source/migrations/003_bundle_content_types.clj b/src/source/migrations/003_bundle_content_types.clj new file mode 100644 index 00000000..f2b55db7 --- /dev/null +++ b/src/source/migrations/003_bundle_content_types.clj @@ -0,0 +1,30 @@ +(ns source.migrations.003-bundle-content-types + (:require [source.db.master] + [source.db.tables :as tables] + [source.services.interface :as services])) + +(defn run-up! [context] + (let [ds-master (:db-master context) + bundles (services/bundles ds-master)] + + (tables/create-tables! + ds-master + :source.db.master + [:bundle-content-types]) + + (run! (fn [{:keys [id content-type-id]}] + (when (some? content-type-id) + (services/insert-bundle-content-types! ds-master {:bundle-id id + :content-types [{:id content-type-id}]}))) + bundles))) + +(defn run-down! [context] + (let [ds-master (:db-master context) + bundle-content-types (services/bundles ds-master)] + + (run! (fn [{:keys [bundle-id content-type-id]}] + (services/update-bundle! ds-master {:id bundle-id + :data {:content-type-id content-type-id}})) + bundle-content-types) + + (tables/drop-table! ds-master :bundle-content-types))) diff --git a/src/source/migrations/004_category_images.clj b/src/source/migrations/004_category_images.clj new file mode 100644 index 00000000..8f90b2d8 --- /dev/null +++ b/src/source/migrations/004_category_images.clj @@ -0,0 +1,15 @@ +(ns source.migrations.004-category-images + (:require [source.db.master] + [source.db.honey :as hon])) + +(defn run-up! [context] + (let [ds-master (:db-master context)] + (->> {:alter-table :categories + :add-column [:display-picture :text]} + (hon/execute! ds-master)))) + +(defn run-down! [context] + (let [ds-master (:db-master context)] + (->> {:alter-table :categories + :drop-column :display-picture} + (hon/execute! ds-master)))) diff --git a/src/source/migrations/005_filtered_feeds_posts.clj b/src/source/migrations/005_filtered_feeds_posts.clj new file mode 100644 index 00000000..ab626a1b --- /dev/null +++ b/src/source/migrations/005_filtered_feeds_posts.clj @@ -0,0 +1,18 @@ +(ns source.migrations.005-filtered-feeds-posts + (:require [source.db.master] + [source.db.tables :as tables])) + +(defn run-up! [context] + (let [ds-master (:db-master context)] + (tables/create-tables! + ds-master + :source.db.master + [:filtered-feeds + :filtered-posts]))) + +(defn run-down! [context] + (let [ds-master (:db-master context)] + (tables/drop-tables! + ds-master + [:filtered-feeds + :filtered-posts]))) diff --git a/src/source/migrations/006_events.clj b/src/source/migrations/006_events.clj new file mode 100644 index 00000000..f8defec0 --- /dev/null +++ b/src/source/migrations/006_events.clj @@ -0,0 +1,17 @@ +(ns source.migrations.006-events + (:require [source.db.master] + [source.db.tables :as tables])) + +(defn run-up! [context] + (let [ds-master (:db-master context)] + (tables/create-tables! + ds-master + :source.db.master + [:events + :event-categories]))) +(defn run-down! [context] + (let [ds-master (:db-master context)] + (tables/drop-tables! + ds-master + [:events + :event-categories]))) diff --git a/src/source/migrations/007_business_fields.clj b/src/source/migrations/007_business_fields.clj new file mode 100644 index 00000000..b07dba8e --- /dev/null +++ b/src/source/migrations/007_business_fields.clj @@ -0,0 +1,33 @@ +(ns source.migrations.007-business-fields + (:require [source.db.master] + [source.db.honey :as hon] + [source.db.tables :as tables] + [honey.sql.helpers :as hsql])) + +(defn run-up! [context] + (let [ds-master (:db-master context)] + (tables/create-table! + ds-master + :source.db.master + :business-types) + + (hon/execute! + ds-master + (-> (hsql/alter-table :businesses) + (hsql/add-column :business-type-id :integer))) + + (hon/execute! + ds-master + (-> (hsql/alter-table :businesses) + (hsql/add-column :registration :text))))) + +(defn run-down! [context] + (let [ds-master (:db-master context)] + (hon/execute! + ds-master + (-> (hsql/alter-table :businesses) + (hsql/drop-column :business-type-id :registration))) + + (tables/drop-table! + ds-master + :business-types))) diff --git a/src/source/migrations/008_provider_rss_instructions.clj b/src/source/migrations/008_provider_rss_instructions.clj new file mode 100644 index 00000000..c83ec9c1 --- /dev/null +++ b/src/source/migrations/008_provider_rss_instructions.clj @@ -0,0 +1,23 @@ +(ns source.migrations.008-provider-rss-instructions + (:require [source.db.master] + [source.db.honey :as hon] + [honey.sql.helpers :as hsql])) + +(defn run-up! [context] + (let [ds-master (:db-master context)] + (hon/execute! + ds-master + (-> (hsql/alter-table :providers) + (hsql/add-column :instructions :text))) + + (hon/execute! + ds-master + (-> (hsql/alter-table :providers) + (hsql/add-column :placeholder-url :text))))) + +(defn run-down! [context] + (let [ds-master (:db-master context)] + (hon/execute! + ds-master + (-> (hsql/alter-table :providers) + (hsql/drop-column :instructions :placeholder-url))))) diff --git a/src/source/migrations/009_user_removed.clj b/src/source/migrations/009_user_removed.clj new file mode 100644 index 00000000..062f6197 --- /dev/null +++ b/src/source/migrations/009_user_removed.clj @@ -0,0 +1,15 @@ +(ns source.migrations.009-user-removed + (:require [source.db.master] + [source.db.honey :as hon])) + +(defn run-up! [context] + (let [ds-master (:db-master context)] + (->> {:alter-table :users + :add-column [:removed :integer [:default 0]]} + (hon/execute! ds-master)))) + +(defn run-down! [context] + (let [ds-master (:db-master context)] + (->> {:alter-table :users + :drop-column :removed} + (hon/execute! ds-master)))) diff --git a/src/source/migrations/010_selection_schema.clj b/src/source/migrations/010_selection_schema.clj new file mode 100644 index 00000000..921bcd00 --- /dev/null +++ b/src/source/migrations/010_selection_schema.clj @@ -0,0 +1,23 @@ +(ns source.migrations.010-selection-schema + (:require [source.db.master] + [source.db.honey :as hon] + [source.db.tables :as tables])) + +(defn run-up! [context] + (let [ds-master (:db-master context)] + (tables/create-table! + ds-master + :source.db.master + :output-schemas) + + (->> {:alter-table :selection-schemas + :add-column [:schema :text]} + (hon/execute! ds-master)))) + +(defn run-down! [context] + (let [ds-master (:db-master context)] + (->> {:alter-table :selection-schemas + :drop-column :schema} + (hon/execute! ds-master)) + + (tables/drop-table! ds-master :output-schemas))) diff --git a/src/source/migrations/011_indices.clj b/src/source/migrations/011_indices.clj new file mode 100644 index 00000000..7726be96 --- /dev/null +++ b/src/source/migrations/011_indices.clj @@ -0,0 +1,47 @@ +(ns source.migrations.011-indices + (:require [source.db.master] + [pg.core :as pg])) + +(defn run-up! [context] + (let [ds-master (:db-master context)] + (pg/with-connection [ds-master ds-master] + (pg/execute ds-master "CREATE INDEX idx_incoming_posts_feed_id ON incoming_posts (feed_id);") + (pg/execute ds-master "CREATE INDEX idx_incoming_posts_creator_id ON incoming_posts (creator_id);") + (pg/execute ds-master "CREATE INDEX idx_incoming_posts_content_type_id ON incoming_posts (content_type_id);") + + (pg/execute ds-master "CREATE INDEX idx_feeds_user_id ON feeds (user_id);") + (pg/execute ds-master "CREATE INDEX idx_feeds_content_type_id ON feeds (content_type_id);") + + (pg/execute ds-master "CREATE INDEX idx_feed_categories_feed_id ON feed_categories (feed_id);") + (pg/execute ds-master "CREATE INDEX idx_feed_categories_category_id ON feed_categories (category_id);") + + (pg/execute ds-master "CREATE INDEX idx_bundles_user_id ON bundles (user_id);") + (pg/execute ds-master "CREATE INDEX idx_bundles_content_type_id ON bundles (content_type_id);") + + (pg/execute ds-master "CREATE INDEX idx_filtered_feeds_bundle_id ON filtered_feeds (bundle_id);") + (pg/execute ds-master "CREATE INDEX idx_filtered_feeds_feed_id ON filtered_feeds (feed_id);") + + (pg/execute ds-master "CREATE INDEX idx_filtered_posts_bundle_id ON filtered_posts (bundle_id);") + (pg/execute ds-master "CREATE INDEX idx_filtered_posts_post_id ON filtered_posts (post_id);")))) + +(defn run-down! [context] + (let [ds-master (:db-master context)] + (pg/with-connection [ds-master ds-master] + (pg/execute ds-master "DROP INDEX IF EXISTS idx_incoming_posts_feed_id;") + (pg/execute ds-master "DROP INDEX IF EXISTS idx_incoming_posts_creator_id;") + (pg/execute ds-master "DROP INDEX IF EXISTS idx_incoming_posts_content_type_id;") + + (pg/execute ds-master "DROP INDEX IF EXISTS idx_feeds_user_id;") + (pg/execute ds-master "DROP INDEX IF EXISTS idx_feeds_content_type_id;") + + (pg/execute ds-master "DROP INDEX IF EXISTS idx_feed_categories_feed_id;") + (pg/execute ds-master "DROP INDEX IF EXISTS idx_feed_categories_category_id;") + + (pg/execute ds-master "DROP INDEX IF EXISTS idx_bundles_user_id") + (pg/execute ds-master "DROP INDEX IF EXISTS idx_bundles_content_type_id") + + (pg/execute ds-master "DROP INDEX IF EXISTS idx_filtered_feeds_bundle_id") + (pg/execute ds-master "DROP INDEX IF EXISTS idx_filtered_feeds_feed_id") + + (pg/execute ds-master "DROP INDEX IF EXISTS idx_filtered_posts_bundle_id") + (pg/execute ds-master "DROP INDEX IF EXISTS idx_filtered_posts_post_id")))) diff --git a/src/source/migrations/012_feed_descriptions.clj b/src/source/migrations/012_feed_descriptions.clj new file mode 100644 index 00000000..fd468f5f --- /dev/null +++ b/src/source/migrations/012_feed_descriptions.clj @@ -0,0 +1,18 @@ +(ns source.migrations.012-feed-descriptions + (:require [source.db.master] + [source.db.honey :as hon] + [honey.sql.helpers :as hsql])) + +(defn run-up! [context] + (let [ds-master (:db-master context)] + (hon/execute! + ds-master + (-> (hsql/alter-table :feeds) + (hsql/add-column :description :text))))) + +(defn run-down! [context] + (let [ds-master (:db-master context)] + (hon/execute! + ds-master + (-> (hsql/alter-table :feeds) + (hsql/drop-column :description))))) diff --git a/src/source/migrations/013_email_hash.clj b/src/source/migrations/013_email_hash.clj new file mode 100644 index 00000000..1dadb1bc --- /dev/null +++ b/src/source/migrations/013_email_hash.clj @@ -0,0 +1,21 @@ +(ns source.migrations.013-email-hash + (:require [source.db.master] + [source.db.honey :as hon] + [honey.sql.helpers :as hsql])) + +(defn run-up! [context] + (let [ds-master (:db-master context)] + (hon/update! ds-master {:tname :users + :where [:= :type "admin"] + :data {:email-verified 1}}) + (hon/execute! + ds-master + (-> (hsql/alter-table :users) + (hsql/add-column :email-hash :text))))) + +(defn run-down! [context] + (let [ds-master (:db-master context)] + (hon/execute! + ds-master + (-> (hsql/alter-table :users) + (hsql/drop-column :email-hash))))) diff --git a/src/source/migrations/014_business_types.clj b/src/source/migrations/014_business_types.clj new file mode 100644 index 00000000..c8a33273 --- /dev/null +++ b/src/source/migrations/014_business_types.clj @@ -0,0 +1,18 @@ +(ns source.migrations.014-business-types + (:require [source.db.master] + [source.db.honey :as hon])) + +(defn run-up! [context] + (let [ds-master (:db-master context)] + (hon/insert! ds-master {:tname :business-types + :data [{:name "For-Profit"} + {:name "Non-Profit"} + {:name "Creator"}]}))) + +(defn run-down! [context] + (let [ds-master (:db-master context)] + (hon/delete! ds-master {:tname :business-types + :where [:or + [:= :name "For-Profit"] + [:= :name "Non-Profit"] + [:= :name "Creator"]]}))) diff --git a/src/source/migrations/015_integration_types.clj b/src/source/migrations/015_integration_types.clj new file mode 100644 index 00000000..f25f55d9 --- /dev/null +++ b/src/source/migrations/015_integration_types.clj @@ -0,0 +1,32 @@ +(ns source.migrations.015-integration-types + (:require [source.db.master] + [source.db.honey :as hon] + [honey.sql.helpers :as hsql] + [source.db.tables :as tables])) + +(defn run-up! [context] + (let [ds-master (:db-master context)] + (tables/create-table! + ds-master + :source.db.master + :integration-types) + + (hon/insert! ds-master {:tname :integration-types + :data [{:name "Website"} + {:name "App"} + {:name "Plugin"} + {:name "Community"}]}) + + (hon/execute! + ds-master + (-> (hsql/alter-table :bundles) + (hsql/add-column :integration-id :int))))) + +(defn run-down! [context] + (let [ds-master (:db-master context)] + (hon/execute! + ds-master + (-> (hsql/alter-table :bundles) + (hsql/drop-column :integration-id))) + + (tables/drop-table! ds-master :integration-types))) diff --git a/src/source/migrations/016_integration_type_ids.clj b/src/source/migrations/016_integration_type_ids.clj new file mode 100644 index 00000000..5d3335ba --- /dev/null +++ b/src/source/migrations/016_integration_type_ids.clj @@ -0,0 +1,26 @@ +(ns source.migrations.016-integration-type-ids + (:require [source.db.master] + [source.db.honey :as hon] + [honey.sql.helpers :as hsql])) + +(defn run-up! [context] + (let [ds-master (:db-master context)] + (hon/execute! + ds-master + (-> (hsql/alter-table :bundles) + (hsql/drop-column :integration-id))) + (hon/execute! + ds-master + (-> (hsql/alter-table :bundles) + (hsql/add-column :integration-type-id :int))))) + +(defn run-down! [context] + (let [ds-master (:db-master context)] + (hon/execute! + ds-master + (-> (hsql/alter-table :bundles) + (hsql/add-column :integration-id))) + (hon/execute! + ds-master + (-> (hsql/alter-table :bundles) + (hsql/drop-column :integration-type-id))))) diff --git a/src/source/migrations/017_category_seeding.clj b/src/source/migrations/017_category_seeding.clj new file mode 100644 index 00000000..539b453c --- /dev/null +++ b/src/source/migrations/017_category_seeding.clj @@ -0,0 +1,70 @@ +(ns source.migrations.017-category-seeding + (:require [source.db.master] + [source.db.honey :as hon] + [source.workers.categories :as categories])) + +(defn run-up! [context] + (let [ds-master (:db-master context)] + (hon/insert! ds-master {:tname :categories + :data [{:name "Regenerative finance and investment"} + {:name "Regenerative business"} + {:name "Sustainable fashion and textiles"} + {:name "Regenerative agriculture and food"} + {:name "Energy and clean technology"} + {:name "Circularity and waste"} + {:name "Built environment"} + {:name "Transport and mobility"} + {:name "Travel and regenerative tourism"} + {:name "Technology and AI for good"} + {:name "Wellbeing and mental health"} + {:name "Social justice and equity"} + {:name "Indigenous wisdom and land rights"} + {:name "Leadership and inner development"} + {:name "Policy and systems change"} + {:name "Water and freshwater systems"} + {:name "Biodiversity and conservation"} + {:name "Ocean and marine"} + {:name "Bioregionalism and local economies"} + {:name "Community and culture"} + {:name "Arts and artivism"} + {:name "Education and climate literacy"} + {:name "Media and communications"} + {:name "Nonprofits and philanthropy"} + {:name "Consumer goods and conscious living"} + {:name "Spirituality and consciousness"} + {:name "Gender and women's leadership"} + {:name "Health and planetary health"}]}))) + +(defn run-down! [context] + (let [ds-master (:db-master context)] + (run! + #(categories/delete-category! ds-master (:id %)) + (hon/find ds-master {:tname :categories + :where [:in :name ["Regenerative finance and investment" + "Regenerative business" + "Sustainable fashion and textiles" + "Regenerative agriculture and food" + "Energy and clean technology" + "Circularity and waste" + "Built environment" + "Transport and mobility" + "Travel and regenerative tourism" + "Technology and AI for good" + "Wellbeing and mental health" + "Social justice and equity" + "Indigenous wisdom and land rights" + "Leadership and inner development" + "Policy and systems change" + "Water and freshwater systems" + "Biodiversity and conservation" + "Ocean and marine" + "Bioregionalism and local economies" + "Community and culture" + "Arts and artivism" + "Education and climate literacy" + "Media and communications" + "Nonprofits and philanthropy" + "Consumer goods and conscious living" + "Spirituality and consciousness" + "Gender and women's leadership" + "Health and planetary health"]]})))) diff --git a/src/source/migrations/018_sector_seeding.clj b/src/source/migrations/018_sector_seeding.clj new file mode 100644 index 00000000..29e27d46 --- /dev/null +++ b/src/source/migrations/018_sector_seeding.clj @@ -0,0 +1,21 @@ +(ns source.migrations.018-sector-seeding + (:require [source.db.master] + [source.db.honey :as hon])) + +(defn run-up! [context] + (let [ds-master (:db-master context) + sectors (->> (hon/find ds-master {:tname :categories}) + (mapv #(dissoc % :display-picture)))] + (hon/delete! ds-master {:tname :user-sectors}) + (hon/delete! ds-master {:tname :sectors}) + (hon/insert! ds-master {:tname :sectors + :data sectors}))) + +(defn run-down! [context] + (let [ds-master (:db-master context)] + (hon/delete! ds-master {:tname :user-sectors}) + (hon/delete! ds-master {:tname :sectors}) + (hon/insert! ds-master {:tname :sectors + :data [{:name "renewable energy"} + {:name "conservation ecology"} + {:name "recycling"}]}))) diff --git a/src/source/routes/admin.clj b/src/source/routes/admin.clj index a598f8f7..af3aacab 100644 --- a/src/source/routes/admin.clj +++ b/src/source/routes/admin.clj @@ -1,27 +1,38 @@ (ns source.routes.admin - (:require [source.services.users :as users] - [source.db.util :as db.util] - [source.password :as pw])) + (:require [source.password :as pw] + [ring.util.response :as res] + [source.db.honey :as hon])) -(defn post [{:keys [body] :as _request}] - (let [ds (db.util/conn :master) - user (users/user - ds - {:where [:= :email (:email body)]}) +(defn post + {:summary "registers an admin user" + :parameters {:body [:map + [:email :string] + [:password :string] + [:confirm-password :string]]} + :responses {201 {:body [:map [:message :string]]} + 401 {:body [:map [:message :string]]} + 403 {:body [:map [:message :string]]}}} + + [{:keys [ds body] :as _request}] + + (let [user (hon/find-one ds {:tname :users + :where [:= :email (:email body)]}) {:keys [password confirm-password]} body] (cond (not (= password confirm-password)) - {:status 400 :body {:message "passwords do not match!"}} + (-> (res/response {:message "passwords do not match!"}) + (res/status 400)) (some? user) - {:status 400 :body {:message "an account for this email already exists!"}} + (-> (res/response {:message "an account for this email already exists!"}) + (res/status 400)) :else (let [pw (pw/hash-password password) new-user (-> (assoc body - :password pw - :type "admin") + :password pw + :type "admin") (dissoc :confirm-password))] - (users/insert-user! ds {:data new-user}) - {:status 200 :body {:message "successfully created user"}})))) - + (hon/insert! ds {:tname :users + :data new-user}) + (res/response {:message "successfully created user"}))))) diff --git a/src/source/routes/admin_feeds.clj b/src/source/routes/admin_feeds.clj new file mode 100644 index 00000000..a4f7a0e9 --- /dev/null +++ b/src/source/routes/admin_feeds.clj @@ -0,0 +1,27 @@ +(ns source.routes.admin-feeds + (:require [ring.util.response :as res] + [source.db.honey :as hon])) + +(defn get + {:summary "get all feeds" + :responses {200 {:body [:vector + [:map + [:id :int] + [:title :string] + [:display-picture [:maybe :string]] + [:url [:maybe :string]] + [:rss-url :string] + [:user-id :int] + [:provider-id [:maybe :int]] + [:created-at :string] + [:updated-at [:maybe :string]] + [:content-type-id :int] + [:cadence-id :int] + [:baseline-id :int] + [:ts-and-cs [:maybe :int]] + [:state [:enum "live" "not live" "pending"]]]]}}} + + [{:keys [ds] :as _request}] + (-> (hon/find ds {:tname :feeds + :ret :*}) + (res/response))) diff --git a/src/source/routes/analytics/bundle/posts/_id_/views.clj b/src/source/routes/analytics/bundle/posts/_id_/views.clj new file mode 100644 index 00000000..a0993e8d --- /dev/null +++ b/src/source/routes/analytics/bundle/posts/_id_/views.clj @@ -0,0 +1,16 @@ +(ns source.routes.analytics.bundle.posts.-id-.views + (:require [ring.util.response :as res] + [source.services.analytics.interface :as analytics] + [source.services.interface :as services])) + +(defn post + {:summary "Explicitly insert a view event for the post with the given id for the purpose of analytics" + :parameters {:query [:map [:uuid :string]] + :path [:map [:id {:title "id" + :description "post id"} :int]]} + :responses {200 {:body [:map [:message :string]]}}} + + [{:keys [ds bundle-id path-params] :as _request}] + (let [post (services/incoming-post ds {:id (:id path-params)})] + (analytics/insert-post-view! ds post bundle-id) + (res/response {:message "Successfully inserted view event"}))) diff --git a/src/source/routes/analytics/creator/deltas.clj b/src/source/routes/analytics/creator/deltas.clj new file mode 100644 index 00000000..cc024129 --- /dev/null +++ b/src/source/routes/analytics/creator/deltas.clj @@ -0,0 +1,35 @@ +(ns source.routes.analytics.creator.deltas + (:require [clojure.walk :as w] + [ring.util.response :as res] + [java-time.api :as jt] + [source.services.analytics.interface :as analytics])) + +(defn get + {:summary "Returns the percentage of growth in impressions, clicks and views per week, over the given time period. Optionally filtered by feed. + Date must be in the format yyyy-MM-dd" + :parameters {:query [:map + [:mindate :string] + [:maxdate :string] + [:feed {:optional true} [:maybe :int]]]} + :responses {200 {:body [:vector + [:map + [:week :string] + [:impressions :float] + [:clicks :float] + [:views :float]]]} + 400 {:body [:map [:message :string]]}}} + + [{:keys [ds user query-params] :as _request}] + (let [{:keys [mindate maxdate feed]} (w/keywordize-keys query-params) + {:keys [parsed-mindate parsed-maxdate]} (try + {:parsed-mindate (jt/local-date mindate) + :parsed-maxdate (jt/local-date maxdate)} + (catch Exception _ (-> (res/response {:message "Invalid date format. Date must be in the format yyyy-MM-dd."}) + (res/status 400)))) + days-between (.until parsed-mindate parsed-maxdate java.time.temporal.ChronoUnit/DAYS) + results (analytics/weekly-growth-averages ds mindate maxdate {:creator-id (:id user) + :feed-id feed})] + (if (>= days-between 14) + (res/response results) + (-> (res/response {:message "Date range too small. Date range must include at least 2 weeks."}) + (res/status 400))))) diff --git a/src/source/routes/analytics/creator/general.clj b/src/source/routes/analytics/creator/general.clj new file mode 100644 index 00000000..984084f3 --- /dev/null +++ b/src/source/routes/analytics/creator/general.clj @@ -0,0 +1,23 @@ +(ns source.routes.analytics.creator.general + (:require [source.services.analytics.interface :as analytics] + [ring.util.response :as res] + [clojure.walk :as w])) + +(defn get + {:summary "Gets the number of impressions, clicks and views per day for a creator over the given time period. Optionally filtered by feed. + Date must be in the format yyyy-MM-dd" + :parameters {:query [:map + [:mindate :string] + [:maxdate :string] + [:feed {:optional true} [:maybe :int]]]} + :responses {200 {:body [:vector + [:map + [:day :string] + [:impressions :int] + [:clicks :int] + [:views :int]]]}}} + + [{:keys [ds user query-params] :as _request}] + (let [{:keys [mindate maxdate feed]} (w/keywordize-keys query-params)] + (res/response (analytics/interval-statistics-query ds :daily mindate maxdate {:creator-id (:id user) + :feed-id feed})))) diff --git a/src/source/routes/analytics/creator/top.clj b/src/source/routes/analytics/creator/top.clj new file mode 100644 index 00000000..1825d808 --- /dev/null +++ b/src/source/routes/analytics/creator/top.clj @@ -0,0 +1,50 @@ +(ns source.routes.analytics.creator.top + (:require [source.services.analytics.interface :as analytics] + [ring.util.response :as res] + [clojure.set :as set] + [source.services.interface :as services])) + +(defn record-names [ds top-field ids] + (if (= top-field :post-id) + (mapv (fn [{:keys [id title]}] + {:id id + :name title}) + (services/incoming-posts ds {:where [:in :id ids]})) + (mapv (fn [{:keys [id name]}] + {:id id + :name name}) + (services/bundles ds {:where [:in :id ids]})))) + +(defn get + {:summary "Get the top n records with the highest number of impressions, clicks and views, in terms of the given top field. Optionally filtered by content type. + Date must be in the format yyyy-MM-dd" + :parameters {:query [:map + [:n :int] + [:mindate :string] + [:maxdate :string] + [:top [:enum "post" "bundle"]] + [:contenttype {:optional true} [:maybe :int]]]} + :responses {200 {:body [:vector + [:map + [:top :string] + [:impressions :int] + [:clicks :int] + [:views :int]]]}}} + + [{:keys [ds user query-params] :as _request}] + (let [{:keys [n mindate maxdate top contenttype]} query-params + top-field (if (= top "post") :post-id :bundle-id) + results (->> {:creator-id (:id user) + :content-type-id contenttype} + (analytics/top-statistics-query ds mindate maxdate n top-field) + (mapv (fn [result] + (set/rename-keys result {top-field :top})))) + ids (mapv :top results) + names (record-names ds top-field ids) + juxted (->> names + (mapv (juxt :id :name)) + (into {})) + named-results (mapv (fn [{:keys [top] :as r}] + (assoc r :top (clojure.core/get juxted top (str top)))) + results)] + (res/response named-results))) diff --git a/src/source/routes/analytics/creator/top_average.clj b/src/source/routes/analytics/creator/top_average.clj new file mode 100644 index 00000000..8a63451a --- /dev/null +++ b/src/source/routes/analytics/creator/top_average.clj @@ -0,0 +1,18 @@ +(ns source.routes.analytics.creator.top-average + (:require [source.services.analytics.interface :as analytics] + [ring.util.response :as res])) + +(defn get + {:summary "Get the average engagement (clicks and views) for a creator, optionally filtered by content type. + Date must be in the format yyyy-MM-dd" + :parameters {:query [:map + [:mindate :string] + [:maxdate :string] + [:contenttype {:optional true} [:maybe :int]]]} + :responses {200 {:body [:map [:average :float]]}}} + + [{:keys [ds user query-params] :as _request}] + (let [{:keys [mindate maxdate contenttype]} query-params + result (analytics/average-engagement ds mindate maxdate {:creator-id (:id user) + :content-type-id contenttype})] + (res/response {:average result}))) diff --git a/src/source/routes/analytics/distributor/general.clj b/src/source/routes/analytics/distributor/general.clj new file mode 100644 index 00000000..8a0c8a68 --- /dev/null +++ b/src/source/routes/analytics/distributor/general.clj @@ -0,0 +1,22 @@ +(ns source.routes.analytics.distributor.general + (:require [source.services.analytics.interface :as analytics] + [ring.util.response :as res])) + +(defn get + {:summary "Gets the number of impressions, clicks and views per day for a distributor over the given time period. Optionally filtered by bundle. + Date must be in the format yyyy-MM-dd" + :parameters {:query [:map + [:mindate :string] + [:maxdate :string] + [:bundle {:optional true} [:maybe :int]]]} + :responses {200 {:body [:vector + [:map + [:day :string] + [:impressions :int] + [:clicks :int] + [:views :int]]]}}} + + [{:keys [ds user query-params] :as _request}] + (let [{:keys [mindate maxdate bundle]} query-params] + (res/response (analytics/interval-statistics-query ds :daily mindate maxdate {:distributor-id (:id user) + :bundle-id bundle})))) diff --git a/src/source/routes/analytics/distributor/top.clj b/src/source/routes/analytics/distributor/top.clj new file mode 100644 index 00000000..6d3a5cd8 --- /dev/null +++ b/src/source/routes/analytics/distributor/top.clj @@ -0,0 +1,49 @@ +(ns source.routes.analytics.distributor.top + (:require [source.services.analytics.interface :as analytics] + [ring.util.response :as res] + [clojure.set :as set] + [source.db.honey :as hon])) + +(defn record-names [ds top-field ids] + (if (= top-field :post-id) + (hon/find ds {:tname :incoming-posts + :where [:in :id ids]}) + (hon/find ds {:tname :feeds + :where [:in :id ids]}))) + +(defn get + {:summary "Get the top n records with the highest number of impressions, clicks and views, in terms of the given top field. Optionally filtered by content type. + Date must be in the format yyyy-MM-dd" + :parameters {:query [:map + [:n :int] + [:mindate :string] + [:maxdate :string] + [:top [:enum "post" "feed"]] + [:contenttype {:optional true} [:maybe :int]]]} + :responses {200 {:body [:vector + [:map + [:top :string] + [:impressions :int] + [:clicks :int] + [:views :int]]]}}} + + [{:keys [ds user query-params] :as _request}] + (let [{:keys [n mindate maxdate top contenttype]} query-params + top-field (if (= top "post") :post-id :feed-id) + results (->> {:distributor-id (:id user) + :content-type-id contenttype} + (analytics/top-statistics-query ds mindate maxdate n top-field) + (mapv (fn [result] + (set/rename-keys result {top-field :top})))) + ids (mapv :top results) + names (record-names ds top-field ids) + juxted (->> names + (mapv (fn [{:keys [id title]}] + {:id id + :name title})) + (mapv (juxt :id :name)) + (into {})) + named-results (mapv (fn [{:keys [top] :as r}] + (assoc r :top (clojure.core/get juxted top (str top)))) + results)] + (res/response named-results))) diff --git a/src/source/routes/analytics/distributor/top_average.clj b/src/source/routes/analytics/distributor/top_average.clj new file mode 100644 index 00000000..b0778dbd --- /dev/null +++ b/src/source/routes/analytics/distributor/top_average.clj @@ -0,0 +1,18 @@ +(ns source.routes.analytics.distributor.top-average + (:require [source.services.analytics.interface :as analytics] + [ring.util.response :as res])) + +(defn get + {:summary "Get the average engagement (clicks and views) for a distributor, optionally filtered by content type. + Date must be in the format yyyy-MM-dd" + :parameters {:query [:map + [:mindate :string] + [:maxdate :string] + [:contenttype {:optional true} [:maybe :int]]]} + :responses {200 {:body [:map [:average :float]]}}} + + [{:keys [ds user query-params] :as _request}] + (let [{:keys [mindate maxdate contenttype]} query-params + result (analytics/average-engagement ds mindate maxdate {:distributor-id (:id user) + :content-type-id contenttype})] + (res/response {:average result}))) diff --git a/src/source/routes/approve_feed.clj b/src/source/routes/approve_feed.clj new file mode 100644 index 00000000..90ead7c2 --- /dev/null +++ b/src/source/routes/approve_feed.clj @@ -0,0 +1,29 @@ +(ns source.routes.approve-feed + (:require [source.email.gmail :as gmail] + [source.email.templates :as templates] + [ring.util.response :as res] + [source.db.honey :as hon])) + +(defn post + {:summary "approve the feed with the given feed-id and allow it to go live" + :parameters {:path [:map [:id {:title "id" + :description "feed id"} :int]]} + :responses {200 {:body [:map [:message :string]]} + 401 {:body [:map [:message :string]]} + 403 {:body [:map [:message :string]]}}} + + [{:keys [ds path-params] :as _request}] + (let [{:keys [id user-id title]} (hon/find-one ds {:tname :feeds + :where [:= :id (:id path-params)]}) + {:keys [email firstname]} (hon/find-one ds {:tname :users + :where [:= :id user-id]})] + (hon/update! ds {:tname :feeds + :where [:= :id (:id path-params)] + :data {:state "live"}}) + (gmail/send-email {:to email + :subject "Feed Approval" + :body (templates/feed-approval {:creator-name firstname + :feed-title title + :feed-id id}) + :type :text/html}) + (res/response {:message "successfully approved feed"}))) diff --git a/src/source/routes/authorized.clj b/src/source/routes/authorized.clj index bb8658cd..0356a2df 100644 --- a/src/source/routes/authorized.clj +++ b/src/source/routes/authorized.clj @@ -1,6 +1,13 @@ (ns source.routes.authorized) -(defn get [{:keys [user] :as _request}] +(defn get + {:summary "checks if authenticated" + :responses {200 {:body [:map + [:user + [:map + [:id :int] + [:type [:enum "creator" "distributor" "admin"]]]]]}}} + + [{:keys [user] :as _request}] {:status 200 :body {:user user}}) - diff --git a/src/source/routes/baselines.clj b/src/source/routes/baselines.clj new file mode 100644 index 00000000..2f69a561 --- /dev/null +++ b/src/source/routes/baselines.clj @@ -0,0 +1,15 @@ +(ns source.routes.baselines + (:require [ring.util.response :as res] + [source.db.honey :as hon])) + +(defn get + {:summary "get all baselines" + :responses {200 {:body [:vector + [:map + [:id :int] + [:label :string] + [:min :int] + [:max :int]]]}}} + [{:keys [ds] :as _request}] + (->> (hon/find ds {:tname :baselines}) + (res/response))) diff --git a/src/source/routes/bundle.clj b/src/source/routes/bundle.clj new file mode 100644 index 00000000..c36c5388 --- /dev/null +++ b/src/source/routes/bundle.clj @@ -0,0 +1,35 @@ +(ns source.routes.bundle + (:require [ring.util.response :as res] + [source.db.honey :as hon] + [pg.core :as pg])) + +(defn get + {:summary "Get metadata for the associated uuid-authorized bundle." + :parameters {:query [:map [:uuid {:description "Bundle UUID"} :string]]} + :responses {200 {:body [:map + [:id :int] + [:name :string] + [:uuid :string] + [:user-id :int] + [:video :int] + [:podcast :int] + [:blog :int] + [:hash [:maybe :string]] + [:content-type-id :int] + [:ts-and-cs [:maybe :int]]]} + 404 {:body [:map [:message :string]]}}} + + [{:keys [ds bundle-id] :as _request}] + (res/response (hon/find-one ds {:tname :bundles + :where [:= :id bundle-id]}))) + +(defn exists + {:summary "Check for the existence of a bundle with the bundle UUID provided" + :parameters {:query [:map [:uuid {:description "Bundle UUID"} :string]]} + :responses {200 {:body [:map [:exists :boolean]]}}} + [{:keys [ds query-params] :as _request}] + (res/response + (-> ds + (pg/execute "SELECT EXISTS(SELECT 1 FROM bundles WHERE uuid = $1) AS exists" {:params [(:uuid query-params)]}) + (first)))) + diff --git a/src/source/routes/bundle_categories.clj b/src/source/routes/bundle_categories.clj new file mode 100644 index 00000000..5a0cd8ba --- /dev/null +++ b/src/source/routes/bundle_categories.clj @@ -0,0 +1,20 @@ +(ns source.routes.bundle-categories + (:require [source.workers.bundles :as bundles] + [ring.util.response :as res])) + +(defn get + {:summary "Get all categories for which content is present in the uuid-authorized bundle (RSS feeds / posts)." + :parameters {:query [:map + [:uuid {:description "Bundle UUID"} :string] + [:type {:optional true + :description "Filters by content type ID"} :int]]} + :responses {200 {:body [:vector + [:map + [:id :int] + [:name :string] + [:display-picture {:optional true} [:maybe :string]]]]} + 404 {:body [:map [:message :string]]}}} + + [{:keys [ds bundle-id query-params] :as _request}] + (res/response (bundles/get-bundle-categories ds {:bundle-id bundle-id + :content-type-id (:type query-params)}))) diff --git a/src/source/routes/bundle_feed.clj b/src/source/routes/bundle_feed.clj new file mode 100644 index 00000000..f4471a8b --- /dev/null +++ b/src/source/routes/bundle_feed.clj @@ -0,0 +1,25 @@ +(ns source.routes.bundle-feed + (:require [ring.util.response :as res] + [source.db.honey :as hon] + [source.services.analytics.interface :as analytics] + [source.routes.openapi :as api] + [source.workers.schemas :as schemas] + [taoensso.telemere :as t])) + +(defn get + {:summary "Get a single RSS feed by id from RSS feeds within the uuid-authorized bundle. + This endpoint will update click analytics for the returned RSS feed." + :parameters {:query [:map [:uuid {:description "Bundle UUID"} :string]] + :path [:map [:id {:title "id" + :description "Feed ID"} :int]]} + :responses (-> (api/success schemas/Feed) + (api/not-found))} + + [{:keys [ds bundle-id path-params] :as _request}] + (let [feed (hon/find-one ds {:tname :feeds + :where [:= :id (:id path-params)]})] + (try + (analytics/insert-feed-click! ds feed bundle-id) + (catch Exception e (t/log! {:level :error + :msg (str "Failed to insert feed click for bundle feed: " (.getMessage e))}))) + (res/response feed))) diff --git a/src/source/routes/bundle_feed_post.clj b/src/source/routes/bundle_feed_post.clj new file mode 100644 index 00000000..c6d885f5 --- /dev/null +++ b/src/source/routes/bundle_feed_post.clj @@ -0,0 +1,41 @@ +(ns source.routes.bundle-feed-post + (:require [ring.util.response :as res] + [source.db.honey :as hon] + [source.services.analytics.interface :as analytics] + [source.db.util :as db.util] + [honey.sql.helpers :as hsql] + [taoensso.telemere :as t])) + +(defn get + {:summary "Get a single post by post id belonging to an RSS feed in the associated uuid-authorized bundle. + This endpoint updates click analytics for the returned post." + :description "This endpoint will fetch a single post belonging to the given feed, regardless of whether the post made it into the bundle during post selection." + :parameters {:query [:map [:uuid {:description "Bundle UUID"} :string]] + :path [:map [:post-id {:title "postId" + :description "Post ID"} :int]]} + :responses {200 {:body + [:map + [:id :int] + [:post-id :string] + [:feed-id :int] + [:creator-id :int] + [:content-type-id :int] + [:title :string] + [:thumbnail [:maybe :string]] + [:info [:maybe :string]] + [:url [:maybe :string]] + [:stream-url [:maybe :string]] + [:season [:maybe :int]] + [:episode [:maybe :int]] + [:redacted {:optional true} [:maybe :int]] + [:posted-at [:maybe :string]]]} + 404 {:body [:map [:message :string]]}}} + + [{:keys [ds bundle-id path-params] :as _request}] + (let [post (hon/find-one ds (-> (db.util/tname :outgoing-posts bundle-id) + (hsql/where [:= :id (:post-id path-params)])))] + (try + (analytics/insert-post-click! ds post bundle-id) + (catch Exception e (t/log! {:level :error + :msg (str "Failed to insert post click on bundle feed post: " (.getMessage e))}))) + (res/response post))) diff --git a/src/source/routes/bundle_feed_posts.clj b/src/source/routes/bundle_feed_posts.clj new file mode 100644 index 00000000..ecd20762 --- /dev/null +++ b/src/source/routes/bundle_feed_posts.clj @@ -0,0 +1,41 @@ +(ns source.routes.bundle-feed-posts + (:require [ring.util.response :as res] + [source.db.honey :as hon] + [source.services.analytics.interface :as analytics] + [taoensso.telemere :as t])) + +(defn get + {:summary "Get all posts present within a given RSS feed by feed id, within the uuid-authorized bundle. + This endpoint will update impressions analytics for the returned posts." + :description "This endpoint will fetch all posts within the given feed, regardless of whether these posts made it into this bundle during post selection." + :parameters {:query [:map [:uuid {:description "Bundle UUID"} :string]] + :path [:map [:id {:title "id" + :description "Feed ID"} :int]]} + :responses {200 {:body [:vector + [:map + [:id :int] + [:post-id :string] + [:feed-id :int] + [:creator-id :int] + [:content-type-id :int] + [:title :string] + [:thumbnail [:maybe :string]] + [:info [:maybe :string]] + [:url [:maybe :string]] + [:stream-url [:maybe :string]] + [:season [:maybe :int]] + [:episode [:maybe :int]] + [:redacted {:optional true} [:maybe :int]] + [:posted-at [:maybe :string]]]]} + 401 {:body [:map [:message :string]]} + 403 {:body [:map [:message :string]]}}} + + [{:keys [ds bundle-id path-params] :as _request}] + (let [posts (hon/find ds {:tname :incoming-posts + :where [:= :feed-id (:id path-params)] + :ret :*})] + (try + (analytics/insert-post-impressions! ds posts bundle-id) + (catch Exception e (t/log! {:level :error + :msg (str "Failed to insert post impressions for bundle feed posts: " (.getMessage e))}))) + (res/response posts))) diff --git a/src/source/routes/bundle_feeds.clj b/src/source/routes/bundle_feeds.clj new file mode 100644 index 00000000..6c18a607 --- /dev/null +++ b/src/source/routes/bundle_feeds.clj @@ -0,0 +1,43 @@ +(ns source.routes.bundle-feeds + (:require [ring.util.response :as res] + [source.db.util :as db.util] + [source.workers.bundles :as bundles] + [source.services.analytics.interface :as analytics] + [source.routes.openapi :as api] + [source.workers.schemas :as schemas] + [taoensso.telemere :as t])) + +(defn post + {:summary "Get all RSS feeds present in the bundle authorised by uuid. + This endpoint will update impressions analytics for the returned RSS feeds." + :description "This endpoint can be filtered by content type ID, category IDs, or most recently added feeds." + :parameters {:query [:map + [:uuid {:description "Bundle UUID"} :string] + [:type {:optional true + :description "Filters by content type ID"} :int] + [:latest {:optional true + :description "Filters by most recently uploaded feeds"} :boolean] + [:nonfiltered {:optional true + :description "Marking this field as true will disable all filters"} :boolean]] + :body [:map [:category-ids [:vector :int]]]} + :responses (-> (api/success schemas/Feeds) + (api/not-found))} + + [{:keys [ds bundle-id query-params body] :as _request}] + (let [{:keys [type latest nonfiltered]} query-params + feeds (->> {:bundle-id bundle-id + :type type + :latest latest + :category-ids (:category-ids body) + :nonfiltered nonfiltered} + (bundles/get-outgoing-feeds ds))] + (try + (analytics/insert-feed-impressions! ds feeds bundle-id) + (catch Exception e (t/log! {:level :error + :msg (str "Failed to insert feed impressions for bundle feeds: " (.getMessage e))}))) + (res/response feeds))) + +(comment + (def ds (db.util/conn :master)) + + ()) diff --git a/src/source/routes/bundle_post.clj b/src/source/routes/bundle_post.clj new file mode 100644 index 00000000..9acbfc27 --- /dev/null +++ b/src/source/routes/bundle_post.clj @@ -0,0 +1,34 @@ +(ns source.routes.bundle-post + (:require [ring.util.response :as res] + [source.services.analytics.interface :as analytics] + [source.db.honey :as hon] + [source.db.util :as db.util] + [honey.sql.helpers :as hsql] + [source.routes.openapi :as api] + [source.workers.schemas :as schemas] + [malli.util :as mu] + [taoensso.telemere :as t])) + +(defn get + {:summary "Get a single post by post id in the uuid-authorized bundle. + Used to return a single post present in the bundle. + This endpoint updates click analytics for the returned post." + :description "This endpoint will pull a single post by ID that made it into the bundle during post selection." + :parameters {:query [:map [:uuid {:description "Bundle UUID"} :string]] + :path [:map [:id {:title "id" + :description "Post ID"} :int]]} + :responses (-> (api/success (-> schemas/Post + (mu/assoc :feed-title :string))) + (api/not-found))} + + [{:keys [ds bundle-id path-params] :as _request}] + (let [post (hon/execute! ds (-> (hsql/select-distinct :p.* [:f.title :feed-title]) + (hsql/from [(:tname (db.util/tname :outgoing-posts bundle-id)) :p]) + (hsql/join [:feeds :f] [:= :p.feed-id :f.id]) + (hsql/where [:= :p.id (:id path-params)])) + {:ret :1})] + (try + (analytics/insert-post-click! ds post bundle-id) + (catch Exception e (t/log! {:level :error + :msg (str "Failed to insert post click on bundle post: " (.getMessage e))}))) + (res/response post))) diff --git a/src/source/routes/bundle_posts.clj b/src/source/routes/bundle_posts.clj new file mode 100644 index 00000000..d714149b --- /dev/null +++ b/src/source/routes/bundle_posts.clj @@ -0,0 +1,55 @@ +(ns source.routes.bundle-posts + (:require [clojure.walk :as walk] + [ring.util.response :as res] + [source.services.analytics.interface :as analytics] + [source.workers.bundles :as bundles] + [source.routes.openapi :as api] + [source.workers.schemas :as schemas] + [malli.util :as mu] + [taoensso.telemere :as t])) + +(def QueryTruncatePosts + [:truncate + {:optional true + :description "Truncates text in posts to a maximum of 100 characters. Defaults to true."} + [:enum "true" "false"]]) + +(def QuerySeed + [:seed {:optional true} [:maybe :string]]) + +(defn post + {:summary "Get a list of posts in the uuid-authorized bundle, determined by analytics. + This endpoint updates impression analytics for the returned posts." + :description "This endpoint pulls a curated list of content (determined by analytics) of the posts that made it into the bundle during post selection. This can be filtered by content type ID, category IDs, or latest (most recently added posts). If results are filtered by latest, they will not be curated by analytics. + + Results can be paginated using the `start` and `limit` query parameters." + :parameters (api/params + :body [:map [:category-ids [:vector :int]]] + :query [:map + schemas/QueryUUID + schemas/QueryLimit + schemas/QueryStart + schemas/QueryContentType + schemas/QueryLatest + QueryTruncatePosts + QuerySeed]) + :responses (api/success (api/paginated [:vector (-> schemas/Post + (mu/assoc :feed-title :string))]))} + + [{:keys [ds bundle-id query-params body] :as _request}] + (let [{:keys [limit start type latest seed truncate]} (walk/keywordize-keys query-params) + {:keys [data] :as posts} (bundles/get-outgoing-posts + ds + {:bundle-id bundle-id + :limit limit + :start start + :type type + :latest latest + :seed seed + :truncate truncate + :category-ids (:category-ids body)})] + (try + (analytics/insert-post-impressions! ds data bundle-id) + (catch Exception e (t/log! {:level :error + :msg (str "Failed to insert post impressions on bundle posts: " (.getMessage e))}))) + (res/response posts))) diff --git a/src/source/routes/business.clj b/src/source/routes/business.clj index f63c3644..3730f73a 100644 --- a/src/source/routes/business.clj +++ b/src/source/routes/business.clj @@ -1,13 +1,44 @@ (ns source.routes.business - (:require [source.services.businesses :as businesses] - [ring.util.response :as res])) + (:require [ring.util.response :as res] + [source.util :as utils] + [source.db.honey :as hon])) -(defn post [{:keys [ds body] :as _request}] - (businesses/insert-business! ds {:data body}) +(defn post + {:summary "insert a business" + :parameters {:body [:map + [:name :string] + [:address {:optional true} :string] + [:url {:optional true} :string] + [:linkedin {:optional true} :string] + [:twitter {:optional true} :string]]} + :responses {201 {:body [:map [:message :string]]}}} + + [{:keys [ds body] :as _request}] + (hon/insert! ds {:tname :businesses + :data body}) (res/response {:message "successfully added business"})) -(defn patch [{:keys [ds body path-params] :as _request}] - (businesses/update-business! ds {:id (:id path-params) - :values body}) +(defn patch + {:summary "update a business by id" + :parameters {:path [:map [:id {:title "id" + :description "business id"} :int]] + :body [:map + [:name :string] + [:address {:optional true} :string] + [:url {:optional true} :string] + [:linkedin {:optional true} :string] + [:twitter {:optional true} :string]]} + :responses {200 {:body [:map [:message :string]]}}} + + [{:keys [ds body path-params] :as _request}] + (hon/update! ds {:tname :businesses + :where [:= :id (:id path-params)] + :data body}) (res/response {:message "successfully updated business"})) +(comment + (require '[source.db.util :as db.util]) + (post {:ds (db.util/conn :master) + :body {:name "modulr" + :url "https://modulr.com"}}) + ()) diff --git a/src/source/routes/business_types.clj b/src/source/routes/business_types.clj new file mode 100644 index 00000000..3a5fca27 --- /dev/null +++ b/src/source/routes/business_types.clj @@ -0,0 +1,49 @@ +(ns source.routes.business-types + (:require [source.db.honey :as db] + [ring.util.response :as res])) + +(defn get + {:summary "Get all business types" + :responses {200 {:body [:vector + [:map + [:id :int] + [:name :string]]]}}} + [{:keys [ds] :as _request}] + (res/response (db/find ds {:tname :business-types + :ret :*}))) + +(defn post + {:summary "add one or more new business types" + :parameters {:body [:vector + [:map + [:name :string]]]} + :responses {201 {:body [:map [:message :string]]}}} + + [{:keys [ds body] :as _request}] + (db/insert! ds {:tname :business-types + :data body}) + (res/response {:message "successfully added business type(s)"})) + +(defn patch + {:summary "update business type with the given id" + :parameters {:path [:map [:id {:title "id" + :description "business type id"} :int]] + :body [:map [:name :string]]} + :responses {200 {:body [:map [:message :string]]}}} + + [{:keys [ds path-params body] :as _request}] + (db/update! ds {:tname :business-types + :data body + :where [:= :id (:id path-params)]}) + (res/response {:message "successfully updated business type"})) + +(defn delete + {:summary "delete business type with the given id" + :parameters {:path [:map [:id {:title "id" + :description "business type id"} :int]]} + :responses {200 {:body [:map [:message :string]]}}} + + [{:keys [ds path-params] :as _request}] + (db/delete! ds {:tname :business-types + :where [:= :id (:id path-params)]}) + (res/response {:message "successfully deleted business type"})) diff --git a/src/source/routes/businesses.clj b/src/source/routes/businesses.clj index d502ff20..c4ece13b 100644 --- a/src/source/routes/businesses.clj +++ b/src/source/routes/businesses.clj @@ -1,9 +1,22 @@ (ns source.routes.businesses - (:require [source.services.businesses :as businesses] - [ring.util.response :as res])) + (:require [ring.util.response :as res] + [source.db.honey :as hon])) -(defn get [{:keys [ds] :as _request}] - (res/response {:businesses (businesses/businesses ds)})) +(defn get + {:summary "get all businesses" + :responses {200 {:body [:map + [:businesses + [:map + [:id :int] + [:name :string] + [:address [:maybe :string]] + [:url [:maybe :string]] + [:linkedin [:maybe :string]] + [:twitter [:maybe :string]]]]]}}} + + [{:keys [ds] :as _request}] + (res/response {:businesses (hon/find ds {:tname :businesses + :ret :*})})) (comment (require '[source.db.util :as db.util]) diff --git a/src/source/routes/cadences.clj b/src/source/routes/cadences.clj new file mode 100644 index 00000000..9fb98240 --- /dev/null +++ b/src/source/routes/cadences.clj @@ -0,0 +1,14 @@ +(ns source.routes.cadences + (:require [ring.util.response :as res] + [source.db.honey :as hon])) + +(defn get + {:summary "get all cadences" + :responses {200 {:body [:vector + [:map + [:id :int] + [:label :string] + [:days :int]]]}}} + [{:keys [ds] :as _request}] + (->> (hon/find ds {:tname :cadences}) + (res/response))) diff --git a/src/source/routes/categories.clj b/src/source/routes/categories.clj new file mode 100644 index 00000000..c5060924 --- /dev/null +++ b/src/source/routes/categories.clj @@ -0,0 +1,59 @@ +(ns source.routes.categories + (:require [ring.util.response :as res] + [source.db.honey :as hon] + [source.workers.categories :as categories] + [source.routes.openapi :as api] + [source.workers.schemas :as schemas] + [malli.util :as mu])) + +(defn get + {:summary "get all categories" + :responses {200 {:body [:vector + [:map + [:id :int] + [:name :string] + [:display-picture {:optional true} [:maybe :string]]]]}}} + [{:keys [ds] :as _request}] + (->> (hon/find ds {:tname :categories}) + (res/response))) + +(defn used-categories + {:summary "get all categories for which feeds and posts exist in the system" + :responses {200 {:body [:vector + [:map + [:id :int] + [:name :string] + [:display-picture {:optional true} [:maybe :string]]]]}}} + [{:keys [ds] :as _request}] + (res/response (categories/used-categories ds))) + +(defn add-category + {:summary "add a new category to the system" + :parameters (api/params :body (-> schemas/Category + (mu/dissoc :id))) + :responses (api/success (api/response-schema))} + [{:keys [ds body]}] + (hon/insert! ds {:tname :categories + :data body}) + (res/response {:message "successfully added new category"})) + +(defn update-category + {:summary "add a new category to the system" + :parameters (api/params + :path [:map [:id :int]] + :body (-> schemas/Category + (mu/dissoc :id))) + :responses (api/success (api/response-schema))} + [{:keys [ds body path-params]}] + (hon/update! ds {:tname :categories + :where [:= :id (:id path-params)] + :data body}) + (res/response {:message "successfully updated category"})) + +(defn delete-category + {:summary "delete a category from the system" + :parameters (api/params :path [:map [:id :int]]) + :responses (api/success (api/response-schema))} + [{:keys [ds path-params]}] + (categories/delete-category! ds (:id path-params)) + (res/response {:message "successfully deleted category"})) diff --git a/src/source/routes/category.clj b/src/source/routes/category.clj new file mode 100644 index 00000000..dfaf6b08 --- /dev/null +++ b/src/source/routes/category.clj @@ -0,0 +1,15 @@ +(ns source.routes.category + (:require [ring.util.response :as res] + [source.db.honey :as hon])) + +(defn get + {:summary "get category by id" + :parameters {:path [:map [:id {:title "id" + :description "category id"} :int]]} + :responses {200 {:body [:map + [:id :int] + [:name :string]]}}} + + [{:keys [ds path-params] :as _request}] + (res/response (hon/find-one ds {:tname :categories + :where [:= :id (:id path-params)]}))) diff --git a/src/source/routes/content_type.clj b/src/source/routes/content_type.clj new file mode 100644 index 00000000..b6b26cf4 --- /dev/null +++ b/src/source/routes/content_type.clj @@ -0,0 +1,16 @@ +(ns source.routes.content-type + (:require [ring.util.response :as res] + [source.db.honey :as hon])) + +(defn get + {:summary "get content type by id" + :parameters {:path [:map [:id {:title "id" + :description "content type id"} :int]]} + :responses {200 {:body [:map + [:id :int] + [:name :string]]}}} + + [{:keys [ds path-params] :as _request}] + (->> (hon/find-one ds {:tname :content-types + :where [:= :id (:id path-params)]}) + (res/response))) diff --git a/src/source/routes/content_types.clj b/src/source/routes/content_types.clj new file mode 100644 index 00000000..c025c98f --- /dev/null +++ b/src/source/routes/content_types.clj @@ -0,0 +1,14 @@ +(ns source.routes.content-types + (:require [ring.util.response :as res] + [source.db.honey :as hon])) + +(defn get + {:summary "get all content types" + :responses {200 {:body [:vector + [:map + [:id :int] + [:name :string]]]}}} + + [{:keys [ds] :as _request}] + (-> (hon/find ds {:tname :content-types}) + (res/response))) diff --git a/src/source/routes/data.clj b/src/source/routes/data.clj new file mode 100644 index 00000000..8c70ac0a --- /dev/null +++ b/src/source/routes/data.clj @@ -0,0 +1,16 @@ +(ns source.routes.data + (:require [source.workers.xml-schemas :as xml] + [ring.util.response :as res])) + +(defn post [{:keys [ds body] :as _request}] + (let [{:keys [schema-id url]} body] + (-> (xml/extract-data ds schema-id url) + (res/response)))) + +(comment + (require '[source.rss.youtube :as yt]) + + (def url (->> "https://www.youtube.com/@ThePrimeTimeagen" + (yt/find-channel-id) + (str "https://www.youtube.com/feeds/videos.xml?channel_id="))) + ()) diff --git a/src/source/routes/feed.clj b/src/source/routes/feed.clj new file mode 100644 index 00000000..2421cab9 --- /dev/null +++ b/src/source/routes/feed.clj @@ -0,0 +1,58 @@ +(ns source.routes.feed + (:require [ring.util.response :as res] + [source.db.honey :as hon] + [source.workers.feeds :as feeds] + [source.jobs.handlers :as handlers] + [source.routes.openapi :as api] + [source.workers.schemas :as schemas])) + +(defn get + {:summary "get feed by id" + :parameters {:path [:map [:id {:title "id" + :description "feed id"} :int]]} + :responses (api/success schemas/Feed)} + + [{:keys [ds path-params] :as _request}] + (-> (hon/find-one ds {:tname :feeds + :where [:= :id (:id path-params)]}) + (res/response))) + +(defn post + {:summary "update feed by id" + :parameters {:path [:map [:id {:title "id" + :description "feed id"} :int]] + :body (-> (api/maybe-keys schemas/Feed) + (api/missoc :id :user-id :provider-id :created-at :updated-at :state :rss-url))} + :responses {200 {:body [:map [:message :string]]} + 401 {:body [:map [:message :string]]} + 403 {:body [:map [:message :string]]}}} + + [{:keys [ds path-params body] :as _request}] + (feeds/update-feed! ds {:feed-id (:id path-params) + :feed-metadata body}) + (res/response {:message "successfully updated feed"})) + +(defn delete + {:summary "delete feed by id" + :parameters {:path [:map [:id {:title "id" + :description "feed id"} :int]]} + :responses {200 {:body [:map [:message :string]]} + 401 {:body [:map [:message :string]]} + 403 {:body [:map [:message :string]]}}} + + [{:keys [ds js user path-params] :as _request}] + (let [id (:id path-params) + feed (hon/find-one ds {:tname :feeds + :where [:and + [:= :user-id (:id user)] + [:= :id id]]}) + {:keys [email]} (hon/find-one ds {:tname :users + :where [:= :id (:id user)]}) + job-id (handlers/update-feed-posts-job-id email id)] + (if (some? feed) + (do + (feeds/hard-delete-feed! ds id) + (feeds/deregister-feed-job! js job-id) + (res/response {:message "successfully deleted feed"})) + (-> (res/response {:message "unauthorized"}) + (res/status 403))))) diff --git a/src/source/routes/feed_categories.clj b/src/source/routes/feed_categories.clj new file mode 100644 index 00000000..6605d525 --- /dev/null +++ b/src/source/routes/feed_categories.clj @@ -0,0 +1,32 @@ +(ns source.routes.feed-categories + (:require [source.services.interface :as services] + [ring.util.response :as res] + [source.workers.feeds :as feeds])) + +(defn get + {:summary "get all categories belonging to the feed with the given id" + :parameters {:path [:map [:id {:title "id" + :description "feed id"} :int]]} + :responses {200 {:body [:vector + [:map + [:id :int] + [:name :string] + [:feed-id :int] + [:category-id :int]]]}}} + [{:keys [ds path-params] :as _request}] + (->> (services/categories-by-feed ds {:feed-id (:id path-params)}) + (res/response))) + +(defn post + {:summary "update categories belonging to the feed with the given id" + :parameters {:path [:map [:id {:title "id" + :description "feed id"} :int]] + :body [:vector + [:map + [:id :int] + [:name :string]]]} + :responses {200 {:body [:map [:message :string]]}}} + [{:keys [ds path-params body] :as _request}] + (feeds/update-feed-categories! ds {:feed-id (:id path-params) + :categories body}) + (res/response {:message "successfully updated feed categories"})) diff --git a/src/source/routes/feeds.clj b/src/source/routes/feeds.clj new file mode 100644 index 00000000..3768d258 --- /dev/null +++ b/src/source/routes/feeds.clj @@ -0,0 +1,77 @@ +(ns source.routes.feeds + (:require [source.workers.feeds :as feeds] + [congest.jobs :as congest] + [ring.util.response :as res] + [source.db.honey :as hon] + [source.util :as util] + [source.jobs.core :as jobs] + [source.routes.openapi :as api] + [source.workers.schemas :as schemas])) + +(defn get + {:summary "get all feeds" + :responses (api/success schemas/Feeds)} + + [{:keys [ds user] :as _request}] + (-> (hon/find ds {:tname :feeds + :where [:= :user-id (:id user)]}) + (res/response))) + +(defn post + {:summary "adds a feed and extracts data from RSS feed URL to create incoming posts and schedules a job to keep them updated" + :parameters {:body (-> schemas/Feed + (api/missoc :id :title :user-id :created-at :updated-at :state))} + :responses (api/success schemas/Feed)} + + [{:keys [js ds user body] :as _request}] + (let [exists (hon/exists? ds {:tname :feeds + :where [:= :rss-url (:rss-url body)] + :ret :1})] + (if exists + (-> (res/response {:message "There is already a feed with the given RSS feed"}) + (res/status 400)) + + (let [{:keys [provider-id rss-url content-type-id]} body + new-feed (feeds/create-feed! ds {:user-id (:id user) + :feed-metadata body}) + {:keys [email]} (hon/find-one ds {:tname :users + :where [:= :id (:id user)]})] + (if new-feed + (do + ;TODO: service needed + (->> (jobs/prepare-congest-metadata + ds + {:id (str email "-" (:id new-feed)) + :initial-delay (* 1000 60 60 24) + :auto-start true + :stop-after-fail false, + :interval (* 1000 60 60 24) + :recurring? true + :args {:feed-id (:id new-feed) + :creator-id (:id user) + :content-type-id content-type-id + :provider-id provider-id + :url rss-url} + :handler :update-feed-posts + :created-at (util/get-utc-timestamp-string) + :sleep false}) + (congest/register! js)) + (res/response new-feed)) + + (-> (res/response {:message "Failed to parse RSS feed"}) + (res/status 422))))))) + +(comment + (require '[source.db.util :as db.util]) + + (get {:ds (db.util/conn) :user {:id 3}}) + (post {:ds (db.util/conn) + :js (congest/create-job-service []) + :user {:id 5} + :body {:rss-url "https://www.youtube.com/feeds/videos.xml?channel_id=UCUyeluBRhGPCW4rPe_UvBZQ" + :provider-id 1 + :content-type-id 1 + :cadence-id 1 + :baseline-id 1}}) + + ()) diff --git a/src/source/routes/google_launch.clj b/src/source/routes/google_launch.clj index efccfd4b..a179ebfc 100644 --- a/src/source/routes/google_launch.clj +++ b/src/source/routes/google_launch.clj @@ -2,6 +2,11 @@ (:require [source.oauth2.google.interface :as google] [ring.util.response :as response])) -(defn get [_req] - (response/response (google/auth-uri))) +(defn get + {:summary "begins google federated login flow" + :responses {200 {:body [:map + [:uuid :string] + [:uri :string]]}}} + [_req] + (response/response (google/auth-uri))) diff --git a/src/source/routes/google_redirect.clj b/src/source/routes/google_redirect.clj index 873a1709..d0da42ca 100644 --- a/src/source/routes/google_redirect.clj +++ b/src/source/routes/google_redirect.clj @@ -1,26 +1,8 @@ (ns source.routes.google-redirect - (:require [source.oauth2.google.interface :as google] - [source.middleware.auth.core :as auth] - [source.services.users :as users] - [source.db.util :as db.util] - [ring.util.response :as res] + (:require [ring.util.response :as res] [source.config :as conf])) -(defn get [req] - (let [{:keys [uuid _uri]} (:body req) - email (google/google-session-user uuid (:params req)) - ds (db.util/conn :master) - user (users/user ds {:where [:= :email email]}) - user-type (get-in req [:cookies "user_type" :value])] - - (if (some? user) - (let [payload (dissoc user :password) - {:keys [access-token]} (auth/create-session payload)] - (res/redirect (str (conf/read-value :cors-origin) "/api/oauth/google?token=" access-token))) - (do - (users/insert-user! ds {:data {:email email - :type user-type}}) - (let [new-user (users/user ds {:where [:= :email email]}) - payload (dissoc new-user :password) - {:keys [access-token]} (auth/create-session payload)] - (res/redirect (str (conf/read-value :cors-origin) "/api/oauth/google?token=" access-token))))))) +(defn get [{:keys [query-string] :as req}] + (-> (conf/read-value :cors-origin) + (str "/oauth?" query-string) + (res/redirect))) diff --git a/src/source/routes/google_user.clj b/src/source/routes/google_user.clj new file mode 100644 index 00000000..e24c3112 --- /dev/null +++ b/src/source/routes/google_user.clj @@ -0,0 +1,54 @@ +(ns source.routes.google-user + (:require [source.oauth2.google.interface :as google] + [source.middleware.auth.core :as auth] + [ring.util.response :as res] + [source.db.honey :as hon] + [source.password :as pw] + [source.email.templates :as templates] + [source.email.gmail :as gmail])) + +(defn get + {:summary "completes the google oauth2 flow and returns the authenticated user" + :parameters {:query [:map + [:code :string] + [:scope :string]]} + :responses {200 {:body [:map + [:user + [:map + [:id :int] + [:address [:maybe :string]] + [:profile-image [:maybe :string]] + [:email :string] + [:firstname [:maybe :string]] + [:lastname [:maybe :string]] + [:type [:enum "creator" "distributor" "admin"]] + [:email-verified [:maybe :int]] + [:onboarded [:maybe :int]] + [:mobile [:maybe :string]]]] + [:access-token :string] + [:refresh-token :string]]}}} + [{:keys [ds body] :as req}] + + (let [{:keys [uuid _uri]} body + email (google/google-session-user uuid (:params req)) + user (hon/find-one ds {:tname :users + :where [:= :email email]}) + user-type (get-in req [:cookies "user_type" :value])] + + (if (some? user) + (let [payload (dissoc user :password) + session (auth/create-session payload)] + (res/response (merge {:user payload} session))) + (do + (hon/insert! ds {:tname :users + :data {:email email + :type user-type}}) + (let [new-user (hon/find-one ds {:tname :users + :where [:= :email email]}) + payload (dissoc new-user :password) + session (auth/create-session payload)] + (gmail/send-email {:to email + :subject "Source - Verify your email" + :body (templates/email-verification {:email-hash (pw/hash-password email)}) + :type :text/html}) + (res/response (merge {:user payload} session))))))) diff --git a/src/source/routes/integration.clj b/src/source/routes/integration.clj new file mode 100644 index 00000000..41692373 --- /dev/null +++ b/src/source/routes/integration.clj @@ -0,0 +1,89 @@ +(ns source.routes.integration + (:require [ring.util.response :as res] + [source.services.interface :as services] + [source.services.bundles :as bundles] + [source.workers.integrations :as integrations] + [congest.jobs :as congest] + [source.jobs.core :as jobs] + [source.util :as util] + [source.jobs.handlers :as handlers] + [source.workers.schemas :as schemas] + [malli.util :as mu] + [source.routes.openapi :as api])) + +(defn get + {:summary "Get metadata of integration by ID" + :parameters {:path [:map [:id {:title "id" + :description "Integration ID"} :int]]} + :responses {200 {:body (-> schemas/Bundle + (mu/assoc :content-types schemas/ContentTypes))} + 401 {:body [:map [:message :string]]} + 403 {:body [:map [:message :string]]}}} + + [{:keys [ds path-params] :as _request}] + (let [integration (services/bundle ds {:id (:id path-params)}) + content-types (services/content-types-by-bundle ds {:bundle-id (:id path-params)})] + (res/response (assoc integration :content-types content-types)))) + +(defn post + {:summary "Update metadata of the given integration by ID" + :description "When the integration is updated, post selection is rerun based on the newly set desired categories and content types." + :parameters {:path [:map [:id {:title "id" + :description "Integration ID"} :int]] + :body (-> [:map + [:name :string] + [:integration-type-id :int]] + (mu/assoc :content-types [:vector schemas/ConstantSchema]) + (mu/assoc :categories [:vector schemas/ConstantSchema]))} + :responses {200 {:body [:map [:message :string]]} + 401 {:body [:map [:message :string]]} + 403 {:body [:map [:message :string]]}}} + + [{:keys [js ds path-params body] :as _request}] + (let [bundle-id (:id path-params) + job-id (str "bundle_" bundle-id) + categories-by-bundle (bundles/categories-in-bundle ds bundle-id)] + (integrations/update-integration! ds {:bundle-id bundle-id + :bundle-metadata (dissoc body :categories :content-types) + :categories (:categories body) + :content-types (:content-types body)}) + ;TODO: service needed + (congest/deregister! js job-id) + (->> (jobs/prepare-congest-metadata + ds + {:id job-id + :initial-delay (* 1000 60 60 24) + :auto-start true + :stop-after-fail false, + :interval (* 1000 60 60 24) + :recurring? true + :ds ds + :args {:bundle-id bundle-id + :categories categories-by-bundle} + :handler :update-bundle + :created-at (util/get-utc-timestamp-string) + :sleep false}) + (congest/register! js))) + (res/response {:message "successfully updated integration"})) + +(defn delete + {:summary "Delete the given integration by ID" + :description "Deletes the integration, bundle and kills the associated post selection job. This action cannot be undone." + :parameters {:path [:map [:id {:title "id" + :description "Integration ID"} :int]]} + :responses {200 {:body (api/response-schema)} + 403 {:body (api/response-schema)}}} + + [{:keys [ds js user path-params] :as _request}] + (let [bundle-id (:id path-params) + bundle (bundles/bundle ds {:where [:and + [:= :id bundle-id] + [:= :user-id (:id user)]]}) + job-id (handlers/update-bundle-job-id bundle-id)] + (if (some? bundle) + (do + (integrations/hard-delete-bundle! ds bundle-id) + (integrations/deregister-bundle-job! js job-id) + (res/response {:message "successfully deleted integration"})) + (-> (res/response {:message "unauthorized"}) + (res/status 403))))) diff --git a/src/source/routes/integration_categories.clj b/src/source/routes/integration_categories.clj new file mode 100644 index 00000000..d3e3cf0a --- /dev/null +++ b/src/source/routes/integration_categories.clj @@ -0,0 +1,33 @@ +(ns source.routes.integration-categories + (:require [ring.util.response :as res] + [source.services.bundles :as bundles] + [source.services.bundle-categories :as bundle-categories])) + +(defn get + {:summary "Get all categories belonging to the given integration by ID" + :description "This endpoint pulls all the categories on which post selection is based." + :parameters {:path [:map [:id {:title "id" + :description "integration id"} :int]]} + :responses {200 {:body [:vector + [:map + [:id :int] + [:name :string]]]}}} + + [{:keys [ds path-params] :as _request}] + (res/response (bundles/categories-in-bundle ds (:id path-params)))) + +(defn post + {:summary "Update categories belonging to the given integration by ID" + :description "This endpoint updates the list of categories for the given integration on which post selection is based; however, post selection will not immediately be rerun. Post selection will be rerun 24 hours from the previous execution. If you want post selection to be rerun immediately upon update, please use `POST: /integrations{id}`." + :parameters {:path [:map [:id {:title "id" + :description "integration id"} :int]] + :body [:vector + [:map + [:id :int] + [:name :string]]]} + :responses {200 {:body [:map [:message :string]]}}} + + [{:keys [ds path-params body] :as _request}] + (bundle-categories/update-bundle-categories! ds {:bundle-id (:id path-params) + :categories body}) + (res/response {:message "successfully updated integration categories"})) diff --git a/src/source/routes/integration_filter_feed.clj b/src/source/routes/integration_filter_feed.clj new file mode 100644 index 00000000..685ddb64 --- /dev/null +++ b/src/source/routes/integration_filter_feed.clj @@ -0,0 +1,45 @@ +(ns source.routes.integration-filter-feed + (:require [ring.util.response :as res] + [source.workers.integrations :as integrations] + [source.db.honey :as hon])) + +(defn get + {:summary "Returns true if the feed with the given id is filtered out by the given integration by ID" + :parameters {:path [:map + [:id {:title "id" + :description "Integration ID"} :int] + [:feed-id {:title "FeedId" + :description "feed ID"} :int]]} + :responses {200 {:body [:map [:filtered :boolean]]} + 401 {:body [:map [:message :string]]} + 403 {:body [:map [:message :string]]}}} + + [{:keys [ds path-params] :as _request}] + + (let [returned (hon/find ds {:tname :filtered-feeds + :where [:and + [:= :bundle-id (:id path-params)] + [:= :feed-id (:feed-id path-params)]]}) + blocked (if (seq returned) true false)] + (res/response {:filtered blocked}))) + +(defn post + {:summary "Filters out the feed with the given id from the given integration by ID" + :description "Filtering a feed out from the integration means that the given feed and all its posts will not appear when pulling content from the bundle." + :parameters {:path [:map + [:id {:title "id" + :description "Integration ID"} :int] + [:feed-id {:title "feedId" + :description "Feed ID"} :int]] + :body [:map + [:filtered :boolean]]} + :responses {:body {200 [:map [:message :string]] + 401 [:map [:message :string]] + 403 [:map [:message :string]]}}} + + [{:keys [ds path-params body] :as _request}] + (->> {:filtered (:filtered body) + :bundle-id (:id path-params) + :feed-id (:feed-id path-params)} + (integrations/update-filtered-feeds! ds)) + (res/response {:message "Successfully updated feed filtering."})) diff --git a/src/source/routes/integration_filter_feeds.clj b/src/source/routes/integration_filter_feeds.clj new file mode 100644 index 00000000..2cce847e --- /dev/null +++ b/src/source/routes/integration_filter_feeds.clj @@ -0,0 +1,20 @@ +(ns source.routes.integration-filter-feeds + (:require [ring.util.response :as res] + [source.db.honey :as hon])) + +(defn get + {:summary "Gets all filtered feed IDs by integration ID" + :description "Returns a list of all feed IDs that have been filtered out from the given integration. If a feed appears in this list, it will not be returned when pulling content from the bundle." + :parameters {:path [:map [:id {:title "id" + :description "integration id"} :int]]} + :responses {200 {:body [:vector + [:map + [:feed-id :int] + [:bundle-id :int]]]} + 401 {:body [:map [:message :string]]} + 403 {:body [:map [:message :string]]}}} + + [{:keys [ds path-params] :as _request}] + + (res/response (hon/find ds {:tname :filtered-feeds + :where [:= :bundle-id (:id path-params)]}))) diff --git a/src/source/routes/integration_filter_post.clj b/src/source/routes/integration_filter_post.clj new file mode 100644 index 00000000..c8891631 --- /dev/null +++ b/src/source/routes/integration_filter_post.clj @@ -0,0 +1,44 @@ +(ns source.routes.integration-filter-post + (:require [ring.util.response :as res] + [source.workers.integrations :as integrations] + [source.db.honey :as hon])) + +(defn get + {:summary "Returns true if the post with the given ID is filtered out from the given integration by ID" + :parameters {:path [:map + [:id {:title "id" + :description "Integration ID"} :int] + [:post-id {:title "postId" + :description "Post ID"} :int]]} + :responses {200 {:body [:map [:filtered :boolean]]} + 401 {:body [:map [:message :string]]} + 403 {:body [:map [:message :string]]}}} + + [{:keys [ds path-params] :as _request}] + + (let [returned (hon/find ds {:tname :filtered-feeds + :where [:and + [:= :bundle-id (:id path-params)] + [:= :post-id (:post-id path-params)]]}) + blocked (if (seq returned) true false)] + (res/response {:filtered blocked}))) + +(defn post + {:summary "Filters out the post with the given id from the bundle with the given bundle ID" + :parameters {:path [:map + [:id {:title "id" + :description "bundle id"} :int] + [:post-id {:title "post-id" + :description "post id"} :int]] + :body [:map + [:filtered :boolean]]} + :responses {:body {200 [:map [:message :string]] + 401 [:map [:message :string]] + 403 [:map [:message :string]]}}} + + [{:keys [ds path-params body] :as _request}] + (->> {:filtered (:filtered body) + :bundle-id (:id path-params) + :post-id (:post-id path-params)} + (integrations/update-filtered-posts! ds)) + (res/response {:message "successfully updated post filtering"})) diff --git a/src/source/routes/integration_filter_posts.clj b/src/source/routes/integration_filter_posts.clj new file mode 100644 index 00000000..0912839f --- /dev/null +++ b/src/source/routes/integration_filter_posts.clj @@ -0,0 +1,20 @@ +(ns source.routes.integration-filter-posts + (:require [ring.util.response :as res] + [source.db.honey :as hon])) + +(defn get + {:summary "Gets all filtered post ids for the given integration by ID" + :description "If a post appears in this list, it will not be returned when pulling content from the bundle." + :parameters {:path [:map [:id {:title "id" + :description "Integration ID"} :int]]} + :responses {200 {:body [:vector + [:map + [:post-id :int] + [:bundle-id :int]]]} + 401 {:body [:map [:message :string]]} + 403 {:body [:map [:message :string]]}}} + + [{:keys [ds path-params] :as _request}] + + (res/response (hon/find ds {:tname :filtered-posts + :where [:= :bundle-id (:id path-params)]}))) diff --git a/src/source/routes/integrations.clj b/src/source/routes/integrations.clj new file mode 100644 index 00000000..129e59ad --- /dev/null +++ b/src/source/routes/integrations.clj @@ -0,0 +1,64 @@ +(ns source.routes.integrations + (:require [ring.util.response :as res] + [source.services.bundles :as bundles] + [source.workers.integrations :as integrations] + [source.util :as util] + [source.jobs.core :as jobs] + [source.jobs.handlers :as handlers] + [congest.jobs :as congest] + [source.routes.openapi :as api] + [source.workers.schemas :as schemas] + [source.db.honey :as hon] + [malli.util :as mu])) + +(defn get + {:summary "Get metadata of all integrations on the user account" + :responses {200 {:body schemas/Bundles} + 401 {:body [:map [:message :string]]} + 403 {:body [:map [:message :string]]}}} + + [{:keys [ds user] :as _request}] + (res/response (bundles/bundles ds {:where [:= :user-id (:id user)]}))) + +(defn post + {:summary "Creates an integration and the associated bundle in which content is stored" + :description "When an integration is created, a job is scheduled to periodically run post selection every 24 hours. During post selection, the bundle is filled with relevant content according to desired categories, content types and analytics." + :parameters {:body (-> [:map + [:name :string] + [:ts-and-cs {:optional true} :int] + [:integration-type-id :int]] + (mu/assoc :content-types [:vector schemas/ConstantSchema]) + (mu/assoc :categories [:vector schemas/ConstantSchema]))} + :responses {201 {:body schemas/Bundle} + 401 {:body [:map [:message :string]]} + 403 {:body [:map [:message :string]]}}} + + [{:keys [js ds user body] :as _request}] + (let [new-bundle (->> {:user-id (:id user) + :bundle-metadata (dissoc body :categories :content-types) + :categories (:categories body) + :content-types (:content-types body)} + (integrations/create-integration! ds)) + categories-by-bundle (bundles/categories-in-bundle ds (:id new-bundle))] + ;TODO: service needed + (->> (jobs/prepare-congest-metadata + ds + {:id (handlers/update-bundle-job-id (:id new-bundle)) + :initial-delay 0 + :auto-start true + :stop-after-fail false, + :interval (* 1000 60 60 24) + :recurring? true + :args {:bundle-id (:id new-bundle) + :categories categories-by-bundle} + :handler :update-bundle + :created-at (util/get-utc-timestamp-string) + :sleep false}) + (congest/register! js)) + (res/response new-bundle))) + +(defn get-integration-types + {:summary "get all integration types" + :responses (api/success schemas/IntegrationTypes)} + [{:keys [ds]}] + (res/response (hon/find ds {:tname :integration-types}))) diff --git a/src/source/routes/interface.clj b/src/source/routes/interface.clj index 605252ee..86c8ccc5 100644 --- a/src/source/routes/interface.clj +++ b/src/source/routes/interface.clj @@ -1,6 +1,6 @@ (ns source.routes.interface (:require [source.routes.reitit :as reitit])) -(defn create-app [] - (reitit/create-app)) +(defn create-app [{:keys [ds js] :as opts}] + (reitit/create-app opts)) diff --git a/src/source/routes/job.clj b/src/source/routes/job.clj new file mode 100644 index 00000000..1255b73d --- /dev/null +++ b/src/source/routes/job.clj @@ -0,0 +1,26 @@ +(ns source.routes.job + (:require [source.services.interface :as services] + [ring.util.response :as res] + [source.routes.openapi :as api] + [source.workers.schemas :as schemas])) + +(defn get + {:summary "View a single job's raw metadata by id" + :parameters (api/params :path [:map [:id :int]]) + :responses (api/success schemas/JobWithMetadata)} + + [{:keys [ds path-params] :as _req}] + (let [job (services/job ds path-params) + metadata (services/job-metadata ds {:id (:job-metadata-id job)})] + (if (some? job) + (res/response (merge (dissoc job :job-metadata-id) + {:metadata metadata})) + (res/response {})))) + +(comment + (require '[source.db.util :as db.util]) + (def ds (db.util/conn)) + + (get {:ds ds :path-params {:id 5}}) + + ()) diff --git a/src/source/routes/job_deregister.clj b/src/source/routes/job_deregister.clj new file mode 100644 index 00000000..7a5e1552 --- /dev/null +++ b/src/source/routes/job_deregister.clj @@ -0,0 +1,14 @@ +(ns source.routes.job-deregister + (:require [congest.jobs :as jobs] + [ring.util.response :as res] + [source.services.interface :as services] + [source.routes.openapi :as api])) + +(defn get + {:summary "Deregister a job by id" + :params (api/params :path [:map [:id :int]]) + :responses (api/success (api/response-schema))} + [{:keys [js ds path-params] :as _req}] + (let [job (services/job ds path-params)] + (jobs/deregister! js (:job-id job)) + (res/response {:message "successfully deregistered job"}))) diff --git a/src/source/routes/job_start.clj b/src/source/routes/job_start.clj new file mode 100644 index 00000000..b00f3120 --- /dev/null +++ b/src/source/routes/job_start.clj @@ -0,0 +1,14 @@ +(ns source.routes.job-start + (:require [source.jobs.core :as jobs] + [ring.util.response :as res] + [source.services.interface :as services] + [source.routes.openapi :as api])) + +(defn get + {:summary "start a job by id" + :params (api/params :path [:map [:id :int]]) + :responses (api/success (api/response-schema))} + [{:keys [js ds path-params]}] + (let [job (services/job ds path-params)] + (jobs/start! js ds (:job-id job)) + (res/response {:message "successfully started job"}))) diff --git a/src/source/routes/job_stop.clj b/src/source/routes/job_stop.clj new file mode 100644 index 00000000..5b01c050 --- /dev/null +++ b/src/source/routes/job_stop.clj @@ -0,0 +1,14 @@ +(ns source.routes.job-stop + (:require [congest.jobs :as jobs] + [ring.util.response :as res] + [source.services.interface :as services] + [source.routes.openapi :as api])) + +(defn get + {:summary "stop a job by id" + :params (api/params :path [:map [:id :int]]) + :responses (api/success (api/response-schema))} + [{:keys [js ds path-params]}] + (let [job (services/job ds path-params)] + (jobs/stop! js (:job-id job) false) + (res/response {:message "successfully stopped job"}))) diff --git a/src/source/routes/jobs.clj b/src/source/routes/jobs.clj new file mode 100644 index 00000000..fbaadd9f --- /dev/null +++ b/src/source/routes/jobs.clj @@ -0,0 +1,39 @@ +(ns source.routes.jobs + (:require [source.services.interface :as services] + [congest.jobs :as congest] + [source.jobs.core :as jobs] + [ring.util.response :as res] + [source.routes.openapi :as api] + [source.workers.schemas :as schemas] + [source.util :as util] + [malli.util :as mu])) + +(defn get + {:summary "get list of raw job metadata" + :responses (api/success schemas/JobsWithMetadata)} + [{:keys [ds] :as _req}] + (->> + (services/jobs ds) + (mapv (fn [job] + (let [metadata (services/job-metadata ds {:id (:job-metadata-id job)})] + (merge (dissoc job :job-metadata-id) + {:metadata metadata})))) + (res/response))) + +(defn post + {:summary "Register a new job with metadata" + :params (api/params :body [:map + [:metadata + (-> schemas/JobMetadata + (api/missoc :recurring :num-calls) + (mu/assoc :auto-start :boolean) + (mu/assoc :stop-after-fail :boolean) + (mu/assoc :recurring? :boolean) + (mu/assoc :sleep :boolean) + (mu/assoc :handler :string))]]) + :responses (api/success (api/response-schema))} + [{:keys [js ds body] :as _req}] + (let [{:keys [metadata]} body + metadata (assoc metadata :created-at (util/get-utc-timestamp-string))] + (congest/register! js (jobs/prepare-congest-metadata ds metadata)) + (res/response {:message "successfully registered job"}))) diff --git a/src/source/routes/jobs_view.clj b/src/source/routes/jobs_view.clj new file mode 100644 index 00000000..a576dde6 --- /dev/null +++ b/src/source/routes/jobs_view.clj @@ -0,0 +1,35 @@ +(ns source.routes.jobs-view + (:require [ring.util.response :as res] + [congest.jobs :as congest] + [clojure.walk :as walk] + [source.routes.openapi :as api] + [source.workers.schemas :as schemas] + [malli.util :as mu])) + +(defn stringify-unknowns [x] + (walk/postwalk + (fn [v] + (if (or (map? v) + (sequential? v) + (string? v) + (number? v) + (boolean? v) + (keyword? v) + (nil? v)) + v + (str v))) x)) + +(defn get + {:summary "gets a list of all jobs" + :responses (api/success [:vector + (-> schemas/JobMetadata + (api/missoc :created-at :recurring) + (mu/assoc :args [:map-of :string :any]) + (mu/assoc :auto-start :boolean) + (mu/assoc :handler-name :string) + (mu/assoc :recurring? :boolean) + (mu/assoc :sleep :boolean))])} + [{:keys [js]}] + (let [raw-jobs (congest/view js) + formatted (mapv (fn [[_ v]] v) (stringify-unknowns raw-jobs))] + (res/response formatted))) diff --git a/src/source/routes/login.clj b/src/source/routes/login.clj index 7bc08fee..5ba54030 100644 --- a/src/source/routes/login.clj +++ b/src/source/routes/login.clj @@ -1,15 +1,40 @@ (ns source.routes.login (:require [source.services.auth :as auth] [ring.util.response :as res] - [source.services.users :as users] - [source.password :as pw])) + [source.password :as pw] + [source.db.honey :as hon])) + +(defn post + {:summary "get user data and access token provided valid credentials" + :parameters {:body [:map + [:email :string] + [:password :string]]} + :responses {200 {:body [:map + [:user + [:map + [:id :int] + [:address [:maybe :string]] + [:profile-image [:maybe :string]] + [:email :string] + [:firstname [:maybe :string]] + [:lastname [:maybe :string]] + [:type [:enum "creator" "distributor" "admin"]] + [:email-verified [:maybe :int]] + [:onboarded [:maybe :int]] + [:mobile [:maybe :string]]]] + [:access-token :string] + [:refresh-token :string]]} + 401 {:body [:map [:message :string]]}}} + + [{:keys [ds body] :as _request}] -(defn post [{:keys [ds body] :as _request}] (let [{:keys [email password]} body - user (users/user ds {:where [:= :email email]})] + user (hon/find-one ds {:tname :users + :where [:= :email email]})] + (if - (or (not (pw/verify-password password (:password user))) - (not (some? user))) + (or (not (some? user)) + (not (pw/verify-password password (:password user)))) {:status 401 :body {:message "Invalid username or password!"}} (res/response (auth/login ds {:user user}))))) diff --git a/src/source/routes/me.clj b/src/source/routes/me.clj new file mode 100644 index 00000000..e1721c13 --- /dev/null +++ b/src/source/routes/me.clj @@ -0,0 +1,102 @@ +(ns source.routes.me + (:require [ring.util.response :as res] + [source.util :as util] + [source.db.honey :as hon] + [source.jobs.core :as jobs] + [source.jobs.handlers :as handlers] + [congest.jobs :as congest] + [source.workers.users :as users] + [source.email.gmail :as gmail] + [source.email.templates :as templates] + [source.routes.openapi :as api])) + +(defn get + {:summary "get logged in user by access token" + :responses {200 {:body [:map + [:id :int] + [:address [:maybe :string]] + [:profile-image [:maybe :string]] + [:email :string] + [:firstname [:maybe :string]] + [:lastname [:maybe :string]] + [:type [:enum "creator" "distributor" "admin"]] + [:email-verified [:maybe :int]] + [:onboarded [:maybe :int]] + [:mobile [:maybe :string]]]} + 401 {:body [:map [:message :string]]} + 403 {:body [:map [:message :string]]}}} + + [{:keys [ds user] :as _request}] + (let [user (hon/find-one ds {:tname :users + :where [:= :id (:id user)]})] + (->> (dissoc user :password :email-hash) + (res/response)))) + +(defn post + {:summary "update logged-in user by access token" + :parameters {:body [:map + [:address {:optional true} [:maybe :string]] + [:profile-image {:optional true} [:maybe :string]] + [:firstname {:optional true} :string] + [:lastname {:optional true} :string] + [:email-verified {:optional true} :int] + [:onboarded {:optional true} :int] + [:mobile {:optional true} [:maybe :string]]]} + :responses {200 {:body [:map [:message :string]]} + 400 {:body [:map [:message :string]]}}} + + [{:keys [ds user body] :as _request}] + (hon/update! ds {:tname :users + :where [:= :id (:id user)] + :data body}) + (res/response {:message "successfully updated user"})) + +(defn delete-user + {:summary "delete logged-in user by access token" + :responses {200 {:body [:map [:message :string]]}}} + + [{:keys [ds js user] :as _request}] + (let [{:keys [id type]} user + job-id (handlers/user-deletion-job-id type id)] + (users/soft-delete-user! ds js id) + + ; TODO: service needed + (->> (jobs/prepare-congest-metadata + ds + {:id job-id + :initial-delay (* 1000 60 60 24 24) + :auto-start true + :stop-after-fail false, + :interval (* 1000 60 60 24 24) + :recurring? false + :kill-after 1 + :args {:user-type type + :user-id id} + :handler :delete-user + :created-at (util/get-utc-timestamp-string) + :sleep false}) + (congest/register! js)) + (res/response {:message "successfully scheduled user deletion"}))) + +(defn cancel-deletion + {:summary "cancel deletion of logged-in user by access token" + :responses {200 {:body [:map [:message :string]]}}} + + [{:keys [ds js user] :as _request}] + (let [{:keys [id type]} user + job-id (handlers/user-deletion-job-id type id)] + (users/cancel-soft-user-deletion! ds id) + (congest/deregister! js job-id) + (res/response {:message "successfully cancelled user deletion"}))) + +(defn resend-email + {:summary "Resend verification email" + :responses (api/success (api/response-schema))} + [{:keys [ds user]}] + (let [{:keys [email email-hash]} (hon/find-one ds {:tname :users + :where [:= :id (:id user)]})] + (gmail/send-email {:to email + :subject "Source - Verify your email" + :body (templates/email-verification {:email-hash email-hash}) + :type :text/html}) + (res/response {:message "successfully resent email-verification email"}))) diff --git a/src/source/routes/me_business.clj b/src/source/routes/me_business.clj new file mode 100644 index 00000000..639f5b5e --- /dev/null +++ b/src/source/routes/me_business.clj @@ -0,0 +1,57 @@ +(ns source.routes.me-business + (:require [ring.util.response :as res] + [source.db.honey :as hon] + [source.routes.openapi :as api])) + +(defn get + {:summary "get business for logged-in user" + :responses {200 {:body [:map + [:id :int] + [:name :string] + (api/sometimes :address :string) + (api/sometimes :url :string) + (api/sometimes :linkedin :string) + (api/sometimes :twitter :string) + (api/sometimes :registration :string) + (api/sometimes :business-type-id :int)]} + 404 {:body [:map]} + 401 {:body [:map [:message :string]]} + 403 {:body [:map [:message :string]]}}} + + [{:keys [ds user] :as _request}] + (let [{:keys [business-id]} (hon/find-one ds {:tname :users + :where [:= :id (:id user)]})] + (if business-id + (res/response (hon/find-one ds {:tname :businesses + :where [:= :id business-id]})) + (res/response {})))) + +(defn post + {:summary "add or update business for logged-in user" + :parameters {:body [:map + [:name {:optional true} [:maybe :string]] + [:address {:optional true} [:maybe :string]] + [:url {:optional true} [:maybe :string]] + [:linkedin {:optional true} [:maybe :string]] + [:twitter {:optional true} [:maybe :string]] + [:registration {:optional true} [:maybe :string]] + [:business-type-id {:optional true} [:maybe :int]]]} + :responses {200 {:body [:map [:message :string]]} + 400 {:body [:map [:message :string]]}}} + + [{:keys [ds user body] :as _request}] + (let [{:keys [business-id]} (hon/find-one ds {:tname :users + :where [:= :id (:id user)]}) + business (when (nil? business-id) + (hon/insert! ds {:tname :businesses + :data body + :ret :1}))] + (if (nil? business-id) + (hon/update! ds {:tname :users + :where [:= :id (:id user)] + :data {:business-id (:id business)}}) + (hon/update! ds {:tname :businesses + :where [:= :id business-id] + :data body})) + + (res/response {:message "successfully added or updated business"}))) diff --git a/src/source/routes/me_sectors.clj b/src/source/routes/me_sectors.clj new file mode 100644 index 00000000..f1132ead --- /dev/null +++ b/src/source/routes/me_sectors.clj @@ -0,0 +1,26 @@ +(ns source.routes.me-sectors + (:require [ring.util.response :as res] + [source.services.user-sectors :as user-sectors])) + +(defn get + {:summary "get sectors for the logged-in user" + :responses {200 {:body [:vector + [:map + [:id :int] + [:name :string]]]} + 401 {:body [:map [:message :string]]} + 403 {:body [:map [:message :string]]}}} + [{:keys [ds user] :as _request}] + (res/response (user-sectors/sectors-by-user ds {:user-id (:id user)}))) + +(defn post + {:summary "update sectors for the logged-in user" + :parameters {:body [:vector + [:map + [:id :int] + [:name :string]]]} + :responses {200 {:body [:map [:message :string]]}}} + [{:keys [ds user body] :as _request}] + (user-sectors/update-user-sectors! ds {:user-id (:id user) + :sectors body}) + (res/response {:message "successfully updated user sectors"})) diff --git a/src/source/routes/openapi.clj b/src/source/routes/openapi.clj new file mode 100644 index 00000000..1067dcd1 --- /dev/null +++ b/src/source/routes/openapi.clj @@ -0,0 +1,182 @@ +(ns source.routes.openapi + (:require [malli.util :as mu])) + +(defn response-schema + ([] (response-schema nil)) + ([data-schema] + (vec + (cond-> [:map [:message :string]] + (some? data-schema) + (concat [[:data data-schema]]))))) + +(defn error + "A function that wraps an optional error data schema in a standard error message schema" + ([] (error nil)) + ([data-schema] + (response-schema data-schema))) + +(defn- assoc-response [acc [key schema]] + (assoc acc key {:body schema})) + +(defn- assoc-param [acc [key schema]] + (assoc acc key schema)) + +;; what i want: (api/response {} 200 (api/success) 404 (api/error)...) => {200 {:body openapi/VocabSearchResult}} +(defn response + [& opts] + (let [map-first? (map? (first opts)) + responses (if map-first? (first opts) {}) + opts (if map-first? (rest opts) opts)] + (merge responses (reduce assoc-response {} (partition 2 opts))))) + +(defn success + "Returns a map that can be used as the responses field + for an openapi handler meta data. Optionally takes an existing + openapi responses map as the first param and updates its 200 field + and then returns it." + ([schema] (success {} schema)) + ([responses data-schema] + (response (if (map? responses) responses {}) + 200 + data-schema))) + +(defn not-found + "Retuns a map that can be used as the responses field for an + openapi handler meta data" + ([] (not-found {} nil)) + ([schema-or-responses] + (let [responses? (map? schema-or-responses) + schema? (not responses?) + responses (if responses? schema-or-responses nil) + schema (if schema? schema-or-responses nil)] + (not-found responses schema))) + ([responses data-schema] + (response (if (map? responses) responses {}) + 404 (error data-schema)))) + +(defn bad-request + "Retuns a map that can be used as the responses field for an + openapi handler meta data" + ([] (bad-request {} nil)) + ([schema-or-responses] + (let [responses? (map? schema-or-responses) + schema? (not responses?) + responses (if responses? schema-or-responses nil) + schema (if schema? schema-or-responses nil)] + (bad-request responses schema))) + ([responses data-schema] + (response (if (map? responses) responses {}) + 400 (error data-schema)))) + +(defn unauthenticated + ([schema] (unauthenticated {} schema)) + ([responses schema] + (response (if (map? responses) responses {}) + 401 (error schema)))) + +(defn unauthorized + ([schema] (unauthorized {} schema)) + ([responses schema] + (response (if (map? responses) responses {}) + 401 (error schema)))) + +(defn params + "Returns a map of the openapi parameter schemas path, body, query. + Optionally accepts an openapi parameters map as the first argument + updating it and returning the result." + [& opts] + (let [map-first? (map? (first opts)) + parameters (if map-first? (first opts) {}) + opts (if map-first? (rest opts) opts)] + (merge parameters (reduce assoc-param {} (partition 2 opts))))) + +(defn- sometimes-entry [[k _ s]] [k {:optional true} [:maybe s]]) +(defn maybe-keys [schema] + (mu/transform-entries + schema + #(mapv sometimes-entry %))) + +(defn missoc + "Executes mu/dissoc with multiple keys" + [schema & ks] + (reduce (fn [acc k] + (mu/dissoc acc k)) schema ks)) + +;; MALLI SCHEMAS + +(defn optional [key type] + [key {:optional true} type]) + +(defn maybe [type] + [:maybe type]) + +(defn sometimes [key type] + (optional key (maybe type))) + +(defn paginated [data-schema] + [:map + [:pagination [:map + [:page-size :int] + [:total-size :int] + [:current-index :int] + (sometimes :next-index :int)]] + [:data data-schema]]) + +(def RegisterParams + [:map + [:email :string] + [:password :string] + [:confirm-password :string] + [:type [:enum "creator" "distributor"]]]) + +(def LoginParams + [:map + [:email :string] + [:password :string]]) + +(def InsertVocab + [:map + [:xhosa :string] + [:english :string] + (sometimes :illustration :string) + (sometimes :noun-class :string) + [:type :string]]) + +(def InsertVocabParams + [:vector InsertVocab]) + +(def CreateUnitParam + [:map + [:name :string] + [:description :string] + [:level :int]]) + +(def CreateUnitsParam + [:vector CreateUnitParam]) + +(def AnswerParam + [:or [:vector :string] + :string]) + +(def AnswerParams + [:vector AnswerParam]) + +(def ExerciseParam + [:map + [:question-type [:enum ["translation" "multiple-choice"]]] + [:question :string] + [:options [:vector :string]] + (sometimes :answers AnswerParams)]) + +(def ExerciseParams + [:vector ExerciseParam]) + +(def GetExerciseResult + [:map + [:id :int] + [:unit-id :int] + [:question-type [:enum ["translation" "multiple-choice"]]] + [:question :string] + [:options [:vector :string]] + [:answers AnswerParams]]) + diff --git a/src/source/routes/output_schema.clj b/src/source/routes/output_schema.clj new file mode 100644 index 00000000..9306a3e8 --- /dev/null +++ b/src/source/routes/output_schema.clj @@ -0,0 +1,14 @@ +(ns source.routes.output-schema + (:require [source.workers.xml-schemas :as xml] + [ring.util.response :as res])) + +(defn get [{:keys [ds path-params] :as _request}] + (let [id (try + (Integer/parseInt (:id path-params)) + (catch Exception _ nil))] + (if (some? id) + (->> id + (xml/output-schema ds) + (res/response)) + (-> (res/response {:message "invalid id"}) + (res/status 400))))) diff --git a/src/source/routes/output_schemas.clj b/src/source/routes/output_schemas.clj new file mode 100644 index 00000000..6b5691ae --- /dev/null +++ b/src/source/routes/output_schemas.clj @@ -0,0 +1,12 @@ +(ns source.routes.output-schemas + (:require [source.workers.xml-schemas :as xml] + [ring.util.response :as res])) + +(defn get [{:keys [ds] :as _request}] + (-> (xml/output-schemas ds) + (res/response))) + +(defn post [{:keys [ds body] :as _request}] + (let [{:keys [schema]} body] + (xml/insert-output-schema! ds schema) + (res/response {:message "successfully added output schema"}))) diff --git a/src/source/routes/post.clj b/src/source/routes/post.clj new file mode 100644 index 00000000..5fd9f6f4 --- /dev/null +++ b/src/source/routes/post.clj @@ -0,0 +1,30 @@ +(ns source.routes.post + (:require [ring.util.response :as res] + [source.db.honey :as hon])) + +(defn get + {:summary "get post by id" + :parameters {:path [:map [:post-id {:title "post-id" + :description "post id"} :int]]} + :responses {200 {:body [:map + [:id :int] + [:post-id :string] + [:feed-id :int] + [:creator-id :int] + [:content-type-id :int] + [:title :string] + [:thumbnail [:maybe :string]] + [:info [:maybe :string]] + [:url [:maybe :string]] + [:stream-url [:maybe :string]] + [:season [:maybe :int]] + [:episode [:maybe :int]] + [:redacted {:optional true} [:maybe :int]] + [:posted-at [:maybe :string]]]}}} + + [{:keys [ds user path-params] :as _request}] + (-> (hon/find-one ds {:tname :incoming-posts + :where [:and + [:= :id (:post-id path-params)] + [:= :creator-id (:id user)]]}) + (res/response))) diff --git a/src/source/routes/post_prune.clj b/src/source/routes/post_prune.clj new file mode 100644 index 00000000..79f198c6 --- /dev/null +++ b/src/source/routes/post_prune.clj @@ -0,0 +1,21 @@ +(ns source.routes.post-prune + (:require [ring.util.response :as res] + [source.db.honey :as hon])) + +(defn post + {:summary "Update redacted status of post with the given id" + :parameters {:path [:map [:post-id {:title "post-id" + :description "post id"} :int]] + :body [:map + [:redacted :boolean]]} + :responses {200 {:body [:map [:message :string]]} + 401 {:body [:map [:message :string]]} + 403 {:body [:map [:message :string]]}}} + + [{:keys [ds path-params user body] :as _request}] + (hon/update! ds {:tname :incoming-posts + :where [:and + [:= :id (:post-id path-params)] + [:= :creator-id (:id user)]] + :data {:redacted (if (:redacted body) 1 0)}}) + (res/response {:message "successfully updated post"})) diff --git a/src/source/routes/posts.clj b/src/source/routes/posts.clj new file mode 100644 index 00000000..8de11a8f --- /dev/null +++ b/src/source/routes/posts.clj @@ -0,0 +1,32 @@ +(ns source.routes.posts + (:require [ring.util.response :as res] + [source.db.honey :as hon])) + +(defn get + {:summary "get all posts by feed id" + :parameters {:path [:map [:id {:title "id" + :description "feed id"} :int]]} + :responses {200 {:body [:vector + [:map + [:id :int] + [:post-id :string] + [:feed-id :int] + [:creator-id :int] + [:content-type-id :int] + [:title :string] + [:thumbnail [:maybe :string]] + [:info [:maybe :string]] + [:url [:maybe :string]] + [:stream-url [:maybe :string]] + [:season [:maybe :int]] + [:episode [:maybe :int]] + [:redacted {:optional true} [:maybe :int]] + [:posted-at [:maybe :string]]]]} + 401 {:body [:map [:message :string]]} + 403 {:body [:map [:message :string]]}}} + + [{:keys [ds path-params] :as _request}] + (-> (hon/find ds {:tname :incoming-posts + :where [:= :feed-id (:id path-params)] + :ret :*}) + (res/response))) diff --git a/src/source/routes/provider.clj b/src/source/routes/provider.clj new file mode 100644 index 00000000..47894287 --- /dev/null +++ b/src/source/routes/provider.clj @@ -0,0 +1,51 @@ +(ns source.routes.provider + (:require [source.workers.xml-schemas :as xml] + [ring.util.response :as res] + [source.db.honey :as hon])) + +(defn get + {:summary "get provider by id" + :parameters {:path [:map [:id {:title "id" + :description "provider id"} :int]]} + :responses {200 {:body [:map + [:id :int] + [:name :string] + [:domain [:maybe :string]] + [:content-type-id :int] + [:instructions [:maybe :string]] + [:placeholder-url [:maybe :string]]]}}} + + [{:keys [ds path-params] :as _request}] + (->> (hon/find-one ds {:tname :providers + :where [:= :id (:id path-params)]}) + (res/response))) + +(defn post + {:summary "update provider by id" + :parameters {:path [:map [:id {:title "id" + :description "provider id"} :int]] + :body [:map + [:name :string] + [:domain {:optional true} [:maybe :string]] + [:content-type-id {:optional true} [:maybe :int]] + [:instructions {:optional true} [:maybe :string]] + [:placeholder-url {:optional true} [:maybe :string]]]} + :responses {200 {:body [:map [:message :string]]}}} + + [{:keys [ds path-params body] :as _request}] + (hon/update! ds {:tname :providers + :where [:= :id (:id path-params)] + :data body}) + (res/response {:message "successfully updated provider"})) + +(defn delete + {:summary "given a provider id, delete provider and associated selection schemas" + :parameters {:path [:map [:id {:title "id" + :description "provider id"} :int]]} + :responses {200 {:body [:map [:message :string]]}}} + + [{:keys [ds path-params] :as _request}] + (hon/delete! ds {:tname :providers + :where [:= :id (:id path-params)]}) + (xml/delete-selection-schemas-by-provider! ds (:id path-params)) + (res/response {:message "successfully deleted provider"})) diff --git a/src/source/routes/provider_selection_schemas.clj b/src/source/routes/provider_selection_schemas.clj new file mode 100644 index 00000000..59bb1f61 --- /dev/null +++ b/src/source/routes/provider_selection_schemas.clj @@ -0,0 +1,15 @@ +(ns source.routes.provider-selection-schemas + (:require [ring.util.response :as res] + [source.workers.xml-schemas :as xml])) + +(defn get [{:keys [ds path-params] :as _request}] + (let [selection-schemas (xml/selection-schemas + ds + {:where [:= :provider-id (Integer/parseInt (:id path-params))]}) + results (mapv (fn [{:keys [output-schema-id] :as ss}] + (merge + ss + {:output-schema + (xml/output-schema ds output-schema-id)})) + selection-schemas)] + (res/response results))) diff --git a/src/source/routes/providers.clj b/src/source/routes/providers.clj new file mode 100644 index 00000000..1b8d39f8 --- /dev/null +++ b/src/source/routes/providers.clj @@ -0,0 +1,35 @@ +(ns source.routes.providers + (:require [ring.util.response :as res] + [source.db.honey :as hon])) + +(defn get + {:summary "get all providers" + :responses {200 {:body [:vector + [:map + [:id :int] + [:name :string] + [:domain [:maybe :string]] + [:content-type-id :int] + [:instructions [:maybe :string]] + [:placeholder-url [:maybe :string]]]]}}} + + [{:keys [ds] :as _request}] + (-> (hon/find ds {:tname :providers + :ret :*}) + (res/response))) + +(defn post + {:summary "add a provider" + :parameters {:body [:map + [:name :string] + [:domain {:optional true} [:maybe :string]] + [:content-type-id :int] + [:instructions {:optional true} [:maybe :string]] + [:placeholder-url {:optional true} [:maybe :string]]]} + :responses {200 {:body [:map [:message :string]]}}} + + [{:keys [ds body] :as _request}] + (hon/insert! ds {:tname :providers + :data body + :ret :1}) + (res/response {:message "successfully added provider"})) diff --git a/src/source/routes/register.clj b/src/source/routes/register.clj index dacb5532..a6d8d382 100644 --- a/src/source/routes/register.clj +++ b/src/source/routes/register.clj @@ -1,11 +1,36 @@ (ns source.routes.register (:require [source.services.interface :as services] - [ring.util.response :as res])) + [ring.util.response :as res] + [source.db.honey :as hon])) + +(defn post + {:summary "register a new user" + :parameters {:body [:map + [:email :string] + [:password :string] + [:confirm-password :string] + [:type [:enum "creator" "distributor"]]]} + :responses {200 {:body [:map + [:user + [:map + [:id :int] + [:address [:maybe :string]] + [:profile-image [:maybe :string]] + [:email :string] + [:firstname [:maybe :string]] + [:lastname [:maybe :string]] + [:type [:enum "creator" "distributor" "admin"]] + [:email-verified [:maybe :int]] + [:onboarded [:maybe :int]] + [:mobile [:maybe :string]]]] + [:access-token :string] + [:refresh-token :string]]}}} + + [{:keys [ds body] :as _request}] -(defn post [{:keys [ds body] :as _request}] - ;;TODO: needs schema validation here (let [{:keys [email password confirm-password]} body - existing-user (services/user ds {:where [:= :email email]})] + existing-user (hon/find-one ds {:tname :users + :where [:= :email email]})] (cond (not (= password confirm-password)) (-> (res/response {:error "Passwords do not match!"})) @@ -19,8 +44,9 @@ (comment (require '[source.db.interface :as db]) - (post {:ds (db/ds :master) :body {:email "test@test.com" + (post {:ds (db/ds :master) :body {:email "poop@test.com" :password "test" :type "distributor" :confirm-password "test"}}) ()) + diff --git a/src/source/routes/reitit.clj b/src/source/routes/reitit.clj index 7c00aa45..878d1a19 100644 --- a/src/source/routes/reitit.clj +++ b/src/source/routes/reitit.clj @@ -1,65 +1,304 @@ (ns source.routes.reitit (:require [reitit.ring :as ring] + [reitit.coercion.malli] + [reitit.ring.malli] [source.middleware.interface :as mw] - [source.db.interface :as db] [clojure.data.json :as json] + [source.routes.util :refer [get patch post delete] :as rutil] [source.routes.user :as user] [source.routes.users :as users] + [source.routes.me :as me] + [source.routes.me-business :as me-business] + [source.routes.me-sectors :as me-sectors] [source.routes.login :as login] [source.routes.register :as register] [source.routes.google-launch :as google-launch] [source.routes.google-redirect :as google-redirect] + [source.routes.google-user :as google-user] [source.routes.admin :as admin] [source.routes.authorized :as authorized] [source.routes.business :as business] [source.routes.businesses :as businesses] - [source.routes.sectors :as sectors])) - -(defn create-app [] - (let [ds (db/ds :master)] - (ring/ring-handler - (ring/router - [["/" {:middleware [[mw/apply-generic :ds ds]]} - ["" (fn [_request] {:status 200 :body {:message "success"}})] - ["users" {:middleware [[mw/apply-auth {:required-type :admin}]]} - ["" {:get users/get}] - ["/:id" {:get user/get - :patch user/patch}]] - ["businesses" - ["" {:get businesses/get - :post business/post}] - ["/:id" {:patch business/patch}]] - ["sectors" - ["" {:get sectors/get}]] - ["login" {:post login/post}] - ["register" {:post register/post}] - ["oauth2" - ["/google" - ["" {:get google-launch/get}] - ["/callback" {:get google-redirect/get}]]] - ["protected" {:middleware [[mw/apply-auth]]} - ["/authorized" {:get authorized/get}]] - ["admin" {:middleware [[mw/apply-auth {:required-type :admin}]]} - ["/add-admin" {:post admin/post}]]]])))) + [source.routes.business-types :as business-types] + [source.routes.sectors :as sectors] + [source.routes.selection-schemas :as selection-schemas] + [source.routes.selection-schema :as selection-schema] + [source.routes.provider-selection-schemas :as provider-selection-schemas] + [source.routes.output-schemas :as output-schemas] + [source.routes.output-schema :as output-schema] + [source.routes.providers :as providers] + [source.routes.provider :as provider] + [source.routes.cadences :as cadences] + [source.routes.categories :as categories] + [source.routes.category :as category] + [source.routes.baselines :as baselines] + [source.routes.content-types :as content-types] + [source.routes.content-type :as content-type] + [source.routes.feeds :as feeds] + [source.routes.feed :as feed] + [source.routes.feed-categories :as feed-categories] + [source.routes.analytics.creator.general :as analytics-creator-general] + [source.routes.analytics.creator.deltas :as analytics-creator-deltas] + [source.routes.analytics.creator.top :as analytics-creator-top] + [source.routes.analytics.creator.top-average :as analytics-creator-top-average] + [source.routes.analytics.distributor.general :as analytics-distributor-general] + [source.routes.analytics.distributor.top :as analytics-distributor-top] + [source.routes.analytics.distributor.top-average :as analytics-distributor-top-average] + [source.routes.analytics.bundle.posts.-id-.views :as analytics-bundle-posts-id-views] + [source.routes.integrations :as integrations] + [source.routes.integration :as integration] + [source.routes.integration-categories :as integration-categories] + [source.routes.integration-filter-feeds :as integration-filter-feeds] + [source.routes.integration-filter-feed :as integration-filter-feed] + [source.routes.integration-filter-posts :as integration-filter-posts] + [source.routes.integration-filter-post :as integration-filter-post] + [source.routes.bundle :as bundle] + [source.routes.bundle-categories :as bundle-categories] + [source.routes.bundle-feeds :as bundle-feeds] + [source.routes.bundle-feed :as bundle-feed] + [source.routes.bundle-feed-posts :as bundle-feed-posts] + [source.routes.bundle-feed-post :as bundle-feed-post] + [source.routes.bundle-posts :as bundle-posts] + [source.routes.bundle-post :as bundle-post] + [source.routes.posts :as posts] + [source.routes.post :as post] + [source.routes.post-prune :as post-prune] + [source.routes.admin-feeds :as admin-feeds] + [source.routes.xml :as xml] + [source.routes.data :as data] + [source.routes.jobs :as jobs] + [source.routes.jobs-view :as jobs-view] + [source.routes.job :as job] + [source.routes.job-deregister :as job-deregister] + [source.routes.job-start :as job-start] + [source.routes.job-stop :as job-stop] + [source.routes.report :as report] + [source.routes.approve-feed :as approve-feed] + [source.routes.reject-feed :as reject-feed])) + +(defn create-app [{:keys [ds js]}] + (ring/ring-handler + (ring/router + [(rutil/swagger-route) + (rutil/openapi-route) + + ["/users" {:middleware [[mw/apply-auth {:required-type :admin}]] + :tags #{"users"} + :swagger {:security [{"auth" []}]} + :openapi {:security [{:bearerAuth []}]}} + + ["" (get users/get)] + ["/:id" (-> (get user/get) + (patch user/patch))]] + + ["/email/verify/:hash" (get user/verify-email)] + + ["/me" {:middleware [[mw/apply-auth]] + :tags #{"me"} + :swagger {:security [{"auth" []}]} + :openapi {:security [{:bearerAuth []}]}} + + ["" (-> (get me/get) + (post me/post) + (delete me/delete-user))] + ["/email/resend" (get me/resend-email)] + ["/deletion/cancel" (get me/cancel-deletion)] + ["/business" (-> (get me-business/get) + (post me-business/post))] + ["/sectors" (-> (get me-sectors/get) + (post me-sectors/post))]] + + ["/mail" {:tags #{"mail"}} + ["/report" (post report/post)]] + + ["/businesses" {:middleware [[mw/apply-auth {:required-type :admin}]] + :tags #{"businesses"} + :swagger {:security [{"auth" []}]} + :openapi {:security [{:bearerAuth []}]}} + + ["" (-> (get businesses/get) + (post business/post))] + ["/:id" (patch business/patch)]] + + ["/business" {:tags #{"businesses"}} + ["/types" (get business-types/get)]] + + ["/sectors" {:tags #{"sectors"}} + ["" (get sectors/get)]] + + ["/login" {:tags #{"auth"}} + ["" (post login/post)]] + + ["/register" {:tags #{"auth"}} + ["" (post register/post)]] + + ["/oauth2" + ["/google" {:tags #{"google"}} + + ["" (get google-launch/get)] + ["/callback" {:no-doc true} + ["" (get google-redirect/get)]] + ["/user" (get google-user/get)]]] + + ["/protected" {:middleware [[mw/apply-auth]] + :tags #{"protected"} + :swagger {:security [{"auth" []}]} + :openapi {:security [{:bearerAuth []}]}} + + ["/authorized" (get authorized/get)]] + + ["/providers" {:tags #{"providers"}} + ["" (get providers/get)] + ["/:id" (get provider/get)]] + + ["/cadences" {:tags #{"cadences"}} + ["" (get cadences/get)]] + + ["/categories" {:tags #{"categories"}} + ["/all" (get categories/get)] + ["/all/:id" (get category/get)] + ["/used" (get categories/used-categories)]] + + ["/baselines" {:tags #{"baselines"}} + ["" (get baselines/get)]] + + ["/contentTypes" {:tags #{"content types"}} + ["" (get content-types/get)] + ["/:id" (get content-type/get)]] + + ["/integrations" {:middleware [[mw/apply-auth]] + :tags #{"integrations"}} + + ["" (-> (get integrations/get) + (post integrations/post))] + ["/:id" (-> (get integration/get) + (post integration/post) + (delete integration/delete))] + ["/:id/categories" (-> (get integration-categories/get) + (post integration-categories/post))] + ["/:id/filter/feeds" (get integration-filter-feeds/get)] + ["/:id/filter/feeds/:feed-id" (-> (get integration-filter-feed/get) + (post integration-filter-feed/post))] + ["/:id/filter/posts" (get integration-filter-posts/get)] + ["/:id/filter/posts/:post-id" (-> (get integration-filter-post/get) + (post integration-filter-post/post))]] + + ["/integration/types" {:tags #{"integrations"}} + ["" (get integrations/get-integration-types)]] + + ["/feeds" {:middleware [[mw/apply-auth]] + :tags #{"feeds"}} + + ["" (-> (get feeds/get) + (post feeds/post))] + ["/:id" (-> (get feed/get) + (post feed/post) + (delete feed/delete))] + ["/:id/posts" (get posts/get)] + ["/:id/posts/:post-id" (get post/get)] + ["/:id/posts/:post-id/prune" (post post-prune/post)] + ["/:id/categories" (-> (get feed-categories/get) + (post feed-categories/post))]] + + ["/analytics" {:tags #{"analytics"}} + + ["/creator" {:middleware [[mw/apply-auth {:required-type :creator}]]} + ["/general" (get analytics-creator-general/get)] + ["/deltas" (get analytics-creator-deltas/get)] + ["/top" (get analytics-creator-top/get)] + ["/top/average" (get analytics-creator-top-average/get)]] + ["/distributor" {:middleware [[mw/apply-auth {:required-type :distributor}]]} + ["/general" (get analytics-distributor-general/get)] + ["/top" (get analytics-distributor-top/get)] + ["/top/average" (get analytics-distributor-top-average/get)]] + ["/bundle" {:middleware [[mw/apply-bundle]]} + ["/posts/:id/views" (post analytics-bundle-posts-id-views/post)]] + ["admin" {:middleware [[mw/apply-auth {:required-type :admin}]]} + ["/general"] + ["/top"]]] + + ["/bundle" {:middleware [[mw/apply-bundle]] + :tags #{"bundles"}} + ["" (get bundle/get)] + ["/categories" (get bundle-categories/get)] + ["/feeds" (post bundle-feeds/post)] + ["/feeds/:id" (get bundle-feed/get)] + ["/feeds/:id/posts" (get bundle-feed-posts/get)] + ["/feeds/:id/posts/:post-id" (get bundle-feed-post/get)] + ["/posts" (post bundle-posts/post)] + ["/posts/:id" (get bundle-post/get)]] + ["/bundle/exists" {:tags #{"bundles"}} + ["" (get bundle/exists)]] + + ["/admin" {:middleware [[mw/apply-auth {:required-type :admin}]] + :tags #{"admin"} + :swagger {:security [{"auth" []}]} + :openapi {:security [{:bearerAuth []}]}} + + ["/users/:id/verify" (get users/verify-email)] + ["/categories" (post categories/add-category)] + ["/categories/:id" (-> (post categories/update-category) + (delete categories/delete-category))] + ["/business/types" (-> (post business-types/post) + (patch business-types/patch) + (delete business-types/delete))] + ["/feeds" (get admin-feeds/get)] + ["/feeds/:id/approve" (post approve-feed/post)] + ["/feeds/:id/reject" (post reject-feed/post)] + ["/jobs" (get jobs/get)] + ["/jobs/manage/view" (get jobs-view/get)] + ["/jobs/manage/register" (post jobs/post)] + ["/jobs/:id" (get job/get)] + ["/jobs/manage/deregister" (get job-deregister/get)] + ["/jobs/manage/start" (get job-start/get)] + ["/jobs/manage/stop" (get job-stop/get)] + ["/add-admin" (post admin/post)] + ["/selection-schemas" (-> (get selection-schemas/get) + (post selection-schemas/post))] + ["/selection-schemas/:id" (get selection-schema/get)] + ["/selection-schemas/providers/:id" (get provider-selection-schemas/get)] + ["/output-schemas" (-> (get output-schemas/get) + (post output-schemas/post))] + ["/output-schemas/:id" (get output-schema/get)] + ["/providers" (post providers/post)] + ["/providers/:id" (-> (post provider/post) + (delete provider/delete))] + ["/ast" (post xml/post)] + ["/extract-data" (post data/post)]]] + + (rutil/data-map [[mw/apply-generic :ds ds :js js]])) + (ring/routes + (rutil/swagger-ui-handler) + (ring/create-default-handler)))) (comment - (require '[source.middleware.auth.util :as auth.util]) + (require '[source.middleware.auth.util :as auth.util] + '[source.db.util :as db.util] + '[congest.jobs :as js] + '[source.rss.youtube :as yt]) - (let [app (create-app) - request {:uri "/users" :request-method :get}] + (def components {:ds (db.util/conn) + :js (js/create-job-service [])}) + + (let [app (create-app components) + request {:uri "/users" + :request-method :get + :headers {"authorization" (str "Bearer " (auth.util/sign-jwt {:id 1 :type "admin"}))}}] (-> request app :body (json/read-json {:key-fn keyword}))) - (let [app (create-app) - request {:uri "/users/3" :request-method :get}] + (let [app (create-app components) + request {:uri "/users/3" + :request-method :get + :headers {"authorization" (str "Bearer " (auth.util/sign-jwt {:id 1 :type "admin"}))}}] (-> request app :body (json/read-json {:key-fn keyword}))) - (let [app (create-app) + (let [app (create-app components) request {:uri "/users/3" :request-method :patch :body {:firstname "Keagan" @@ -69,7 +308,7 @@ :body (json/read-json {:key-fn keyword}))) - (let [app (create-app) + (let [app (create-app components) request {:uri "/protected/authorized" :request-method :get :headers {"authorization" (str "Bearer " (auth.util/sign-jwt {:id 5 :type "distributor"}))}}] @@ -78,7 +317,7 @@ :body (json/read-json {:key-fn keyword}))) - (let [app (create-app) + (let [app (create-app components) request {:uri "/admin/add-admin" :request-method :post :body {:email "test@test.com" @@ -90,7 +329,7 @@ :body (json/read-json {:key-fn keyword}))) - (let [app (create-app) + (let [app (create-app components) request {:uri "/oauth2/google" :request-method :get}] (-> request @@ -98,7 +337,7 @@ :body (json/read-json {:key-fn keyword}))) - (let [app (create-app) + (let [app (create-app components) request {:uri "/businesses" :request-method :get}] (-> request @@ -106,7 +345,7 @@ :body (json/read-json {:key-fn keyword}))) - (let [app (create-app) + (let [app (create-app components) request {:uri "/businesses" :request-method :post :body {:name "beep" @@ -116,7 +355,7 @@ :body (json/read-json {:key-fn keyword}))) - (let [app (create-app) + (let [app (create-app components) request {:uri "/businesses/1" :request-method :patch :body {:name "thebest" @@ -126,7 +365,7 @@ :body (json/read-json {:key-fn keyword}))) - (let [app (create-app) + (let [app (create-app components) request {:uri "/sectors" :request-method :get}] (-> request @@ -134,4 +373,176 @@ :body (json/read-json {:key-fn keyword}))) + (let [app (create-app components) + request {:uri "/providers" + :request-method :get}] + (-> request + app + :body + (json/read-json {:key-fn keyword}))) + + (let [app (create-app components) + request {:uri "/providers/1" + :request-method :get}] + (-> request + app + :body + (json/read-json {:key-fn keyword}))) + + (let [app (create-app components) + request {:uri "/content-types" + :request-method :get}] + (-> request + app + :body + (json/read-json {:key-fn keyword}))) + + (let [app (create-app components) + request {:uri "/content-types/1" + :request-method :get}] + (-> request + app + :body + (json/read-json {:key-fn keyword}))) + + (defn get-url [] + (->> "https://www.youtube.com/@ThePrimeTimeagen" + (yt/find-channel-id) + (str "https://www.youtube.com/feeds/videos.xml?channel_id="))) + + (let [app (create-app components) + request {:uri "/admin/feeds" + :request-method :get + :headers {"authorization" (str "Bearer " (auth.util/sign-jwt {:id 5 :type "creator"}))}}] + (-> request + app + :body + (json/read-json {:key-fn keyword}))) + + (let [app (create-app components) + request {:uri "/admin/feeds" + :request-method :post + :body {:title "primeagen test" + :rss-url (get-url) + :provider-id 1 + :content-type-id 1 + :cadence-id 1 + :baseline-id 1} + :headers {"authorization" (str "Bearer " (auth.util/sign-jwt {:id 5 :type "creator"}))}}] + (-> request + app + :body + (json/read-json {:key-fn keyword}))) + + (let [app (create-app components) + request {:uri "/admin/selection-schemas" + :request-method :post + :body {:record {:provider-id 1 + :output-schema-id 1} + :schema {:title {:path ["tag/body" "tag/feed" "tag/title" "content/0"]}}} + :headers {"authorization" (str "Bearer " (auth.util/sign-jwt {:id 1 :type "admin"}))}}] + (-> request + app + :body + (json/read-json {:key-fn keyword}))) + + (let [app (create-app components) + request {:uri "/admin/selection-schemas/1" + :request-method :get + :headers {"authorization" (str "Bearer " (auth.util/sign-jwt {:id 1 :type "admin"}))}}] + (-> request + app + :body + (json/read-json {:key-fn keyword}))) + + (let [app (create-app components) + request {:uri "/admin/selection-schemas" + :request-method :get + :headers {"authorization" (str "Bearer " (auth.util/sign-jwt {:id 1 :type "admin"}))}}] + (-> request + app + :body + (json/read-json {:key-fn keyword}))) + + (defn get-url [] + (->> "https://www.youtube.com/@ThePrimeTimeagen" + (yt/find-channel-id) + (str "https://www.youtube.com/feeds/videos.xml?channel_id="))) + + (let [app (create-app components) + request {:uri "/admin/ast" + :request-method :post + :body {:url (get-url)} + :headers {"authorization" (str "Bearer " (auth.util/sign-jwt {:id 1 :type "admin"}))}}] + (-> request + app + :body + (json/read-json {:key-fn keyword}))) + + (let [app (create-app components) + request {:uri "/admin/extract-data" + :request-method :post + :body {:schema-id 1 + :url (get-url)} + :headers {"authorization" (str "Bearer " (auth.util/sign-jwt {:id 1 :type "admin"}))}}] + (-> request + app + :body + (json/read-json {:key-fn keyword}))) + + (let [app (create-app components) + request {:uri "/admin/jobs" + :request-method :get + :headers {"authorization" (str "Bearer " (auth.util/sign-jwt {:id 1 :type "admin"}))}}] + (-> request + app + :body + (json/read-json {:key-fn keyword}))) + + (let [app (create-app components) + request {:uri "/admin/jobs/1" + :request-method :get + :headers {"authorization" (str "Bearer " (auth.util/sign-jwt {:id 1 :type "admin"}))}}] + (-> request + app + :body + (json/read-json {:key-fn keyword}))) + + (let [app (create-app components) + request {:uri "/admin/jobs/manage/register" + :request-method :post + :body {:metadata {:initial-delay 10 + :auto-start true + :stop-after-fail false, + :interval 1000 + :recurring? true + :args {:name "congest"} + :handler "test" + :created-at nil + :sleep false}} + :headers {"authorization" (str "Bearer " (auth.util/sign-jwt {:id 1 :type "admin"}))}}] + (-> request + app + :body + (json/read-json {:key-fn keyword}))) + + (let [app (create-app components) + request {:uri "/admin/jobs/8/manage/deregister" + :request-method :get + :headers {"authorization" (str "Bearer " (auth.util/sign-jwt {:id 1 :type "admin"}))}}] + (-> request + app + :body + (json/read-json {:key-fn keyword}))) + + (let [app (create-app components) + request {:uri "/mail/report" + :request-method :post + :body {:message "I didn't get my cheesy fries how dare you"} + :headers {"authorization" (str "Bearer " (auth.util/sign-jwt {:id 1 :type "admin"}))}}] + (-> request + app + :body + (json/read-json {:key-fn keyword}))) + ()) diff --git a/src/source/routes/reject_feed.clj b/src/source/routes/reject_feed.clj new file mode 100644 index 00000000..3b010a2e --- /dev/null +++ b/src/source/routes/reject_feed.clj @@ -0,0 +1,30 @@ +(ns source.routes.reject-feed + (:require [source.email.gmail :as gmail] + [source.email.templates :as templates] + [ring.util.response :as res] + [source.db.honey :as hon])) + +(defn post + {:summary "reject the feed with the given feed-id and prevent it from going live" + :parameters {:path [:map [:id {:title "id" + :description "feed id"} :int]] + :body [:map [:message :string]]} + :responses {200 {:body [:map [:message :string]]} + 401 {:body [:map [:message :string]]} + 403 {:body [:map [:message :string]]}}} + + [{:keys [ds path-params body] :as _request}] + (let [{:keys [user-id title]} (hon/find-one ds {:tname :feeds + :where [:= :id (:id path-params)]}) + {:keys [email firstname]} (hon/find-one ds {:tname :users + :where [:= :id user-id]})] + (hon/update! ds {:tname :feeds + :where [:= :id (:id path-params)] + :data {:state "not live"}}) + (gmail/send-email {:to email + :subject "Feed Rejection" + :body (templates/feed-rejection {:creator-name firstname + :feed-title title + :reason (:message body)}) + :type :text/html}) + (res/response {:message "successfully rejected feed"}))) diff --git a/src/source/routes/report.clj b/src/source/routes/report.clj new file mode 100644 index 00000000..a8d43ef1 --- /dev/null +++ b/src/source/routes/report.clj @@ -0,0 +1,35 @@ +(ns source.routes.report + (:require [source.config :as conf] + [source.email.gmail :as gmail] + [source.email.templates :as templates] + [ring.util.response :as res] + [source.db.honey :as hon])) + +(defn post + {:summary "sends us a message to let us know of a problem" + :parameters {:body [:map [:message :string]]} + :responses {200 {:body [:map [:message :string]]} + 401 {:body [:map [:message :string]]} + 403 {:body [:map [:message :string]]}}} + [{:keys [ds user body] :as _req}] + (let [{:keys [email]} (hon/find-one ds {:tname :users + :where [:= :id (:id user)]}) + email-body (templates/admin-reported-problem {:user-id (:id user) + :user-type (:type user) + :user-email email + :message (:message body)})] + (gmail/send-email {:to (conf/read-value :email :address) + :subject (str "Report from " email) + :body email-body + :type :text/html}) + (res/response {:message "successfully sent report"}))) + +(comment + (require '[source.db.util :as db.util]) + + (post {:ds (db.util/conn) + :user {:id 3 + :type "admin"} + :body {:message "my happy meal didn't come with a toy :("}}) + + ()) diff --git a/src/source/routes/sectors.clj b/src/source/routes/sectors.clj index f6aaf7ff..3e77f966 100644 --- a/src/source/routes/sectors.clj +++ b/src/source/routes/sectors.clj @@ -1,9 +1,17 @@ (ns source.routes.sectors - (:require [source.services.sectors :as sectors] - [ring.util.response :as res])) + (:require [ring.util.response :as res] + [source.db.honey :as hon])) -(defn get [{:keys [ds] :as _request}] - (res/response {:sectors (sectors/sectors ds)})) +(defn get + {:summary "get all sectors" + :responses {200 {:body [:vector + [:map + [:id :int] + [:name :string]]]}}} + + [{:keys [ds] :as _request}] + (res/response (hon/find ds {:tname :sectors + :ret :*}))) (comment (require '[source.db.util :as db.util]) diff --git a/src/source/routes/selection_schema.clj b/src/source/routes/selection_schema.clj new file mode 100644 index 00000000..79387288 --- /dev/null +++ b/src/source/routes/selection_schema.clj @@ -0,0 +1,16 @@ +(ns source.routes.selection-schema + (:require [ring.util.response :as res] + [source.workers.xml-schemas :as xml])) + +(defn get [{:keys [ds path-params] :as _request}] + (let [{:keys [output-schema-id] :as selection-schema} + (xml/selection-schema ds path-params) + output-schema (xml/output-schema ds output-schema-id)] + (res/response (merge {:output-schema output-schema} + selection-schema)))) + +(comment + (require '[source.db.util :as db.util]) + (get {:ds (db.util/conn) + :path-params {:id 1}}) + ()) diff --git a/src/source/routes/selection_schemas.clj b/src/source/routes/selection_schemas.clj new file mode 100644 index 00000000..43df3cd2 --- /dev/null +++ b/src/source/routes/selection_schemas.clj @@ -0,0 +1,16 @@ +(ns source.routes.selection-schemas + (:require [ring.util.response :as res] + [source.workers.xml-schemas :as xml])) + +(defn get [{:keys [ds] :as _request}] + (-> (xml/selection-schemas ds) + (res/response))) + +(defn post [{:keys [ds body] :as _request}] + (-> (xml/insert-selection-schema! ds body) + (res/response))) + +(comment + (require '[source.db.util :as db.util]) + (get {:ds (db.util/conn)}) + ()) diff --git a/src/source/routes/user.clj b/src/source/routes/user.clj index a35761f8..173f48c7 100644 --- a/src/source/routes/user.clj +++ b/src/source/routes/user.clj @@ -1,19 +1,93 @@ (ns source.routes.user - (:require [source.services.interface :as services] - [ring.util.response :as res])) + (:require [ring.util.response :as res] + [source.db.honey :as hon] + [source.routes.openapi :as api] + [source.config :as conf])) -(defn get [{:keys [ds path-params] :as _request}] - (->> path-params - (services/user ds) - (assoc {} :user) - (res/response))) +(defn get + {:summary "get user by id" + :parameters {:path [:map [:id {:title "id" + :description "user id"} :int]]} + :responses {200 {:body [:map + [:user + [:map + [:id :int] + [:address [:maybe :string]] + [:profile-image [:maybe :string]] + [:email :string] + [:firstname [:maybe :string]] + [:lastname [:maybe :string]] + [:type [:enum "creator" "distributor" "admin"]] + [:email-verified [:maybe :int]] + [:onboarded [:maybe :int]] + [:mobile [:maybe :string]]]]]} + 401 {:body [:map [:message :string]]} + 403 {:body [:map [:message :string]]}}} -(defn patch [{:keys [ds body path-params] :as _request}] - (services/update-user! ds - {:id (:id path-params) - :values body}) + [{:keys [ds path-params] :as _request}] + (let [user (hon/find-one ds {:tname :users + :where [:= :id (:id path-params)]})] + (->> (dissoc user :password) + (assoc {} :user) + (res/response)))) + +(defn patch + {:summary "update user by id" + :parameters {:path [:map [:id {:title "id" + :description "user id"} :int]] + :body [:map + [:address {:optional true} :string] + [:profile-image {:optional true} :string] + [:email :string] + [:firstname {:optional true} :string] + [:lastname {:optional true} :string] + [:type [:enum "creator" "distributor" "admin"]] + [:email-verified {:optional true} :int] + [:onboarded {:optional true} :int] + [:mobile {:optional true} :string]]} + :responses {200 {:body [:map + [:user + [:map + [:id :int] + [:address [:maybe :string]] + [:profile-image [:maybe :string]] + [:email :string] + [:firstname [:maybe :string]] + [:lastname [:maybe :string]] + [:type [:enum "creator" "distributor" "admin"]] + [:email-verified [:maybe :int]] + [:onboarded [:maybe :int]] + [:mobile [:maybe :string]]]]]} + 401 {:body [:map [:message :string]]} + 403 {:body [:map [:message :string]]}}} + + [{:keys [ds body path-params] :as _request}] + (hon/update! ds {:tname :users + :where [:= :id (:id path-params)] + :data body}) (res/response {:message "successfully updated user"})) +(defn verify-email + {:summary "verify user email with email hash" + :parameters (api/params :path [:map [:hash :string]]) + :responses {302 {:body (api/response-schema)} + 403 {:body (api/response-schema)}}} + [{:keys [ds path-params]}] + (let [email-hash (:hash path-params) + user (hon/find-one ds {:tname :users + :where [:= :email-hash email-hash]})] + (if (some? user) + (do + (hon/update! ds {:tname :users + :where [:= :id (:id user)] + :data {:email-verified 1 + :email-hash ""}}) + (-> (conf/read-value :cors-origin) + (str "/dashboard/onboarding") + (res/redirect))) + (-> (res/response {:message "unauthorized"}) + (res/status 403))))) + (comment (require '[source.db.interface :as db]) (get {:ds (db/ds :master) :path-params {:id 5}}) @@ -21,4 +95,6 @@ :path-params {:id 5} :body {:firstname "kiigan" :lastname "korinzu"}}) + + (meta #'get) ()) diff --git a/src/source/routes/users.clj b/src/source/routes/users.clj index f883bfdb..841fed03 100644 --- a/src/source/routes/users.clj +++ b/src/source/routes/users.clj @@ -1,9 +1,41 @@ (ns source.routes.users (:require [ring.util.response :as res] - [source.services.users :as users])) + [source.db.honey :as hon] + [source.routes.openapi :as api])) -(defn get [{:keys [ds] :as _request}] - (res/response {:users (users/users ds)})) +(defn get + {:summary "get all users" + :responses {200 {:body [:map + [:users + [:vector + [:map + [:id :int] + [:address [:maybe :string]] + [:profile-image [:maybe :string]] + [:email :string] + [:firstname [:maybe :string]] + [:lastname [:maybe :string]] + [:type [:enum "creator" "distributor" "admin"]] + [:email-verified [:maybe :int]] + [:onboarded [:maybe :int]] + [:mobile [:maybe :int]]]]]]} + 401 {:body [:map [:message :string]]} + 403 {:body [:map [:message :string]]}}} + + [{:keys [ds] :as _request}] + (res/response {:users (hon/find ds {:tname :users + :ret :*})})) + +(defn verify-email + {:summary "verify user email with email hash" + :parameters (api/params :path [:map [:id :int]]) + :responses (api/success (api/response-schema))} + [{:keys [ds path-params]}] + (hon/update! ds {:tname :users + :where [:= :id (:id path-params)] + :data {:email-verified 1 + :email-hash ""}}) + (res/response {:message "successfully verified user"})) (comment (require '[source.db.interface :as db]) diff --git a/src/source/routes/util.clj b/src/source/routes/util.clj new file mode 100644 index 00000000..03569cb9 --- /dev/null +++ b/src/source/routes/util.clj @@ -0,0 +1,119 @@ +(ns source.routes.util + (:require [source.util :as util] + [reitit.swagger :as swagger] + [reitit.swagger-ui :as swagger-ui] + [reitit.openapi :as openapi] + [reitit.coercion.malli :as coercion] + [source.middleware.interface :as mw] + [malli.util :as mu] + [source.routes.openapi :as api] + [taoensso.telemere :as t])) + +(defn- extract-openapi-meta [handler] + (-> (util/metadata handler) + (dissoc :arglists :line :column :file :name :ns))) + +(defn- attach-handler [handler validation-mw openapi-meta] + (cond-> openapi-meta + (some? (:parameters openapi-meta)) + (assoc :middleware [[validation-mw openapi-meta]] + :responses (merge (:responses openapi-meta) + (api/bad-request))) + true (assoc :handler handler))) + +(defn- merge-route-map [validation-mw acc [method handler]] + (->> (extract-openapi-meta handler) + (attach-handler handler validation-mw) + (assoc {} method) + (merge acc))) + +(defn- parse-route-opts [& opts] + (let [map-first? (map? (first opts)) + route-map (if map-first? (first opts) {}) + opts (if map-first? (rest opts) opts)] + [route-map (vec opts)])) + +(defn route + [& opts] + (let [[route-map opts] (apply parse-route-opts opts)] + (->> (partition 2 opts) + (reduce (partial merge-route-map mw/apply-validation) {}) + (merge route-map)))) + +(defn- resolve-route-map [method] + (fn + ([handler] (route {} method handler)) + ([route-map handler] + (when (not (map? route-map)) + (throw + (t/error! + ::route-map-resolution + (ex-info "Invalid argument for resolve-route-map: route-map must be a map" + {:panic? "not really, but you should never get this in prod" + :possible-cause "something is wrong in one of the endpoints' swagger docs, we're expecting a map here" + :next-steps (str "check the swagger docs for something looking like this: " route-map)})))) + (route route-map method handler)))) + +(defn get [& opts] + (apply (resolve-route-map :get) (vec opts))) + +(defn post [& opts] + (apply (resolve-route-map :post) (vec opts))) + +(defn patch [& opts] + (apply (resolve-route-map :patch) (vec opts))) + +(defn delete [& opts] + (apply (resolve-route-map :delete) (vec opts))) + +(defn swagger-ui-handler [] + (swagger-ui/create-swagger-ui-handler {:path "/" + :config {:validatorUrl nil + :urls [{:name "swagger", :url "swagger.json"} + {:name "openapi", :url "openapi.json"}] + :urls.primaryName "swagger" + :operationsSorter "alpha"}})) + +(defn tag-definitions [] + [{:name "integrations" + :description "Management endpoints for integrations and the associated bundle"} + {:name "bundles" + :description "Content endpoints for the bundle associated with an integration"}]) + +(defn swagger-route [] + ["/swagger.json" {:get {:no-doc true + :swagger {:info {:title "source-api" + :description "Swagger docs for Source API with Malli and Reitit-Ring. For usage examples, check out our postman collection on our GitHub Repository: https://github.com/modulr-software/source-be" + :version "0.0.2"} + :tags (tag-definitions) + :securityDefinitions {"auth" {:type :apiKey + :in :header + :name "Authorization"} + "apiKey" {:type :apiKey + :in :header + :name "Authorization"}}} + :handler (swagger/create-swagger-handler)}}]) + +(defn openapi-route [] + ["/openapi.json" {:get {:no-doc true + :openapi {:info {:title "source-api" + :description "openapi3 docs for source api with malli and reitit-ring" + :version "0.0.2"} + :tags (tag-definitions) + :components {:securitySchemes {"bearerAuth" {:type :http + :scheme :bearer + :bearerFormat "JWT" + :description "JWT Authorization using the Bearer scheme"} + "apiKey" {:type :http + :scheme :bearer + :description "API Key authorization using the Bearer scheme"}}}} + :handler (openapi/create-openapi-handler)}}]) + +(defn data-map [middleware] + {:data {:coercion (reitit.coercion.malli/create + {:error-keys #{#_:type :coercion :in :schema :value :errors :humanized #_:transformed} + :compile mu/closed-schema + :strip-extra-keys true + :default-values true + :options nil}) + :middleware middleware}}) diff --git a/src/source/routes/xml.clj b/src/source/routes/xml.clj new file mode 100644 index 00000000..37793cce --- /dev/null +++ b/src/source/routes/xml.clj @@ -0,0 +1,8 @@ +(ns source.routes.xml + (:require [source.workers.xml-schemas :as xml] + [ring.util.response :as res])) + +(defn post [{:keys [body] :as _request}] + (let [{:keys [url]} body] + (-> (xml/ast url) + (res/response)))) diff --git a/src/source/rss/core.clj b/src/source/rss/core.clj index e6a1aa1a..5ca8a7dd 100644 --- a/src/source/rss/core.clj +++ b/src/source/rss/core.clj @@ -1 +1,145 @@ -(ns source.rss.core) +(ns source.rss.core + (:require [hickory.core :as h] + [hickory.select :as s])) + +(defn get-ast + "Constructs a hickory tree representation from an xml string." + [xml] + (-> xml h/parse h/as-hickory)) + +(def cdata-prefix (count "")) + +(defn cdata-content [cdata-node] + (let [front-removed (subs cdata-node cdata-prefix)] + (subs front-removed 0 (- (count front-removed) cdata-postfix)))) + +(defn unwrap-cdata [maybe-cdata-node] + (if (and + (string? maybe-cdata-node) + (> (count maybe-cdata-node) 9) + (= (subs maybe-cdata-node 0 9) " (count content) 1) ;; filter out erroneous string children + (filterv #(not= (type %) java.lang.String) content) + content)] + (mapv #(collect-leaf-paths % new-path) nodes)))] + (-> node + (assoc :path new-path) + (dissoc :content) + (assoc :content content-with-paths)))))) + +(defn build-child-selector + "given a sequence of tag keywords this will return a hickory selector + that hickory.select/select can use to get the node you want" + [tags] + (let [ts (mapv #(last %) tags) + root (first ts)] + (reduce (fn [sel tag] + (s/child sel (s/tag tag))) + (s/tag root) + (rest ts)))) + +(defn split + "splits path segment into [namespace name] + i.e. \"attr/attr-name\" becomes [:attr :attr-name]" + [path-segment] + (let [path-keyword (keyword path-segment)] + (mapv + #(keyword %) + [(namespace path-keyword) + (name path-keyword)]))) + +(defn leaf? + "Returns true if :seg-type from namespaced keyword path segment's type is content or attribute." + [kw-path-seg] + (let [seg-type (first kw-path-seg)] + (or (= seg-type :content) (= seg-type :attr)))) + +(defn extract-leaf + "Get's the content from a leaf node using the leaf path segment." + [node kw-path-seg] + (let [[seg-type seg-val] kw-path-seg] + (if (= seg-type :attr) + (-> (:attrs node) + (get seg-val)) + + (unwrap-cdata (-> (:content node) + (get (Integer/parseInt (name seg-val)))))))) + +(defn extract-data + "Recursively extracts data from a hickory xml tree according to the input selection schema. + The input selection schema contains paths to each field in the schema. These paths are relative to their parent node + with any number of tag/tag-name segments and end with a attr/attribute-name or content/n-th segment if the field is a string." + [schema ast] + (reduce-kv + (fn [result field {:keys [type path schema]}] + (let [keyword-path (mapv #(split %) path) + is-leaf? (leaf? (last keyword-path)) + selector-path (if is-leaf? (butlast keyword-path) keyword-path) + selector (build-child-selector selector-path)] + + (cond + (= type "map") + (let [child-node (first (s/select selector ast))] + (assoc result field + (extract-data schema child-node))) + + (= type "vector") + (let [child-nodes (s/select selector ast)] + (assoc result field + (mapv #(extract-data schema %) child-nodes))) + + is-leaf? + (assoc result field + (-> (s/select selector ast) + first + (extract-leaf (last keyword-path)))) + + :else + result))) + {} + schema)) + +(comment + (let [ast (get-ast (slurp "https://www.youtube.com/feeds/videos.xml?channel_id=UCWI-ohtRu8eEeDj93hmUsUQ")) + schema {:title + {:type "string", + :required true, + :path ["html" "body" "feed" "title" "content/0"]}, + :url + {:type "string", + :required true, + :path ["html" "body" "feed" "author" "uri" "content/0"]}, + :posts + {:type "vector", + :required true, + :schema + {:title {:type "string", :required true, :path ["title" "content/0"]}, + :stream-url + {:type "string", :required true, :path ["link" "attr/href"]}, + :description + {:type "string", + :required false, + :path ["media:group" "media:description" "content/0"]}, + :posted-at {:type "string", :required false, :path ["published" "content/0"]}}, + :path ["html" "body" "feed" "entry"]}}] + (extract-data schema ast))) diff --git a/src/source/rss/youtube.clj b/src/source/rss/youtube.clj index 965a7616..3d3aecc7 100644 --- a/src/source/rss/youtube.clj +++ b/src/source/rss/youtube.clj @@ -1,7 +1,9 @@ (ns source.rss.youtube (:require [clojure.string :as s] [source.rss.squash :as squash] - [clojure.xml :as xml])) + [clojure.xml :as xml] + [org.httpkit.client :as http] + [hickory.core :as h])) (defn find-channel-id "a rudimentary first implementation, we should probably consider using @@ -23,5 +25,22 @@ (xml/parse) (squash/squash))) +(defn channel-image [channel-url] + (->> @(http/request {:url channel-url}) + (:body) + (h/parse) + (h/as-hickory) + (:content) + (second) + (:content) + (first) + (:content) + (filterv (fn [{:keys [type attrs]}] + (and (= type :element) (= (:rel attrs) "image_src")))) + (first) + (:attrs) + (:href))) + (comment - (find-channel-id "https://www.youtube.com/@CodingWithLewis")) + (find-channel-id "https://www.youtube.com/@Veritasium") + (channel-image "https://www.youtube.com/@Veritasium")) diff --git a/src/source/server.clj b/src/source/server.clj index 2eef3cb9..d1bfe0e6 100644 --- a/src/source/server.clj +++ b/src/source/server.clj @@ -1,38 +1,98 @@ (ns source.server - (:require - [org.httpkit.server :as http] - [source.routes.interface :as routes] - [source.util :as util])) + (:require [org.httpkit.server :as http] + [source.db.interface :as db] + [congest.jobs :as congest] + [source.jobs.core :as jobs] + [source.routes.interface :as routes] + [source.util :as util] + [taoensso.telemere :as t] + [source.config :as conf])) -(defonce ^:private *server (atom nil)) +(defonce ^:private *components (atom nil)) + +(defn initialise-server! [{:keys [ds js]}] + (http/run-server + (routes/create-app {:ds ds + :js js}) + {:port (conf/read-value :port)})) + +(defn initialise-job-service! [{:keys [ds] :as _deps}] + (->> (jobs/interrupted-jobs ds) + (congest/create-job-service))) + +(defn component-on? [component] + (if (some? (get @*components component)) + true + false)) + +(defn deps-on? [deps] + (every? component-on? deps)) + +(defn initialise! + "executes the init-fn on the provided component and, if successful, updates the components atom with the new component" + [{:keys [name init-fn deps] :as _component}] + (try + (when (deps-on? deps) + (swap! *components assoc name (init-fn @*components))) + (catch Exception e (t/log! {:level :error + :msg (str "Failed to initialise " name ": " e)})))) + +(defn initialise-components! [components] + (run! initialise! components)) (defn running? [] - (some? @*server)) + (some? (:server @*components))) (defn start-server [] - (cond (not (some? @*server)) + (cond (not (some? (:server @*components))) (do - (println "Starting server on port 3000...") - (reset! *server (http/run-server - (routes/create-app) - {:port 3000}))) + (t/log! (str "Starting server on port " (conf/read-value :port) "...")) + (initialise-components! [{:name :ds + :init-fn (fn [_deps] (db/ds :master)) + :deps []} + {:name :js + :deps [:ds] + :init-fn initialise-job-service!} + {:name :server + :deps [:ds :js] + :init-fn initialise-server!}])) :else - (println "Server already running!"))) + (t/log! "Server already running!"))) (defn stop-server [] - (println "Stopping server...") - (when (some? @*server) - (@*server)) - (reset! *server nil)) + (t/log! "Stopping server...") + (when (some? (:js @*components)) + (congest/kill! (:js @*components))) + (when (some? (:server @*components)) + (let [server-stop (:server @*components)] + (server-stop))) + (reset! *components nil)) -(defn restart-server [] - (stop-server) - (start-server)) +(defn restart-server [& {:keys [keep-js]}] + (if keep-js + (do + (when (some? (:server @*components)) + (let [server-stop (:server @*components)] + (server-stop))) + (swap! *components select-keys [:js]) + (initialise-components! [{:name :ds + :init-fn (fn [_deps] (db/ds :master)) + :deps []} + {:name :server + :deps [:ds :js] + :init-fn initialise-server!}])) + (do + (stop-server) + (start-server)))) (comment + (start-server) + (stop-server) + (restart-server) + (restart-server :keep-js true) (def test-wrapper (util/wrap-json (fn [request] request))) (test-wrapper {:status 200 :body "{\"value\":\"Hello, Source!\"}" - :headers {"Content-Type" "application/json"}})) - + :headers {"Content-Type" "application/json"}}) + ()) diff --git a/src/source/services/analytics.clj b/src/source/services/analytics.clj deleted file mode 100644 index fc21e704..00000000 --- a/src/source/services/analytics.clj +++ /dev/null @@ -1,7 +0,0 @@ -(ns source.services.analytics - (:require [source.db.interface :as db])) - -(defn insert-event! [ds {:keys [_values _ret] :as opts}] - (->> {:tname :analytics} - (merge opts) - (db/insert! ds))) diff --git a/src/source/services/analytics/core.clj b/src/source/services/analytics/core.clj new file mode 100644 index 00000000..6a348dd4 --- /dev/null +++ b/src/source/services/analytics/core.clj @@ -0,0 +1,448 @@ +(ns source.services.analytics.core + (:require [honey.sql.helpers :as hsql] + [source.db.honey :as hon] + [source.services.bundles :as bundles] + [source.util :as util] + [source.services.feed-categories :as feed-categories] + [honey.sql :as sql] + [clojure.walk :as walk] + [pg.core :as pg])) + +(defn convert-all-datetimes-to-string + "Uses postwalk to convert all instances of java.time.LocalDate into string" + [m] + (walk/postwalk (fn [v] + (if (instance? java.time.LocalDate v) + (.toString v) + v)) m)) + +(defn metric-query + "Generic select query function for returning analytics data from the events table" + [ds {:keys [select order-by group-by limit metric feed-id post-id content-type-id bundle-id creator-id distributor-id min-date max-date category-ids where ret]}] + (let [clauses (cond-> {} + (some? where) (merge where) + (some? metric) (hsql/where [:= :event metric]) + (some? feed-id) (hsql/where [:= :feed-id feed-id]) + (some? post-id) (hsql/where [:= :post-id post-id]) + (some? content-type-id) (hsql/where [:= :content-type-id content-type-id]) + (some? bundle-id) (hsql/where [:= :bundle-id bundle-id]) + (some? creator-id) (hsql/where [:= :creator-id creator-id]) + (some? distributor-id) (hsql/where [:= :distributor-id distributor-id]) + (and (some? min-date) (nil? max-date)) (hsql/where [:>= :timestamp min-date]) + (and (some? max-date) (nil? min-date)) (hsql/where [:<= :timestamp max-date]) + (and (some? min-date) (some? max-date)) (hsql/where [:between :timestamp min-date max-date]) + (some? select) (merge select) + (nil? select) (merge {:select [[[:count :*] :total]]}) + (some? limit) (hsql/limit limit) + (some? order-by) (merge order-by) + (some? group-by) (merge group-by) + (seq category-ids) (-> (hsql/join [:event-categories :ec] [:= :events.id :ec.event-id]) + (hsql/where [:in :ec.category-id category-ids])))] + (-> ds + (hon/execute! + (merge {:from [:events]} + clauses) + {:ret (if ret ret :*)}) + (convert-all-datetimes-to-string)))) + +(defn statistics-query + "Returns the number of impressions, clicks and views, filtered by any other arguments accepted by metric-query. + If ret is not given, returns a single record." + [ds {:keys [ret] :as opts}] + (metric-query ds (merge {:select (hsql/select [(hsql/filter :%count.* (hsql/where := :event "impression")) :impressions] + [(hsql/filter :%count.* (hsql/where := :event "click")) :clicks] + [(hsql/filter :%count.* (hsql/where := :event "view")) :views]) + :ret (if ret ret :1)} + opts))) + +(defn interval-statistics-query + "returns the number of impressions, clicks and views per interval (:daily, :weekly, :monthly or :yearly) over the given time period, + filtered by any other arguments accepted by metric-query. + + Date parameters must be in the format YYYY-MM-DD." + [ds interval min-date max-date opts] + (let [select (cond + (= interval :daily) [[:date :timestamp] :day] + (= interval :weekly) [[:strftime "%W" :timestamp] :week] + (= interval :monthly) [[:strftime "%m" :timestamp] :month] + (= interval :yearly) [[:strftime "%Y" :timestamp] :year] + :else [[:date :timestamp] :day]) + column (cond + (= interval :daily) :day + (= interval :weekly) :week + (= interval :monthly) :month + (= interval :yearly) :year + :else :day)] + + (metric-query ds (merge {:select (hsql/select select + [(hsql/filter :%count.* (hsql/where := :event "impression")) :impressions] + [(hsql/filter :%count.* (hsql/where := :event "click")) :clicks] + [(hsql/filter :%count.* (hsql/where := :event "view")) :views]) + :min-date (str min-date) + :max-date (str max-date "T23:59:59Z") + :group-by (hsql/group-by column) + :order-by (hsql/order-by column)} + opts)))) + +(defn top-statistics-query + "Returns the top n of the given top field in order of the number of their impressions, clicks and views within the given time period" + [ds min-date max-date n top-field opts] + (metric-query ds (merge {:select (hsql/select top-field + [(hsql/filter :%count.* (hsql/where := :event "impression")) :impressions] + [(hsql/filter :%count.* (hsql/where := :event "click")) :clicks] + [(hsql/filter :%count.* (hsql/where := :event "view")) :views]) + :min-date min-date + :max-date max-date + :where (hsql/where [:!= top-field nil]) + :group-by (hsql/group-by top-field) + :order-by (hsql/order-by [:impressions :desc] [:clicks :desc] [:views :desc]) + :limit n + :ret :*} + opts))) + +(defn weekly-growth-averages + "Returns the percentage of growth in impressions, clicks and views per week, over the given time period. + Uses the first week as a basis for comparison. + Can be filtered by any other arguments accepted by metric-query." + [ds min-date max-date opts] + (let [days (interval-statistics-query ds :daily min-date max-date opts) + parts (partition-all 7 days) + weeks (mapv (fn [week i] + {:week (str "week " i) + :impressions (apply + (mapv :impressions week)) + :clicks (apply + (mapv :clicks week)) + :views (apply + (mapv :views week))}) + parts (range 1 (inc (count parts)))) + {:keys [impressions clicks views]} (first weeks) + denom #(if (= % 0) 1 %)] + (mapv (fn [w] + {:week (:week w) + :impressions (float (* (/ (- (:impressions w) impressions) (denom impressions)) 100)) + :clicks (float (* (/ (- (:clicks w) clicks) (denom clicks)) 100)) + :views (float (* (/ (- (:views w) views) (denom views)) 100))}) + weeks))) + +(defn average-engagement + "Returns the average engagement (average clicks and views) over the given time period. + Can be filtered by any other arguments accepted by metric-query." + [ds min-date max-date opts] + (let [{:keys [clicks views]} (statistics-query ds (merge {:min-date min-date + :max-date max-date} + opts))] + (float (/ (+ clicks views) 2)))) + +(defn click-through-rate + "Returns the click-through rate based on impressions and clicks filtered by any arguments accepted by metric-query" + [ds opts] + (let [{:keys [impressions clicks]} (statistics-query ds opts)] + (float (* (/ clicks impressions) 100)))) + +(defn insert-event! [ds {:keys [data ret] :as opts}] + (->> {:tname :events + :data data + :ret ret} + (merge opts) + (hon/insert! ds))) + +(defn insert-event-categories! [ds {:keys [data ret] :as opts}] + (->> {:tname :event-categories + :data data + :ret ret} + (merge opts) + (hon/insert! ds))) + +(defn delete-event! [ds {:keys [id where] :as opts}] + (->> {:tname :events + :where (if (some? id) + [:= :id id] + where) + :ret :1} + (merge opts) + (hon/delete! ds))) + +(defn insert-feed-event-categories! + "Given a list of events and a list of feeds (or a single event/feed), + inserts an event category record for each event and each category + associated with the given feeds." + [ds events feeds] + (let [multi-events? (vector? events) + multi-feeds? (vector? feeds) + events' (if multi-events? events [events]) + feeds' (if multi-feeds? feeds [feeds]) + category-ids (->> {:where [:in :feed-id (mapv :id feeds')]} + (feed-categories/feed-categories ds) + (mapv :category-id)) + event-categories (->> events' + (mapv (fn [{:keys [id]}] + (mapv (fn [c] + {:event-id id + :category-id c}) + category-ids))) + (flatten) + (vec))] + (when (seq event-categories) (insert-event-categories! ds {:data event-categories})))) + +(defn insert-post-event-categories! + "Given a list of events and a list of posts (or a single event/post), + inserts an event category record for each event and each category + associated with the given posts" + [ds events posts] + (let [multi-events? (vector? events) + multi-posts? (vector? posts) + events' (if multi-events? events [events]) + posts' (if multi-posts? posts [posts]) + feed-ids (mapv :feed-id posts') + category-ids (->> {:where [:in :feed-id feed-ids]} + (feed-categories/feed-categories ds) + (mapv :category-id)) + event-categories (->> events' + (mapv (fn [{:keys [id]}] + (mapv (fn [c] + {:event-id id + :category-id c}) + category-ids))) + (flatten) + (vec))] + (when (seq event-categories) + (insert-event-categories! ds {:data event-categories})))) + +(defn insert-feed-impressions! + "Given a list of feeds and a bundle id, inserts impression event reconds + for each given feed. Inserts event categories for each feed." + [ds feeds bundle-id] + (pg/with-transaction [ds ds] + (let [bundle (bundles/bundle ds {:id bundle-id}) + events (mapv (fn [{:keys [id content-type-id user-id]}] + {:timestamp (util/get-utc-timestamp-string) + :event "impression" + :feed-id id + :content-type-id content-type-id + :creator-id user-id + :bundle-id bundle-id + :distributor-id (:user-id bundle)}) feeds) + events' (insert-event! ds {:data events + :ret :*})] + (insert-feed-event-categories! ds events' feeds)))) + +(defn insert-post-impressions! + "Given a list of posts and a bundle id, inserts impression event reconds + for each given post. Inserts event categories for each post." + [ds posts bundle-id] + (pg/with-transaction [ds ds] + (let [bundle (bundles/bundle ds {:id bundle-id}) + events (mapv (fn [{:keys [id feed-id content-type-id creator-id]}] + {:timestamp (util/get-utc-timestamp-string) + :event "impression" + :feed-id feed-id + :post-id id + :content-type-id content-type-id + :creator-id creator-id + :bundle-id bundle-id + :distributor-id (:user-id bundle)}) posts) + events' (when (seq posts) (insert-event! ds {:data events + :ret :*}))] + (when (seq events') (insert-post-event-categories! ds events' posts))))) + +(defn insert-feed-click! + "Given a feed and a bundle id, inserts a click event record + for the given feed" + [ds {:keys [id content-type-id user-id] :as feed} bundle-id] + (pg/with-transaction [ds ds] + (let [bundle (bundles/bundle ds {:id bundle-id}) + event {:timestamp (util/get-utc-timestamp-string) + :event "click" + :feed-id id + :content-type-id content-type-id + :creator-id user-id + :bundle-id bundle-id + :distributor-id (:user-id bundle)} + event' (insert-event! ds {:data event + :ret :*})] + (insert-feed-event-categories! ds event' feed)))) + +(defn insert-post-click! + "Given a post and a bundle id, inserts a click event record + for the given post" + [ds {:keys [id feed-id content-type-id creator-id] :as post} bundle-id] + (pg/with-transaction [ds ds] + (let [bundle (bundles/bundle ds {:id bundle-id}) + event {:timestamp (util/get-utc-timestamp-string) + :event "click" + :feed-id feed-id + :post-id id + :content-type-id content-type-id + :creator-id creator-id + :bundle-id bundle-id + :distributor-id (:user-id bundle)} + event' (insert-event! ds {:data event + :ret :*})] + (insert-post-event-categories! ds event' post)))) + +(defn insert-post-view! + "Given a post and a bundle id, inserts a view event record + for the given post" + [ds {:keys [id feed-id content-type-id creator-id] :as post} bundle-id] + (pg/with-transaction [ds ds] + (let [bundle (bundles/bundle ds {:id bundle-id}) + event {:timestamp (util/get-utc-timestamp-string) + :event "view" + :feed-id feed-id + :post-id id + :content-type-id content-type-id + :creator-id creator-id + :bundle-id bundle-id + :distributor-id (:user-id bundle)} + event' (insert-event! ds {:data event + :ret :*})] + (insert-post-event-categories! ds event' post)))) + +(comment + (require '[source.db.util :as db.util]) + + (defonce ds (db.util/conn)) + + (def maximums {:creators 100 + :distributors 50 + :feeds 150 + :posts 6000 + :bundles 75}) + + (defn seed-event! [{:keys [creators distributors feeds posts bundles]}] + (let [creator-id (inc (rand-int creators)) + distributor-id (inc (rand-int distributors)) + feed-id (inc (rand-int feeds)) + post-id (inc (rand-int posts)) + bundle-id (inc (rand-int bundles)) + event-type (rand-int 3) + category-ids [(inc (rand-int 50)) (inc (rand-int 50))] + + new-event (hon/insert! ds {:tname :events + :ret :1 + :data {:timestamp (util/get-utc-timestamp-string) + :event (cond + (= event-type 0) "impression" + (= event-type 1) "click" + (= event-type 2) "view") + :feed-id feed-id + :post-id (when (> (rand-int 10) 3) post-id) + :content-type-id (inc (rand-int 3)) + :creator-id creator-id + :bundle-id bundle-id + :distributor-id distributor-id}}) + + event-categories (mapv (fn [x] {:event-id (:id new-event) + :category-id x}) category-ids)] + + (hon/insert! ds {:tname :event-categories + :data event-categories}))) + + (defn seed-events! [num-records] + (dotimes [_ num-records] + (seed-event! maximums))) + + (time (metric-query ds {:min-date "2025-11-25T15:00:00Z" + :feed-id 2})) + + (time (statistics-query ds {:ret :*})) + + (time (hon/find ds {:tname :events + :where [:< :id 500] + :ret :*})) + + (time (hon/update! ds {:tname :events + :where [:between :id 5000000 5500000] + :data {:timestamp (str "2025-11-22" "T13:00:00Z")} + :ret :*})) + + (time (hon/execute! ds (-> (hsql/select [[:date :timestamp] :day] + [(hsql/filter :%count.* (hsql/where := :event "impression")) :impressions] + [(hsql/filter :%count.* (hsql/where := :event "click")) :clicks] + [(hsql/filter :%count.* (hsql/where := :event "view")) :views]) + (hsql/from :events) + (hsql/where [:between :timestamp "2025-11-17T00:00:00Z" "2025-11-24T23:59:59Z"]) + (hsql/group-by :day) + (hsql/order-by :day)) {:ret :*})) + + (time (interval-statistics-query ds :daily "2025-11-17" "2025-11-24" {})) + + (time (hon/execute! ds (-> (hsql/select [[:strftime "%W" :timestamp] :week] + [(hsql/filter :%count.* (hsql/where := :event "impression")) :impressions] + [(hsql/filter :%count.* (hsql/where := :event "click")) :clicks] + [(hsql/filter :%count.* (hsql/where := :event "view")) :views]) + (hsql/from :events) + (hsql/where [:between :timestamp "2025-11-01T00:00:00Z" "2025-11-30T23:59:00Z"]) + (hsql/group-by :week) + (hsql/order-by :week)) + {:ret :*})) + + (time (interval-statistics-query ds :weekly "2025-11-01" "2025-11-30" {:feed-id 4})) + (time (interval-statistics-query ds :monthly "2025-10-01" "2025-12-01" {:feed-id 4})) + (time (interval-statistics-query ds :yearly "2024-01-01" "2026-01-01" {:feed-id 4})) + + (time (weekly-growth-averages ds "2025-11-01" "2025-11-30" {:feed-id 4})) + + (time (average-engagement ds "2025-11-24" "2025-11-30" {:feed-id 4})) + + (time (click-through-rate ds {:min-date "2025-11-24T00:00:00Z" + :max-date "2025-11-24T23:59:59Z" + :feed-id 4})) + + (time + (hon/execute! + ds + {:select [[[:count :*] :total]] + :from [:events]} + {:ret :*})) + + (time (hon/find ds {:tname :event-categories + :ret :*})) + + (time (seed-event! maximums)) + + (time (seed-events! 3000000)) + + (time (sql/format (hsql/select [(hsql/filter :%count.* (hsql/where := :event "impression")) :impressions] + [(hsql/filter :%count.* (hsql/where := :event "click")) :clicks] + [(hsql/filter :%count.* (hsql/where := :event "view")) :views]))) + + (time (insert-event! ds {:data {:timestamp (util/get-utc-timestamp-string) + :event "impression" + :feed-id 1 + :content-type-id 1 + :creator-id 1 + :bundle-id 1 + :distributor-id 1}})) + + (time (top-statistics-query ds "2025-11-17" "2025-11-24" 10 :post-id {:content-type-id 3})) + + (def data [{:a 1 :n 64} + {:a 2 :n 65} + {:a 3 :n 66} + {:a 4 :n 67} + {:a 5 :n 68} + {:a 6 :n 69} + {:a 7 :n 70} + {:a 8 :n 71} + {:a 9 :n 72} + {:a 10 :n 73}]) + + (mapv (fn [week i] + {:week i + :n (apply + (mapv :n week))}) + (partition-all 7 data) (range 1 (inc (count (partition-all 7 data))))) + + (metric-query ds (merge {:select (hsql/select :post-id + [(hsql/filter :%count.* (hsql/where := :event "impression")) :impressions] + [(hsql/filter :%count.* (hsql/where := :event "click")) :clicks] + [(hsql/filter :%count.* (hsql/where := :event "view")) :views]) + :min-date "2025-11-17" + :max-date "2025-11-24" + :where (hsql/where [:!= :post-id nil]) + :group-by (hsql/group-by :post-id) + :order-by (hsql/order-by [:impressions :desc] [:clicks :desc] [:views :desc]) + :limit 10 + :ret :*} + {:content-type-id 3})) + + ()) + diff --git a/src/source/services/analytics/interface.clj b/src/source/services/analytics/interface.clj new file mode 100644 index 00000000..ff581175 --- /dev/null +++ b/src/source/services/analytics/interface.clj @@ -0,0 +1,96 @@ +(ns source.services.analytics.interface + (:require [source.services.analytics.core :as core])) + +(defn metric-query + "Generic select query function for returning analytics data from the events table" + [ds {:keys [_select _order-by _group-by _metric _feed-id _post-id _content-type-id _bundle-id _creator-id _distributor-id _min-date _max-date _category-ids _ret] :as opts}] + (core/metric-query ds opts)) + +(defn statistics-query + "returns the number of impressions, clicks and views, filtered by any other arguments accepted by metric-query" + [ds opts] + (core/statistics-query ds opts)) + +(defn interval-statistics-query + "returns the number of impressions, clicks and views per interval (:daily, :weekly, :monthly or :yearly) over the given time period, + filtered by any other arguments accepted by metric-query. + + Date parameters must be in the format YYYY-mm-dd." + [ds interval min-date max-date opts] + (core/interval-statistics-query ds interval min-date max-date opts)) + +(defn weekly-growth-averages + "Returns the percentage of growth in impressions, clicks and views per week, over the given time period. + Uses the first week as a basis for comparison, not included in results. + Can be filtered by any other arguments accepted by metric-query." + [ds min-date max-date opts] + (core/weekly-growth-averages ds min-date max-date opts)) + +(defn top-statistics-query + "Returns the top n of the given top field in order of the number of their impressions, clicks and views within the given time period." + [ds min-date max-date n top-field opts] + (core/top-statistics-query ds min-date max-date n top-field opts)) + +(defn average-engagement + "Returns the average engagement (average clicks and views) over the given time period. + Can be filtered by any other arguments accepted by metric-query." + [ds min-date max-date opts] + (core/average-engagement ds min-date max-date opts)) + +(defn click-through-rate + "Returns the click-through rate based on impressions and clicks filtered by any arguments accepted by metric-query" + [ds opts] + (core/click-through-rate ds opts)) + +(defn insert-event! [ds {:keys [_data _ret] :as opts}] + (core/insert-event! ds opts)) + +(defn insert-event-categories! [ds {:keys [_data _ret] :as opts}] + (core/insert-event-categories! ds opts)) + +(defn delete-event! [ds {:keys [_id _where] :as opts}] + (core/delete-event! ds opts)) + +(defn insert-feed-event-categories! + "Given a list of events and a list of feeds (or a single event/feed), + inserts an event category record for each event and each category + associated with the given feeds." + [ds events feeds] + (core/insert-feed-event-categories! ds events feeds)) + +(defn insert-post-event-categories! + "Given a list of events and a list of posts (or a single event/post), + inserts an event category record for each event and each category + associated with the given posts" + [ds events posts] + (core/insert-post-event-categories! ds events posts)) + +(defn insert-feed-impressions! + "Given a list of feeds and a bundle id, inserts impression event reconds + for each given feed. Inserts event categories for each feed." + [ds feeds bundle-id] + (core/insert-feed-impressions! ds feeds bundle-id)) + +(defn insert-post-impressions! + "Given a list of posts and a bundle id, inserts impression event reconds + for each given post. Inserts event categories for each post." + [ds posts bundle-id] + (core/insert-post-impressions! ds posts bundle-id)) + +(defn insert-feed-click! + "Given a feed and a bundle id, inserts a click event record + for the given feed" + [ds {:keys [_id _content-type-id _user-id] :as feed} bundle-id] + (core/insert-feed-click! ds feed bundle-id)) + +(defn insert-post-click! + "Given a post and a bundle id, inserts a click event record + for the given post" + [ds {:keys [_id _feed-id _content-type-id _creator-id] :as post} bundle-id] + (core/insert-post-click! ds post bundle-id)) + +(defn insert-post-view! + "Given a post and a bundle id, inserts a view event record + for the given post" + [ds {:keys [_id _feed-id _content-type-id _creator-id] :as post} bundle-id] + (core/insert-post-view! ds post bundle-id)) diff --git a/src/source/services/auth.clj b/src/source/services/auth.clj index a96270a3..f8dc8f49 100644 --- a/src/source/services/auth.clj +++ b/src/source/services/auth.clj @@ -1,21 +1,32 @@ (ns source.services.auth (:require [source.password :as pw] - [source.services.users :as users] - [source.middleware.auth.core :as auth])) + [source.middleware.auth.core :as auth] + [source.db.honey :as hon] + [pg.core :as pg] + [source.email.gmail :as gmail] + [source.email.templates :as templates])) (defn login [ds {:keys [user] :as _login}] (merge - {:user (dissoc user :password)} + {:user (dissoc user :password :email-hash)} (auth/create-session (select-keys user [:id :type])))) (defn register [ds {:keys [email password] :as user}] - (users/insert-user! ds {:data (-> user - (dissoc :confirm-password) - (assoc :password (pw/hash-password password)))}) - (let [user (users/user ds {:where [:= :email email]})] - (merge - {:user user} - (auth/create-session (select-keys user [:id :type]))))) + (pg/with-transaction [ds ds] + (hon/insert! ds {:tname :users + :data (-> user + (dissoc :confirm-password) + (assoc :password (pw/hash-password password) + :email-hash (pw/hash-password email)))}) + (gmail/send-email {:to email + :subject "Source - Verify your email" + :body (templates/email-verification {:email-hash (pw/hash-password email)}) + :type :text/html}) + (let [user (hon/find-one ds {:tname :users + :where [:= :email email]})] + (merge + {:user (dissoc user :password :email-hash)} + (auth/create-session (select-keys user [:id :type])))))) (comment (require '[source.db.interface :as db]) diff --git a/src/source/services/bundle_categories.clj b/src/source/services/bundle_categories.clj new file mode 100644 index 00000000..88e605ae --- /dev/null +++ b/src/source/services/bundle_categories.clj @@ -0,0 +1,32 @@ +(ns source.services.bundle-categories + (:require [source.db.interface :as db] + [source.db.bundle :as bundle] + [source.db.util :as db.util] + [honey.sql.helpers :as hsql] + [pg.core :as pg])) + +(defn category-id [ds {:keys [bundle-id where] :as opts}] + (->> {:tname :bundle-categories + :where (if (some? bundle-id) + [:= :bundle-id bundle-id] + where) + :ret :*} + (merge opts) + (db/find ds))) + +(defn insert-bundle-categories! [ds {:keys [bundle-id categories]}] + (let [bundle-categories (mapv (fn [{:keys [id]}] + {:bundle-id bundle-id + :category-id id}) categories)] + (db/insert! ds (-> (db.util/tname :bundle-categories bundle-id) + (assoc :data bundle-categories))))) + +(defn update-bundle-categories! [ds {:keys [bundle-id categories]}] + (pg/with-transaction [ds ds] + (let [bundle-categories (mapv (fn [{:keys [id]}] + {:bundle-id bundle-id + :category-id id}) categories)] + (db/delete! ds (-> (db.util/tname :bundle-categories bundle-id) + (hsql/where [:= :bundle-id bundle-id]))) + (db/insert! ds (-> (db.util/tname :bundle-categories bundle-id) + (assoc :data bundle-categories)))))) diff --git a/src/source/services/bundle_content_types.clj b/src/source/services/bundle_content_types.clj new file mode 100644 index 00000000..4c795370 --- /dev/null +++ b/src/source/services/bundle_content_types.clj @@ -0,0 +1,43 @@ +(ns source.services.bundle-content-types + (:require [source.db.interface :as db] + [source.db.honey :as hon] + [pg.core :as pg])) + +;;NEW +(defn insert-bundle-content-types! [ds {:keys [bundle-id content-types]}] + (let [content-types (mapv (fn [{:keys [id]}] + {:bundle-id bundle-id + :content-type-id id}) content-types)] + (->> {:tname :bundle-content-types + :data content-types + :ret :*} + (db/insert! ds)))) + +(defn delete-bundle-content-types! [ds {:keys [id where] :as opts}] + (->> {:tname :bundle-content-types + :where (if (some? id) + [:= :id id] + where) + :ret :1} + (merge opts) + (db/delete! ds))) + +(defn content-types-by-bundle [ds {:keys [bundle-id where] :as _opts}] + (hon/execute! ds + {:select [[:bundle-content-types.content-type-id :id] :name] + :from :content-types + :join [:bundle-content-types [:= :bundle-content-types.content-type-id :content-types.id]] + :where (if (some? bundle-id) + [:= :bundle-id bundle-id] + where)} + {:ret :*})) + +;;NEW +(defn update-bundle-content-types! [ds {:keys [bundle-id content-types]}] + (pg/with-transaction [ds ds] + (let [content-types (mapv (fn [{:keys [id]}] + {:bundle-id bundle-id + :content-type-id id}) content-types)] + (delete-bundle-content-types! ds {:where [:= :bundle-id bundle-id]}) + (hon/insert! ds {:tname :bundle-content-types + :data content-types})))) diff --git a/src/source/services/bundles.clj b/src/source/services/bundles.clj index 27d7f946..0180dec4 100644 --- a/src/source/services/bundles.clj +++ b/src/source/services/bundles.clj @@ -1,5 +1,38 @@ (ns source.services.bundles - (:require [source.db.interface :as db])) + (:require [source.db.interface :as db] + [source.util :as utils] + [source.db.util :as db.util] + [honey.sql.helpers :as hsql])) + +(defn insert-bundle! [ds {:keys [_values _ret] :as opts}] + (->> {:tname :bundles} + (merge opts) + (db/insert! ds))) + +;;NEW +(defn create-bundle! [ds {:keys [user-id bundle-metadata]}] + (insert-bundle! ds {:data (merge {:user-id user-id + :content-type-id 1 ; temporarily assign garbo id + :uuid (utils/uuid)} + bundle-metadata) + :ret :1})) + +(defn update-bundle! [ds {:keys [id data where] :as opts}] + (->> {:tname :bundles + :values data + :where (if (some? id) [:= :id id] where) + :ret :1} + (merge opts) + (db/update! ds))) + +(defn bundles + ([ds] (bundles ds {})) + ([ds {:keys [where] :as opts}] + (->> {:tname :bundles + :where where + :ret :*} + (merge opts) + (db/find ds)))) (defn bundle [ds {:keys [id where] :as opts}] (->> {:tname :bundles @@ -10,12 +43,10 @@ (merge opts) (db/find ds))) -(defn insert-bundle! [ds {:keys [_values _ret] :as opts}] - (->> {:tname :bundles} - (merge opts) - (db/insert! ds))) - -(comment - (require '[source.db.util :as db.util]) - (db/find (db.util/conn :master) {:tname ""}) - ()) +;;NEW +(defn categories-in-bundle [ds bundle-id] + (let [category-ids (db/find ds (-> (db.util/tname :bundle-categories bundle-id) + (hsql/where [:= :bundle-id bundle-id]))) + id-vec (mapv (fn [{:keys [category-id]}] category-id) category-ids)] + (db/find ds {:tname :categories + :where [:in :id id-vec]}))) diff --git a/src/source/services/businesses.clj b/src/source/services/businesses.clj deleted file mode 100644 index 7b8dffc6..00000000 --- a/src/source/services/businesses.clj +++ /dev/null @@ -1,22 +0,0 @@ -(ns source.services.businesses - (:require [source.db.interface :as db])) - -(defn businesses - ([ds] (businesses ds {})) - ([ds opts] - (->> {:tname :businesses - :ret :*} - (merge opts) - (db/find ds)))) - -(defn insert-business! [ds {:keys [_values _ret] :as opts}] - (->> {:tname :businesses} - (merge opts) - (db/insert! ds))) - -(defn update-business! [ds {:keys [id values where] :as opts}] - (->> {:tname :businesses - :values values - :where (if (some? id) [:= :id id] where)} - (merge opts) - (db/update! ds))) diff --git a/src/source/services/event_categories.clj b/src/source/services/event_categories.clj deleted file mode 100644 index 852a423e..00000000 --- a/src/source/services/event_categories.clj +++ /dev/null @@ -1,8 +0,0 @@ -(ns source.services.event-categories - (:require [source.db.interface :as db])) - -(defn insert-event-category! [ds {:keys [_values _ret] :as opts}] - (->> {:tname :event-categories} - (merge opts) - (db/insert! ds))) - diff --git a/src/source/services/feed_categories.clj b/src/source/services/feed_categories.clj index d58b13de..7a16cdd9 100644 --- a/src/source/services/feed_categories.clj +++ b/src/source/services/feed_categories.clj @@ -1,5 +1,39 @@ (ns source.services.feed-categories - (:require [source.db.interface :as db])) + (:require [source.db.interface :as db] + [source.db.honey :as hon] + [honey.sql.helpers :as hsql])) + +(defn feed-categories + ([ds] (feed-categories ds {})) + ([ds {:keys [where] :as opts}] + (->> {:tname :feed-categories + :where where + :ret :*} + (merge opts) + (db/find ds)))) + +(defn insert-feed-category! [ds {:keys [_data _ret] :as opts}] + (->> {:tname :feed-categories} + (merge opts) + (db/insert! ds))) + +(defn upsert-feed-categories! [ds {:keys [data]}] + (hon/execute! + ds + (-> (hsql/insert-into :feed-categories) + (hsql/values data) + (assoc :on-conflict [:feed-id :category-id]) + (assoc :do-update-set {:category-id :excluded.category-id})))) + +(defn categories-by-feed [ds {:keys [feed-id where] :as _opts}] + (hon/execute! ds + {:select [[:feed-categories.category-id :id] :name] + :from :categories + :join [:feed-categories [:= :feed-categories.category-id :categories.id]] + :where (if (some? feed-id) + [:= :feed-id feed-id] + where)} + {:ret :*})) (defn category-id [ds {:keys [feed-id where] :as opts}] (->> {:tname :feed-categories @@ -10,3 +44,15 @@ (merge opts) (db/find ds))) +(comment + (require '[source.db.util :as db.util]) + (def ds (db.util/conn)) + + (db/delete! ds {:tname :feed-categories}) + (feed-categories ds) + (insert-feed-category! ds {:data {:feed-id 1 + :category-id 2}}) + (categories-by-feed ds {:feed-id 1}) + (upsert-feed-categories! ds {:data [{:feed-id 1 + :category-id 6}]}) + ()) diff --git a/src/source/services/incoming_posts.clj b/src/source/services/incoming_posts.clj new file mode 100644 index 00000000..d4df9b4a --- /dev/null +++ b/src/source/services/incoming_posts.clj @@ -0,0 +1,42 @@ +(ns source.services.incoming-posts + (:require [source.db.interface :as db] + [source.db.honey :as hon])) + +(defn incoming-posts + ([ds] (incoming-posts ds {})) + ([ds {:keys [where] :as opts}] + (->> {:tname :incoming-posts + :where where + :ret :*} + (merge opts) + (db/find ds)))) + +(defn incoming-posts-with-feeds + [ds {:keys [_where] :as opts}] + (hon/execute! ds + (merge + {:select [[:incoming-posts.id :id] :incoming-posts.post-id :feed-id] + :from :incoming-posts + :join [:feeds [:= :incoming-posts.feed-id :feeds.id]]} + opts) + {:ret :*})) + +(defn incoming-post [ds {:keys [id where] :as opts}] + (->> {:tname :incoming-posts + :where (if (some? id) + [:= :id id] + where) + :ret :1} + (merge opts) + (db/find ds))) + +(defn categories-by-posts [ds {:keys [_where] :as opts}] + (hon/execute! ds + (merge + {:select [[:incoming-posts.id :post-id] [:categories.id :id] :categories.name] + :from :incoming-posts + :join [:feeds [:= :incoming-posts.feed-id :feeds.id]] + :left-join [:feed-categories [:= :feed-categories.feed-id :feeds.id]] + :right-join [:categories [:= :categories.id :feed-categories.category-id]]} + opts) + {:ret :*})) diff --git a/src/source/services/interface.clj b/src/source/services/interface.clj index c27a8428..cfa3db3a 100644 --- a/src/source/services/interface.clj +++ b/src/source/services/interface.clj @@ -1,33 +1,79 @@ (ns source.services.interface - (:require [source.services.users :as users] - [source.db.interface :as db] - [source.services.auth :as auth] - [source.services.bundles :as bundles])) + (:require [source.services.auth :as auth] + [source.services.bundles :as bundles] + [source.services.bundle-content-types :as bundle-content-types] + [source.services.post-heuristics :as post-heuristics] + [source.services.incoming-posts :as incoming-posts] + [source.services.feed-categories :as feed-categories] + [source.services.jobs :as jobs])) -(defn users - [& args] - (apply users/users args)) +(defn register [ds user] + (auth/register ds user)) -(defn user [ds {:keys [_id] :as opts}] - (users/user ds opts)) +(defn update-bundle! [ds {:keys [_id _data _where] :as opts}] + (bundles/update-bundle! ds opts)) -(defn insert-user! [ds {:keys [_values _ret] :as opts}] - (users/insert-user! ds opts)) +(defn bundles + ([ds] (bundles ds {})) + ([ds {:keys [_where] :as opts}] + (bundles/bundles ds opts))) -(defn update-user! [ds {:keys [_id _values _where] :as opts}] - (users/update-user! ds opts)) +(defn bundle [ds {:keys [_id _where] :as opts}] + (bundles/bundle ds opts)) -(defn login [ds {:keys [_email] :as opts}] - (auth/login ds opts)) +(defn insert-bundle-content-types! [ds {:keys [_data _ret] :as opts}] + (bundle-content-types/insert-bundle-content-types! ds opts)) -(defn register [ds user] - (auth/register ds user)) +(defn content-types-by-bundle [ds {:keys [_bundle-id _where] :as opts}] + (bundle-content-types/content-types-by-bundle ds opts)) -(defn bundle [ds {:keys [_id _where] :as opts}] - (bundles/bundle ds opts)) +(defn upsert-post-heuristics! [ds {:keys [_data] :as opts}] + (post-heuristics/upsert-post-heuristics! ds opts)) + +(defn top-posts-by-heuristic [ds {:keys [_select _limit _heuristic] :as opts}] + (post-heuristics/top-posts-by-heuristic ds opts)) + +(defn incoming-posts + ([ds] + (incoming-posts/incoming-posts ds)) + ([ds {:keys [_where] :as opts}] + (incoming-posts/incoming-posts ds opts))) + +(defn incoming-posts-with-feeds + [ds {:keys [_where] :as opts}] + (incoming-posts/incoming-posts-with-feeds ds opts)) + +(defn incoming-post [ds opts] + (incoming-posts/incoming-post ds opts)) + +(defn categories-by-feed [ds {:keys [_feed-id _where] :as opts}] + (feed-categories/categories-by-feed ds opts)) + +(defn insert-job! [ds {:keys [_data _ret] :as opts}] + (jobs/insert-job! ds opts)) + +(defn update-job! [ds {:keys [_id _data _where] :as opts}] + (jobs/update-job! ds opts)) + +(defn delete-job! [ds {:keys [_id _where] :as opts}] + (jobs/delete-job! ds opts)) + +(defn jobs + ([ds] (jobs ds {})) + ([ds opts] + (jobs/jobs ds opts))) + +(defn job [ds {:keys [_id _where] :as opts}] + (jobs/job ds opts)) + +(defn insert-job-metadata! [ds {:keys [_data _ret] :as opts}] + (jobs/insert-job-metadata! ds opts)) + +(defn update-job-metadata! [ds {:keys [_id _data _where] :as opts}] + (jobs/update-job-metadata! ds opts)) + +(defn delete-job-metadata! [ds {:keys [_id _where] :as opts}] + (jobs/delete-job-metadata! ds opts)) -(comment - (users (db/ds :master)) - (user (db/ds :master) {:id 2}) - (login (db/ds :master) {:email "merveillevaneck@gmail.com"}) - ()) +(defn job-metadata [ds {:keys [_id _where] :as opts}] + (jobs/job-metadata ds opts)) diff --git a/src/source/services/jobs.clj b/src/source/services/jobs.clj new file mode 100644 index 00000000..7a7b1574 --- /dev/null +++ b/src/source/services/jobs.clj @@ -0,0 +1,72 @@ +(ns source.services.jobs + (:require [source.db.interface :as db])) + +(defn insert-job! [ds {:keys [data ret] :as opts}] + (->> {:tname :jobs + :data data + :ret ret} + (merge opts) + (db/insert! ds))) + +(defn update-job! [ds {:keys [id data where] :as opts}] + (->> {:tname :jobs + :values data + :where (if (some? id) [:= :id id] where)} + (merge opts) + (db/update! ds))) + +(defn delete-job! [ds {:keys [id where] :as opts}] + (->> {:tname :jobs + :where (if (some? id) + [:= :id id] + where) + :ret :1} + (merge opts) + (db/delete! ds))) + +(defn jobs + ([ds] (jobs ds {})) + ([ds opts] + (->> {:tname :jobs + :ret :*} + (merge opts) + (db/find ds)))) + +(defn job [ds {:keys [id where] :as opts}] + (->> {:tname :jobs + :where (if (some? id) [:= :id id] where) + :ret :1} + (merge opts) + (db/find ds))) + +(defn insert-job-metadata! [ds {:keys [data ret] :as opts}] + (->> {:tname :job-metadata + :data data + :ret ret} + (merge opts) + (db/insert! ds))) + +(defn update-job-metadata! [ds {:keys [id data where] :as opts}] + (->> {:tname :job-metadata + :values data + :where (if (some? id) [:= :id id] where)} + (merge opts) + (db/update! ds))) + +(defn delete-job-metadata! [ds {:keys [id where] :as opts}] + (->> {:tname :job-metadata + :where (if (some? id) + [:= :id id] + where) + :ret :1} + (merge opts) + (db/delete! ds))) + +(defn job-metadata [ds {:keys [id where] :as opts}] + (->> {:tname :job-metadata + :where (if (some? id) + [:= :id id] + where) + :ret :1} + (merge opts) + (db/find ds))) diff --git a/src/source/services/outgoing_posts.clj b/src/source/services/outgoing_posts.clj deleted file mode 100644 index 6ca27f7c..00000000 --- a/src/source/services/outgoing_posts.clj +++ /dev/null @@ -1,13 +0,0 @@ -(ns source.services.outgoing-posts - (:require [source.db.interface :as db] - [source.db.util :as db.util])) - -(defn outgoing-post [ds {:keys [id where] :as opts}] - (->> {:tname :outgoing-posts - :where (if (some? id) - [:= :id id] - where) - :ret :1} - (merge opts) - (db/find ds))) - diff --git a/src/source/services/post_heuristics.clj b/src/source/services/post_heuristics.clj new file mode 100644 index 00000000..d295a04c --- /dev/null +++ b/src/source/services/post_heuristics.clj @@ -0,0 +1,21 @@ +(ns source.services.post-heuristics + (:require [source.db.honey :as hon] + [honey.sql.helpers :as hsql] + [source.db.util :as db.util])) + +(defn upsert-post-heuristics! [ds {:keys [bundle-id data]}] + (hon/execute! + ds + (-> (hsql/insert-into (:tname (db.util/tname :post-heuristics bundle-id))) + (hsql/values data) + (assoc :on-conflict [:post-id]) + (assoc :do-update-set {:long-heuristic :excluded.long-heuristic + :short-heuristic :excluded.short-heuristic})))) + +(defn top-posts-by-heuristic [ds {:keys [select limit heuristic bundle-id] :as _opts}] + (hon/execute! ds + (merge {:select (or select :*) + :from (:tname (db.util/tname :post-heuristics bundle-id)) + :order-by [[heuristic :desc]] + :limit limit}) + {:ret :*})) diff --git a/src/source/services/sectors.clj b/src/source/services/sectors.clj deleted file mode 100644 index 82979c7b..00000000 --- a/src/source/services/sectors.clj +++ /dev/null @@ -1,10 +0,0 @@ -(ns source.services.sectors - (:require [source.db.interface :as db])) - -(defn sectors - ([ds] (sectors ds {})) - ([ds opts] - (->> {:tname :sectors - :ret :*} - (merge opts) - (db/find ds)))) diff --git a/src/source/services/user_sectors.clj b/src/source/services/user_sectors.clj new file mode 100644 index 00000000..8fce6216 --- /dev/null +++ b/src/source/services/user_sectors.clj @@ -0,0 +1,37 @@ +(ns source.services.user-sectors + (:require [source.db.interface :as db] + [source.db.honey :as hon] + [pg.core :as pg])) + +(defn insert-user-sector! [ds {:keys [_data _ret] :as opts}] + (->> {:tname :user-sectors} + (merge opts) + (db/insert! ds))) + +(defn delete-user-sector! [ds {:keys [id where] :as opts}] + (->> {:tname :user-sectors + :where (if (some? id) + [:= :id id] + where) + :ret :1} + (merge opts) + (db/delete! ds))) + +(defn sectors-by-user [ds {:keys [user-id where] :as _opts}] + (hon/execute! ds + {:select [[:user-sectors.sector-id :id] :name] + :from :sectors + :join [:user-sectors [:= :user-sectors.sector-id :sectors.id]] + :where (if (some? user-id) + [:= :user-id user-id] + where)} + {:ret :*})) + +;;NEW +(defn update-user-sectors! [ds {:keys [user-id sectors]}] + (pg/with-transaction [ds ds] + (let [update-data (reduce (fn [acc {:keys [id]}] + (conj acc {:user-id user-id + :sector-id id})) [] sectors)] + (delete-user-sector! ds {:where [:= :user-id user-id]}) + (insert-user-sector! ds {:data update-data})))) diff --git a/src/source/services/users.clj b/src/source/services/users.clj deleted file mode 100644 index aa4e7a0f..00000000 --- a/src/source/services/users.clj +++ /dev/null @@ -1,53 +0,0 @@ -(ns source.services.users - (:require [source.db.interface :as db] - [source.password :as pw])) - -(defn users - ([ds] (users ds {})) - ([ds opts] - (->> {:tname :users - :ret :*} - (merge opts) - (db/find ds) - (mapv #(dissoc % :password))))) - -(defn user [ds {:keys [id where] :as opts}] - (->> {:tname :users - :where (if (some? id) - [:= :id id] - where) - :ret :1} - (merge opts) - (db/find ds))) - -(defn insert-user! [ds {:keys [_values _ret] :as opts}] - (->> {:tname :users} - (merge opts) - (db/insert! ds))) - -(defn delete-user! [ds {:keys [id where] :as opts}] - (->> {:tname :users - :where (if (some? id) [:= :id id] where)} - (merge opts) - (db/delete! ds))) - -(defn update-user! [ds {:keys [id values where] :as opts}] - (->> {:tname :users - :values values - :where (if (some? id) [:= :id id] where)} - (merge opts) - (db/update! ds))) - -(comment - (users (db/ds :master)) - (insert-user! (db/ds :master) {:data {:email "chonkin@bonkin.com" - :password (pw/hash-password "test") - :firstname "merv" - :lastname "vaneck" - :type "creator"} - :ret :*}) - (user (db/ds :master) {:id 5}) - (update-user! (db/ds :master) {:id 5 - :values {:firstname "kiigan" - :lastname "korinzu"}}) - ()) diff --git a/src/source/util.clj b/src/source/util.clj index 504d046d..73871d82 100644 --- a/src/source/util.clj +++ b/src/source/util.clj @@ -1,6 +1,18 @@ (ns source.util (:require [buddy.core.codecs :as codecs] - [buddy.core.nonce :as nonce])) + [buddy.core.nonce :as nonce] + [clojure.main :refer [demunge]] + [malli.core :as m] + [malli.transform :as mt] + [clojure.string :as string]) + (:import (java.math BigInteger) + (java.security MessageDigest))) + +(defn vectors? + "Returns true if coll is a 2d vector" + [coll] + (and (vector? coll) + (vector? (first coll)))) (defn content-type [request] (or (get-in request [:headers "Content-Type"]) @@ -25,3 +37,114 @@ (-> (nonce/random-bytes 8) (codecs/bytes->hex))) + +(defn metadata [func] + (-> (class func) + (print-str) + (demunge) + (symbol) + (find-var) + (meta))) + +(defn sha256 + "Computes SHA256 hash of given string and returns it as a hex string" + [s] + (let [digest (MessageDigest/getInstance "SHA-256") + bytes (.getBytes s "UTF-8") + hash-bytes (.digest digest bytes)] + (format "%064x" (BigInteger. 1 hash-bytes)))) + +(defn parse-type-from-schema [schema] + (cond + (keyword? schema) + (name schema) + + (and (seq schema) (= (first schema) :vector)) + (try + (str "Array[" (string/join " " (mapv name (rest schema))) "]") + (catch Exception _ "Array[]")) + + (and (seq schema) (= (first schema) :map)) + (try + (str "Object { " (reduce (fn [acc pair] + (str acc (string/join ": " (mapv name pair)) "; ")) + "" (rest schema)) "}") + (catch Exception _ "Object {}")))) + +(defn append-humanised-error [acc error-map] + (let [path (string/join "." (mapv (fn [p] + (if (keyword? p) + (name p) + (str "[" p "]"))) (:path error-map))) + k (if (or (nil? path) (= path "")) "" (str path ": ")) + value (:value error-map) + value (if (nil? value) "null" (:value error-map)) + schema (m/form (:schema error-map)) + error-type (cond + (= (:type error-map) :malli.core/missing-key) :missing + (= (:type error-map) :malli.core/invalid-type) :invalid + :else :type-error) + expected (parse-type-from-schema schema)] + (cond + (= error-type :missing) (str acc "Missing required key: '" path "'\n") + (= error-type :invalid) (str acc k "Expected '" expected "', found '" value "'\n") + :else (str acc k "Expected '" expected "', found '" value "'\n")))) + +(defn humanise [{:keys [errors] :as _error}] + (reduce append-humanised-error "" errors)) + +(defn validate + [data schema] + (let [transformed (m/decode schema data mt/string-transformer) + success (m/validate schema transformed)] + {:data (when success transformed) + :success success + :error (when-not success (->> transformed + (m/explain schema) + (humanise)))})) + +; (defn validate +; ([handler data] +; (validate handler data :body)) +; ([handler data schema-type] +; (let [schema (get-in (metadata handler) [:parameters schema-type]) +; transformed (m/decode schema data mt/string-transformer) +; success (m/validate schema transformed)] +; {:data (when success transformed) +; :success success +; :error (when-not success (->> transformed +; (m/explain schema) +; (me/humanize)))}))) + +(defn format-rss-date + "Takes a date as a string in RFC 1123 format and returns it in a format that meets ISO 8601 standards for SQLite. + This is necessary because some RSS feeds use a different date than what is accepted by SQLite. + Returns the original string if it is not in this format." + [s] + (try + (let [zdt (java.time.ZonedDateTime/parse s java.time.format.DateTimeFormatter/RFC_1123_DATE_TIME) + zdt-utc (.withZoneSameInstant zdt (java.time.ZoneId/of "UTC")) + out (.format zdt-utc (java.time.format.DateTimeFormatter/ofPattern "yyyy-MM-dd HH:mm:ss"))] + out) + (catch Exception _ s))) + +(comment + (sha256 "1") + (validate {:a "1"} [:map [:a {:title "aoeu" + :description "aoeu"} :int]]) + + (validate {:message "yeet" + :b 1 + :array {} + :test [] + :units [{:name "cheese"}]} + [:map + [:message :string] + [:a :int] + [:b :string] + [:array [:vector :int]] + [:test [:map [:a :int]]] + [:units [:vector [:map + [:name [:maybe :string]] + [:description [:maybe :string]]]]]]) + ()) diff --git a/src/source/workers/auth.clj b/src/source/workers/auth.clj new file mode 100644 index 00000000..3035af53 --- /dev/null +++ b/src/source/workers/auth.clj @@ -0,0 +1,3 @@ +(ns source.workers.auth + (:require [source.workers.schemas :as schemas])) + diff --git a/src/source/workers/bundles.clj b/src/source/workers/bundles.clj new file mode 100644 index 00000000..9ec696cf --- /dev/null +++ b/src/source/workers/bundles.clj @@ -0,0 +1,122 @@ +(ns source.workers.bundles + (:require [source.db.honey :as hon] + [honey.sql.helpers :as hsql] + [source.db.util :as db.util] + [source.prandom.core :as prandom]) + (:import [java.time LocalDateTime] + [java.time.format DateTimeFormatter])) + +(defn get-bundle-categories + "Get all categories for feeds/posts in bundle" + [ds {:keys [bundle-id content-type-id]}] + (->> (-> (hsql/select-distinct :c.*) + (hsql/from [:categories :c]) + (hsql/join [:feed-categories :fc] [:= :c.id :fc.category-id]) + (hsql/join [(:tname (db.util/tname :outgoing-posts bundle-id)) :p] [:= :fc.feed-id :p.feed-id]) + (hsql/group-by :c.id :c.name :c.display-picture) + (hsql/where (when content-type-id [:= :p.content-type-id content-type-id])) + (hsql/having (when (nil? content-type-id) + [:= + [:count [:distinct :p.content-type-id]] + (-> (hsql/select [[:count :id]]) + (hsql/from :content-types))]))) + (hon/execute! ds))) + +(defn get-outgoing-feeds + "Gets a filtered list of outgoing feeds for the associated bundle." + [ds {:keys [bundle-id type latest category-ids nonfiltered]}] + (let [filtered-query (-> (hsql/select-distinct :f.*) + (hsql/from [:feeds :f]) + (hsql/join [(:tname (db.util/tname :outgoing-posts bundle-id)) :p] [:= :f.id :p.feed-id]) + (hsql/join [:feed-categories :fc] [:= :fc.feed-id :f.id]) + (hsql/where + (when (nil? nonfiltered) [:not-in :f.id (-> (hsql/select :feed-id) + (hsql/from :filtered-feeds) + (hsql/where [:= :bundle-id bundle-id]))]) + (when type [:= :f.content-type-id type]) + (when (seq category-ids) [:in :fc.category-id category-ids]))) + filtered-feeds (hon/execute! ds (if (some? latest) + (assoc filtered-query :order-by [[:created-at :desc]]) + filtered-query))] + filtered-feeds)) + +(defn- select-outgoing-post [truncate] + (hsql/select-distinct + :p.id + :p.post-id + :p.feed-id + :p.creator-id + :p.content-type-id + :p.title + :p.thumbnail + (if (= truncate "false") :p.info [[:|| [:left :p.info 100] "..."] :info]) + :p.url + :p.stream-url + :p.season + :p.episode + :p.posted-at + [:f.title :feed-title])) + +(defn get-outgoing-posts + "Get outgoing posts based on short heuristics and update analytics impressions" + [ds {:keys [bundle-id limit start type latest category-ids seed truncate]}] + (let [filtered-posts (hon/execute! + ds + (-> (select-outgoing-post truncate) + (hsql/from [(:tname (db.util/tname :outgoing-posts bundle-id)) :p]) + (hsql/join [:feeds :f] [:= :p.feed-id :f.id]) + (hsql/join [:feed-categories :fc] [:= :p.feed-id :fc.feed-id]) + (hsql/join [:categories :c] [:= :fc.category-id :c.id]) + (hsql/where + (when type [:= :p.content-type-id type]) + [:not-in :p.id (-> (hsql/select :post-id) + (hsql/from :filtered-posts) + (hsql/where [:= :bundle-id bundle-id]))] + [:not-in :p.feed-id (-> (hsql/select :feed-id) + (hsql/from :filtered-feeds) + (hsql/where [:= :bundle-id bundle-id]))] + (when (seq category-ids) [:in :c.id category-ids])) + (hsql/order-by [:p.posted-at :desc]))) + + order-map (->> (if (or (nil? seed) (= seed "")) + (.format (LocalDateTime/now) (DateTimeFormatter/ofPattern "yyyy-MM-dd HH")) + seed) + (prandom/seeded-shuffle (count filtered-posts)) + (map-indexed (fn [i item] [item i])) + (into {})) + + shuffled-posts (if (= latest "true") + filtered-posts + (->> (zipmap (-> filtered-posts count inc range) filtered-posts) + (sort-by #(get order-map (first %))) + (mapv last))) + + total-size (count shuffled-posts) + + valid-start? (and (some? start) (>= start 0) (< start total-size)) + started-posts (if valid-start? + (subvec shuffled-posts start) + shuffled-posts) + + valid-limit? (and (some? limit) (> (count started-posts) limit)) + limited-posts (if valid-limit? + (subvec started-posts 0 limit) + started-posts) + + next-index (when (and start limit) (+ start limit))] + + {:pagination {:page-size (count limited-posts) + :total-size total-size + :current-index start + :next-index (when (and next-index (< next-index total-size)) next-index)} + :data limited-posts})) + +(comment + + (time (get-outgoing-posts (db.util/conn) {:bundle-id 14 + :category-ids [50 52 54] + :latest "false"})) + + (time (get-bundle-categories (db.util/conn) {:bundle-id 14})) + + ()) diff --git a/src/source/workers/categories.clj b/src/source/workers/categories.clj new file mode 100644 index 00000000..82c787ee --- /dev/null +++ b/src/source/workers/categories.clj @@ -0,0 +1,33 @@ +(ns source.workers.categories + (:require [source.db.honey :as hon] + [pg.core :as pg] + [source.db.util :as db.util] + [honey.sql.helpers :as hsql])) + +(defn used-categories + "returns all categories for which feeds exist." + [ds] + (let [category-ids (->> (hon/find ds {:tname :feed-categories}) + (mapv :category-id))] + (hon/find ds {:tname :categories + :where (if (seq category-ids) + [:in :id category-ids] + [:= :id -1])}))) + +(defn delete-category! [ds category-id] + (pg/with-transaction [ds ds] + (let [bundle-ids (mapv :id (hon/find ds {:tname :bundles}))] + (hon/delete! ds {:tname :feed-categories + :where [:= :category-id category-id]}) + (hon/delete! ds {:tname :event-categories + :where [:= :category-id category-id]}) + (run! + #(hon/delete! ds (-> (db.util/tname :bundle-categories %) + (hsql/where [:= :category-id category-id]))) + bundle-ids) + (hon/delete! ds {:tname :categories + :where [:= :id category-id]})))) + +(comment + (used-categories (db.util/conn)) + ()) diff --git a/src/source/workers/content_types.clj b/src/source/workers/content_types.clj new file mode 100644 index 00000000..e2cdcb29 --- /dev/null +++ b/src/source/workers/content_types.clj @@ -0,0 +1,4 @@ +(ns source.workers.content-types + (:require [source.workers.schemas :as schemas])) + +;; Service schemas diff --git a/src/source/workers/feeds.clj b/src/source/workers/feeds.clj new file mode 100644 index 00000000..f4f8fb74 --- /dev/null +++ b/src/source/workers/feeds.clj @@ -0,0 +1,117 @@ +(ns source.workers.feeds + (:require [source.util :as utils] + [source.workers.xml-schemas :as xml] + [source.db.honey :as hon] + [source.rss.youtube :as yt] + [clojure.string :as string] + [taoensso.telemere :as t] + [pg.core :as pg] + [congest.jobs :as congest])) + +(defn create-feed! + "Creates feed with incoming posts pulled from RSS feed and starts associated job" + [ds {:keys [user-id feed-metadata]}] + (pg/with-transaction [ds ds] + (let [{:keys [provider-id rss-url content-type-id]} feed-metadata + datetime (utils/get-utc-timestamp-string) + youtube? (= provider-id 1) + rss-url (if (and youtube? (not (string/includes? rss-url "/feeds/videos.xml?"))) + (->> (yt/find-channel-id rss-url) + (str "https://www.youtube.com/feeds/videos.xml?channel_id=")) + rss-url) + selection-schemas (->> [:= :provider-id provider-id] + (assoc {} :where) + (xml/selection-schemas ds)) + latest-ss (->> selection-schemas + (reduce (fn [acc {:keys [id]}] + (conj acc id)) []) + (apply max -1)) + extracted (try + (when-not (= latest-ss -1) + (xml/extract-data ds latest-ss rss-url)) + (catch Exception e + (throw + (t/error! + ::data-extraction + (ex-info (str "Data extraction failed for feed creation - RSS feed url: " rss-url " creator-id " user-id) + {:panic? "Not a huge deal, possibly just user error - but panic if it's not user error" + :possible-cause "RSS feed url might be incorrect or provider is unsupported" + :next-steps (str + "Check if the RSS feed url is correct. If it is, test data extraction in the admin panel with provider-id " + provider-id) + :raw-error (.getMessage e)}))))) + extracted-posts (get-in extracted [:feed :posts]) + display-picture (if youtube? + (yt/channel-image (get-in extracted [:feed :url])) + (get-in extracted [:feed :display-picture])) + new-feed (hon/insert! + ds + {:tname :feeds + :data (merge feed-metadata {:title (get-in extracted [:feed :title]) + :display-picture display-picture + :description (get-in extracted [:feed :description]) + :user-id user-id + :created-at datetime + :state "pending"}) + :ret :1}) + + _ (when (or (nil? display-picture) (= display-picture "")) + (t/log! {:level :error + :msg (str "Failed to pull display picture for feed [ " (:id new-feed) " " (:title new-feed) "] provider-id " provider-id)})) + + extended-posts (mapv (fn [post] + (merge post + {:feed-id (:id new-feed) + :creator-id user-id + :content-type-id content-type-id + :thumbnail (or (:thumbnail post) (:display-picture new-feed))})) + extracted-posts)] + (if (some? extracted-posts) + (do + (hon/insert! ds {:tname :incoming-posts + :data extended-posts}) + new-feed) + false)))) + +(defn update-feed! [ds {:keys [feed-id feed-metadata]}] + (hon/update! ds {:tname :feeds + :where [:= :id feed-id] + :data feed-metadata + :ret :1})) + +(defn hard-delete-feed! [ds feed-id] + (pg/with-transaction [ds ds] + (let [post-ids (mapv :id (hon/find ds {:tname :incoming-posts + :where [:= :feed-id feed-id]})) + event-ids (mapv :id (hon/find ds {:tname :events + :where [:= :feed-id feed-id]}))] + (hon/delete! ds {:tname :filtered-feeds + :where [:= :feed-id feed-id]}) + (hon/delete! ds {:tname :filtered-posts + :where [:in :post-id post-ids]}) + (hon/delete! ds {:tname :incoming-posts + :where [:= :feed-id feed-id]}) + (hon/delete! ds {:tname :feed-categories + :where [:= :feed-id feed-id]}) + (when (seq event-ids) + (hon/delete! ds {:tname :event-categories + :where [:in :event-id event-ids]})) + (hon/delete! ds {:tname :events + :where [:= :feed-id feed-id]}) + (hon/delete! ds {:tname :feeds + :where [:= :id feed-id]})))) + +(defn deregister-feed-job! [js job-id] + (congest/deregister! js job-id)) + +(defn update-feed-categories! [ds {:keys [feed-id categories]}] + (let [update-data (mapv (fn [{:keys [id]}] + {:feed-id feed-id + :category-id id}) categories)] + (pg/with-transaction [ds ds] + (when (seq update-data) + (hon/delete! ds {:tname :feed-categories + :where [:= :feed-id feed-id]}) + (hon/insert! ds {:tname :feed-categories + :data update-data + :ret :*}))))) diff --git a/src/source/workers/incoming_posts.clj b/src/source/workers/incoming_posts.clj new file mode 100644 index 00000000..f2482f35 --- /dev/null +++ b/src/source/workers/incoming_posts.clj @@ -0,0 +1,2 @@ +(ns source.workers.incoming-posts + (:require [source.workers.schemas :as schemas])) diff --git a/src/source/workers/integrations.clj b/src/source/workers/integrations.clj new file mode 100644 index 00000000..9f058eed --- /dev/null +++ b/src/source/workers/integrations.clj @@ -0,0 +1,90 @@ +(ns source.workers.integrations + (:require [source.services.bundles :as bundles] + [source.migrate :as migrate] + [source.db.util :as db.util] + [source.services.bundle-categories :as bundle-categories] + [source.services.bundle-content-types :as bundle-content-types] + [source.db.tables :as tables] + [source.db.honey :as hon] + [taoensso.telemere :as t] + [pg.core :as pg] + [congest.jobs :as congest])) + +(defn create-integration! [ds {:keys [user-id bundle-metadata categories content-types]}] + (pg/with-transaction [ds ds] + (let [new-bundle (bundles/create-bundle! ds {:user-id user-id + :bundle-metadata bundle-metadata})] + + (try + (migrate/migrate-bundle (:id new-bundle) ["up"]) + (catch Exception e + (throw + (t/error! + ::create-bundle + (ex-info (str "Migration for new bundle with id " (:id new-bundle) " failed. This bundle belongs to the user with id " user-id) + {:panic? "Yes, if this isn't working, it's likely no one will be able to create new integrations" + :possible-cause "Something could be wrong in one of the bundle migrations" + :next-steps "Go see what's going on in bundle migrations ASAP, you may be able to see the problem in the raw error message" + :raw-error (.getMessage e)}))))) + + (bundle-categories/insert-bundle-categories! ds {:bundle-id (:id new-bundle) + :categories categories}) + (bundle-content-types/insert-bundle-content-types! ds {:bundle-id (:id new-bundle) + :content-types content-types}) + new-bundle))) + +(defn update-integration! [ds {:keys [bundle-id bundle-metadata categories content-types]}] + (pg/with-transaction [ds ds] + (bundles/update-bundle! ds {:id bundle-id + :data bundle-metadata}) + (bundle-categories/update-bundle-categories! ds {:bundle-id bundle-id + :categories categories}) + (bundle-content-types/update-bundle-content-types! ds {:bundle-id bundle-id + :content-types content-types}))) + +(defn hard-delete-bundle! [ds bundle-id] + (pg/with-transaction [ds ds] + (let [event-ids (mapv :id (hon/find ds {:tname :events + :where [:= :bundle-id bundle-id]}))] + (hon/delete! ds {:tname :filtered-feeds + :where [:= :bundle-id bundle-id]}) + (hon/delete! ds {:tname :filtered-posts + :where [:= :bundle-id bundle-id]}) + (hon/delete! ds {:tname :bundle-content-types + :where [:= :bundle-id bundle-id]}) + (when (seq event-ids) + (hon/delete! ds {:tname :event-categories + :where [:in :event-id event-ids]})) + (hon/delete! ds {:tname :events + :where [:= :bundle-id bundle-id]}) + (tables/drop-tables! ds (db.util/tnames [:outgoing-posts + :bundle-categories + :post-heuristics] + bundle-id)) + (hon/delete! ds {:tname :bundles + :where [:= :id bundle-id]})))) + +(defn deregister-bundle-job! [js job-id] + (congest/deregister! js job-id)) + +(defn update-filtered-feeds! [ds {:keys [filtered bundle-id feed-id]}] + (pg/with-transaction [ds ds] + (if filtered + (hon/insert! ds {:tname :filtered-feeds + :data {:feed-id feed-id + :bundle-id bundle-id}}) + (hon/delete! ds {:tname :filtered-feeds + :where [:and + [:= :feed-id feed-id] + [:= :bundle-id bundle-id]]})))) + +(defn update-filtered-posts! [ds {:keys [filtered bundle-id post-id]}] + (pg/with-transaction [ds ds] + (if filtered + (hon/insert! ds {:tname :filtered-posts + :data {:post-id post-id + :bundle-id bundle-id}}) + (hon/delete! ds {:tname :filtered-posts + :where [:and + [:= :post-id post-id] + [:= :bundle-id bundle-id]]})))) diff --git a/src/source/workers/schemas.clj b/src/source/workers/schemas.clj new file mode 100644 index 00000000..d03f46d1 --- /dev/null +++ b/src/source/workers/schemas.clj @@ -0,0 +1,294 @@ +(ns source.workers.schemas + (:require [malli.util :as mu] + [source.routes.openapi :as api])) + +(def Business + [:map + [:id :int] + [:name :string] + [:address :string] + [:url :string] + [:linkedin :string] + [:twitter :string]]) + +(def UserType [:enum ["creator" "distributor" "admin"]]) + +(def User + [:map + [:id :int] + [:email :string] + [:firstname :string] + [:lastname :string] + [:type UserType] + [:email-verified :int] + [:onboarded :int] + [:address :string] + [:mobile :string] + [:profile-image :string]]) + +(def UserWithBusiness + (-> User + (mu/assoc :business Business))) + +(def SessionCredentials + [:map + [:access-token :string] + [:refresh-token :string]]) + +(def Login + (-> SessionCredentials + (mu/assoc :user User))) + +(def Register + (-> SessionCredentials + (mu/assoc :user User))) + +(def ConstantSchema + [:map + [:id :int] + [:name :string]]) + +(def Sector + ConstantSchema) + +(def Category + (-> ConstantSchema + (mu/assoc :display-picture :string))) + +(def Categories + [:vector Category]) + +(def ContentType + ConstantSchema) + +(def ContentTypes + [:vector ContentType]) + +(def Provider + (-> ConstantSchema + (mu/assoc :domain :string))) + +(def ProviderWithContentType + (-> Provider + (mu/assoc :content-type ContentType))) + +(def BusinessType + ConstantSchema) + +(def Cadence + [:map + [:id :int] + [:label :string] + [:days :int]]) + +(def Baseline + [:map + [:id :int] + [:label :string] + [:min :int] + [:max :int]]) + +(def FeedStatus [:enum "live" "not live" "pending"]) + +(def FeedRecord + [:map + [:id :int] + [:title :string] + (api/sometimes :display-picture :string) + (api/sometimes :description :string) + (api/sometimes :url :string) + [:rss-url :string] + [:created-at :string] + (api/sometimes :updated-at :string) + (api/sometimes :ts-and-cs :int) + [:state FeedStatus]]) + +(def Feed + (-> FeedRecord + (mu/assoc :user-id :int) + (mu/assoc :content-type-id :int) + (mu/assoc :cadence-id :int) + (mu/assoc :baseline-id :int) + (mu/assoc :provider-id :int))) + +(def Feeds + [:vector Feed]) + +(defn paginated [data-schema] + [:map + [:pagination [:map + [:page-size :int] + [:total-size :int] + [:current-index :int] + (api/sometimes :next-index :int)]] + [:data data-schema]]) + +(def IncomingPostRecord + [:map + [:id :int] + [:post-id :string] + [:title :string] + [:thumbnail :string] + [:info :string] + [:url :string] + [:stream-url :string] + [:season :int] + [:episode :int] + [:redacted :int] + [:posted-at :string]]) + +(def IncomingPost + (-> IncomingPostRecord + (mu/assoc :feed FeedRecord))) + +;; This is exactly the same as IncomingPost except without redacted +;; We could probably just make a Post schema with (sometimes :redacted :int) +(def OutgoingPost + [:map + [:id :int] + [:post-id :string] + [:title :string] + [:thumbnail :string] + [:info :string] + [:url :string] + [:stream-url :string] + [:season :int] + [:episode :int] + [:posted-at :string]]) + +(def Post + [:map + [:id :int] + [:post-id :string] + [:feed-id :int] + [:creator-id :int] + [:content-type-id :int] + [:title :string] + [:thumbnail (api/maybe :string)] + [:info (api/maybe :string)] + [:url (api/maybe :string)] + [:stream-url (api/maybe :string)] + [:season (api/maybe :int)] + [:episode (api/maybe :int)] + (api/sometimes :redacted :int) + [:posted-at (api/maybe :string)]]) + +(def Posts + [:vector Post]) + +(def QueryUUID + [:uuid {:description "Bundle UUID"} :string]) + +(def QueryStart + [:start + {:optional true + :description "Used for pagination. Specifies the starting point for the returned items, incremented by the limit."} + :int]) + +(def QueryLimit + [:limit + {:optional true + :description "Used for pagination. Specifies a number of items to be returned."} + :int]) + +(def QueryContentType + [:type {:optional true + :description "Filters by content type ID"} :int]) + +(def QueryLatest + [:latest + {:optional true + :description "Filters by most recently published"} + [:enum "true" "false"]]) + +(def JobStatus [:enum ["running" "stopped"]]) + +(def Job + [:map + [:id :int] + [:job-id :string] + [:status JobStatus] + [:args :string] + [:handler :string] + [:last-heartbeat :string]]) + +(def JobMetadata + [:map + [:id :string] + [:initial-delay :int] + [:stop-after-fail :int] + (api/sometimes :auto-start :int) + (api/sometimes :kill-after :int) + [:num-calls :int] + [:interval :int] + [:recurring :int] + (api/sometimes :created-at :string) + [:sleep :int]]) + +(def JobWithMetadata + (-> Job + (mu/assoc :job-metadata JobMetadata))) + +(def JobsWithMetadata + [:vector JobWithMetadata]) + +(def IntegrationType + ConstantSchema) + +(def IntegrationTypes + [:vector IntegrationType]) + +(def Bundle + [:map + [:id :int] + [:name :string] + [:uuid :string] + [:user-id :int] + [:video :int] + [:podcast :int] + [:blog :int] + (api/sometimes :hash :string) + [:content-type-id :int] + (api/sometimes :integration-type-id :int) + [:ts-and-cs {:optional true} :int]]) + +(def Bundles + [:vector Bundle]) + +(def BundleWithUser + (-> Bundle + (mu/assoc :user User))) + +(def GeneralStatistic + [:map + [:day :string] + [:impressions :int] + [:clicks :int] + [:views :int]]) + +(def GeneralAnalytics + [:vector GeneralStatistic]) + +(def DeltasStatistic + [:map + [:week :string] + [:impressions :float] + [:clicks :float] + [:views :float]]) + +(def DeltasAnalytics + [:vector DeltasStatistic]) + +(def TopStatistic + [:map + [:top :string] + [:impressions :int] + [:clicks :int] + [:views :int]]) + +(def TopAnalytics + [:vector TopStatistic]) + +(def AverageEngagementAnalytics + [:map + [:average :float]]) diff --git a/src/source/workers/sectors.clj b/src/source/workers/sectors.clj new file mode 100644 index 00000000..9e0fce29 --- /dev/null +++ b/src/source/workers/sectors.clj @@ -0,0 +1,2 @@ +(ns source.workers.sectors + (:require [source.workers.schemas :as schemas])) diff --git a/src/source/workers/users.clj b/src/source/workers/users.clj new file mode 100644 index 00000000..5abeb636 --- /dev/null +++ b/src/source/workers/users.clj @@ -0,0 +1,70 @@ +(ns source.workers.users + (:require [source.workers.feeds :as feeds] + [source.workers.integrations :as integrations] + [source.db.honey :as hon] + [pg.core :as pg])) + +(defn hard-delete-creator! [ds user-id] + (let [feed-ids (mapv :id (hon/find ds {:tname :feeds + :where [:= :user-id user-id]}))] + (run! #(feeds/hard-delete-feed! ds %) feed-ids) + (hon/delete! ds {:tname :events + :where [:= :creator-id user-id]}))) + +(defn deregister-creator-jobs! [ds js user-id email] + (let [feed-ids (mapv :id (hon/find ds {:tname :feeds + :where [:= :user-id user-id]}))] + (run! #(feeds/deregister-feed-job! js (str email "-" %)) feed-ids))) + +(defn hard-delete-distributor! [ds user-id] + (let [bundle-ids (mapv :id (hon/find ds {:tname :bundles + :where [:= :user-id user-id]}))] + (run! #(integrations/hard-delete-bundle! ds %) bundle-ids) + (hon/delete! ds {:tname :events + :where [:= :distributor-id user-id]}))) + +(defn deregister-distributor-jobs! [ds js user-id] + (let [bundle-ids (mapv :id (hon/find ds {:tname :bundles + :where [:= :user-id user-id]}))] + (run! #(integrations/hard-delete-bundle! js (str "bundle_" %)) bundle-ids))) + +(defn hard-delete-user! [ds user-type user-id] + (pg/with-transaction [ds ds] + (let [{:keys [business-id]} (hon/find-one ds {:tname :users + :where [:= :id user-id]})] + (cond + (= user-type :creator) + (hard-delete-creator! ds user-id) + (= user-type :distributor) + (hard-delete-distributor! ds user-id)) + + (hon/delete! ds {:tname :user-sectors + :where [:= :user-id user-id]}) + (when (some? business-id) (hon/delete! ds {:tname :businesses + :where [:= :id business-id]})) + (hon/delete! ds {:tname :users + :where [:= :id user-id]})))) + +(defn soft-delete-user! [ds js user-id] + (let [{:keys [email type]} (hon/find-one ds {:tname :users + :where [:= :id user-id]})] + (cond + (= (keyword type) :creator) + (deregister-creator-jobs! ds js user-id email) + (= (keyword type) :distributor) + (deregister-distributor-jobs! ds js user-id)) + + (hon/update! ds {:tname :users + :where [:= :id user-id] + :data {:removed 1}}))) + +(defn cancel-soft-user-deletion! [ds user-id] + (hon/update! ds {:tname :users + :where [:= :id user-id] + :data {:removed 0}})) + +(defn removed? [ds user-id] + (let [removed? (-> (hon/find ds {:tname :users + :where [:= :id user-id]}) + (:removed))] + (when (or (nil? removed?) (= removed? 0)) true))) diff --git a/src/source/workers/xml_schemas.clj b/src/source/workers/xml_schemas.clj new file mode 100644 index 00000000..39573bfe --- /dev/null +++ b/src/source/workers/xml_schemas.clj @@ -0,0 +1,67 @@ +(ns source.workers.xml-schemas + (:require [clojure.data.json :as json] + [source.db.honey :as hon] + [source.rss.core :as rss])) + +(defn insert-output-schema! [ds schema] + (hon/insert! ds {:tname :output-schemas + :data {:schema (json/write-str schema)}})) + +(defn output-schemas [ds] + (->> (hon/find ds {:tname :output-schemas}) + (mapv #(assoc % :schema (json/read-str (:schema %) {:key-fn keyword}))))) + +(defn output-schema [ds id] + (let [{:keys [schema] :as os} (hon/find-one ds {:tname :output-schemas + :where [:= :id id]})] + (assoc os :schema (json/read-str (or schema "{}") {:key-fn keyword})))) + +(defn insert-selection-schema! + [ds {:keys [schema record]}] + (let [{:keys [output-schema-id provider-id]} record] + (hon/insert! ds {:tname :selection-schemas + :data {:schema (json/write-str schema) + :output-schema-id output-schema-id + :provider-id provider-id} + :ret :1}))) + +(defn delete-selection-schemas-by-provider! + [ds provider-id] + (hon/delete! ds {:tname :selection-schemas + :where [:= :provider-id provider-id]})) + +(defn selection-schemas + ([ds] (selection-schemas ds {})) + ([ds opts] + (->> {:tname :selection-schemas + :ret :*} + (merge opts) + (hon/find ds) + (mapv #(assoc % :schema (json/read-str (or (:schema %) "{}") {:key-fn keyword})))))) + +(defn selection-schema [ds {:keys [id] :as opts}] + (let [{:keys [schema] :as selection-schema} (->> {:tname :selection-schemas + :where [:= :id id]} + (merge opts) + (hon/find-one ds)) + schema' (json/read-str (or schema "{}") {:key-fn keyword})] + (assoc selection-schema :schema schema'))) + +(defn ast + [url] + (-> url + (slurp) + (rss/get-ast) + (rss/collect-leaf-paths))) + +(defn extract-data + [ds schema-id url] + (let [json (->> {:tname :selection-schemas + :where [:= :id schema-id]} + (hon/find-one ds) + (:schema)) + schema (json/read-str (or json "{}") {:key-fn keyword})] + (->> url + (slurp) + (rss/get-ast) + (rss/extract-data schema)))) diff --git a/start.sh b/start.sh index d16c0e08..91693940 100755 --- a/start.sh +++ b/start.sh @@ -1,5 +1,8 @@ -#!/usr/bin/env bash +#!/bin/bash +export $(grep '.*' .env | xargs) +echo "Running migrations before startup..." clojure -M:migrate -java -jar target/source-be-standalone.jar +echo "Starting server..." +$JAVA_CMD -jar target/source-be-standalone.jar