Asset based container generation and AI context resolution refinement

apps/backend/asset/asset.model.d.ts

@@ -1,10 +1,5 @@
 import type { Model, ModelStatic } from 'sequelize';
-import type {
-  Asset as AssetAttributes,
-  FileAsset as FileAssetAttrs,
-  LinkAsset as LinkAssetAttrs,
-  MediaAsset as MediaAssetAttrs,
-} from '@tailor-cms/interfaces/asset.ts';
+import type { Asset as AssetAttributes } from '@tailor-cms/interfaces/asset.ts';
 
 export type { AssetAttributes };
 export {
@@ -17,10 +12,26 @@ export {
   Uploader,
 } from '@tailor-cms/interfaces/asset.ts';
 
-// Sequelize model variants of the interface discriminated unions
-export type FileAsset = FileAssetAttrs & Model<AssetAttributes>;
-export type MediaAsset = MediaAssetAttrs & Model<AssetAttributes>;
-export type LinkAsset = LinkAssetAttrs & Model<AssetAttributes>;
+type AssetBase = AssetAttributes & Model<AssetAttributes>;
+
+export interface FileAsset extends AssetBase {
+  type: 'IMAGE' | 'DOCUMENT' | 'OTHER';
+  storageKey: string;
+  meta: FileAssetMeta;
+}
+
+export interface MediaAsset extends AssetBase {
+  type: 'VIDEO' | 'AUDIO';
+  storageKey: string;
+  meta: MediaAssetMeta;
+}
+
+export interface LinkAsset extends AssetBase {
+  type: 'LINK';
+  storageKey: null;
+  meta: LinkAssetMeta;
+}
+
 export type Asset = FileAsset | MediaAsset | LinkAsset;
 
 declare const Asset: ModelStatic<Asset>;

apps/backend/asset/asset.service.ts

@@ -56,10 +56,7 @@ const IS_NOT_VIDEO_LINK = {
 export const UPLOADER_INCLUDE = {
   model: User,
   as: 'uploader',
-  attributes: [
-    'id', 'email', 'firstName', 'lastName',
-    'fullName', 'label', 'imgUrl',
-  ],
+  attributes: ['id', 'email', 'firstName', 'lastName', 'fullName', 'label', 'imgUrl'],
 };
 
 /** Wraps an async fn so it logs warnings instead of throwing. */

apps/backend/asset/discovery/discovery.service.ts

@@ -90,8 +90,8 @@ const strategies: Record<ContentFilter, Strategy> = {
     serper.newsSearch(q, Math.ceil(n * 0.15) + DEDUP_BUFFER),
   ],
   [Image]: (q, n) => [
-    unsplash.search(q, Math.ceil(n * 0.6)),
-    serper.imageSearch(q, Math.ceil(n * 0.4) + DEDUP_BUFFER),
+    unsplash.search(q, n),
+    serper.imageSearch(q, Math.ceil(n * 0.5) + DEDUP_BUFFER),
   ],
   [Video]: (q, n) => [serper.videoSearch(q, n + DEDUP_BUFFER)],
   [Pdf]: (q, n) => [serper.webSearch(`${q} filetype:pdf`, n + DEDUP_BUFFER)],

apps/backend/asset/extraction/synthetic-content.ts

@@ -22,13 +22,22 @@ export function hasUsefulDescription(asset: Asset): boolean {
   return true;
 }
 
-// Builds a markdown document from asset metadata and optional body text.
-// Returns null when there's nothing meaningful to index (name-only).
+// Builds a markdown document from asset metadata
+// and optional body text (captions, page content, etc.).
+// Asset ID and type are included so the AI can discover
+// and reference relevant media via vector store search.
 export function buildSyntheticContent(
   asset: Asset, bodyText = '',
 ): string | null {
+  const meta = asset.meta as any;
   const parts: string[] = [`# ${asset.name}`];
-  const meta = asset.meta as Record<string, any>;
+  parts.push(`Asset ID: ${asset.id} | Type: ${asset.type}`);
+  if (meta?.contentType) {
+    parts[1] += ` | Content: ${meta.contentType}`;
+  }
+  if (meta?.provider) {
+    parts[1] += ` | Provider: ${meta.provider}`;
+  }
   // Attribution and provenance
   if (meta?.source?.author) parts.push(`Author: ${meta.source.author}`);
   if (meta?.siteName || meta?.domain) {
@@ -43,8 +52,6 @@ export function buildSyntheticContent(
   if (meta?.analysis) parts.push(`Analysis: ${meta.analysis}`);
   if (meta?.contentSuggestion) parts.push(`Suggested use: ${meta.contentSuggestion}`);
   if (meta?.tags?.length) parts.push(`Tags: ${meta.tags.join(', ')}`);
-  // Full content (page text, captions, vision description)
   if (bodyText) parts.push(bodyText);
-  // Name-only content isn't worth indexing
   return parts.length > 1 ? parts.join('\n\n') : null;
 }

apps/backend/asset/indexing/indexing.service.ts

@@ -121,7 +121,16 @@ async function indexDocument(ctx: IndexingContext<FileAsset>) {
     originalname: asset.name,
     mimetype: asset.meta.mimeType,
   };
-  const result = await AIService.vectorStore!.upload([file], storeId);
+  const result = await AIService.vectorStore!.upload(
+    [file], storeId,
+  );
+  // Companion metadata doc so AI can discover this
+  // document's asset ID via file_search
+  const meta = buildSyntheticContent(asset);
+  if (meta) {
+    indexSynthetic(storeId, meta, `${asset.id}-meta.md`)
+      .catch(() => {});
+  }
   if (asset.meta.mimeType === mime.lookup('pdf')) {
     // Non-critical side effect - don't let image extraction fail the indexing
     extractAndSaveImages(asset, buffer).catch((err) =>

apps/backend/package.json

@@ -121,6 +121,7 @@
   },
   "devDependencies": {
     "@types/bluebird": "^3.5.42",
+    "@types/common-tags": "^1.8.4",
     "@types/express": "^5.0.6",
     "@types/html-to-text": "^9.0.4",
     "@types/lodash": "^4.17.24",

apps/backend/repository/repository.model.js

@@ -1,4 +1,4 @@
-import { literal, Model, Op } from 'sequelize';
+import { Model, Op } from 'sequelize';
 import first from 'lodash/first.js';
 import intersection from 'lodash/intersection.js';
 import map from 'lodash/map.js';
@@ -162,24 +162,18 @@ class Repository extends Model {
     return false;
   }
 
-  /**
-   * Atomically sets the AI vector store ID in the repository's
-   * data JSONB. Returns true if the value was written, false if
-   * another request already set it (concurrent indexing race).
-   */
   async setVectorStoreId(storeId) {
-    const path = `{$$,ai,storeId}`;
-    const [count] = await Repository.update(
-      { data: literal(`jsonb_set(COALESCE(data,'{}'),'${path}','"${storeId}"')`) },
-      {
-        where: {
-          id: this.id,
-          [Op.and]: literal(`data->'$$'->'ai'->'storeId' IS NULL`),
-        },
+    const current = this.data || {};
+    if (current.$$?.ai?.storeId) return false;
+    const merged = {
+      ...current,
+      $$: {
+        ...current.$$,
+        ai: { ...current.$$?.ai, storeId },
       },
-    );
-    if (count > 0) await this.reload();
-    return count > 0;
+    };
+    await this.update({ data: merged });
+    return true;
   }
 
   getVectorStoreId() {

apps/backend/shared/ai/ai.service.ts

@@ -2,15 +2,15 @@ import type { AiContext, ImageDescription } from '@tailor-cms/interfaces/ai.ts';
 import OpenAI from 'openai';
 
 import { ai as aiConfig } from '#config';
-import { createLogger } from '#logger';
 import { AiPrompt } from './lib/AiPrompt.ts';
+import { createAiLogger } from './logger.ts';
 import { VectorStoreService } from './lib/VectorStoreService.ts';
 import {
   PROMPT as IMAGE_PROMPT,
   Schema as IMAGE_SCHEMA,
 } from './schemas/ImageDescription.ts';
 
-const logger = createLogger('ai:service');
+const logger = createAiLogger('service');
 
 class AiService {
   #openai;

apps/backend/shared/ai/lib/AiPrompt.ts

@@ -1,18 +1,19 @@
-import type { AiContext, AiInput } from '@tailor-cms/interfaces/ai.ts';
+import type {
+  AiContext,
+  AiInput,
+} from '@tailor-cms/interfaces/ai.ts';
 import type { OpenAI } from 'openai';
-import { ContentElementType } from '@tailor-cms/content-element-collection/types.js';
-import StorageService from '../../storage/storage.service.js';
 
 import getContentSchema from '../schemas/index.ts';
 import RepositoryContext from './RepositoryContext.ts';
 
 import { ai as aiConfig } from '#config';
-import { createLogger } from '#logger';
+import { createAiLogger, formatPrompt } from '../logger.ts';
 
-const logger = createLogger('ai:prompt');
+const logger = createAiLogger('prompt');
 
 const systemPrompt = `
-  Assistant is a bot desinged to help authors to create content for
+  Assistant is a bot designed to help authors create content for
   Courses, Q&A content, Knowledge base, etc.
   Rules:
   - Use the User rules to generate the content
@@ -23,20 +24,20 @@ const documentPrompt = `
   The user has provided source documents indexed in a vector store.
   Use the file_search tool to find relevant information from these documents.
   Base ALL generated content on information found in the documents.
-  Do not invent information not present in the documents.`;
+  Do not invent information not present in the documents.
+  If any assets are marked as PRIMARY SOURCE, they represent the core knowledge
+  base. Structure and model your content primarily after these sources.
+  Supplementary assets provide additional context but should not override
+  the core sources.`;
 
 export class AiPrompt {
   // OpenAI client
   private client: OpenAI;
-  // Context of the request
-  private context: AiContext;
   // Information about the repository, content location, topic, etc.
   private repositoryContext: RepositoryContext;
-  // User, assistant or system generated inputs
+  private requestContext: AiContext;
   private inputs: AiInput[];
-  // Existing content, relevant for the context
   private content: string;
-  // Response
   private response: any;
 
   constructor(client: OpenAI, context: AiContext) {
@@ -45,26 +46,46 @@ export class AiPrompt {
     this.content = context.content || '';
     this.inputs = context.inputs;
     this.client = client;
-    this.context = context;
+    this.requestContext = context;
+  }
+
+  get vectorStoreId() {
+    return this.repositoryContext.vectorStoreId;
+  }
+
+  get context(): AiContext {
+    return {
+      ...this.requestContext,
+      assets: this.repositoryContext.assets,
+      repository: {
+        ...this.requestContext.repository,
+        vectorStoreId: this.vectorStoreId,
+      },
+    };
   }
 
   async execute() {
     try {
+      await this.repositoryContext.resolve(this.requestContext.repository);
+      const input = this.toOpenAiInput();
       const params: any = {
         model: aiConfig.modelId,
-        input: this.toOpenAiInput(),
+        input,
         text: { format: this.format },
       };
-      // Add file_search tool when source documents are available
-      const { vectorStoreId } = this.context.repository;
-      if (vectorStoreId) {
-        params.tools = [
-          { type: 'file_search', vector_store_ids: [vectorStoreId] },
-        ];
+      if (this.vectorStoreId) {
+        params.tools = [{
+          type: 'file_search',
+          vector_store_ids: [this.vectorStoreId],
+        }];
       }
+      logger.debug(`Final prompt:\n${formatPrompt(input)}`);
       const response = await this.client.responses.create(params);
       this.response = this.responseProcessor(response.output_text);
-      this.response = await this.applyImageTool();
+      logger.info(
+        { schema: this.prompt.responseSchema, type: this.prompt.type },
+        'Generation complete',
+      );
       return this.response;
     } catch (err) {
       logger.error(err, 'Generation failed');
@@ -99,7 +120,7 @@ export class AiPrompt {
     const processor = this.isCustomPrompt
       ? noop
       : getContentSchema(this.prompt.responseSchema)?.processResponse || noop;
-    return (val) => processor(JSON.parse(val));
+    return (val: string) => processor(JSON.parse(val), this.context);
   }
 
   // TODO: Add option to control the size of the output
@@ -108,7 +129,7 @@ export class AiPrompt {
     const res: OpenAI.Responses.ResponseInputItem[] = [];
     res.push({ role: 'developer', content: systemPrompt });
     res.push({ role: 'developer', content: this.repositoryContext.toString() });
-    if (this.context.repository.vectorStoreId) {
+    if (this.vectorStoreId) {
       res.push({ role: 'developer', content: documentPrompt });
     }
     if (this.prevPrompt) res.push(this.processUserInput(this.prevPrompt));
@@ -135,41 +156,4 @@ export class AiPrompt {
       content: `${base} ${text}. ${responseSchemaDescription} ${target}`,
     };
   }
-
-  async applyImageTool() {
-    if (
-      !this.prompt.useImageGenerationTool ||
-      this.prompt.responseSchema !== 'HTML'
-    )
-      return this.response;
-    // output needs to be sliced to avoid exceeding the max length
-    const userPrompt = `
-      ${this.repositoryContext.toString()}
-      Generate appropriate image for the given topic and content:
-      ${JSON.stringify(this.response).slice(0, 1000)}`;
-    const imgUrl = await this.generateImage(userPrompt);
-    const imgInternalUrl = await StorageService.downloadToStorage(imgUrl);
-    const imageElement = {
-      type: ContentElementType.Image,
-      data: {
-        assets: { url: imgInternalUrl },
-      },
-    };
-    const [firstElement, ...restElements] = this.response;
-    return [firstElement, imageElement, ...restElements];
-  }
-
-  private async generateImage(prompt) {
-    const { data } = await this.client.images.generate({
-      model: 'dall-e-3',
-      prompt,
-      // amount of images, max 1 for dall-e-3
-      n: 1,
-      // 'standard' | 'hd'
-      quality: 'hd',
-      size: '1024x1024',
-      style: 'natural',
-    });
-    if (data) return new URL(data[0].url as string);
-  }
 }