diff --git a/apps/backend/asset/asset.model.d.ts b/apps/backend/asset/asset.model.d.ts index 29272b89b..b8ea71c47 100644 --- a/apps/backend/asset/asset.model.d.ts +++ b/apps/backend/asset/asset.model.d.ts @@ -1,10 +1,5 @@ import type { Model, ModelStatic } from 'sequelize'; -import type { - Asset as AssetAttributes, - FileAsset as FileAssetAttrs, - LinkAsset as LinkAssetAttrs, - MediaAsset as MediaAssetAttrs, -} from '@tailor-cms/interfaces/asset.ts'; +import type { Asset as AssetAttributes } from '@tailor-cms/interfaces/asset.ts'; export type { AssetAttributes }; export { @@ -17,10 +12,26 @@ export { Uploader, } from '@tailor-cms/interfaces/asset.ts'; -// Sequelize model variants of the interface discriminated unions -export type FileAsset = FileAssetAttrs & Model; -export type MediaAsset = MediaAssetAttrs & Model; -export type LinkAsset = LinkAssetAttrs & Model; +type AssetBase = AssetAttributes & Model; + +export interface FileAsset extends AssetBase { + type: 'IMAGE' | 'DOCUMENT' | 'OTHER'; + storageKey: string; + meta: FileAssetMeta; +} + +export interface MediaAsset extends AssetBase { + type: 'VIDEO' | 'AUDIO'; + storageKey: string; + meta: MediaAssetMeta; +} + +export interface LinkAsset extends AssetBase { + type: 'LINK'; + storageKey: null; + meta: LinkAssetMeta; +} + export type Asset = FileAsset | MediaAsset | LinkAsset; declare const Asset: ModelStatic; diff --git a/apps/backend/asset/asset.service.ts b/apps/backend/asset/asset.service.ts index 526cfb748..bd571e79e 100644 --- a/apps/backend/asset/asset.service.ts +++ b/apps/backend/asset/asset.service.ts @@ -56,10 +56,7 @@ const IS_NOT_VIDEO_LINK = { export const UPLOADER_INCLUDE = { model: User, as: 'uploader', - attributes: [ - 'id', 'email', 'firstName', 'lastName', - 'fullName', 'label', 'imgUrl', - ], + attributes: ['id', 'email', 'firstName', 'lastName', 'fullName', 'label', 'imgUrl'], }; /** Wraps an async fn so it logs warnings instead of throwing. */ diff --git a/apps/backend/asset/discovery/discovery.service.ts b/apps/backend/asset/discovery/discovery.service.ts index 54aaf27d7..05ebdff93 100644 --- a/apps/backend/asset/discovery/discovery.service.ts +++ b/apps/backend/asset/discovery/discovery.service.ts @@ -90,8 +90,8 @@ const strategies: Record = { serper.newsSearch(q, Math.ceil(n * 0.15) + DEDUP_BUFFER), ], [Image]: (q, n) => [ - unsplash.search(q, Math.ceil(n * 0.6)), - serper.imageSearch(q, Math.ceil(n * 0.4) + DEDUP_BUFFER), + unsplash.search(q, n), + serper.imageSearch(q, Math.ceil(n * 0.5) + DEDUP_BUFFER), ], [Video]: (q, n) => [serper.videoSearch(q, n + DEDUP_BUFFER)], [Pdf]: (q, n) => [serper.webSearch(`${q} filetype:pdf`, n + DEDUP_BUFFER)], diff --git a/apps/backend/asset/extraction/synthetic-content.ts b/apps/backend/asset/extraction/synthetic-content.ts index 15dbbbab3..82cc59d25 100644 --- a/apps/backend/asset/extraction/synthetic-content.ts +++ b/apps/backend/asset/extraction/synthetic-content.ts @@ -22,13 +22,22 @@ export function hasUsefulDescription(asset: Asset): boolean { return true; } -// Builds a markdown document from asset metadata and optional body text. -// Returns null when there's nothing meaningful to index (name-only). +// Builds a markdown document from asset metadata +// and optional body text (captions, page content, etc.). +// Asset ID and type are included so the AI can discover +// and reference relevant media via vector store search. export function buildSyntheticContent( asset: Asset, bodyText = '', ): string | null { + const meta = asset.meta as any; const parts: string[] = [`# ${asset.name}`]; - const meta = asset.meta as Record; + parts.push(`Asset ID: ${asset.id} | Type: ${asset.type}`); + if (meta?.contentType) { + parts[1] += ` | Content: ${meta.contentType}`; + } + if (meta?.provider) { + parts[1] += ` | Provider: ${meta.provider}`; + } // Attribution and provenance if (meta?.source?.author) parts.push(`Author: ${meta.source.author}`); if (meta?.siteName || meta?.domain) { @@ -43,8 +52,6 @@ export function buildSyntheticContent( if (meta?.analysis) parts.push(`Analysis: ${meta.analysis}`); if (meta?.contentSuggestion) parts.push(`Suggested use: ${meta.contentSuggestion}`); if (meta?.tags?.length) parts.push(`Tags: ${meta.tags.join(', ')}`); - // Full content (page text, captions, vision description) if (bodyText) parts.push(bodyText); - // Name-only content isn't worth indexing return parts.length > 1 ? parts.join('\n\n') : null; } diff --git a/apps/backend/asset/indexing/indexing.service.ts b/apps/backend/asset/indexing/indexing.service.ts index 634092031..d26197fb1 100644 --- a/apps/backend/asset/indexing/indexing.service.ts +++ b/apps/backend/asset/indexing/indexing.service.ts @@ -121,7 +121,16 @@ async function indexDocument(ctx: IndexingContext) { originalname: asset.name, mimetype: asset.meta.mimeType, }; - const result = await AIService.vectorStore!.upload([file], storeId); + const result = await AIService.vectorStore!.upload( + [file], storeId, + ); + // Companion metadata doc so AI can discover this + // document's asset ID via file_search + const meta = buildSyntheticContent(asset); + if (meta) { + indexSynthetic(storeId, meta, `${asset.id}-meta.md`) + .catch(() => {}); + } if (asset.meta.mimeType === mime.lookup('pdf')) { // Non-critical side effect - don't let image extraction fail the indexing extractAndSaveImages(asset, buffer).catch((err) => diff --git a/apps/backend/package.json b/apps/backend/package.json index d20eed279..9e2e155c3 100644 --- a/apps/backend/package.json +++ b/apps/backend/package.json @@ -121,6 +121,7 @@ }, "devDependencies": { "@types/bluebird": "^3.5.42", + "@types/common-tags": "^1.8.4", "@types/express": "^5.0.6", "@types/html-to-text": "^9.0.4", "@types/lodash": "^4.17.24", diff --git a/apps/backend/repository/repository.model.js b/apps/backend/repository/repository.model.js index af48624c0..bd105a143 100644 --- a/apps/backend/repository/repository.model.js +++ b/apps/backend/repository/repository.model.js @@ -1,4 +1,4 @@ -import { literal, Model, Op } from 'sequelize'; +import { Model, Op } from 'sequelize'; import first from 'lodash/first.js'; import intersection from 'lodash/intersection.js'; import map from 'lodash/map.js'; @@ -162,24 +162,18 @@ class Repository extends Model { return false; } - /** - * Atomically sets the AI vector store ID in the repository's - * data JSONB. Returns true if the value was written, false if - * another request already set it (concurrent indexing race). - */ async setVectorStoreId(storeId) { - const path = `{$$,ai,storeId}`; - const [count] = await Repository.update( - { data: literal(`jsonb_set(COALESCE(data,'{}'),'${path}','"${storeId}"')`) }, - { - where: { - id: this.id, - [Op.and]: literal(`data->'$$'->'ai'->'storeId' IS NULL`), - }, + const current = this.data || {}; + if (current.$$?.ai?.storeId) return false; + const merged = { + ...current, + $$: { + ...current.$$, + ai: { ...current.$$?.ai, storeId }, }, - ); - if (count > 0) await this.reload(); - return count > 0; + }; + await this.update({ data: merged }); + return true; } getVectorStoreId() { diff --git a/apps/backend/shared/ai/ai.service.ts b/apps/backend/shared/ai/ai.service.ts index e0bed5a5d..f062c5eb2 100644 --- a/apps/backend/shared/ai/ai.service.ts +++ b/apps/backend/shared/ai/ai.service.ts @@ -2,15 +2,15 @@ import type { AiContext, ImageDescription } from '@tailor-cms/interfaces/ai.ts'; import OpenAI from 'openai'; import { ai as aiConfig } from '#config'; -import { createLogger } from '#logger'; import { AiPrompt } from './lib/AiPrompt.ts'; +import { createAiLogger } from './logger.ts'; import { VectorStoreService } from './lib/VectorStoreService.ts'; import { PROMPT as IMAGE_PROMPT, Schema as IMAGE_SCHEMA, } from './schemas/ImageDescription.ts'; -const logger = createLogger('ai:service'); +const logger = createAiLogger('service'); class AiService { #openai; diff --git a/apps/backend/shared/ai/lib/AiPrompt.ts b/apps/backend/shared/ai/lib/AiPrompt.ts index 5db5f43b4..bf85c5960 100644 --- a/apps/backend/shared/ai/lib/AiPrompt.ts +++ b/apps/backend/shared/ai/lib/AiPrompt.ts @@ -1,18 +1,19 @@ -import type { AiContext, AiInput } from '@tailor-cms/interfaces/ai.ts'; +import type { + AiContext, + AiInput, +} from '@tailor-cms/interfaces/ai.ts'; import type { OpenAI } from 'openai'; -import { ContentElementType } from '@tailor-cms/content-element-collection/types.js'; -import StorageService from '../../storage/storage.service.js'; import getContentSchema from '../schemas/index.ts'; import RepositoryContext from './RepositoryContext.ts'; import { ai as aiConfig } from '#config'; -import { createLogger } from '#logger'; +import { createAiLogger, formatPrompt } from '../logger.ts'; -const logger = createLogger('ai:prompt'); +const logger = createAiLogger('prompt'); const systemPrompt = ` - Assistant is a bot desinged to help authors to create content for + Assistant is a bot designed to help authors create content for Courses, Q&A content, Knowledge base, etc. Rules: - Use the User rules to generate the content @@ -23,20 +24,20 @@ const documentPrompt = ` The user has provided source documents indexed in a vector store. Use the file_search tool to find relevant information from these documents. Base ALL generated content on information found in the documents. - Do not invent information not present in the documents.`; + Do not invent information not present in the documents. + If any assets are marked as PRIMARY SOURCE, they represent the core knowledge + base. Structure and model your content primarily after these sources. + Supplementary assets provide additional context but should not override + the core sources.`; export class AiPrompt { // OpenAI client private client: OpenAI; - // Context of the request - private context: AiContext; // Information about the repository, content location, topic, etc. private repositoryContext: RepositoryContext; - // User, assistant or system generated inputs + private requestContext: AiContext; private inputs: AiInput[]; - // Existing content, relevant for the context private content: string; - // Response private response: any; constructor(client: OpenAI, context: AiContext) { @@ -45,26 +46,46 @@ export class AiPrompt { this.content = context.content || ''; this.inputs = context.inputs; this.client = client; - this.context = context; + this.requestContext = context; + } + + get vectorStoreId() { + return this.repositoryContext.vectorStoreId; + } + + get context(): AiContext { + return { + ...this.requestContext, + assets: this.repositoryContext.assets, + repository: { + ...this.requestContext.repository, + vectorStoreId: this.vectorStoreId, + }, + }; } async execute() { try { + await this.repositoryContext.resolve(this.requestContext.repository); + const input = this.toOpenAiInput(); const params: any = { model: aiConfig.modelId, - input: this.toOpenAiInput(), + input, text: { format: this.format }, }; - // Add file_search tool when source documents are available - const { vectorStoreId } = this.context.repository; - if (vectorStoreId) { - params.tools = [ - { type: 'file_search', vector_store_ids: [vectorStoreId] }, - ]; + if (this.vectorStoreId) { + params.tools = [{ + type: 'file_search', + vector_store_ids: [this.vectorStoreId], + }]; } + logger.debug(`Final prompt:\n${formatPrompt(input)}`); const response = await this.client.responses.create(params); this.response = this.responseProcessor(response.output_text); - this.response = await this.applyImageTool(); + logger.info( + { schema: this.prompt.responseSchema, type: this.prompt.type }, + 'Generation complete', + ); return this.response; } catch (err) { logger.error(err, 'Generation failed'); @@ -99,7 +120,7 @@ export class AiPrompt { const processor = this.isCustomPrompt ? noop : getContentSchema(this.prompt.responseSchema)?.processResponse || noop; - return (val) => processor(JSON.parse(val)); + return (val: string) => processor(JSON.parse(val), this.context); } // TODO: Add option to control the size of the output @@ -108,7 +129,7 @@ export class AiPrompt { const res: OpenAI.Responses.ResponseInputItem[] = []; res.push({ role: 'developer', content: systemPrompt }); res.push({ role: 'developer', content: this.repositoryContext.toString() }); - if (this.context.repository.vectorStoreId) { + if (this.vectorStoreId) { res.push({ role: 'developer', content: documentPrompt }); } if (this.prevPrompt) res.push(this.processUserInput(this.prevPrompt)); @@ -135,41 +156,4 @@ export class AiPrompt { content: `${base} ${text}. ${responseSchemaDescription} ${target}`, }; } - - async applyImageTool() { - if ( - !this.prompt.useImageGenerationTool || - this.prompt.responseSchema !== 'HTML' - ) - return this.response; - // output needs to be sliced to avoid exceeding the max length - const userPrompt = ` - ${this.repositoryContext.toString()} - Generate appropriate image for the given topic and content: - ${JSON.stringify(this.response).slice(0, 1000)}`; - const imgUrl = await this.generateImage(userPrompt); - const imgInternalUrl = await StorageService.downloadToStorage(imgUrl); - const imageElement = { - type: ContentElementType.Image, - data: { - assets: { url: imgInternalUrl }, - }, - }; - const [firstElement, ...restElements] = this.response; - return [firstElement, imageElement, ...restElements]; - } - - private async generateImage(prompt) { - const { data } = await this.client.images.generate({ - model: 'dall-e-3', - prompt, - // amount of images, max 1 for dall-e-3 - n: 1, - // 'standard' | 'hd' - quality: 'hd', - size: '1024x1024', - style: 'natural', - }); - if (data) return new URL(data[0].url as string); - } } diff --git a/apps/backend/shared/ai/lib/RepositoryContext.ts b/apps/backend/shared/ai/lib/RepositoryContext.ts index eb015d090..7a1e46ce0 100644 --- a/apps/backend/shared/ai/lib/RepositoryContext.ts +++ b/apps/backend/shared/ai/lib/RepositoryContext.ts @@ -1,33 +1,110 @@ -import type { AiRepositoryContext } from '@tailor-cms/interfaces/ai.ts'; +import type { + AiRepositoryContext, + AssetReference, +} from '@tailor-cms/interfaces/ai.ts'; import { schema as schemaAPI } from '@tailor-cms/config'; +import pick from 'lodash/pick.js'; + +import db from '#shared/database/index.js'; +import { createAiLogger } from '../logger.ts'; + +const logger = createAiLogger('context'); export default class RepositoryContext { schemaId: string; name: string; description: string; - outlineLocation?: string; outlineActivityType?: string; containerType?: string; topic?: string; tags?: string[]; + vectorStoreId?: string; + assets: AssetReference[] = []; + // Resolved outline: ancestors + current activity + private outlineLocation: any[] = []; constructor(context: AiRepositoryContext) { this.schemaId = context.schemaId; this.name = context.name; this.description = context.description; - this.outlineLocation = context.outlineLocation; this.outlineActivityType = context.outlineActivityType; this.containerType = context.containerType; this.topic = context.topic; + // Fallback tags from frontend (e.g. outline generation before repo exists) this.tags = context.tags; + this.vectorStoreId = context.vectorStoreId; + } + + async resolve(context: AiRepositoryContext) { + const { repositoryId, activityId } = context; + if (!repositoryId) return; + await Promise.all([ + this.resolveOutlineContext(activityId), + this.resolveRepository(repositoryId), + this.resolveAssets(repositoryId), + ]); + } + + private async resolveOutlineContext(activityId?: number) { + if (!activityId) return; + try { + const { Activity } = db; + const activity = await Activity.findByPk(activityId); + if (!activity) return; + const ancestors = await activity.predecessors(); + this.outlineLocation = [...ancestors, activity]; + } catch (err) { + logger.warn(err, 'Failed to resolve outline context'); + } + } + + private async resolveRepository(repositoryId: number) { + try { + const { Repository } = db; + const repo = await Repository.findByPk(repositoryId); + if (!repo) return; + // Vector store ID + if (!this.vectorStoreId) { + this.vectorStoreId = repo.getVectorStoreId() || undefined; + } + // Tags from AI meta + const aiMeta = repo.data?.$$?.ai; + if (!aiMeta) return; + const tags = [ + ...(aiMeta.topicTags || []), + ...(aiMeta.styleTags || []), + ].filter(Boolean); + if (tags.length) this.tags = tags; + } catch (err) { + logger.warn(err, 'Failed to resolve repository'); + } + } + + private async resolveAssets(repositoryId: number) { + try { + const { Asset } = db; + const rows = await Asset.findAll({ where: { repositoryId } }); + await Asset.resolvePublicUrls(rows); + this.assets = rows.map((a: any) => ({ + ...pick(a, ['id', 'name', 'type', 'storageKey', 'publicUrl']), + meta: { + ...pick(a.meta || {}, ['contentType', 'url', 'description', 'tags']), + isCoreSource: !!a.meta?.isCoreSource, + }, + })); + logger.info( + { repositoryId, count: this.assets.length }, + 'Loaded repository assets', + ); + } catch (err) { + logger.warn(err, 'Failed to load assets'); + } } get schema() { const { schemaId } = this; if (!schemaId) return null; - const schema = schemaAPI.getSchema(schemaId); - if (!schema) return null; - return schema; + return schemaAPI.getSchema(schemaId) || null; } get baseContext() { @@ -39,9 +116,13 @@ export default class RepositoryContext { } get locationContext() { - const { outlineLocation } = this; - if (!outlineLocation) return ''; - return `The content is located in "${outlineLocation}".`; + const location = this.outlineLocation + .slice(0, -1) + .map((a: any) => a.data?.name) + .filter(Boolean) + .join(', '); + if (!location) return ''; + return `The content is located in "${location}".`; } get contentContainerDescription() { @@ -66,12 +147,22 @@ export default class RepositoryContext { return `${base} ${tags.join(', ')}.`; } + get instructionContext() { + const instructions = this.outlineLocation + .map((a: any) => a.data?.aiPrompt?.trim()) + .filter(Boolean) + .join('\n'); + if (!instructions) return ''; + return `Author instructions:\n${instructions}`; + } + toString() { return ` ${this.baseContext} ${this.topicContext} ${this.locationContext} ${this.contentContainerDescription} - ${this.tagContext}`.trim(); + ${this.tagContext} + ${this.instructionContext}`.trim(); } } diff --git a/apps/backend/shared/ai/lib/VectorStoreService.ts b/apps/backend/shared/ai/lib/VectorStoreService.ts index fdc9c45dc..173247052 100644 --- a/apps/backend/shared/ai/lib/VectorStoreService.ts +++ b/apps/backend/shared/ai/lib/VectorStoreService.ts @@ -2,9 +2,9 @@ import type OpenAI from 'openai'; import { toFile } from 'openai'; import { ai as aiConfig } from '#config'; -import { createLogger } from '#logger'; +import { createAiLogger } from '../logger.ts'; -const logger = createLogger('ai:vector-store'); +const logger = createAiLogger('vector-store'); const { name: STORE_NAME, expiresAfter: STORE_EXPIRY } = aiConfig.vectorStore; interface FileStatus { diff --git a/apps/backend/shared/ai/logger.ts b/apps/backend/shared/ai/logger.ts new file mode 100644 index 000000000..92ada3ac9 --- /dev/null +++ b/apps/backend/shared/ai/logger.ts @@ -0,0 +1,18 @@ +import { createLogger } from '#logger'; + +const SEPARATOR = '-'.repeat(60); + +export const formatPrompt = (input: any[]) => { + const sections = input.map((item: any) => { + const role = (item.role || 'unknown').toUpperCase(); + const content = Array.isArray(item.content) + ? item.content + .map((p: any) => (p.type === 'text' ? p.text : `[${p.type}]`)) + .join('\n') + : item.content || ''; + return `${SEPARATOR}\n[${role}]\n${SEPARATOR}\n${content}`; + }); + return `\n${sections.join('\n\n')}\n${SEPARATOR}`; +}; + +export const createAiLogger = (name: string) => createLogger(`ai:${name}`); diff --git a/apps/backend/shared/ai/schemas/CcStructuredContent.ts b/apps/backend/shared/ai/schemas/CcStructuredContent.ts deleted file mode 100644 index 310a69572..000000000 --- a/apps/backend/shared/ai/schemas/CcStructuredContent.ts +++ /dev/null @@ -1,296 +0,0 @@ -import type { AiContext } from '@tailor-cms/interfaces/ai.ts'; -import { - getSchema as getMetaInputSchema, -} from '@tailor-cms/meta-element-collection/schema.js'; -import { schema as schemaAPI } from '@tailor-cms/config'; - -import type { AiResponseSpec, OpenAISchema } from './interfaces.ts'; -import { HTML_TYPE } from './CeHtml.ts'; -import elementRegistry from '../../content-plugins/elementRegistry.js'; - -// Resolve AI config (Schema + processResponse) for an element type. -const getAiSpec = (type: string) => elementRegistry.getAiConfig(type); - -// Filter to element types with AI support, fall back to HTML. -const resolveSupportedTypes = (types: string[]): string[] => { - const supported = types.filter((t) => getAiSpec(t)?.Schema); - return supported.length ? supported : [HTML_TYPE]; -}; - -interface MetaField { - key: string; - label: string; - // JSON Schema inferred from meta-input manifest at runtime. - // e.g. { type: 'string' }, { type: 'array', items: { type: 'number' } } - // null when meta-input has no schema (e.g. FILE) - excluded from AI output. - schema: { type: string; items?: { type: string } } | null; - options?: { value: string | number; label: string }[]; -} - -interface SubcontainerConfig { - label: string; - metaInputs: MetaField[]; - elementTypes: string[]; -} -type SubcontainerConfigs = Record; - -interface ParsedConfig { - subcontainers: SubcontainerConfigs; - ai?: { - definition?: string; - outputRules?: { prompt?: string }; - }; -} - -const obj = (properties: any, required: string[]) => ({ - type: 'object' as const, - properties, - ...(required.length && { required }), - additionalProperties: false, -}); - -// Server packages export raw content schemas (e.g. { content: 'string' } for HTML). -// Wrap into full element format with type discriminator and data envelope. -const buildElementSchema = (type: string) => { - const contentSchema = getAiSpec(type).Schema.schema; - return { - type: 'object' as const, - properties: { - type: { enum: [type] }, - ...contentSchema.properties, - }, - required: ['type', ...(contentSchema.required || [])], - additionalProperties: false, - }; -}; - -const getElementsSchema = (types: string[]) => { - const resolved = resolveSupportedTypes(types); - const schemas = resolved.map(buildElementSchema); - const items = schemas.length === 1 ? schemas[0] : { anyOf: schemas }; - return { type: 'array', items }; -}; - -// Processed contentElementConfig format: -// [{ name: 'Group', items: [{ id: 'TYPE' }] }] -const getElementTypeIds = (config?: any[]): string[] => - config?.flatMap((group: any) => - (group.items || []).map((it: any) => it.id || it), - ) ?? []; - -// Subcontainer meta can be a static array or a factory fn. -// e.g. meta: () => [{ key: 'title', type: 'TEXT_FIELD' }] -const getMetaDefinitions = (val: any): any[] => - typeof val.meta === 'function' - ? val.meta() - : val.meta || []; - -// Map raw meta definitions to MetaField with resolved schema. -// Options come from m.options (select) or m.items (radio). -const toMetaFields = (meta: any[]): MetaField[] => - meta.map((m: any) => ({ - key: m.key, - label: m.label, - schema: getMetaInputSchema(m.type, m), - ...((m.options || m.items) && { - options: m.options || m.items, - }), - })); - -// Parse container schema config into per-subcontainer configs. -// Resolves element types and meta field schemas for each -// subcontainer type defined in the container config. -const getConfigs = (context: AiContext): ParsedConfig => { - const empty: ParsedConfig = { subcontainers: {} }; - const { outlineActivityType, containerType } = - context.repository; - if (!outlineActivityType || !containerType) return empty; - const containers = schemaAPI.getSupportedContainers( - outlineActivityType, - ); - const container = containers.find( - (c: any) => c.type === containerType, - ); - if (!container?.config) return empty; - // Container-level element types as default fallback - const defaultElementTypes = getElementTypeIds( - container.contentElementConfig, - ); - const subcontainers: SubcontainerConfigs = {}; - for (const [type, val] of Object.entries( - container.config as Record, - )) { - // Subcontainer config overrides container-level - const elementTypes = val.contentElementConfig - ? getElementTypeIds(val.contentElementConfig) - : defaultElementTypes; - subcontainers[type] = { - label: val.label || type, - elementTypes, - metaInputs: toMetaFields(getMetaDefinitions(val)), - }; - } - return { subcontainers, ai: container.ai }; -}; - -// Build JSON schema for a single subcontainer type: -// discriminated by type enum, with per-type elements and data. -const buildSubcontainerSchema = ( - type: string, - config: SubcontainerConfig, -) => { - const { metaInputs, elementTypes } = config; - const props: Record = { - type: { enum: [type] }, - elements: getElementsSchema(elementTypes), - }; - const required = ['type', 'elements']; - const dataProps: Record = {}; - const dataRequired: string[] = []; - for (const field of metaInputs) { - if (!field.schema) continue; - const values = field.options?.map((o) => o.value); - dataProps[field.key] = values?.length - ? { ...field.schema, enum: values } - : field.schema; - dataRequired.push(field.key); - } - if (Object.keys(dataProps).length) { - props.data = obj(dataProps, dataRequired); - required.push('data'); - } - return obj(props, required); -}; - -// Build OpenAI structured output schema from container config. -// Each subcontainer type becomes a discriminated union variant -// with its own allowed element types and metadata fields. -export const Schema = (context: AiContext): OpenAISchema => { - const { subcontainers } = getConfigs(context); - // Default to generic section when no config is defined - const entries = Object.entries(subcontainers); - if (!entries.length) { - entries.push(['SECTION', { - label: 'Section', metaInputs: [], elementTypes: [], - }]); - } - // Build per-subcontainer schema with type discriminator - const schemas = entries.map(([type, config]) => - buildSubcontainerSchema(type, config), - ); - const subcontainerSchema = schemas.length === 1 - ? schemas[0] - : { anyOf: schemas }; - return { - type: 'json_schema', - name: 'cc_structured_content', - schema: obj( - { subcontainers: { type: 'array', items: subcontainerSchema } }, - ['subcontainers'], - ), - }; -}; - -const describeField = ({ key, label, options }: MetaField): string => { - const base = `"${key}" (${label})`; - const opts = options?.map((o) => o.value).join(', '); - return opts ? `${base} [options: ${opts}]` : base; -}; - -const describeElementTypes = (types: string[]): string => - resolveSupportedTypes(types) - .map((type) => { - const prompt = getAiSpec(type)?.getPrompt?.() || ''; - // Extract element description from server package prompt. - // e.g. "Generate a accordion content element as an object..." - // -> "a accordion content element" - const match = prompt.match(/generate\s+(.+?)\s+as\s+an/i); - return ` - "${type}": ${match?.[1] || type}`; - }) - .join('\n'); - -const describeSubcontainerTypes = (configs: SubcontainerConfigs): string => { - const entries = Object.entries(configs); - if (!entries.length) return ' - Type "SECTION" (Section)'; - return entries - .map(([type, { label, metaInputs = [] }]) => { - const fields = metaInputs.map(describeField).join(', '); - const suffix = fields ? `: metadata fields: ${fields}` : ''; - return ` - Type "${type}" (${label})${suffix}`; - }) - .join('\n'); -}; - -// Build prompt with available element types, subcontainer types, -// metadata fields, and container-level AI instructions. -export const getPrompt = (context: AiContext) => { - const { subcontainers, ai } = getConfigs(context); - // Collect all unique element types across subcontainers - const allElementTypes = [ - ...new Set(Object.values(subcontainers).flatMap((c) => c.elementTypes)), - ]; - const guidelines: string[] = [ - '- Fill in ALL metadata fields with values relevant to each subcontainer\'s content', - '- Each subcontainer should focus on a distinct topic or aspect', - '- Choose the best element type for each piece of content', - '- Skip media elements (images, videos, audio, files)', - '- Include at most one question element per subcontainer', - ]; - // Container ai.definition describes the content purpose - // e.g. "Learning Bit content is organized into sections" - if (ai?.definition) { - guidelines.push(`- Context: ${ai.definition}`); - } - if (context.repository.vectorStoreId) { - guidelines.push( - '- Base ALL content on the provided source documents', - '- Reference specific information, data, and examples from the documents', - '- Do not invent information not present in the documents', - ); - } - if (ai?.outputRules?.prompt) { - guidelines.push(ai.outputRules.prompt.trim()); - } - return ` - Response should be a JSON object with a "subcontainers" array. - Each subcontainer has: - - "type": one of the available subcontainer types - - "data": metadata object with the described fields filled in - - "elements": array of content elements (format defined by the schema) - - Available element types: - ${describeElementTypes(allElementTypes)} - - Available subcontainer types: - ${describeSubcontainerTypes(subcontainers)} - - Guidelines: - ${guidelines.join('\n ')}`; -}; - -// AI returns flat elements matching the content schema (e.g. { type, content } for HTML). -// Server processResponse transforms raw content into the element's data format. -const processElement = (el: any) => { - const { type, ...rawContent } = el; - const spec = getAiSpec(type); - const data = spec?.processResponse - ? spec.processResponse(rawContent) - : rawContent; - return { type, data }; -}; - -const processResponse = (data: any = {}) => { - const subcontainers = data?.subcontainers || []; - return subcontainers.map((sc: any) => ({ - ...sc, - elements: (sc.elements || []).map(processElement), - })); -}; - -const spec: AiResponseSpec = { - getPrompt, - Schema, - processResponse, -}; - -export default spec; diff --git a/apps/backend/shared/ai/schemas/CcStructuredContent/config.ts b/apps/backend/shared/ai/schemas/CcStructuredContent/config.ts new file mode 100644 index 000000000..8d1c88d2a --- /dev/null +++ b/apps/backend/shared/ai/schemas/CcStructuredContent/config.ts @@ -0,0 +1,58 @@ +// Container config resolution. +// Parses schema config into per-subcontainer configs +// with element types and metadata field schemas. +import { + getSchema as getMetaInputSchema, +} from '@tailor-cms/meta-element-collection/schema.js'; +import { schema as schemaAPI } from '@tailor-cms/config'; +import type { AiContext } from '@tailor-cms/interfaces/ai.ts'; + +import type { + MetaField, + ParsedConfig, + SubcontainerConfigs, +} from './types.ts'; + +const { getSupportedElementTypes } = schemaAPI; + +const getMetaDefinitions = (val: any): any[] => + typeof val.meta === 'function' + ? val.meta() + : val.meta || []; + +const toMetaFields = (meta: any[]): MetaField[] => + meta.map((m: any) => ({ + key: m.key, + label: m.label, + schema: getMetaInputSchema(m.type, m), + ...((m.options || m.items) && { + options: m.options || m.items, + }), + })); + +export const getConfigs = (context: AiContext): ParsedConfig => { + const empty: ParsedConfig = { subcontainers: {} }; + const { repository } = context; + const { outlineActivityType, containerType } = repository; + if (!outlineActivityType || !containerType) return empty; + const containers = schemaAPI.getSupportedContainers( + outlineActivityType, + ); + const container = containers.find((c: any) => c.type === containerType); + if (!container?.config) return empty; + const defaultTypes = getSupportedElementTypes(container.contentElementConfig); + const subcontainers: SubcontainerConfigs = {}; + for (const [type, val] of Object.entries( + container.config as Record, + )) { + const elementTypes = val.contentElementConfig + ? getSupportedElementTypes(val.contentElementConfig) + : defaultTypes; + subcontainers[type] = { + label: val.label || type, + elementTypes, + metaInputs: toMetaFields(getMetaDefinitions(val)), + }; + } + return { subcontainers, ai: container.ai }; +}; diff --git a/apps/backend/shared/ai/schemas/CcStructuredContent/index.ts b/apps/backend/shared/ai/schemas/CcStructuredContent/index.ts new file mode 100644 index 000000000..52a561bf2 --- /dev/null +++ b/apps/backend/shared/ai/schemas/CcStructuredContent/index.ts @@ -0,0 +1,13 @@ +import type { AiResponseSpec } from '../interfaces.ts'; +import { Schema } from './schema.ts'; +import { getPrompt } from './prompt.ts'; +import { processResponse } from './response.ts'; + +const spec: AiResponseSpec = { + getPrompt, + Schema, + processResponse, +}; + +export default spec; +export { Schema, getPrompt, processResponse }; diff --git a/apps/backend/shared/ai/schemas/CcStructuredContent/media.ts b/apps/backend/shared/ai/schemas/CcStructuredContent/media.ts new file mode 100644 index 000000000..0808dc08e --- /dev/null +++ b/apps/backend/shared/ai/schemas/CcStructuredContent/media.ts @@ -0,0 +1,148 @@ +// Media element schemas and processing. +// IMAGE, VIDEO, and EMBED packages have no AI specs - +// they're simple media containers. Schemas here use +// assetId to map vector store asset references to +// elements. processMediaElement then resolves assetId +// → native element data (storage:// URLs, alt text, +// embed transforms). +import type { AssetReference } from '@tailor-cms/interfaces/ai.ts'; +import { AssetType, LinkContentType } from '@tailor-cms/interfaces/asset.ts'; +import { ContentElementType } from '@tailor-cms/content-element-collection/types.js'; +import { oneLine } from 'common-tags'; +import { toEmbedUrl } from '@tailor-cms/common/asset'; + +import { createAiLogger } from '../../logger.ts'; + +const logger = createAiLogger('cc-structured-content'); + +const obj = (properties: any, required: string[]) => ({ + type: 'object' as const, + properties, + ...(required.length && { required }), + additionalProperties: false, +}); + +// JSON schemas for OpenAI structured output. +// assetId maps to vector store catalog entries; +// processMediaElement resolves to native element data. +export const MEDIA_SCHEMAS: Record = { + [ContentElementType.Image]: obj( + { + type: { enum: [ContentElementType.Image] }, + assetId: { type: 'integer' }, + alt: { type: 'string' }, + }, + ['type', 'assetId', 'alt'], + ), + [ContentElementType.Video]: obj( + { + type: { enum: [ContentElementType.Video] }, + assetId: { type: 'integer' }, + }, + ['type', 'assetId'], + ), + [ContentElementType.Embed]: obj( + { + type: { enum: [ContentElementType.Embed] }, + assetId: { type: 'integer' }, + }, + ['type', 'assetId'], + ), +}; + +export const MEDIA_DESCRIPTIONS: Record = { + [ContentElementType.Image]: oneLine` + a standalone image element for photos, diagrams. + NEVER use tags inside HTML.`, + [ContentElementType.Video]: oneLine` + a video player for uploaded video files. + Only for assets marked "→ use as VIDEO".`, + [ContentElementType.Embed]: oneLine` + an embedded resource (video, interactive content, + web page). Only for assets marked "→ use as EMBED".`, +}; + +// Uses asset.meta.contentType set by detectLinkProvider +// at creation — no URL parsing needed here. +export const isVideoLink = (a: AssetReference) => + a.type === AssetType.Link && a.meta?.contentType === LinkContentType.Video; + +// TODO: Figure out MUX video support +export const isVideoFile = (a: AssetReference) => + a.type === AssetType.Video || + (a.meta?.contentType === LinkContentType.Video && !!a.storageKey); + +// Resolve what element type an asset maps to. +// Returns the element type and a display label. +export const resolveAssetElementType = ( + asset: AssetReference, +): { elementType: string; label: string } | null => { + if (asset.type === AssetType.Image) { + return { + elementType: ContentElementType.Image, + label: 'image', + }; + } + if (isVideoFile(asset)) { + return { + elementType: ContentElementType.Video, + label: 'video', + }; + } + if (isVideoLink(asset)) { + return { + elementType: ContentElementType.Embed, + label: 'video link', + }; + } + return null; +}; + +// Resolve assetId to its reference from context +const resolveAsset = (assetId: number, assets: AssetReference[]) => { + const asset = assets.find((a) => a.id === assetId); + if (!asset) logger.warn({ assetId }, 'Asset not found'); + return asset || null; +}; + +// Transform media AI output into element data. +// Uses storage:// URLs so Tailor resolves signed URLs. +export const processMediaElement = (el: any, assets: AssetReference[]) => { + const asset = resolveAsset(el.assetId, assets); + if (!asset) return null; + if (el.type === ContentElementType.Image) { + const url = asset.storageKey + ? `storage://${asset.storageKey}` + : asset.publicUrl || asset.meta?.url || ''; + const isInternal = !!asset.storageKey; + return { + type: ContentElementType.Image, + data: { + url: isInternal ? '' : url, + alt: el.alt || asset.meta?.description || '', + assets: isInternal ? { url } : {}, + }, + }; + } + if (el.type === ContentElementType.Video) { + const url = asset.storageKey + ? `storage://${asset.storageKey}` + : asset.publicUrl || asset.meta?.url || ''; + const isInternal = !!asset.storageKey; + return { + type: ContentElementType.Video, + data: { + url: isInternal ? '' : url, + assets: isInternal ? { url } : {}, + }, + }; + } + if (el.type === ContentElementType.Embed) { + const url = asset.meta?.url || asset.publicUrl || ''; + return { + type: ContentElementType.Embed, + data: { url: toEmbedUrl(url) || url, height: 400 }, + }; + } + return el; +}; diff --git a/apps/backend/shared/ai/schemas/CcStructuredContent/prompt.ts b/apps/backend/shared/ai/schemas/CcStructuredContent/prompt.ts new file mode 100644 index 000000000..3f4f797ec --- /dev/null +++ b/apps/backend/shared/ai/schemas/CcStructuredContent/prompt.ts @@ -0,0 +1,300 @@ +// Prompt builder for structured content generation. +// Assembles element descriptions, subcontainer types, +// guidelines, and asset catalog into the system prompt. +import type { AiContext, AssetReference } from '@tailor-cms/interfaces/ai.ts'; + +import elementRegistry from '../../../content-plugins/elementRegistry.js'; +import { getConfigs } from './config.ts'; +import { + MEDIA_DESCRIPTIONS, + isVideoFile, + isVideoLink, + resolveAssetElementType, +} from './media.ts'; +import { getAiSpec, resolveSupportedTypes } from './schema.ts'; +import type { MetaField, ParsedConfig, SubcontainerConfigs } from './types.ts'; + +const describeField = ({ key, label, options }: MetaField): string => { + const base = `"${key}" (${label})`; + const opts = options?.map((o) => o.value).join(', '); + return opts ? `${base} [options: ${opts}]` : base; +}; + +// Extract a human-readable description from an element's +// getPrompt(). Convention: prompts start with +// "Generate a as an object...". +// Falls back to the raw type ID if no prompt or no match. +const extractElementDescription = (type: string): string => { + const spec = getAiSpec(type); + if (!spec?.getPrompt) return type; + const prompt = spec.getPrompt(); + if (!prompt) return type; + // "Generate a accordion content element as an" → "a accordion...." + const match = prompt.match(/generate\s+(.+?)\s+as\s+an/i); + return match?.[1] || type; +}; + +// Build element type descriptions for the prompt. +// When hasAssets is true, IMAGE and EMBED types are +// appended so the AI knows it can reference media. +const describeElementTypes = ( + types: string[], + hasAssets = false, +): string => { + const resolved = resolveSupportedTypes(types); + const lines = resolved.map((type) => { + const desc = extractElementDescription(type); + return ` - "${type}": ${desc}`; + }); + if (hasAssets) { + for (const [type, desc] of Object.entries(MEDIA_DESCRIPTIONS)) { + if (!resolved.includes(type)) { + lines.push(` - "${type}": ${desc}`); + } + } + } + return lines.join('\n'); +}; + +const describeSubcontainerTypes = (configs: SubcontainerConfigs): string => { + const entries = Object.entries(configs); + if (!entries.length) { + return ' - Type "SECTION" (Section)'; + } + return entries + .map(([type, { label, metaInputs = [] }]) => { + const fields = metaInputs.map(describeField).join(', '); + const suffix = fields ? `: metadata fields: ${fields}` : ''; + return ` - Type "${type}" (${label})${suffix}`; + }) + .join('\n'); +}; + +// Build asset catalog for the prompt. +// Includes ALL usable assets — vector store file_search +// handles relevance, this just lists valid IDs for the AI. +const buildAssetCatalog = (assets: AssetReference[]): string => { + const usable = assets.filter((it) => it.publicUrl || it.meta?.url || it.storageKey); + if (!usable.length) return ''; + const lines = usable.map((it) => { + const media = resolveAssetElementType(it); + const hint = media ? ` → use as ${media.elementType}` : ''; + const label = media?.label || it.type; + return ` - ID:${it.id} [${label}] "${it.name}"${hint}`; + }); + return ['', 'Assets available (reference by assetId):', ...lines].join( + '\n ', + ); +}; + +const buildGuidelines = ( + context: AiContext, + ai: ParsedConfig['ai'], + hasAssets: boolean, + elementTypes: string[], +): string[] => { + const hasQuestions = elementTypes.some( + (t) => elementRegistry.isQuestion(t), + ); + const guidelines = [ + '- Fill in ALL metadata fields', + '- Each subcontainer: distinct topic or aspect', + ]; + // Perspective and depth + guidelines.push( + '- Write from an educator/teacher perspective:', + ' clear explanations, progressive complexity,', + ' practical examples, learning objectives', + '- Each subcontainer must thoroughly cover its', + ' topic — substantive, not superficial', + '- Structure content for effective learning:', + ' introduce concepts, explain, illustrate, assess', + ); + // HTML element formatting + guidelines.push( + '- HTML elements: use text-body-2 mb-5 on

,', + ' text-h3 mb-7 on headings', + '- Use

    /
      ,
      , for variety', + '- Accent important sections with CSS classes:', + ' "ce-highlight" for key takeaways,', + ' "ce-callout" for tips/warnings,', + ' "ce-example" for worked examples.', + ' Add minimal inline style as default fallback', + ' (e.g. border-left, background) — presentation', + ' layer can override these classes', + '- Each HTML element: focused content block,', + ' 300-600 words per element', + '- Mix element types coherently: text for concepts,', + ' questions to reinforce learning, media to', + ' illustrate — each element should serve a', + ' pedagogical purpose, not just variety for its own sake', + ); + // Question element guidance + if (hasQuestions) { + guidelines.push( + '- Place a question after teaching a concept —', + ' it should check understanding of what was', + ' just explained, not test random knowledge', + '- Max one question per subcontainer', + '- Pick the question type best suited to the', + ' concept being assessed (e.g. true/false for', + ' facts, multiple choice for distinctions)', + '- Write clear, unambiguous answer options', + '- Include plausible distractors in choices', + ); + } + if (hasAssets) { + const hasVideos = context.assets?.some( + (a) => isVideoLink(a) || isVideoFile(a), + ); + guidelines.push( + '- Use assets as SEPARATE elements', + '- Reference assets by their assetId number', + '- Images: IMAGE element with assetId,', + ' NEVER in HTML', + '- Uploaded videos: VIDEO element with assetId.', + ' Only for assets marked "→ use as VIDEO"', + '- Video links (YouTube, Vimeo): EMBED element.', + ' Only for assets marked "→ use as EMBED"', + '- NEVER use ,