diff --git a/packages/database/features/addContent.feature b/packages/database/features/addContent.feature index 9148b6328..35e8b237a 100644 --- a/packages/database/features/addContent.feature +++ b/packages/database/features/addContent.feature @@ -133,3 +133,52 @@ Feature: Content access And a user logged in space s1 should see 3 Content in the database And a user logged in space s1 should see 1 ContentEmbedding_openai_text_embedding_3_small_1536 in the database And a user logged in space s1 should see 1 Document in the database + + Scenario: Full content representations with different content types coexist + When user user1 upserts these documents to space s1: + """json + [ + { + "source_local_id": "page1_uid", + "created": "2000/01/01", + "last_modified": "2001/01/02", + "author_local_id": "user1", + "content_type": "text/markdown" + } + ] + """ + And user user1 upserts this content to space s1: + """json + [ + { + "author_local_id": "user1", + "document_local_id": "page1_uid", + "source_local_id": "page1_uid", + "variant": "full", + "scale": "document", + "created": "2000/01/01", + "last_modified": "2001/01/02", + "text": "# Markdown", + "content_type": "text/markdown" + } + ] + """ + And user user1 upserts this content to space s1: + """json + [ + { + "author_local_id": "user1", + "document_local_id": "page1_uid", + "source_local_id": "page1_uid", + "variant": "full", + "scale": "document", + "created": "2000/01/01", + "last_modified": "2001/01/02", + "text": "{\"type\":\"root\",\"children\":[]}", + "content_type": "application/vnd.discourse-graph.atjson+json; version=1" + } + ] + """ + Then a user logged in space s1 should see 2 Content in the database + And a user logged in space s1 should see 1 content rows with variant "full" and content type "text/markdown" + And a user logged in space s1 should see 1 content rows with variant "full" and content type "application/vnd.discourse-graph.atjson+json; version=1" diff --git a/packages/database/features/step-definitions/stepdefs.ts b/packages/database/features/step-definitions/stepdefs.ts index 676ff52aa..4f0f087e8 100644 --- a/packages/database/features/step-definitions/stepdefs.ts +++ b/packages/database/features/step-definitions/stepdefs.ts @@ -21,6 +21,7 @@ import { } from "@repo/database/lib/contextFunctions"; type Platform = Enums<"Platform">; +type ContentVariant = Enums<"ContentVariant">; type TableName = keyof Database["public"]["Tables"]; type LocalRefsType = Record; const PLATFORMS: readonly Platform[] = Constants.public.Enums.Platform; @@ -307,6 +308,30 @@ Then( }, ); +/* eslint-disable max-params -- Cucumber inspects function.length for step arity. */ +Then( + "a user logged in space {word} should see {int} content rows with variant {string} and content type {string}", + async ( + spaceName: string, + expectedCount: number, + variant: ContentVariant, + contentType: string, + ) => { + const localRefs = (world.localRefs || {}) as LocalRefsType; + const spaceId = localRefs[spaceName]; + if (typeof spaceId !== "number") assert.fail("spaceId not a number"); + const client = await getLoggedinDatabase(spaceId); + const response = await client + .from("my_contents") + .select("*", { count: "exact", head: true }) + .eq("variant", variant) + .eq("content_type", contentType); + assert.equal(response.error, null); + assert.equal(response.count, expectedCount); + }, +); +/* eslint-enable max-params */ + // invoke the upsert_accounts_in_space function, expects json Given( "user {word} upserts these accounts to space {word}:", diff --git a/packages/database/src/dbTypes.ts b/packages/database/src/dbTypes.ts index eff6b23a5..a94c55b25 100644 --- a/packages/database/src/dbTypes.ts +++ b/packages/database/src/dbTypes.ts @@ -587,6 +587,7 @@ export type Database = { } FileReference: { Row: { + content_type: string created: string filehash: string filepath: string @@ -596,6 +597,7 @@ export type Database = { variant: Database["public"]["Enums"]["ContentVariant"] | null } Insert: { + content_type?: string created: string filehash: string filepath: string @@ -605,6 +607,7 @@ export type Database = { variant?: Database["public"]["Enums"]["ContentVariant"] | null } Update: { + content_type?: string created?: string filehash?: string filepath?: string @@ -616,24 +619,39 @@ export type Database = { Relationships: [ { foreignKeyName: "FileReference_content_fkey" - columns: ["space_id", "source_local_id", "variant"] + columns: ["space_id", "source_local_id", "variant", "content_type"] isOneToOne: false referencedRelation: "Content" - referencedColumns: ["space_id", "source_local_id", "variant"] + referencedColumns: [ + "space_id", + "source_local_id", + "variant", + "content_type", + ] }, { foreignKeyName: "FileReference_content_fkey" - columns: ["space_id", "source_local_id", "variant"] + columns: ["space_id", "source_local_id", "variant", "content_type"] isOneToOne: false referencedRelation: "my_contents" - referencedColumns: ["space_id", "source_local_id", "variant"] + referencedColumns: [ + "space_id", + "source_local_id", + "variant", + "content_type", + ] }, { foreignKeyName: "FileReference_content_fkey" - columns: ["space_id", "source_local_id", "variant"] + columns: ["space_id", "source_local_id", "variant", "content_type"] isOneToOne: false referencedRelation: "my_contents_with_embedding_openai_text_embedding_3_small_1536" - referencedColumns: ["space_id", "source_local_id", "variant"] + referencedColumns: [ + "space_id", + "source_local_id", + "variant", + "content_type", + ] }, ] } @@ -1164,6 +1182,7 @@ export type Database = { text: string | null variant: Database["public"]["Enums"]["ContentVariant"] | null vector: string | null + content_type: string | null } Relationships: [ { diff --git a/packages/database/supabase/migrations/20260704120000_update_content_uniqueness.sql b/packages/database/supabase/migrations/20260704120000_update_content_uniqueness.sql new file mode 100644 index 000000000..b459a6f90 --- /dev/null +++ b/packages/database/supabase/migrations/20260704120000_update_content_uniqueness.sql @@ -0,0 +1,239 @@ +ALTER TABLE ONLY public."FileReference" +DROP CONSTRAINT IF EXISTS "FileReference_content_fkey"; + +ALTER TABLE public."FileReference" +ADD COLUMN IF NOT EXISTS content_type character varying NOT NULL DEFAULT 'text/obsidian+markdown'; + +UPDATE public."FileReference" AS fr +SET content_type = ct.content_type +FROM public."Content" AS ct +WHERE fr.space_id = ct.space_id + AND fr.source_local_id = ct.source_local_id + AND fr.variant = ct.variant; + +DROP INDEX IF EXISTS public.content_space_local_id_variant_idx; + +CREATE UNIQUE INDEX IF NOT EXISTS content_space_local_id_variant_content_type_idx ON public."Content" USING btree ( + space_id, source_local_id, variant, content_type +) NULLS DISTINCT; + +ALTER TABLE ONLY public."FileReference" +ADD CONSTRAINT "FileReference_content_fkey" FOREIGN KEY ( + space_id, source_local_id, variant, content_type +) REFERENCES public."Content" (space_id, source_local_id, variant, content_type) ON DELETE CASCADE; + +CREATE OR REPLACE FUNCTION public._local_content_to_db_content(data public.content_local_input) +RETURNS public."Content" STABLE +SET search_path = '' +LANGUAGE plpgsql AS $$ +DECLARE + content public."Content"%ROWTYPE; + reference_content JSONB := jsonb_build_object(); + key varchar; + value JSONB; + ref_single_val BIGINT; + ref_array_val BIGINT[]; +BEGIN + content := jsonb_populate_record(NULL::public."Content", to_jsonb(data)); + IF data.document_local_id IS NOT NULL THEN + SELECT id FROM public."Document" + WHERE source_local_id = data.document_local_id INTO content.document_id; + END IF; + IF data.creator_local_id IS NOT NULL THEN + SELECT id FROM public."PlatformAccount" + WHERE account_local_id = data.creator_local_id INTO content.creator_id; + ELSIF account_local_id(creator_inline(data)) IS NOT NULL THEN + SELECT id FROM public."PlatformAccount" + WHERE account_local_id = account_local_id(creator_inline(data)) INTO content.creator_id; + END IF; + IF data.author_local_id IS NOT NULL THEN + SELECT id FROM public."PlatformAccount" + WHERE account_local_id = data.author_local_id INTO content.author_id; + ELSIF account_local_id(author_inline(data)) IS NOT NULL THEN + SELECT id FROM public."PlatformAccount" + WHERE account_local_id = account_local_id(author_inline(data)) INTO content.author_id; + END IF; + IF data.part_of_local_id IS NOT NULL THEN + SELECT id FROM public."Content" + WHERE source_local_id = data.part_of_local_id INTO content.part_of_id; + END IF; + IF data.space_url IS NOT NULL THEN + SELECT id FROM public."Space" + WHERE url = data.space_url INTO content.space_id; + END IF; + -- now avoid null defaults + IF content.metadata IS NULL then + content.metadata := '{}'; + END IF; + IF content.content_type IS NULL THEN + content.content_type := 'text/plain'; + END IF; + RETURN content; +END; +$$; + +COMMENT ON FUNCTION public._local_content_to_db_content IS 'utility function so we have the option to use platform identifiers for content upsert'; + +CREATE OR REPLACE FUNCTION public.upsert_content(v_space_id bigint, data jsonb, v_creator_id BIGINT, content_as_document boolean DEFAULT TRUE) +RETURNS SETOF BIGINT +SET search_path = '' +LANGUAGE plpgsql +AS $$ +DECLARE + v_platform public."Platform"; + db_document public."Document"%ROWTYPE; + document_id BIGINT; + local_content public.content_local_input; + db_content public."Content"%ROWTYPE; + content_row JSONB; + upsert_id BIGINT; +BEGIN + SELECT platform INTO STRICT v_platform FROM public."Space" WHERE id=v_space_id; + FOR content_row IN SELECT * FROM jsonb_array_elements(data) + LOOP + local_content := jsonb_populate_record(NULL::public.content_local_input, content_row); + local_content.space_id := v_space_id; + db_content := public._local_content_to_db_content(local_content); + IF account_local_id(author_inline(local_content)) IS NOT NULL THEN + SELECT public.create_account_in_space( + v_space_id, + account_local_id(author_inline(local_content)), + name(author_inline(local_content)) + ) INTO STRICT upsert_id; + db_content.author_id := upsert_id; + END IF; + IF account_local_id(creator_inline(local_content)) IS NOT NULL THEN + SELECT public.create_account_in_space( + v_space_id, + account_local_id(creator_inline(local_content)), + name(creator_inline(local_content)) + ) INTO STRICT upsert_id; + db_content.creator_id := upsert_id; + END IF; + IF content_as_document THEN + db_content.scale = 'document'; + END IF; + IF content_as_document AND document_id(db_content) IS NULL AND source_local_id(document_inline(local_content)) IS NULL THEN + local_content.document_inline.space_id := v_space_id; + local_content.document_inline.source_local_id := db_content.source_local_id; + local_content.document_inline.last_modified := db_content.last_modified; + local_content.document_inline.created := db_content.created; + local_content.document_inline.author_id := db_content.author_id; + END IF; + IF source_local_id(document_inline(local_content)) IS NOT NULL THEN + IF content_type(document_inline(local_content)) IS NULL THEN + local_content.document_inline.content_type := CASE + WHEN v_platform='Roam' THEN 'text/roam+markdown' + WHEN v_platform='Obsidian' THEN 'text/obsidian+markdown' + ELSE 'text/plain' END; + END IF; + db_document := public._local_document_to_db_document(document_inline(local_content)); + IF (db_document.author_id IS NULL AND author_inline(local_content) IS NOT NULL) THEN + db_document.author_id := upsert_account_in_space(v_space_id, author_inline(local_content)); + END IF; + INSERT INTO public."Document" ( + space_id, + source_local_id, + url, + created, + metadata, + last_modified, + author_id, + contents, + content_type + ) VALUES ( + COALESCE(db_document.space_id, v_space_id), + db_document.source_local_id, + db_document.url, + db_document.created, + COALESCE(db_document.metadata, '{}'::jsonb), + db_document.last_modified, + db_document.author_id, + db_document.contents, + db_document.content_type + ) + ON CONFLICT (space_id, source_local_id) DO UPDATE SET + url = COALESCE(db_document.url, EXCLUDED.url), + created = COALESCE(db_document.created, EXCLUDED.created), + metadata = COALESCE(db_document.metadata, EXCLUDED.metadata), + last_modified = COALESCE(db_document.last_modified, EXCLUDED.last_modified), + author_id = COALESCE(db_document.author_id, EXCLUDED.author_id), + contents = COALESCE(db_document.contents, EXCLUDED.contents), + content_type = COALESCE(db_document.content_type, EXCLUDED.content_type) + RETURNING id INTO STRICT document_id; + db_content.document_id := document_id; + END IF; + INSERT INTO public."Content" ( + document_id, + source_local_id, + variant, + author_id, + creator_id, + created, + text, + metadata, + scale, + space_id, + last_modified, + part_of_id, + content_type + ) VALUES ( + db_content.document_id, + db_content.source_local_id, + COALESCE(db_content.variant, 'direct'::public."ContentVariant"), + db_content.author_id, + db_content.creator_id, + db_content.created, + db_content.text, + COALESCE(db_content.metadata, '{}'::jsonb), + db_content.scale, + db_content.space_id, + db_content.last_modified, + db_content.part_of_id, + db_content.content_type + ) + ON CONFLICT (space_id, source_local_id, variant, content_type) DO UPDATE SET + document_id = COALESCE(db_content.document_id, EXCLUDED.document_id), + author_id = COALESCE(db_content.author_id, EXCLUDED.author_id), + creator_id = COALESCE(db_content.creator_id, EXCLUDED.creator_id), + created = COALESCE(db_content.created, EXCLUDED.created), + text = COALESCE(db_content.text, EXCLUDED.text), + metadata = COALESCE(db_content.metadata, EXCLUDED.metadata), + scale = COALESCE(db_content.scale, EXCLUDED.scale), + last_modified = COALESCE(db_content.last_modified, EXCLUDED.last_modified), + part_of_id = COALESCE(db_content.part_of_id, EXCLUDED.part_of_id) + RETURNING id INTO STRICT upsert_id; + IF model(embedding_inline(local_content)) IS NOT NULL THEN + PERFORM public.upsert_content_embedding(upsert_id, model(embedding_inline(local_content)), vector(embedding_inline(local_content))); + END IF; + RETURN NEXT upsert_id; + END LOOP; +END; +$$; + +COMMENT ON FUNCTION public.upsert_content IS 'batch content upsert'; + +CREATE OR REPLACE VIEW public.my_contents_with_embedding_openai_text_embedding_3_small_1536 AS +SELECT +ct.id, +ct.document_id, +ct.source_local_id, +ct.variant, +ct.author_id, +ct.creator_id, +ct.created, +ct.text, +ct.metadata, +ct.scale, +ct.space_id, +ct.last_modified, +ct.part_of_id, +emb.model, +emb.vector, +ct.content_type +FROM public."Content" AS ct +JOIN public."ContentEmbedding_openai_text_embedding_3_small_1536" AS emb ON (ct.id = emb.target_id) +LEFT OUTER JOIN public.my_accessible_resources () AS ra USING (space_id, source_local_id) +WHERE (ct.space_id = any (public.my_space_ids ('reader')) +OR (ct.space_id = any (public.my_space_ids ('partial')) AND ra.space_id IS NOT NULL)) +AND NOT emb.obsolete; diff --git a/packages/database/supabase/schemas/assets.sql b/packages/database/supabase/schemas/assets.sql index 001fb2977..2e5003be3 100644 --- a/packages/database/supabase/schemas/assets.sql +++ b/packages/database/supabase/schemas/assets.sql @@ -5,6 +5,7 @@ CREATE TABLE IF NOT EXISTS public."FileReference" ( filehash character varying NOT NULL, -- or binary? "created" timestamp without time zone NOT NULL, last_modified timestamp without time zone NOT NULL, + content_type character varying NOT NULL DEFAULT 'text/obsidian+markdown', -- not allowed virtual with user types variant public."ContentVariant" GENERATED ALWAYS AS ('full') STORED ); @@ -13,8 +14,8 @@ ADD CONSTRAINT "FileReference_pkey" PRIMARY KEY (source_local_id, space_id, file ALTER TABLE ONLY public."FileReference" ADD CONSTRAINT "FileReference_content_fkey" FOREIGN KEY ( - space_id, source_local_id, variant -) REFERENCES public."Content" (space_id, source_local_id, variant) ON DELETE CASCADE; + space_id, source_local_id, variant, content_type +) REFERENCES public."Content" (space_id, source_local_id, variant, content_type) ON DELETE CASCADE; -- note the absence of on update ; the generated column forbids cascade, so it will error -- However, update on those columns should never happen. diff --git a/packages/database/supabase/schemas/content.sql b/packages/database/supabase/schemas/content.sql index 54f2a3bc9..a41846c6e 100644 --- a/packages/database/supabase/schemas/content.sql +++ b/packages/database/supabase/schemas/content.sql @@ -131,8 +131,8 @@ CREATE INDEX "Content_part_of" ON public."Content" USING btree ( CREATE INDEX "Content_space" ON public."Content" USING btree (space_id); -CREATE UNIQUE INDEX content_space_local_id_variant_idx ON public."Content" USING btree ( - space_id, source_local_id, variant +CREATE UNIQUE INDEX content_space_local_id_variant_content_type_idx ON public."Content" USING btree ( + space_id, source_local_id, variant, content_type ) NULLS DISTINCT; CREATE INDEX "Content_text" ON public."Content" USING pgroonga (text); @@ -180,7 +180,7 @@ ADD CONSTRAINT "ResourceAccess_account_uid_fkey" FOREIGN KEY ( CREATE INDEX resource_access_content_local_id_idx ON public."ResourceAccess" (source_local_id, space_id); --- note that I cannot have a foreign key for Content because the variant is part of the unique key. +-- note that I cannot have a foreign key for Content because variant and content_type are part of the unique key. GRANT ALL ON TABLE public."ResourceAccess" TO authenticated; GRANT ALL ON TABLE public."ResourceAccess" TO service_role; @@ -429,6 +429,9 @@ BEGIN IF content.metadata IS NULL then content.metadata := '{}'; END IF; + IF content.content_type IS NULL THEN + content.content_type := 'text/plain'; + END IF; RETURN content; END; $$; @@ -644,7 +647,7 @@ BEGIN db_content.part_of_id, db_content.content_type ) - ON CONFLICT (space_id, source_local_id, variant) DO UPDATE SET + ON CONFLICT (space_id, source_local_id, variant, content_type) DO UPDATE SET document_id = COALESCE(db_content.document_id, EXCLUDED.document_id), author_id = COALESCE(db_content.author_id, EXCLUDED.author_id), creator_id = COALESCE(db_content.creator_id, EXCLUDED.creator_id), @@ -653,8 +656,7 @@ BEGIN metadata = COALESCE(db_content.metadata, EXCLUDED.metadata), scale = COALESCE(db_content.scale, EXCLUDED.scale), last_modified = COALESCE(db_content.last_modified, EXCLUDED.last_modified), - part_of_id = COALESCE(db_content.part_of_id, EXCLUDED.part_of_id), - content_type = COALESCE(db_content.content_type, EXCLUDED.content_type) + part_of_id = COALESCE(db_content.part_of_id, EXCLUDED.part_of_id) RETURNING id INTO STRICT upsert_id; IF model(embedding_inline(local_content)) IS NOT NULL THEN PERFORM public.upsert_content_embedding(upsert_id, model(embedding_inline(local_content)), vector(embedding_inline(local_content))); diff --git a/packages/database/supabase/schemas/embedding.sql b/packages/database/supabase/schemas/embedding.sql index 5a5346491..43801cf5d 100644 --- a/packages/database/supabase/schemas/embedding.sql +++ b/packages/database/supabase/schemas/embedding.sql @@ -44,7 +44,8 @@ ct.space_id, ct.last_modified, ct.part_of_id, emb.model, -emb.vector +emb.vector, +ct.content_type FROM public."Content" AS ct JOIN public."ContentEmbedding_openai_text_embedding_3_small_1536" AS emb ON (ct.id = emb.target_id) LEFT OUTER JOIN public.my_accessible_resources () AS ra USING (space_id, source_local_id)