Feature document text

pom.xml

@@ -113,6 +113,11 @@
       <artifactId>pdfbox</artifactId>
       <version>3.0.8</version>
     </dependency>
+    <dependency>
+      <groupId>org.apache.poi</groupId>
+      <artifactId>poi-ooxml</artifactId>
+      <version>5.5.1</version>
+    </dependency>
     <dependency>
       <groupId>commons-validator</groupId>
       <artifactId>commons-validator</artifactId>

src/main/java/com/simisinc/platform/application/cms/PublishContentCommand.java

@@ -26,10 +26,10 @@
 import com.simisinc.platform.domain.model.cms.WebPage;
 import com.simisinc.platform.infrastructure.persistence.cms.ContentRepository;
 import com.simisinc.platform.infrastructure.persistence.cms.WebPageRepository;
-import com.simisinc.platform.infrastructure.scheduler.cms.RefreshWebPageTextIndexJob;
 import com.simisinc.platform.infrastructure.workflow.WorkflowManager;
 import com.zeroio.platform.domain.model.cms.ContentVersion;
 import com.zeroio.platform.infrastructure.persistence.cms.ContentVersionRepository;
+import com.zeroio.platform.infrastructure.scheduler.cms.RefreshWebPageTextIndexJob;
 
 /**
  * Publishes draft content to live

src/main/java/com/simisinc/platform/application/cms/SaveContentCommand.java

@@ -24,9 +24,9 @@
 import com.simisinc.platform.application.DataException;
 import com.simisinc.platform.domain.model.cms.Content;
 import com.simisinc.platform.infrastructure.persistence.cms.ContentRepository;
-import com.simisinc.platform.infrastructure.scheduler.cms.RefreshWebPageTextIndexJob;
 import com.zeroio.platform.domain.model.cms.ContentVersion;
 import com.zeroio.platform.infrastructure.persistence.cms.ContentVersionRepository;
+import com.zeroio.platform.infrastructure.scheduler.cms.RefreshWebPageTextIndexJob;
 
 /**
  * Validates and saves content objects

src/main/java/com/simisinc/platform/domain/model/cms/FileItem.java

@@ -1,4 +1,5 @@
 /*
+ * Copyright 2026 Matt Rajkowski (https://github.com/rajkowski)
  * Copyright 2022 SimIS Inc. (https://www.simiscms.com)
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
@@ -44,6 +45,7 @@ public class FileItem extends Entity {
   private String fileType = null;
   private String mimeType = null;
   private String fileHash = null;
+  private String documentText = null;
   private String webPath = null;
   private int width = -1;
   private int height = -1;
@@ -175,6 +177,14 @@ public void setFileHash(String fileHash) {
     this.fileHash = fileHash;
   }
 
+  public String getDocumentText() {
+    return documentText;
+  }
+
+  public void setDocumentText(String documentText) {
+    this.documentText = documentText;
+  }
+
   public int getWidth() {
     return width;
   }

src/main/java/com/simisinc/platform/domain/model/items/ItemFileItem.java

@@ -1,4 +1,5 @@
 /*
+ * Copyright 2026 Matt Rajkowski (https://github.com/rajkowski)
  * Copyright 2022 SimIS Inc. (https://www.simiscms.com)
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
@@ -44,6 +45,7 @@ public class ItemFileItem extends Entity {
   private String fileType = null;
   private String mimeType = null;
   private String fileHash = null;
+  private String documentText = null;
   private String webPath = null;
   private int width = -1;
   private int height = -1;
@@ -183,6 +185,14 @@ public void setFileHash(String fileHash) {
     this.fileHash = fileHash;
   }
 
+  public String getDocumentText() {
+    return documentText;
+  }
+
+  public void setDocumentText(String documentText) {
+    this.documentText = documentText;
+  }
+
   public int getWidth() {
     return width;
   }
@@ -302,7 +312,8 @@ public String getWebPath() {
   public void setWebPath(String webPath) {
     this.webPath = webPath;
   }
-public String getUrl() {
+
+  public String getUrl() {
     return webPath + "-" + id + "/" + UrlCommand.encodeUri(filename);
-  }  
+  }
 }

src/main/java/com/simisinc/platform/infrastructure/database/DB.java

@@ -419,6 +419,29 @@ public static DataResult selectAllFrom(String tableName, SqlUtils select, SqlWhe
 
   public static DataResult selectAllFrom(String tableName, SqlUtils select, SqlJoins joins, SqlWhere where,
       SqlUtils orderBy, DataConstraints constraints, Function<ResultSet, Entity> buildRecord) {
+    // This method creates a TABLE_NAME.* column automatically (but may be deprecated in the future)
+    if (select == null) {
+      select = DB.SELECT();
+    }
+    // At the front of the columns, add the table's full columns for backwards compatibility
+    select.getValues().add(0, new SqlValue(tableName + ".*"));
+    return selectFrom(tableName, select, joins, where, orderBy, constraints, buildRecord);
+  }
+
+  /**
+   * Select records from the database with the specified parameters.
+   *
+   * @param tableName the name of the table
+   * @param columns the columns values
+   * @param joins the join clauses
+   * @param where the where clause
+   * @param orderBy the order by clause
+   * @param constraints the data constraints
+   * @param buildRecord the function to build an entity from the result set
+   * @return the data result containing the records
+   */
+  public static DataResult selectFrom(String tableName, SqlUtils columns, SqlJoins joins, SqlWhere where,
+      SqlUtils orderBy, DataConstraints constraints, Function<ResultSet, Entity> buildRecord) {
 
     // Determine the max records based on the where conditions
     DataResult dataResult = new DataResult();
@@ -462,10 +485,8 @@ public static DataResult selectAllFrom(String tableName, SqlUtils select, SqlJoi
 
     // Prepare the query
     StringBuilder sb = new StringBuilder();
-    sb.append("SELECT ").append(tableName).append(".*");
-    if (select != null) {
-      sb.append(createAdditionalSelectFields(select));
-    }
+    sb.append("SELECT ");
+    sb.append(createSelectFields(columns));
     sb.append(" FROM ").append(tableName);
     sb.append(joinsSb);
     sb.append(whereSb);
@@ -489,7 +510,7 @@ public static DataResult selectAllFrom(String tableName, SqlUtils select, SqlJoi
     List<Entity> records = null;
     long startQueryTime = System.currentTimeMillis();
     try (Connection connection = getConnection();
-        PreparedStatement pst = createPreparedStatement(connection, sb.toString(), select, where, orderBy);
+        PreparedStatement pst = createPreparedStatement(connection, sb.toString(), columns, where, orderBy);
         ResultSet rs = pst.executeQuery()) {
       records = new ArrayList<>();
       while (rs.next()) {

src/main/java/com/simisinc/platform/infrastructure/database/SqlWhere.java

@@ -35,6 +35,8 @@ public class SqlWhere {
 
   public static final String AND_OPERATOR = "AND";
   public static final String OR_OPERATOR = "OR";
+  public static final String NOT_AND_OPERATOR = "NOT AND";
+  public static final String NOT_OR_OPERATOR = "OR";
 
   private List<SqlValue> values = new ArrayList<>();
 
@@ -296,10 +298,20 @@ private static String arrayToSqlStatementList(String jsonbColumnName, String[] a
     if (StringUtils.isBlank(jsonbColumnName) || arrayValues == null || arrayValues.length == 0) {
       return null;
     }
+    StringBuilder sql = new StringBuilder();
 
     // Default to AND if operator is null or invalid
-    String op = OR_OPERATOR.equalsIgnoreCase(operator) ? OR_OPERATOR : AND_OPERATOR;
-    StringBuilder sql = new StringBuilder("(");
+    String op = (OR_OPERATOR.equalsIgnoreCase(operator) || NOT_OR_OPERATOR.equalsIgnoreCase(operator))
+        ? OR_OPERATOR
+        : AND_OPERATOR;
+
+    // If the operator is a NOT variant, prepend "NOT " to the condition
+    if (NOT_AND_OPERATOR.equalsIgnoreCase(operator) || NOT_OR_OPERATOR.equalsIgnoreCase(operator)) {
+      sql.append("NOT ");
+    }
+
+    // Build the condition with one placeholder per value
+    sql.append("(");
     for (int i = 0; i < arrayValues.length; i++) {
       if (i > 0) {
         sql.append(" ").append(op).append(" ");

src/main/java/com/simisinc/platform/infrastructure/persistence/cms/FileItemRepository.java

@@ -61,11 +61,51 @@ public class FileItemRepository {
   private static String[] PRIMARY_KEY = new String[] { "file_id" };
 
   private static DataResult query(FileSpecification specification, DataConstraints constraints) {
-    SqlUtils select = new SqlUtils();
+
+    SqlUtils select = DB.SELECT(
+        TABLE_NAME + ".file_id",
+        TABLE_NAME + ".folder_id",
+        TABLE_NAME + ".filename",
+        TABLE_NAME + ".title",
+        TABLE_NAME + ".barcode",
+        TABLE_NAME + ".version",
+        TABLE_NAME + ".extension",
+        TABLE_NAME + ".path",
+        TABLE_NAME + ".file_length",
+        TABLE_NAME + ".file_type",
+        TABLE_NAME + ".mime_type",
+        TABLE_NAME + ".file_hash",
+        TABLE_NAME + ".width",
+        TABLE_NAME + ".height",
+        TABLE_NAME + ".summary",
+        TABLE_NAME + ".created_by",
+        TABLE_NAME + ".created",
+        TABLE_NAME + ".modified_by",
+        TABLE_NAME + ".modified",
+        TABLE_NAME + ".processed",
+        TABLE_NAME + ".expiration_date",
+        TABLE_NAME + ".privacy_type",
+        TABLE_NAME + ".default_token",
+        TABLE_NAME + ".version_count",
+        TABLE_NAME + ".download_count",
+        TABLE_NAME + ".sub_folder_id",
+        TABLE_NAME + ".category_id",
+        TABLE_NAME + ".web_path",
+        TABLE_NAME + ".tags");
+
     SqlJoins joins = new SqlJoins();
     SqlWhere where = DB.WHERE();
     SqlUtils orderBy = new SqlUtils();
+
+    final boolean includeDocumentText = specification != null && specification.getIncludeDocumentText();
+
     if (specification != null) {
+
+      // Only include this potentially-large field when requested
+      if (includeDocumentText) {
+        select.add(TABLE_NAME + ".document_text");
+      }
+
       joins.add("LEFT JOIN folders ON (files.folder_id = folders.folder_id)");
       where
           .andAddIfHasValue("file_id = ?", specification.getId(), -1)
@@ -79,6 +119,10 @@ private static DataResult query(FileSpecification specification, DataConstraints
         where.AND("LOWER(files.file_type) = ANY(?)",
             Arrays.stream(specification.getFileType()).map(String::toLowerCase).toArray(String[]::new), Types.ARRAY);
       }
+      if (specification.getFileExtension() != null) {
+        where.AND("LOWER(files.extension) = ANY(?)",
+            Arrays.stream(specification.getFileExtension()).map(String::toLowerCase).toArray(String[]::new), Types.ARRAY);
+      }
       if (specification.getMatchesName() != null) {
         String likeValue = specification.getMatchesName().trim()
             .replace("!", "!!")
@@ -97,6 +141,13 @@ private static DataResult query(FileSpecification specification, DataConstraints
           where.AND("sub_folder_id IS NULL");
         }
       }
+      if (specification.getIsProcessed() != DataConstants.UNDEFINED) {
+        if (specification.getIsProcessed() == DataConstants.TRUE) {
+          where.AND("processed IS NOT NULL");
+        } else {
+          where.AND("processed IS NULL");
+        }
+      }
 
       // For user id
       // User must be in a user group with folder access
@@ -143,6 +194,10 @@ private static DataResult query(FileSpecification specification, DataConstraints
         where.AND("tags", specification.getFilterTags(), SqlWhere.AND_OPERATOR);
       }
 
+      if (specification.getExcludeTags() != null && specification.getExcludeTags().length > 0) {
+        where.AND("tags", specification.getExcludeTags(), SqlWhere.NOT_OR_OPERATOR);
+      }
+
       // Add modified date range filters
       // Some imported files can have a null modified date, so fall back to created.
       if (specification.getModifiedAfter() != null) {
@@ -167,8 +222,10 @@ private static DataResult query(FileSpecification specification, DataConstraints
         where.AND(userCondition.toString(), (Object[]) userIdsBoxed);
       }
     }
-    return DB.selectAllFrom(
-        TABLE_NAME, select, joins, where, orderBy, constraints, FileItemRepository::buildRecord);
+
+    return DB.selectFrom(
+        TABLE_NAME, select, joins, where, orderBy, constraints,
+        rs -> buildRecord(rs, includeDocumentText));
   }
 
   public static FileItem findById(long id) {
@@ -369,6 +426,21 @@ public static FileItem saveVersion(FileItem record) {
     return null;
   }
 
+  public static boolean updateDocumentText(FileItem record, String documentText) {
+    try (Connection connection = DB.getConnection()) {
+      SqlUtils updateValues = new SqlUtils()
+          .add("document_text", documentText)
+          .add("processed", new Timestamp(System.currentTimeMillis()));
+      if (DB.update(connection, TABLE_NAME, updateValues, DB.WHERE("file_id = ?", record.getId()))) {
+        return true;
+      }
+    } catch (SQLException se) {
+      LOG.error("SQLException: " + se.getMessage());
+    }
+    LOG.error("The document text update failed!");
+    return false;
+  }
+
   public static boolean remove(FileItem record) {
     try (Connection connection = DB.getConnection();
         AutoStartTransaction a = new AutoStartTransaction(connection);
@@ -427,6 +499,10 @@ public static long findTotalFileSize() {
   }
 
   private static FileItem buildRecord(ResultSet rs) {
+    return buildRecord(rs, false);
+  }
+
+  private static FileItem buildRecord(ResultSet rs, boolean includeDocumentText) {
     try {
       FileItem record = new FileItem();
       record.setId(rs.getLong("file_id"));
@@ -463,6 +539,9 @@ private static FileItem buildRecord(ResultSet rs) {
       record.setCategoryId(DB.getLong(rs, "category_id", -1L));
       record.setWebPath(rs.getString("web_path"));
       record.setTags(JsonCommand.fromJsonArray(rs.getString("tags")));
+      if (includeDocumentText) {
+        record.setDocumentText(rs.getString("document_text"));
+      }
       return record;
     } catch (SQLException se) {
       LOG.error("buildRecord", se);