diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..d999cf0 --- /dev/null +++ b/.env.example @@ -0,0 +1,4 @@ +# Optional HTTP service settings. Never commit real secrets. +PPTX_EXTRACTION_WORK_DIR=./work +PPTX_EXTRACTION_MAX_UPLOAD_MB=50 +PPTX_EXTRACTION_WORKERS=2 diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..3394f7c --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,13 @@ +version: 2 +updates: + - package-ecosystem: pip + directory: "/" + schedule: + interval: monthly + groups: + python-dependencies: + patterns: ["*"] + - package-ecosystem: github-actions + directory: "/" + schedule: + interval: monthly diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..b5cea2c --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,44 @@ +name: CI + +on: + push: + pull_request: + +permissions: + contents: read + +jobs: + test: + strategy: + matrix: + python-version: ["3.10", "3.11", "3.12"] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: pip + - run: python -m pip install --upgrade pip + - run: python -m pip install -e ".[dev,api]" + - run: ruff check . + - run: ruff format --check . + - run: mypy src/pptx_extraction + - run: pytest + - run: python -m build + + privacy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Reject common private artifacts + shell: bash + run: | + if git ls-files | grep -E '\.(pptx?|pptm|potx|ppsx|mp3|wav|srt)$'; then + echo "Private presentation/audio artifact is tracked." + exit 1 + fi + if git grep -nE '([A-Z]:\\\\|BEGIN (RSA |OPENSSH )?PRIVATE KEY|api[_-]?key[[:space:]]*=)' -- ':!docs/release.md'; then + echo "Potential machine path or secret found." + exit 1 + fi diff --git a/.gitignore b/.gitignore index 866f58c..273485f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,30 +1,45 @@ -# 忽略 Python 字节码文件 +# Python __pycache__/ -*.pyc -*.pyo +*.py[cod] *.pyd +*.egg-info/ +.pytest_cache/ +.mypy_cache/ +.ruff_cache/ +.coverage +htmlcov/ -# 忽略虚拟环境 +# Environments and secrets +.venv/ venv/ .env +.env.* +!.env.example +*.pem +*.key -# 忽略输出目录 +# Builds and runtime data +build/ +dist/ +release/ +work/ output/ -*.txt # 如果只想忽略特定输出文件，可改为 output/*.txt - -# 忽略日志文件 -logs/ +outputs/ +_legacy_local_backup/ *.log - -# 忽略临时文件 *.tmp -*.swp -# 忽略系统文件 -.DS_Store # macOS -Thumbs.db # Windows - -# PPT +# Office lock files and local presentation material +~$*.ppt* *.ppt *.pptx -PPT_SET/ +*.pptm +*.potx +*.ppsx +!tests/fixtures/*.pptx + +# Editor / OS +.idea/ +.vscode/ +.DS_Store +Thumbs.db diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..f5f1770 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,23 @@ +# pptx_extraction changelog + +All notable changes follow [Keep a Changelog](https://keepachangelog.com/) and semantic versioning. + +## [2.0.0] - 2026-08-11 + +### Added + +- Structured extraction for text, tables, chart data, speaker notes, hyperlinks, images and metadata. +- JSON, Markdown and plain-text exporters with a versioned schema. +- Safe OOXML validation, deterministic image deduplication and optional Tesseract OCR. +- Cross-platform CLI, concurrent batch mode, optional LibreOffice legacy conversion and job-based API. +- Unit/integration tests, CI, security policy, architecture documentation and Agent Skill packaging. + +### Changed + +- Rebuilt the prototype as a typed `src/` package with explicit domain models and error boundaries. +- Replaced import-time Spacy/Transformers/PaddleOCR loading with deterministic offline defaults. + +### Removed + +- Hard-coded personal paths, personal contact details and committed user-generated outputs. +- Misleading PDF support and the unrelated audio-caption experiment from the core package. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..66275d5 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,17 @@ +# Contributing + +Use Python 3.10 or newer. Create a virtual environment, install `-e ".[dev]"`, then run: + +```bash +ruff check . +ruff format --check . +mypy src/pptx_extraction +pytest +``` + +Keep extraction deterministic and offline by default. New output fields must remain backward compatible +within the current schema major version. Add a generated fixture or focused unit test for every parser fix; +never commit private decks, extracted media, credentials or machine-specific paths. + +Open a small issue before large changes. Use conventional, imperative commit subjects and include the +observable behavior and test evidence in pull requests. diff --git a/DeepLearning.pptx b/DeepLearning.pptx deleted file mode 100644 index 9bf0c6b..0000000 Binary files a/DeepLearning.pptx and /dev/null differ diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..99fac01 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 pptx_extraction contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..2b03e9f --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,8 @@ +include CHANGELOG.md +include CONTRIBUTING.md +include LICENSE +include README.md +include SECURITY.md +recursive-include docs *.md +recursive-include schemas *.json +recursive-include tests *.py diff --git a/README.md b/README.md index f321516..3d2f71b 100644 --- a/README.md +++ b/README.md @@ -1,231 +1,352 @@ -# 课件内容提取与优化工具 (PPT_Text_Extractor) +
-## 简介 -项目是一个基于 Python 的智能工具，旨在从多种 PPT 文件中提取文本和图片内容，并通过优化处理生成自然流畅的文本输出。利用 `python-pptx`、`win32com` 和 `PaddleOCR` 技术，支持幻灯片文本提取和图片文字识别，适用于教学文档整理或自动化处理。 +# pptx_extraction -![Python](https://img.shields.io/badge/Python-3.8+-blue.svg) -![License](https://img.shields.io/badge/License-MIT-green.svg) -![Build Status](https://img.shields.io/badge/build-passing-brightgreen.svg) -![Version](https://img.shields.io/badge/version-1.0.0-yellow.svg) +### 面向检索、RAG 与 Agent 的 PowerPoint 结构化内容提取工具 ---- +[![CI](https://github.com/BlairCode/pptx_extraction/actions/workflows/ci.yml/badge.svg)](https://github.com/BlairCode/pptx_extraction/actions/workflows/ci.yml) +[![Python](https://img.shields.io/badge/Python-3.10%2B-3776AB?logo=python&logoColor=white)](https://www.python.org/) +[![Release](https://img.shields.io/github/v/release/BlairCode/pptx_extraction?include_prereleases)](https://github.com/BlairCode/pptx_extraction/releases) +[![License](https://img.shields.io/badge/License-MIT-2ea44f)](LICENSE) +[![Schema](https://img.shields.io/badge/Schema-1.0-6f42c1)](schemas/pptx-extraction.presentation.v1.schema.json) -## 功能特点 -- 📄 **PPT 文本提取**：自动提取幻灯片的标题、段落和表格内容，支持 `.pptx` 和旧版格式（`.ppt`、`.pot`、`.pps`）。 -- 🖼️ **图片文本识别**：使用 PaddleOCR 从幻灯片图片中提取文字，支持多种 PowerPoint 文件格式。 -- ✍️ **文本优化**：将提取的内容优化为叙述性文本，便于阅读或后续使用。 -- 📋 **结构化输出**：按幻灯片分隔保存文本和图片内容。 -- ⚙️ **日志记录**：提供详细的处理日志，便于调试。 -- 🌐 **Web 支持**：通过 Flask 提供 Web 接口，可上传文件并获取优化结果。 +**默认离线 · 可追溯 · 跨平台 · CLI / Python API / HTTP API / Agent Skill** ---- +[快速开始](#快速开始) · [常用工作流](#常用工作流) · [输出说明](#输出说明) · [完整文档](#完整文档) -## 技术栈 -- **Python**: 3.8 或更高版本 -- **python-pptx**: 解析 `.pptx` 文件 -- **pywin32**: 处理旧版 PowerPoint 文件（`.ppt`、`.pot`、`.pps`） -- **PaddleOCR**: 图片文本识别 -- **Flask**: Web 服务框架 -- **Spacy**: 自然语言处理（文本优化） -- **Transformers**: AI 文本生成（优化衔接） -- **logging**: 日志记录 +
--- -## 安装步骤 +`pptx_extraction` 将 PowerPoint 文件转换为带来源定位的 JSON、Markdown 和纯文本。它不仅提取 +“看得见的文字”，还保留页码、段落层级、视觉阅读顺序、原始堆叠顺序、shape 信息、坐标、 +表格、图表数据、备注、链接、图片哈希与告警，适合知识库构建、内容迁移、无障碍审计和 Agent +读取等需要可靠引用来源的场景。 + +> 项目保持原仓库名称 `pptx_extraction`。安装包名称为 `pptx-extraction`，Python 导入名为 +> `pptx_extraction`，命令行入口为 `pptx-extraction`。 + +## 核心能力 + +| 能力 | 处理结果 | +|---|---| +| 文本与链接 | 标题/正文、段落层级、超链接、shape ID/name、坐标 | +| 表格与图表 | 原生单元格、图表分类、序列名与数值；不使用 OCR 猜测数据 | +| 图片与 OCR | SHA-256 命名、跨页去重、alt text、可选 Tesseract OCR | +| 备注与隐藏页 | speaker notes 独立输出；隐藏页保留并标记 `hidden: true` | +| 可追溯性 | 每个元素保留页码、阅读顺序、z-order 和归一化位置 | +| 安全与隐私 | 不联网、不执行宏；检查 ZIP 路径穿越、压缩炸弹与异常包 | +| 工程接口 | 单文件、批处理、Python API、异步 HTTP API、Agent Skill | + +```mermaid +flowchart LR + A["PowerPoint OOXML"] --> B["安全校验"] + B --> C["文本 / 表格 / 图表 / 备注 / 图片"] + C --> D["统一结构化模型"] + C -. 可选 .-> O["Tesseract OCR"] + D --> J["JSON"] + D --> M["Markdown"] + D --> T["Text"] + J --> R["Search / RAG / Agent"] + M --> R +``` + +## 支持范围 + +| 文件类型 | 支持方式 | +|---|---| +| `.pptx` / `.pptm` / `.potx` / `.ppsx` | 直接解析；宏只检测、不执行 | +| `.ppt` / `.pot` / `.pps` | 使用 `convert` 命令调用本机 LibreOffice 转换 | +| `.pdf` | 不支持；请先使用 PDF 专用工具 | +| SmartArt / OLE / 音视频 / 动画 | 可能只能得到部分信息，并在可检测时输出告警 | -### 前置条件 -- Python 3.8 或以上版本 -- Git（用于克隆仓库） -- Windows 系统（因使用 `win32com`，需安装 Microsoft PowerPoint） -- 可选：GPU 支持（加速 PaddleOCR） +## 快速开始 + +下面的命令可以直接复制。示例输入为 `slides.pptx`，请替换为你的真实文件路径。 + +### 1. 克隆并进入现有仓库 -### 安装依赖 ```bash -# 1. 克隆项目仓库 -git clone https://github.com/blankboards/pptx_extraction.git +git clone https://github.com/BlairCode/pptx_extraction.git cd pptx_extraction +``` -# 2. 创建并激活虚拟环境 -python -m venv venv -# Unix/macOS -source venv/bin/activate -# Windows -venv\Scripts\activate +### 2. 创建虚拟环境 -# 3. 安装依赖 -pip install -r requirements.txt +Windows PowerShell： + +```powershell +python -m venv .venv +.\.venv\Scripts\Activate.ps1 +python -m pip install --upgrade pip +python -m pip install -e . ``` -### 依赖列表 (requirements.txt 示例) +Linux / macOS： + +```bash +python3 -m venv .venv +source .venv/bin/activate +python -m pip install --upgrade pip +python -m pip install -e . +``` + +确认安装成功： + +```bash +pptx-extraction --version ``` -python-pptx>=0.6.21 -paddlepaddle>=2.5.0 -paddleocr>=2.6.1 -pywin32>=306 -flask>=2.2.5 -flask-cors>=4.0.0 -python-dotenv>=1.0.0 -spacy>=3.7.2 -transformers>=4.35.2 -protobuf==3.20.3 -scikit-image>=0.21.0 + +预期输出： + +```text +pptx_extraction 2.0.0 ``` -### 额外步骤 -- 下载 Spacy 中文模型： +### 3. 校验并提取第一份 PPTX + ```bash -python -m spacy download zh_core_web_sm +pptx-extraction validate "slides.pptx" +pptx-extraction extract "slides.pptx" \ + --output "output/slides" \ + --format json \ + --format markdown \ + --format text \ + --redact-metadata ``` ---- +PowerShell 如果不使用反引号续行，建议直接写成一行： -## 快速开始 +```powershell +pptx-extraction extract "slides.pptx" --output "output/slides" --format json --format markdown --format text --redact-metadata +``` -### 命令行使用 -以下是如何提取和优化 PPT 内容的示例： -```python -from main import process_ppt_file +首次运行后会得到： -# 指定 PPT 文件路径 -ppt_path = 'sample.ppt' # 支持 .ppt 和 .pptx +```text +output/slides/ +├── presentation.json # 完整结构化数据，适合程序、RAG 与 Agent +├── presentation.md # 按页整理，适合阅读和快速检查 +├── presentation.txt # 无 Markdown 标记的纯文本 +└── assets/ # 按内容哈希命名并去重的嵌入图片 +``` -# 处理 PPT 并生成优化文本 -process_ppt_file(ppt_path) -print("优化文本已保存至 output/optimized_output.txt") +再次写入同一非空目录时，程序会保护已有文件并停止。确认该目录可以替换后再加： + +```bash +pptx-extraction extract "slides.pptx" -o "output/slides" --overwrite ``` -### Web 使用 -1. 运行 Web 服务： +## 输出说明 + +`presentation.json` 是最完整的结果。常用字段如下： + +| 字段 | 含义 | +|---|---| +| `schema_version` | 当前数据契约版本，现为 `1.0` | +| `source_sha256` | 输入文件内容哈希，用于区分不同版本 | +| `slides[].number` | 1 开始的幻灯片页码 | +| `slides[].text_blocks` | 标题/正文、层级、链接与来源 shape | +| `slides[].tables` | 表格二维单元格数据 | +| `slides[].charts` | 图表标题、分类、序列和数值 | +| `slides[].images` | 图片哈希、路径、alt text 与可选 OCR | +| `slides[].notes` | 演讲者备注，不与正文混合 | +| `order` / `z_order` | 视觉阅读顺序 / PowerPoint 原始堆叠顺序 | +| `bbox` | points 坐标和 0–1 归一化坐标 | +| `warnings` | 缺少 alt text、宏、未支持对象等限制 | + +完整约束见 [JSON Schema](schemas/pptx-extraction.presentation.v1.schema.json)。 + +## 常用工作流 + +### 只检查内容概况，不生成文件 + ```bash -python app.py +pptx-extraction inspect "slides.pptx" ``` -2. 打开浏览器，访问 `http://localhost:5000/`，上传 PPT 文件获取优化结果。 -### 输出示例 +输出完整 JSON 记录，同时隐藏作者等元数据： + +```bash +pptx-extraction inspect "slides.pptx" --full --redact-metadata ``` -Title: N/A -Author: huawei -现在我们来看看第 1 张幻灯片： -首先是 深度学习基础。 -这里有个重点 本课程将深入讲解深度学习的基本原理。 +### 批量处理目录 + +递归发现目录中的受支持文件，使用 4 个工作线程： + +```bash +pptx-extraction batch "./decks" --output "./output" --workers 4 --redact-metadata ``` ---- +同时传入多个文件或目录： -## 配置参数 +```bash +pptx-extraction batch "deck-a.pptx" "deck-b.pptx" "./more-decks" -o "./output" +``` -### 主函数参数 -| 参数名 | 类型 | 默认值 | 描述 | -|--------------|-------|---------------|--------------------------| -| `file_path` | `str` | 必填 | PPT 文件路径 | -| `output_dir` | `str` | `'output'` | 输出目录（定义在 config.py） | +每个输入会写入独立目录，目录名包含源文件哈希前缀；单个文件失败不会中断其他任务。只要有一项 +失败，命令退出码为 `4`，失败原因会写在终端 JSON 中。 -### 可调整配置 (config.py) -- `OUTPUT_DIR`: 输出文本和图片的目录（命令行模式） -- `OUTPUT_DIR_2`: Web 模式输出目录 -- `PPTX_FILE`: 默认处理的 PPT 文件路径（命令行模式） -- `PPTX_FILE_2`: 备用 PPT 文件路径 +### 识别嵌入图片中的文字 ---- +先安装 Python OCR 适配器： -## 项目结构 -``` -PPT_Text_Extractor/ -├── modules/ # 功能模块目录 -│ ├── __init__.py # 包初始化文件 -│ ├── ppt_text_extraction.py # PPT 文本和元数据提取模块 -│ ├── image_extraction_p.py # 图片文本提取模块（PaddleOCR） -│ ├── image_extraction_t.py # Tesseract 图片文本提取（未使用） -│ ├── ai_optimizer.py # 文本优化模块 -│ ├── utils.py # 工具函数 -│ └── config.py # 配置文件 -├── static/ # Web 静态文件目录 -│ └── index.html # 前端界面 -├── output/ # 输出目录示例（运行时生成） -│ ├── slide_1/ # 幻灯片 1 的输出 -│ │ ├── image/ # 图片文件目录 -│ │ │ ├── image_1.jpg # 提取的图片 -│ │ │ └── slide_1_image_1_text.txt # 图片识别文本 -│ │ └── slide_1_texts.txt # 幻灯片文本 -│ └── optimized_output.txt # 优化后的文本（命令行模式） -├── app.py # Web 服务脚本 -├── main.py # 命令行入口脚本 -├── test.py # 测试脚本 -├── requirements.txt # 依赖列表 -├── .env # 环境变量配置（注意敏感信息） -├── .gitignore # 忽略文件配置 -└── README.md # 项目说明文档 +```bash +python -m pip install -e ".[ocr]" ``` ---- +再安装系统级 Tesseract 和所需语言包，然后运行： + +```bash +pptx-extraction extract "slides.pptx" -o "output/ocr" \ + --ocr tesseract \ + --ocr-language "chi_sim+eng" +``` -## 使用指南 +OCR 只处理 PPTX 中的嵌入图片，不会渲染整页幻灯片。同一图片即使跨页重复，也只识别一次。 + +### 转换旧版 `.ppt` + +先安装 LibreOffice，并确保 `soffice` 在 `PATH` 中： -### 运行步骤（命令行） -1. 准备一个 PPT 文件（如 `sample.pptx` 或 `sample.ppt`）。 -2. 在 `config.py` 中设置 `PPTX_FILE_2` 为你的文件路径。 -3. 运行： ```bash -python main.py +pptx-extraction convert "legacy.ppt" --output "converted" +pptx-extraction extract "converted/legacy.pptx" --output "output/legacy" +``` + +如果 `soffice` 不在 `PATH`，Windows 可显式指定： + +```powershell +pptx-extraction convert "legacy.ppt" -o "converted" --soffice "$env:ProgramFiles\LibreOffice\program\soffice.exe" ``` -4. 查看 `output/optimized_output.txt` 中的结果。 -### 运行步骤（Web） -1. 运行 Web 服务： +### Python API + +```python +from pptx_extraction import ExtractionOptions, extract_file + +result = extract_file( + "slides.pptx", + "output/python-api", + options=ExtractionOptions( + include_assets=True, + include_notes=True, + redact_metadata=True, + ), + formats=("json", "markdown", "text"), +) + +print(result.output_dir) +print(result.record.summary) +``` + +### HTTP API + +安装并启动： + ```bash -python app.py +python -m pip install -e ".[api]" +uvicorn pptx_extraction.api:create_app --factory --host 127.0.0.1 --port 8000 ``` -2. 访问 `http://localhost:5000/`，上传 PPT 文件。 -3. 查看返回的优化文本或 `output/optimized_output_*.txt`。 -### 自定义输出 -- 修改 `ai_optimizer.py` 调整文本优化逻辑。 -- 在 `config.py` 中更改 `OUTPUT_DIR_2` 设置输出路径。 +新开一个 PowerShell 窗口上传文件并轮询结果： ---- +```powershell +$job = curl.exe -s -X POST -F "file=@slides.pptx" http://127.0.0.1:8000/v1/jobs | ConvertFrom-Json +$job -## 常见问题 (FAQ) +$status = $null +do { + Start-Sleep -Seconds 1 + $status = curl.exe -s "http://127.0.0.1:8000/v1/jobs/$($job.id)" | ConvertFrom-Json + $status +} while ($status.status -in @("queued", "running")) -1. **支持哪些文件格式？** -- 支持 `.pptx`、`.pptm`、`.potx`、`.ppsx`（使用 `python-pptx`）。 -- 支持 `.ppt`、`.pot`、`.pps`（使用 `pywin32`，需 Windows 和 PowerPoint）。 -- `.pdf` 暂未完全支持，可后续扩展。 +if ($status.status -ne "succeeded") { + throw "Extraction failed: $($status.error)" +} -2. **图片文本识别不准确怎么办？** -- 确保图片清晰，调整 PaddleOCR 的置信度阈值（`process_image_for_ocr`）。 +curl.exe -s "http://127.0.0.1:8000/v1/jobs/$($job.id)/result" -o presentation.json +``` -3. **处理速度慢怎么办？** -- 启用 GPU（安装 `paddlepaddle-gpu`）。 -- 减少幻灯片中的图片数量。 +状态为 `succeeded` 后才能读取结果。接口说明和生产部署边界见 [docs/api.md](docs/api.md)。 -4. **依赖安装失败怎么办？** -- 检查网络，运行 `pip install --upgrade pip`。 -- 确保 Python 版本 >= 3.8，Protobuf 版本为 3.20.3。 +## CLI 速查 ---- +| 命令 | 用途 | 是否写文件 | +|---|---|---| +| `pptx-extraction validate FILE` | 检查格式、OOXML 结构与安全限制 | 否 | +| `pptx-extraction inspect FILE` | 查看页数和元素统计 | 否 | +| `pptx-extraction extract FILE -o DIR` | 提取单个文件 | 是 | +| `pptx-extraction batch INPUT... -o DIR` | 并发批处理文件/目录 | 是 | +| `pptx-extraction convert FILE.ppt -o DIR` | 通过 LibreOffice 转换旧格式 | 是 | +| `pptx-extraction COMMAND --help` | 查看某个命令的全部参数 | 否 | -## 许可证 -本项目采用 [MIT License](https://opensource.org/licenses/MIT)，欢迎使用和修改。 +稳定退出码：`0` 成功，`2` 参数/输入问题，`3` 提取失败，`4` 批处理部分失败，`5` 缺少可选依赖。 ---- +## Agent Skill -## 贡献指南 -欢迎贡献代码或建议！ -- **报告问题**：提交 GitHub Issue。 -- **提交代码**：Fork 仓库并创建 Pull Request。 -- **规范**：遵循 PEP 8，添加注释。 +可复用 Skill 位于 [`agent-skill/pptx-extraction`](agent-skill/pptx-extraction)： -### 贡献流程 -1. Fork 仓库。 -2. 创建分支（`git checkout -b feature/xxx`）。 -3. 提交更改（`git commit -m "描述"`）。 -4. Push 到远程（`git push origin feature/xxx`）。 -5. 创建 Pull Request。 +```bash +python agent-skill/pptx-extraction/scripts/extract.py \ + "slides.pptx" \ + --output "output/agent-run" +``` ---- +它会默认脱敏作者类元数据，并指导 Agent 区分正文、备注、图表值和 OCR 派生文本。Skill 已通过官方 +`quick_validate.py`，发布脚本会将项目和 Skill 生成两个独立 ZIP。 + +## 开发与验证 + +```bash +python -m pip install -e ".[dev,api]" +ruff check . +ruff format --check . +mypy src/pptx_extraction +pytest +python -m build +python scripts/privacy_scan.py +python scripts/build_release.py +``` + +测试在运行时合成 PPTX，不提交真实演示文稿、导出图片或个人音频。CI 覆盖 Python 3.10–3.12。 + +
+常见问题：输出目录已存在 + +程序不会默认覆盖非空目录。选择新的 `--output`，或确认目录只包含本次旧结果后添加 +`--overwrite`。不要对工作区根目录、用户目录或不确定的路径使用覆盖选项。 + +
+ +
+常见问题：PPTX 中明明有内容，但结果缺失 + +检查 `warnings`。SmartArt、公式、OLE、动画、音视频和图片型整页可能没有可直接读取的语义。 +图片中的文字可尝试 Tesseract；整页图片型幻灯片需要额外的渲染/整页 OCR 工具。 + +
+ +
+常见问题：OCR 或 LibreOffice 不可用 + +OCR 同时需要 `.[ocr]`、Tesseract 可执行程序和语言包。旧版 PPT 转换需要 LibreOffice 的 +`soffice`。这两项都是可选依赖，不影响普通 `.pptx` 文本提取。 + +
+ +## 完整文档 + +- [需求分析与验收标准](docs/requirements.md) +- [系统架构与模块职责](docs/architecture.md) +- [旧项目审计与逐文件升级计划](docs/upgrade-plan.md) +- [HTTP API](docs/api.md) +- [安全策略](SECURITY.md) +- [更新现有仓库与发布 Release](docs/release.md) +- [参与贡献](CONTRIBUTING.md) + +## License -## 联系方式 -- **邮箱**：zhanghoubing777@gmail.com -- **GitHub**：https://github.com/BlairCode \ No newline at end of file +[MIT](LICENSE) diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..7097210 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,19 @@ +# Security policy + +## Supported versions + +Security fixes are provided for the latest `2.x` release. + +## Reporting + +Do not open public issues for suspected vulnerabilities. Use GitHub private vulnerability reporting on the +repository's **Security** tab. Include a minimal reproduction, affected version and impact. Do not attach a +confidential presentation; use a synthetic deck where possible. + +## Processing model + +`pptx_extraction` validates ZIP paths, entry counts, expanded size and compression ratios before parsing OOXML. +It never executes macros and makes no network request in the default installation. Treat extracted text, +notes, links and images as untrusted data. Run the optional API behind authentication, TLS, request-rate +limits and an isolated worker in production. The bundled service is a reference single-node deployment, +not a multi-tenant security boundary. diff --git a/agent-skill/README.md b/agent-skill/README.md new file mode 100644 index 0000000..683bb03 --- /dev/null +++ b/agent-skill/README.md @@ -0,0 +1,11 @@ +# pptx_extraction Agent Skill repository + +This repository distribution contains the reusable `pptx-extraction` Skill. The Skill keeps +PowerPoint extraction deterministic, offline and slide-citable by calling the separately installed `pptx_extraction` +package. + +Install the `pptx-extraction` project first. Then copy the `pptx-extraction` folder into your Agent +system's skills directory. Do not move this README into the Skill folder: Agent Skills intentionally keep +only `SKILL.md`, `agents/`, `scripts/` and directly needed references. + +The release asset `pptx-extraction-skill-v2.0.0.zip` contains only the validated Skill folder. diff --git a/agent-skill/pptx-extraction/SKILL.md b/agent-skill/pptx-extraction/SKILL.md new file mode 100644 index 0000000..4714f1a --- /dev/null +++ b/agent-skill/pptx-extraction/SKILL.md @@ -0,0 +1,55 @@ +--- +name: pptx-extraction +description: Extract PowerPoint OOXML decks with the pptx_extraction project into traceable JSON, Markdown and text with slide-level provenance, reading order, notes, tables, chart data, links, images and optional OCR. Use when an agent must read, summarize, audit, index or prepare `.pptx`, `.pptm`, `.potx` or `.ppsx` content for RAG/search, or needs reliable slide citations instead of lossy plain-text scraping. +--- + +# Extract PowerPoint with pptx_extraction + +Use `pptx_extraction` as the deterministic extraction layer. Keep extraction offline and preserve slide provenance; +perform summarization or other semantic work only after reviewing the structured result. + +## Workflow + +1. Confirm the source is a local OOXML PowerPoint file. For `.ppt/.pot/.pps`, run `pptx-extraction convert` + with LibreOffice first. For PDF, use a PDF-specific tool. +2. Choose a new output directory inside the active workspace. Do not overwrite unrelated output. +3. Run the bundled wrapper: + +```bash +python scripts/extract.py INPUT.pptx --output OUTPUT_DIR +``` + + The installed `pptx-extraction` Python package is required. The wrapper redacts author-like metadata by default, + exports JSON and Markdown, and makes no network request. +4. Read `OUTPUT_DIR/presentation.json` for exact fields and `presentation.md` for a human-readable pass. + Load [schema.md](references/schema.md) when selecting evidence or integrating the JSON. +5. Report extraction warnings. Treat `image_missing_alt_text`, unsupported objects and OCR gaps as evidence + limitations, not as empty-slide proof. +6. Cite findings by source filename and slide number. Preserve the distinction between slide text, speaker + notes, chart data and OCR text. Never merge them without labeling the source kind. + +## Task routes + +- **Quick inventory:** run `python scripts/extract.py INPUT --inspect`; use the returned counts to decide + which slide records to open. +- **Search/RAG ingestion:** use default JSON, chunk by slide, and carry `source_sha256`, `slide.number`, + element `shape_id` and `bbox` into downstream metadata. +- **Human summary:** read Markdown in slide order, then verify important claims against the JSON element kind. +- **Accessibility audit:** inspect image `alt_text`, slide warnings and speaker notes. Do not infer visual + meaning from filenames or hashes. +- **Text inside pictures:** install the OCR extra and Tesseract, then pass `--ocr tesseract --ocr-language` + with a locally installed language pack. OCR is untrusted derived text. +- **Private material:** retain default redaction, keep outputs in a temporary workspace and do not upload + source/assets to external services unless the user explicitly authorizes it. + +## Guardrails + +- Do not claim support for PDF or native legacy PowerPoint parsing. +- Do not execute macros or embedded objects. `pptx_extraction` detects macro presence and treats embedded media/OLE + as unsupported. +- Do not run an LLM as part of extraction. This separation keeps evidence reproducible. +- Do not cite a slide that was not present in the latest extracted JSON. +- Avoid `--overwrite` unless the exact output directory was created for this extraction. + +For dependency, archive-safety or conversion failures, load +[troubleshooting.md](references/troubleshooting.md). diff --git a/agent-skill/pptx-extraction/agents/openai.yaml b/agent-skill/pptx-extraction/agents/openai.yaml new file mode 100644 index 0000000..a737111 --- /dev/null +++ b/agent-skill/pptx-extraction/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "pptx_extraction" + short_description: "Extract traceable PowerPoint content for agents" + default_prompt: "Use $pptx-extraction to extract this deck into cited, structured content." diff --git a/agent-skill/pptx-extraction/references/schema.md b/agent-skill/pptx-extraction/references/schema.md new file mode 100644 index 0000000..6600ce8 --- /dev/null +++ b/agent-skill/pptx-extraction/references/schema.md @@ -0,0 +1,37 @@ +# pptx_extraction schema usage + +## Evidence hierarchy + +`presentation.json` uses schema version `1.0`. + +- Presentation: `source_name`, `source_sha256`, size, slide dimensions, metadata, slides and warnings. +- Slide: 1-based `number`, `title`, `hidden`, `layout_name`, `text_blocks`, `tables`, `charts`, `images` + and `notes`. +- Element provenance: `order` is visual reading order; `z_order` is original stacking order; + `shape_id`/`shape_name` locate the PowerPoint object; `bbox` contains points and normalized ratios. +- Text: `kind` is `title` or `body`, `level` preserves paragraph hierarchy, and `hyperlinks` lists targets. +- Chart: categories and series are source workbook values exposed by PowerPoint, not OCR. +- Image: `sha256` identifies content, `asset_path` may be shared by duplicate images, `alt_text` is author + supplied and `ocr_text` is derived/untrusted. + +## Citation pattern + +For prose answers use: `SourceDeck.pptx, slide 7 (chart: Revenue by segment)` or the host application's +equivalent local-file citation. Include the source hash when results from multiple versions may be confused. +Do not cite `order` as a slide number. Label speaker notes and OCR explicitly. + +## Chunking pattern + +Use one slide as the default chunk. For each downstream chunk carry: + +```json +{ + "source_name": "deck.pptx", + "source_sha256": "...", + "slide_number": 7, + "element_kind": "text|table|chart|note|image_ocr", + "shape_id": 12 +} +``` + +Split a slide further only when it is unusually dense; never discard the slide-level locator. diff --git a/agent-skill/pptx-extraction/references/troubleshooting.md b/agent-skill/pptx-extraction/references/troubleshooting.md new file mode 100644 index 0000000..fcfd4a2 --- /dev/null +++ b/agent-skill/pptx-extraction/references/troubleshooting.md @@ -0,0 +1,28 @@ +# Troubleshooting + +## Unsupported format + +- `.pptx/.pptm/.potx/.ppsx`: process directly. +- `.ppt/.pot/.pps`: install LibreOffice and run `pptx-extraction convert INPUT.ppt -o CONVERTED_DIR`. +- `.pdf`: route to a PDF extraction tool. + +## Optional dependency errors + +Install the project first (`python -m pip install .` from its source checkout). For OCR install +`pptx-extraction[ocr]` plus the native Tesseract executable and requested language pack. OCR remains optional; +normal PowerPoint text never needs it. + +## Unsafe package + +`pptx_extraction` rejects encrypted entries, traversal paths, excessive archive entries, expanded-size limits and +suspicious compression ratios. Do not bypass these checks for an untrusted file. If a trusted large deck +exceeds defaults, use the Python API with a narrowly increased `PackageLimits` value and document why. + +## Missing content + +- SmartArt, equations, diagrams, embedded spreadsheets, OLE objects, audio, video and animations may not + expose semantic content through `python-pptx`. +- Image-only slides require rendering or slide-level OCR from a presentation/PDF tool; embedded-image OCR + does not render the whole slide. +- Hidden slides are extracted and marked `hidden: true`. +- Review the `warnings` array before concluding that a slide is empty. diff --git a/agent-skill/pptx-extraction/scripts/extract.py b/agent-skill/pptx-extraction/scripts/extract.py new file mode 100644 index 0000000..837932d --- /dev/null +++ b/agent-skill/pptx-extraction/scripts/extract.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +"""Thin, deterministic Agent wrapper around the pptx_extraction package.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Extract a PowerPoint deck with pptx_extraction.") + parser.add_argument("source", type=Path) + parser.add_argument("--output", type=Path) + parser.add_argument("--inspect", action="store_true") + parser.add_argument("--format", action="append", choices=("json", "markdown", "text")) + parser.add_argument("--ocr", choices=("none", "tesseract"), default="none") + parser.add_argument("--ocr-language", default="eng") + parser.add_argument("--no-assets", action="store_true") + parser.add_argument("--include-private-metadata", action="store_true") + parser.add_argument("--overwrite", action="store_true") + return parser + + +def main() -> int: + args = build_parser().parse_args() + try: + from pptx_extraction.exceptions import PptxExtractionError + from pptx_extraction.models import ExtractionOptions + from pptx_extraction.pipeline import extract_file, inspect_file + except ImportError: + print( + json.dumps( + { + "error": { + "code": "pptx_extraction_not_installed", + "message": ( + "Install the pptx_extraction project package before using this skill." + ), + } + }, + ensure_ascii=False, + ), + file=sys.stderr, + ) + return 5 + + options = ExtractionOptions( + include_assets=not args.no_assets, + redact_metadata=not args.include_private_metadata, + ocr_backend=args.ocr, + ocr_language=args.ocr_language, + ) + try: + if args.inspect: + record = inspect_file(args.source, options=options) + print(json.dumps(record.summary, ensure_ascii=False, indent=2)) + return 0 + if args.output is None: + raise ValueError("--output is required unless --inspect is used.") + result = extract_file( + args.source, + args.output, + options=options, + formats=tuple(args.format or ("json", "markdown")), + overwrite=args.overwrite, + ) + print( + json.dumps( + { + "status": "ok", + "output_dir": str(result.output_dir), + "summary": result.record.summary, + "warnings": [warning.code for warning in result.record.warnings], + }, + ensure_ascii=False, + indent=2, + ) + ) + return 0 + except (PptxExtractionError, ValueError) as exc: + code = getattr(exc, "code", "invalid_arguments") + print( + json.dumps({"error": {"code": code, "message": str(exc)}}, ensure_ascii=False), + file=sys.stderr, + ) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/app.py b/app.py deleted file mode 100644 index ebe5a86..0000000 --- a/app.py +++ /dev/null @@ -1,238 +0,0 @@ -# app.py -# Author: BLAIR -# Backend interface for PPT content extraction and optimization - -import os -import logging -import time -from flask import Flask, request, jsonify, send_from_directory -from flask_cors import CORS -from modules.ppt_text_extraction import extract_text_from_ppt, extract_metadata, extract_text_from_ppt_legacy, extract_metadata_from_ppt_legacy -from modules.image_extraction_p import extract_images_from_ppt_paddleocr, extract_images_from_ppt_legacy -from modules.ai_optimizer import optimize_text_with_ai -from modules.utils import setup_logger, validate_file_type -from modules.config import OUTPUT_DIR_2 -import re -import warnings -import socket -from dotenv import load_dotenv -from concurrent.futures import ThreadPoolExecutor -import tempfile - -load_dotenv() - -LOG_FILE = os.getenv("LOG_FILE", "ppt_processor.log") -logging.basicConfig( - level=logging.INFO, - format='%(asctime)s - %(levelname)s - %(message)s', - handlers=[logging.FileHandler(LOG_FILE), logging.StreamHandler()] -) -logger = setup_logger() - -USE_GPU = os.getenv("USE_GPU", "False").lower() in ("true", "1", "yes") -logger.info(f"GPU enabled: {USE_GPU}") - -app = Flask(__name__, static_folder='static', static_url_path='') -CORS(app, resources={r"/api/*": {"origins": "*"}}) -executor = ThreadPoolExecutor(max_workers=1) - -PROJECT_ROOT = os.path.abspath(os.path.dirname(__file__)) -OUTPUT_DIR = os.path.abspath(os.getenv("OUTPUT_DIR", OUTPUT_DIR_2)) -if not os.path.exists(OUTPUT_DIR): - os.makedirs(OUTPUT_DIR, exist_ok=True) - logger.info(f"Created output directory: {OUTPUT_DIR}") - -SUPPORTED_FORMATS = ['.ppt', '.pptx', '.pot', '.potx', '.pps', '.ppsx', '.pptm', '.pdf'] - -def clean_text_output(text_list): - try: - cleaned_output = [] - watermark_pattern = re.compile(r'stablediffusionweb\.com') - for slide in text_list: - if not slide or not slide.strip(): - continue - cleaned_slide = watermark_pattern.sub('', slide) - cleaned_slide = re.sub(r'\n\s*\n', '\n', cleaned_slide.strip()) - cleaned_output.append(cleaned_slide) - return cleaned_output - except Exception as e: - logger.error(f"Error cleaning text output: {str(e)}", exc_info=True) - return [] - -@app.route('/') -def serve_index(): - try: - return send_from_directory(app.static_folder, 'index.html') - except Exception as e: - logger.error(f"Error serving index.html: {str(e)}", exc_info=True) - return jsonify({"error": "Failed to load index page"}), 500 - -@app.route('/') -def serve_static(path): - if os.path.exists(os.path.join(app.static_folder, path)): - return send_from_directory(app.static_folder, path) - return jsonify({"error": "File not found"}), 404 - -@app.route('/api/process_ppt', methods=['POST']) -def process_ppt(): - logger.info("Received POST request to /api/process_ppt") - file_path = None - try: - if 'file' not in request.files: - logger.error("No file part in the request") - return jsonify({"error": "No file uploaded"}), 400 - - file = request.files['file'] - if file.filename == '': - logger.error("No file selected") - return jsonify({"error": "No file selected"}), 400 - - # 使用临时文件避免覆盖和权限问题 - temp_dir = tempfile.gettempdir() - file_path = os.path.join(temp_dir, f"temp_{os.urandom(8).hex()}_{file.filename}") - logger.info(f"Saving file to temporary path: {file_path}") - file.save(file_path) - - if not validate_file_type(file_path, SUPPORTED_FORMATS): - os.remove(file_path) - logger.error(f"Invalid file type: {file.filename}. Supported formats: {SUPPORTED_FORMATS}") - return jsonify({"error": f"Invalid file type: {file.filename}. Supported formats: {', '.join(SUPPORTED_FORMATS)}"}), 400 - - if not os.path.exists(file_path): - logger.error(f"File does not exist after saving: {file_path}") - return jsonify({"error": "File save failed"}), 500 - - logger.info(f"Starting processing for file: {file_path}") - warnings.filterwarnings("ignore", category=UserWarning, module="PIL.Image") - - def process_file(file_path): - ext = os.path.splitext(file_path.lower())[1] - is_pptx = ext == '.pptx' - - metadata = extract_metadata(file_path) if is_pptx else extract_metadata_from_ppt_legacy(file_path) - if "Error" in metadata: - logger.error(f"Metadata extraction failed: {metadata['Error']}") - metadata_output = "Metadata extraction failed\n" - else: - metadata_output = "\n".join([f"{key}: {value}" for key, value in metadata.items()]) - logger.info("Metadata processed") - - text_output = extract_text_from_ppt(file_path) if is_pptx else extract_text_from_ppt_legacy(file_path) - if not text_output: - logger.warning("No text extracted from PPT slides") - text_output = [] - - image_output = extract_images_from_ppt_paddleocr(file_path, OUTPUT_DIR, use_gpu=USE_GPU) if is_pptx else extract_images_from_ppt_legacy(file_path, OUTPUT_DIR, use_gpu=USE_GPU) - if not image_output: - logger.warning("No image text extracted") - image_output = [] - - cleaned_text_output = clean_text_output(text_output + image_output) - combined_output = "\n".join([metadata_output] + cleaned_text_output) - if not combined_output.strip(): - logger.warning("No combined output generated") - combined_output = "No content extracted" - - optimized_text = optimize_text_with_ai(combined_output) or combined_output - output_file = os.path.abspath(os.path.join(OUTPUT_DIR, f"optimized_output_{file.filename}.txt")) - with open(output_file, "w", encoding="utf-8") as f: - f.write(optimized_text) - logger.info(f"Optimized text saved to {output_file}") - - return optimized_text, output_file - - future = executor.submit(process_file, file_path) - optimized_text, output_file = future.result(timeout=120) - - # 重试删除文件 - for _ in range(3): # 尝试 3 次 - try: - if os.path.exists(file_path): - os.remove(file_path) - logger.info(f"Temporary file removed: {file_path}") - break - except PermissionError: - logger.warning(f"Retrying file removal: {file_path}") - time.sleep(1) # 等待 1 秒重试 - - return jsonify({ - "message": "File processed successfully", - "output_file": output_file, - "optimized_text": optimized_text - }), 200 - - except PermissionError as e: - logger.error(f"Permission denied: {str(e)}", exc_info=True) - if file_path and os.path.exists(file_path): - for _ in range(3): - try: - os.remove(file_path) - logger.info(f"Temporary file removed after permission error: {file_path}") - break - except PermissionError: - logger.warning(f"Retrying file removal after permission error: {file_path}") - time.sleep(1) - return jsonify({"error": "Permission denied"}), 403 - except TimeoutError: - logger.error(f"Processing timed out for file: {file_path}") - if file_path and os.path.exists(file_path): - os.remove(file_path) - logger.info(f"Temporary file removed after timeout: {file_path}") - return jsonify({"error": "Processing timed out"}), 504 - except Exception as e: - logger.error(f"Error processing file: {str(e)}", exc_info=True) - if file_path and os.path.exists(file_path): - for _ in range(3): - try: - os.remove(file_path) - logger.info(f"Temporary file removed after error: {file_path}") - break - except PermissionError: - logger.warning(f"Retrying file removal after error: {file_path}") - time.sleep(1) - return jsonify({"error": "Processing failed", "details": str(e)}), 500 - -@app.route('/health', methods=['GET']) -def health_check(): - logger.info("Health check requested") - return jsonify({"status": "healthy", "message": "PPT Processor server is running"}), 200 - -def check_port(host, port): - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - s.settimeout(1) - try: - s.bind((host, port)) - return True - except OSError: - logger.error(f"Port {port} is already in use") - return False - -if __name__ == "__main__": - HOST = os.getenv("HOST", "0.0.0.0") - PORT = int(os.getenv("PORT", 5000)) - MAX_PORT_ATTEMPTS = int(os.getenv("MAX_PORT_ATTEMPTS", 5)) - - os.makedirs(OUTPUT_DIR, exist_ok=True) - static_dir = os.path.join(PROJECT_ROOT, 'static') - if not os.path.exists(static_dir): - os.makedirs(static_dir, exist_ok=True) - logger.info(f"Created static directory: {static_dir}") - - if not os.path.exists(os.path.join(static_dir, 'index.html')): - logger.warning("index.html not found in static folder. Please place it there.") - - try: - for attempt in range(MAX_PORT_ATTEMPTS): - if check_port(HOST, PORT): - logger.info(f"Starting Flask server on {HOST}:{PORT}") - app.run(host=HOST, port=PORT, debug=False, threaded=False) - break - else: - PORT += 1 - logger.info(f"Trying next port: {PORT}") - else: - logger.error(f"Failed to find an available port after {MAX_PORT_ATTEMPTS} attempts") - raise SystemExit(f"Startup failed: No available port found after {MAX_PORT_ATTEMPTS} attempts") - except Exception as e: - logger.error(f"Server startup failed: {str(e)}") - raise \ No newline at end of file diff --git a/caption.txt b/caption.txt deleted file mode 100644 index 598b0bd..0000000 --- a/caption.txt +++ /dev/null @@ -1,56 +0,0 @@ -音频转字幕API说明文档 -创建日期: 2025年3月20日 - -1. 音频转字幕接口 -请求方法: POST -请求路径: /audio_to_subtitles -描述: 将上传的音频文件转换为字幕文件（SRT格式） - -请求参数: -- file: 音频文件 - 格式: MP3/WAV等（由ffmpeg支持的格式） - 描述: 需要转换为字幕的源音频文件 -- language: 语音语言 - 默认值: "zh" - 描述: 音频的语言，例如 "zh" (中文), "en" (英文) -- model_size: 模型大小 - 默认值: "medium" - 描述: 使用 faster_whisper 模型的大小，可选值: "tiny", "base", "small", "medium", "large" - -响应: -成功: - 状态码: 200 - 内容类型: text/plain - 返回值: 生成的字幕文本 (SRT 格式) - 示例: - 1 - 00:00:00,000 --> 00:00:02,500 - 你好，这是一个测试 - - 2 - 00:00:02,501 --> 00:00:05,000 - 欢迎使用音频转字幕服务 -失败: -状态码: 400 或 500 -内容类型: application/json -返回值示例: -{ - "error": "音频文件不存在" 或 "发生错误：具体错误描述" -} - -示例请求: -curl -X POST \ --F "file=@sample.mp3" \ --F "language=zh" \ --F "model_size=medium" \ -http://localhost:5000/audio_to_subtitles - -注意事项: -1. 文件上传使用 multipart/form-data 格式 -2. 字幕文本使用简体中文输出（通过opencc t2s转换） -3. 时间戳精度为毫秒级 -4. 每段音频按句号分句，平均分配时间 -5. 处理时间取决于音频长度和模型大小 - -库依赖： -pip install ffmpeg-python faster-whisper opencc-python-reimplemented \ No newline at end of file diff --git a/delete.py b/delete.py deleted file mode 100644 index b8ff038..0000000 --- a/delete.py +++ /dev/null @@ -1,9 +0,0 @@ -from modules.config import OUTPUT_DIR_2, OUTPUT_DIR -import shutil - -def main(): - shutil.rmtree(OUTPUT_DIR_2) - print(f"{OUTPUT_DIR_2} DELETED SUCCESSFULLY!") - -if __name__ == "__main__": - main() diff --git a/docs/api.md b/docs/api.md new file mode 100644 index 0000000..36e667d --- /dev/null +++ b/docs/api.md @@ -0,0 +1,16 @@ +# HTTP API + +Install `pptx-extraction[api]`, then run the reference single-node service: + +```bash +uvicorn pptx_extraction.api:create_app --factory --host 127.0.0.1 --port 8000 +``` + +- `GET /healthz` reports process health. +- `POST /v1/jobs` accepts one multipart `file`, returns `202` and a random job ID. +- `GET /v1/jobs/{id}` returns `queued`, `running`, `succeeded` or `failed`. +- `GET /v1/jobs/{id}/result` returns schema v1 JSON after success. + +The service caps streamed uploads with `PPTX_EXTRACTION_MAX_UPLOAD_MB`, stores only randomized job paths and +redacts author-like metadata. It intentionally enables no wildcard CORS. Add authentication, TLS, reverse- +proxy limits, rate limiting, durable queue/object storage and lifecycle cleanup before multi-user deployment. diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000..888f961 --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,54 @@ +# Architecture + +## System shape + +```mermaid +flowchart LR + A["CLI / Python / HTTP API"] --> B["Input validation"] + B --> C["OOXML extractor"] + B --> L["Optional LibreOffice converter"] + L --> C + C --> D["Domain model"] + C --> O["Optional OCR backend"] + O --> D + D --> E["JSON exporter"] + D --> F["Markdown exporter"] + D --> G["Text exporter"] + E --> H["RAG / search / agents"] + F --> H + G --> H +``` + +The extractor is pure orchestration around `python-pptx`; OOXML safety checks run before it. Optional +dependencies are loaded only when requested. Exporters depend on the domain model, never on PowerPoint +objects, so output behavior can be tested independently. + +## Package map + +| Path | Responsibility | Public entry points | +|---|---|---| +| `src/pptx_extraction/models.py` | Versioned serializable domain records | `PresentationRecord`, `SlideRecord`, `ExtractionOptions` | +| `src/pptx_extraction/security.py` | OOXML/ZIP validation and source hashing | `validate_package`, `sha256_file` | +| `src/pptx_extraction/extractors/pptx.py` | Shape traversal and PowerPoint semantics | `PptxExtractor.extract` | +| `src/pptx_extraction/ocr.py` | Lazy OCR protocol and Tesseract adapter | `create_ocr_backend` | +| `src/pptx_extraction/exporters.py` | JSON/Markdown/text serialization | `export_record` | +| `src/pptx_extraction/pipeline.py` | Atomic output orchestration | `extract_file`, `inspect_file`, `batch_extract` | +| `src/pptx_extraction/converter.py` | Optional legacy Office conversion | `convert_legacy` | +| `src/pptx_extraction/cli.py` | Stable command/exit-code interface | `main` | +| `src/pptx_extraction/api.py` | Optional bounded job service | `create_app` | + +## Data contract + +`schema_version` is currently `1.0`. Every presentation carries a source hash, sanitized source name, +metadata, slide size, slides and warnings. Slide children carry both semantic content and source locators. +Bounding boxes are expressed in points and normalized ratios so consumers are independent of Office EMUs. +Image file names use the first 16 SHA-256 characters and the detected extension. + +Exit codes: `0` success, `2` usage/validation error, `3` extraction failure, `4` partial batch failure and +`5` optional dependency unavailable. + +## Deployment notes + +The CLI/Python library is the primary production surface. The API intentionally stores state on one node and +uses a bounded thread pool. Multi-node deployments should replace the local job store with a durable queue, +object storage and authenticated result URLs without changing the extraction package. diff --git a/docs/release.md b/docs/release.md new file mode 100644 index 0000000..ce79b92 --- /dev/null +++ b/docs/release.md @@ -0,0 +1,137 @@ +# 更新现有 GitHub 仓库与发布 Release + +目标仓库： + +以下命令以 Windows PowerShell 为例。项目已经存在，不要运行 `gh repo create`，也不要修改 +`origin`。默认分支当前为 `master`。 + +## 1. 发布前检查 + +在项目根目录执行： + +```powershell +git remote -v +python scripts/privacy_scan.py +ruff check . +ruff format --check . +mypy src/pptx_extraction +pytest +python -m build +python scripts/build_release.py +git diff --check +``` + +`git remote -v` 应显示： + +```text +origin https://github.com/BlairCode/pptx_extraction.git (fetch) +origin https://github.com/BlairCode/pptx_extraction.git (push) +``` + +生成的发布文件： + +```text +dist/pptx_extraction-2.0.0-py3-none-any.whl +dist/pptx_extraction-2.0.0.tar.gz +release/pptx_extraction-v2.0.0.zip +release/pptx_extraction-skill-v2.0.0.zip +``` + +`dist/`、`release/`、`work/`、`_legacy_local_backup/` 和真实 PPTX/音频均被 Git 忽略。不要使用 +`git add -f` 强制加入这些本地内容。 + +## 2. 在现有仓库创建升级分支 + +当前工作区包含完整重构，可直接从当前提交点创建升级分支： + +```powershell +git switch -c codex/pptx-extraction-v2 +git add -A +git status --short +git diff --cached --check +git diff --cached --name-status +``` + +重点确认： + +- 旧原型脚本、个人样例和运行输出显示为删除； +- `src/pptx_extraction/`、`tests/`、`docs/`、`agent-skill/` 显示为新增； +- 没有 `.env`、PPTX、音频、日志、`work/`、`dist/` 或 `release/` 文件被暂存。 + +确认后提交并推送： + +```powershell +git commit -m "refactor: rebuild pptx_extraction for structured workflows" +git push -u origin codex/pptx-extraction-v2 +``` + +## 3. 创建 Pull Request + +使用 GitHub CLI： + +```powershell +gh auth status +gh pr create ` + --repo BlairCode/pptx_extraction ` + --base master ` + --head codex/pptx-extraction-v2 ` + --title "refactor: pptx_extraction 2.0" ` + --body "Rebuild extraction around structured JSON/Markdown output, safe OOXML validation, batch processing, optional OCR/API, tests, documentation, and an Agent Skill." +``` + +等待 GitHub Actions 全部通过，再在 GitHub 页面合并 PR。不要在 CI 失败时直接给 `master` 打标签。 + +## 4. 合并后发布 v2.0.0 + +回到本地并同步默认分支： + +```powershell +git switch master +git pull --ff-only origin master +git tag -a v2.0.0 -m "pptx_extraction 2.0.0" +git push origin v2.0.0 +``` + +创建 Release，并分别上传项目、Python 包和 Skill： + +```powershell +gh release create v2.0.0 ` + "release/pptx_extraction-v2.0.0.zip" ` + "release/pptx_extraction-skill-v2.0.0.zip" ` + "dist/pptx_extraction-2.0.0-py3-none-any.whl" ` + "dist/pptx_extraction-2.0.0.tar.gz" ` + --repo BlairCode/pptx_extraction ` + --title "pptx_extraction 2.0.0" ` + --notes-file CHANGELOG.md +``` + +这样 Skill 和项目仍是两个独立 Release 文件，但共同发布在既有仓库的同一个版本页面中。 + +## 5. Release 发布后验证 + +下载 wheel 到新的临时目录并测试： + +```powershell +python -m venv release-check +.\release-check\Scripts\Activate.ps1 +python -m pip install "PATH_TO_WHEEL\pptx_extraction-2.0.0-py3-none-any.whl" +pptx-extraction --version +``` + +解压 Skill ZIP，确认根目录为 `pptx-extraction/`，并包含： + +```text +SKILL.md +agents/openai.yaml +scripts/extract.py +references/schema.md +references/troubleshooting.md +``` + +## 关于旧 Git 历史中的隐私内容 + +现有仓库历史过去曾包含个人路径、样例 PPTX、音频和旧 README 联系信息。普通更新只会确保新提交与 +Release 不再包含它们，不会删除旧提交中的对象。如果仓库已经公开，这些内容可能已被克隆或缓存。 + +若确实需要从全部历史中清除，先备份并评估影响，再使用 `git filter-repo` 重写历史；这会改变所有 +提交哈希并要求强制推送和通知协作者，不应与普通 v2.0 更新混在同一次操作中。 diff --git a/docs/requirements.md b/docs/requirements.md new file mode 100644 index 0000000..42ec390 --- /dev/null +++ b/docs/requirements.md @@ -0,0 +1,62 @@ +# Product requirements + +## Product position + +`pptx_extraction` converts PowerPoint material into traceable, machine-readable knowledge for search, RAG, content +migration, accessibility auditing, compliance review and Agent workflows. It is an extraction layer, not a +slide renderer or a generative rewriting system. + +## Primary users and jobs + +| User | Job | Required outcome | +|---|---|---| +| Knowledge engineer | Ingest decks into RAG/search | Stable JSON, page citations, deterministic IDs and no hidden network calls | +| Data/content team | Migrate large slide libraries | Batch execution, partial-failure reports and collision-safe outputs | +| Compliance/accessibility team | Audit notes, metadata, links and image descriptions | Source coordinates, warnings and optional metadata redaction | +| Developer | Embed extraction in a service | Typed Python API, CLI contract, versioned schema and structured errors | +| Agent | Inspect a deck and cite evidence | Concise Markdown/JSON with slide numbers, element kinds and asset paths | + +## Functional requirements + +1. Accept `.pptx`, `.pptm`, `.potx` and `.ppsx` OOXML packages. Convert legacy `.ppt/.pot/.pps` + through an explicitly invoked LibreOffice adapter; reject PDF rather than advertising false support. +2. Extract core metadata, slide dimensions, hidden state, layout name, text hierarchy, hyperlinks, tables, + chart titles/categories/series, speaker notes and embedded pictures. +3. Preserve provenance through slide number, shape name/id, normalized bounding box and source SHA-256. +4. Order visual elements predictably by rows and horizontal position while retaining original z-order. +5. Save media with content hashes, deduplicate repeated blobs and optionally run OCR once per unique image. +6. Export schema-versioned JSON plus readable Markdown and plain text. +7. Support inspect, validate, extract and concurrent batch workflows from a cross-platform CLI. +8. Offer an optional job-based HTTP API with bounded uploads, non-blocking processing and result polling. +9. Emit actionable warnings for unsupported shapes, missing alt text, encrypted/corrupt packages and + extraction degradation without silently claiming success. + +## Non-functional requirements + +- **Safety:** reject path traversal, encrypted ZIP entries, excessive entry counts, expanded-size limits and + suspicious compression ratios before `python-pptx` parses a file. +- **Privacy:** perform no outbound request by default; never log slide content; expose metadata redaction; + use random service job IDs and sanitized filenames. +- **Portability:** core works on Windows, Linux and macOS without PowerPoint installed. +- **Determinism:** identical input/options produce identical semantic output; timestamps are excluded from + content identity and image names are hash-derived. +- **Performance target:** on a modern laptop, parse a normal 100-slide/50 MB deck without OCR in under + 30 seconds and below 1 GB RSS. This is a target to benchmark, not a fabricated guarantee. +- **Maintainability:** typed modules, explicit protocols, no heavyweight import-time side effects, at least + one integration fixture covering text/table/chart/notes/image behavior. +- **Compatibility:** schema breaking changes require a major schema version; CLI exit codes are documented. + +## Deliberate exclusions for v2 + +- Pixel-perfect slide rendering, animation timelines, video/audio transcription and embedded-object execution. +- Handwriting/image-layout OCR beyond embedded pictures. +- Automatic LLM rewriting. Enrichment belongs downstream so extraction remains auditable. +- Native `.ppt` parsing without conversion and PDF parsing (use a dedicated converter/tool first). + +## Acceptance criteria + +- A synthetic deck round-trips through the pipeline with expected text, table, chart, notes and one deduped + image in JSON and Markdown. +- Malformed/non-ZIP input, traversal entries and configured archive-limit violations fail with stable codes. +- Batch mode continues after one failed file and returns a non-zero partial-failure exit. +- The sdist/wheel, Agent Skill validation and repository privacy scan all pass in CI. diff --git a/docs/upgrade-plan.md b/docs/upgrade-plan.md new file mode 100644 index 0000000..d5e35d9 --- /dev/null +++ b/docs/upgrade-plan.md @@ -0,0 +1,36 @@ +# Upgrade plan and traceability + +## Audit summary + +The prototype mixed parsing, OCR, text generation, Flask routing and file cleanup in root scripts. It loaded +large ML models at import time, called OCR twice per image, hard-coded personal Windows paths, returned server +filesystem paths, accepted unsupported PDF input and used tests that referenced another computer. The empty +requirements file made the advertised setup unreproducible. Generated text/images/audio and a personal deck +were committed alongside source. + +## File-level migration + +| Legacy file | Action | Replacement | +|---|---|---| +| `main.py` | remove duplicated synchronous orchestration | `pipeline.py`, `cli.py`, `__main__.py` | +| `app.py`, `static/index.html` | replace blocking Flask demo and wildcard CORS | optional `api.py` job service | +| `modules/config.py` | remove personal absolute paths | CLI args and `ServiceSettings.from_env()` | +| `modules/ppt_text_extraction.py`, `text_extraction.py` | consolidate conflicting parsers | `extractors/pptx.py` | +| `modules/image_extraction_p.py`, `image_extraction_t.py` | replace eager vendor-specific OCR | `ocr.py` lazy backend + hash dedupe | +| `modules/ai_optimizer.py` | remove random, lossy and import-heavy rewriting | deterministic exporters; downstream enrichment boundary | +| `generate_caption.py`, `caption.txt`, audio samples | remove unrelated scope | separate future repository if needed | +| `delete.py` | remove unsafe fixed-directory deletion | atomic pipeline outputs and explicit overwrite flag | +| `requirements.txt` | replace empty file | `pyproject.toml` core/optional/dev dependency groups | +| `tests/test_*.py` | replace machine-specific print scripts | generated fixtures and behavior assertions | +| committed `output2/`, `.srt`, sample deck | remove private/runtime artifacts | ignored local output; tests synthesize fixtures | + +## Delivery phases + +1. **Foundation:** package metadata, models, errors, logging boundary and safe archive inspection. +2. **Extraction:** recursive grouped-shape traversal, reading order, notes/tables/charts/links/images, warnings. +3. **Interfaces:** atomic multi-format pipeline, Python API, CLI, batch execution, legacy conversion, HTTP jobs. +4. **Assurance:** unit/integration/security tests, schema, CI, lint/type/build configuration and privacy scan. +5. **Adoption:** GitHub README, architecture/security/release docs and a separately distributable Agent Skill. + +Each phase is accepted only after executable tests. Optional OCR/API paths must fail with a clear installation +hint rather than breaking import of the core package. diff --git a/generate_caption.py b/generate_caption.py deleted file mode 100644 index 8965c16..0000000 --- a/generate_caption.py +++ /dev/null @@ -1,123 +0,0 @@ -import os -import time -import ffmpeg -from datetime import timedelta -from faster_whisper import WhisperModel -import opencc - -class SubtitleGenerator: - def __init__(self, model_size="medium", device="cpu", compute_type="int8"): - """初始化字幕生成器""" - self.model_size = model_size - self.device = device - self.compute_type = compute_type - self.converter = opencc.OpenCC('t2s') - self.model = None - - def load_model(self): - """加载语音识别模型""" - try: - print(f"加载 {self.model_size} 模型（{self.device} 模式）...") - self.model = WhisperModel( - self.model_size, - device=self.device, - compute_type=self.compute_type - ) - return True - except Exception as e: - print(f"模型加载失败：{str(e)}") - return False - - def generate_subtitles(self, audio_file, language="zh", beam_size=5): - """生成字幕文本""" - try: - if not os.path.exists(audio_file): - raise FileNotFoundError(f"音频文件 {audio_file} 不存在") - - if not self.model: - self.load_model() - - print("正在识别...") - segments, _ = self.model.transcribe( - audio_file, - language=language, - beam_size=beam_size, - word_timestamps=True - ) - - subtitles = "" - segment_id = 1 - - for segment in segments: - text = self.converter.convert(segment.text.strip()) - sentences = text.split("。") - if not sentences[-1]: - sentences.pop() - - duration = segment.end - segment.start - sentence_duration = duration / len(sentences) if sentences else duration - - for i, sentence in enumerate(sentences): - if sentence.strip(): - start_time = segment.start + i * sentence_duration - end_time = start_time + sentence_duration - subtitles += ( - f"{segment_id}\n" - f"{self._format_timestamp(start_time)} --> " - f"{self._format_timestamp(end_time)}\n" - f"{sentence.strip()}\n\n" - ) - segment_id += 1 - - return subtitles - - except FileNotFoundError as e: - return f"错误：{str(e)}" - except Exception as e: - return f"发生错误：{str(e)}" - - def save_to_srt(self, subtitle_text, filename="output.srt"): - """保存字幕到SRT文件""" - try: - with open(filename, "w", encoding="utf-8") as f: - f.write(subtitle_text) - return True - except Exception as e: - print(f"保存字幕失败：{str(e)}") - return False - - # def generate_video_with_subtitles(self, audio_file, subtitle_file, output_file="output_with_subtitles.mp4"): - # """生成带字幕的视频""" - # try: - # command = ( - # f'ffmpeg -f lavfi -i "color=c=black:s=1280x720" ' - # f'-i "{audio_file}" ' - # f'-vf "subtitles={subtitle_file}" ' - # f'-c:a aac -shortest "{output_file}"' - # ) - # os.system(command) - # return True - # except Exception as e: - # print(f"视频生成失败：{str(e)}") - # return False - - @staticmethod - def _format_timestamp(seconds): - """格式化时间戳""" - ms = int((seconds % 1) * 1000) - time_str = str(timedelta(seconds=int(seconds))) - if len(time_str) < 8: - time_str = "0" + time_str - return f"{time_str},{ms:03d}" - -# 使用示例 -if __name__ == "__main__": - generator = SubtitleGenerator(model_size="medium", device="cpu") - audio_file = "G:/我的云端硬盘/PPTX/PPT_Text_Extractor/sample/sample.mp3" - subtitles = generator.generate_subtitles(audio_file, language="zh") - print("字幕结果：") - print(subtitles) - generator.save_to_srt(subtitles, "output.srt") - print("字幕已保存到 output.srt") - # generator.generate_video_with_subtitles(audio_file, "output.srt") - # print("已生成 output_with_subtitles.mp4") \ No newline at end of file diff --git a/main.py b/main.py deleted file mode 100644 index fd9d333..0000000 --- a/main.py +++ /dev/null @@ -1,111 +0,0 @@ -# main.py -from modules.ppt_text_extraction import extract_text_from_ppt, extract_metadata, extract_text_from_ppt_legacy, extract_metadata_from_ppt_legacy -from modules.image_extraction_t import extract_images_from_ppt_tesseract -from modules.image_extraction_p import extract_images_from_ppt_paddleocr, extract_images_from_ppt_legacy -from modules.ai_optimizer import optimize_text_with_ai -from modules.utils import setup_logger, validate_file_type -from modules.config import PPTX_FILE, OUTPUT_DIR, PPTX_FILE_2, OUTPUT_DIR_2 -import warnings -import re -import os -from concurrent.futures import ThreadPoolExecutor -from dotenv import load_dotenv - -# Load environment variables from .env file -load_dotenv() - -logger = setup_logger() - -# GPU option from environment variable (default to False if not set) -USE_GPU = os.getenv("USE_GPU", "False").lower() in ("true", "1", "yes") -logger.info(f"GPU enabled: {USE_GPU}") - -# Thread pool for async processing -executor = ThreadPoolExecutor(max_workers=2) - -def process_ppt_file(file_path): - warnings.filterwarnings("ignore", category=UserWarning, module="PIL.Image") - if not validate_file_type(file_path, ['.ppt', '.pptx', '.pot', '.potx', '.pps', '.ppsx', '.pptm', '.pdf']): - logger.error(f"Invalid file type: {file_path}. Skipping.") - return - - logger.info(f"Processing file: {file_path}") - - try: - # Asynchronous processing function - def process_file(file_path): - ext = os.path.splitext(file_path.lower())[1] - is_pptx = ext == '.pptx' - - # Extract metadata - metadata = extract_metadata(file_path) if is_pptx else extract_metadata_from_ppt_legacy(file_path) - if "Error" in metadata: - logger.error(f"Metadata extraction failed: {metadata['Error']}") - raise Exception(f"Metadata extraction failed: {metadata['Error']}") - metadata_output = "\n".join([f"{key}: {value}" for key, value in metadata.items()]) - logger.info("Metadata extracted successfully.") - - # Extract slide text - text_output = extract_text_from_ppt(file_path) if is_pptx else extract_text_from_ppt_legacy(file_path) - if not text_output: - logger.warning("No text extracted from PPT slides.") - text_output = [] - - # Extract image text (using PaddleOCR with GPU option) - image_output = extract_images_from_ppt_paddleocr(file_path, OUTPUT_DIR_2, use_gpu=USE_GPU) if is_pptx else extract_images_from_ppt_legacy(file_path, OUTPUT_DIR_2, use_gpu=USE_GPU) - if not image_output: - logger.warning("No image text extracted.") - image_output = [] - - # Clean and combine text - cleaned_text_output = clean_text_output(text_output + image_output) - combined_output = "\n".join([metadata_output] + cleaned_text_output) - if not combined_output.strip(): - logger.warning("No combined output generated") - combined_output = "No content extracted" - - # Optimize text with AI - optimized_text = optimize_text_with_ai(combined_output) - if not optimized_text: - logger.warning("Text optimization returned empty, using combined output as fallback") - optimized_text = combined_output - - # Save result - output_file = os.path.abspath(os.path.join(OUTPUT_DIR_2, "optimized_output.txt")) - os.makedirs(OUTPUT_DIR_2, exist_ok=True) - with open(output_file, "w", encoding="utf-8") as f: - f.write(optimized_text) - logger.info(f"File processed successfully: {output_file}") - - return optimized_text - - # Run processing in a thread with timeout - future = executor.submit(process_file, file_path) - optimized_text = future.result(timeout=120) # 2 分钟超时 - - except TimeoutError: - logger.error(f"Processing timed out for file: {file_path}") - except Exception as e: - logger.error(f"Error processing file {file_path}: {str(e)}") - -def clean_text_output(text_list): - """清理提取的文本，去除冗余信息（如水印）并优化结构""" - cleaned_output = [] - watermark_pattern = re.compile(r'stablediffusionweb\.com') # 假设这是常见的水印 - for slide in text_list: - # 跳过空幻灯片 - if not slide.strip(): - continue - # 去除水印 - cleaned_slide = watermark_pattern.sub('', slide) - # 清理多余换行和空格 - cleaned_slide = re.sub(r'\n\s*\n', '\n', cleaned_slide.strip()) - cleaned_output.append(cleaned_slide) - return cleaned_output - -def main(): - # process_ppt_file(PPTX_FILE) - process_ppt_file(PPTX_FILE_2) - -if __name__ == "__main__": - main() \ No newline at end of file diff --git a/modules/__init__.py b/modules/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/modules/ai_optimizer.py b/modules/ai_optimizer.py deleted file mode 100644 index c2d9176..0000000 --- a/modules/ai_optimizer.py +++ /dev/null @@ -1,314 +0,0 @@ -# import google.generativeai as genai -# import os -# from dotenv import load_dotenv -# import logging - -# # 配置日志 -# logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s") -# logger = logging.getLogger(__name__) - -# # 载入环境变量 -# load_dotenv() - -# # 设置 Gemini API Key -# genai.configure(api_key=os.getenv("GOOGLE_GEMINI_API_KEY")) - -# def optimize_text_with_ai(text): -# try: -# return text -# # 选择 Gemini 2 Flash 模型 -# model = genai.GenerativeModel("gemini-1.5-flash") - -# # 发送请求 -# response = model.generate_content(f"请优化以下文本，使其更加通顺，去除乱码，并在每句话末尾加上问号：\n\n{text}") - -# # 返回优化后的文本 -# return response.text if response and response.text else text -# except Exception as e: -# logger.error(f"Error calling Google AI API: {e}") -# return text - -############################################### - -# # 虚假AI（缺少API） -# import re -# import logging -# import random - -# # 配置日志 -# logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s") -# logger = logging.getLogger(__name__) - -# def optimize_text_with_ai(text): -# """ -# 本地优化 PPT 提取的文本，使其更自然、适合有声课件讲解。 -# """ -# try: -# lines = text.split('\n') -# optimized_lines = [] -# in_list = False -# slide_count = 0 -# prev_title = "" - -# # 多样化的引导语和连接词 -# slide_openers = ["现在我们来看看", "接下来聊聊", "这一节我们讲讲", "让我们进入"] -# title_openers = ["首先是", "那么我们说说", "这里有个重点", "值得一提的是"] -# list_openers = ["先来看看", "我们聊聊", "这里包括", "比如说"] -# list_connectors = ["然后是", "接着是", "还有呢", "另外一点"] -# formula_openers = ["这里有个公式", "看看这个计算", "简单解释一下"] -# image_openers = ["这里有张图", "我们看个示例", "这有个图片说明"] - -# for line in lines: -# if not line.strip(): -# continue - -# # 处理元数据 -# if ':' in line and slide_count == 0: -# optimized_lines.append(line) -# continue - -# # 检测幻灯片分隔符 -# if line.startswith('@@@Slide_'): -# slide_count += 1 -# if slide_count > 1: -# optimized_lines.append("\n那我们就先讲到这里，接下来看看新的内容。") -# optimized_lines.append(f"\n{random.choice(slide_openers)}第 {slide_count} 张幻灯片：") -# in_list = False -# continue - -# # 清理水印和乱码 -# cleaned_line = re.sub(r'stablediffusionweb\.com|WHATEV-VEER\.', '', line).strip() -# if not cleaned_line: -# continue - -# # 处理图片文本 -# if 'Image' in cleaned_line: -# if cleaned_line.endswith('Text:'): -# optimized_lines.append(f"{random.choice(image_openers)}，具体内容可以参考幻灯片。") -# else: -# content = cleaned_line.split('Text:')[-1].strip() -# if content: -# optimized_lines.append(f"{random.choice(image_openers)}，上面写着：{content}。") -# continue - -# # 检测标题（非列表、非公式） -# if not (cleaned_line.startswith(('▪', '-', '•')) or '=' in cleaned_line or '∑' in cleaned_line or 'σ' in cleaned_line): -# if in_list: -# optimized_lines.append("这些要点就先讲到这里。") -# in_list = False -# prev_title = cleaned_line -# optimized_lines.append(f"{random.choice(title_openers)} {cleaned_line}。") -# continue - -# # 检测列表项 -# if cleaned_line.startswith(('▪', '-', '•')): -# if not in_list: -# optimized_lines.append(f"{random.choice(list_openers)}：") -# in_list = True -# connector = "" -# else: -# connector = random.choice(list_connectors) -# content = cleaned_line[1:].strip() -# optimized_lines.append(f"{connector} {content}，挺关键的吧？") -# continue - -# # 检测公式 -# if '=' in cleaned_line or '∑' in cleaned_line or 'σ' in cleaned_line: -# if in_list: -# optimized_lines.append("这些要点就先讲到这里。") -# in_list = False -# formula_desc = "它描述了模型的计算过程" if "∑" in cleaned_line else "它让模型更有效" -# optimized_lines.append(f"{random.choice(formula_openers)}：{cleaned_line}，{formula_desc}。") -# continue - -# # 添加结束语 -# if in_list: -# optimized_lines.append("这些要点就先讲到这里。") -# optimized_lines.append("\n好了，今天的内容就到这儿，希望大家收获满满，下次再聊！") - -# optimized_text = "\n".join(optimized_lines) -# logger.info("Text optimization completed successfully.") -# return optimized_text - -# except Exception as e: -# logger.error(f"Error optimizing text: {e}") -# return text - -# # 测试代码 -# if __name__ == "__main__": -# sample_text = """Title: N/A -# @@@Slide_1@@@ -# 深度学习基础 -# @@@Slide_2@@@ -# 课程内容概览 -# ▪ 神经网络基本原理 -# Slide 5, Image 1 Text: -# stablediffusionweb.com -# """ -# result = optimize_text_with_ai(sample_text) -# print(result) - -########################################## - -import re -import logging -import random -from transformers import pipeline -import spacy - -# 加载 Spacy 模型 -nlp = spacy.load("zh_core_web_sm") - -# 加载 Transformers 模型（文本生成） -generator = pipeline("text-generation", model="uer/gpt2-chinese-cluecorpussmall", max_length=50) - -# 配置日志 -logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s") -logger = logging.getLogger(__name__) - -def optimize_text_with_ai(text): - """ - 使用 Transformers 和 Spacy 优化 PPT 文本，生成自然、复杂的讲解内容。 - """ - try: - # 预处理文本 - lines = [line.strip() for line in text.split('\n') if line.strip()] - optimized_lines = [] - slide_count = 0 - current_section = "metadata" - slide_buffer = [] - prev_context = None - - # 多样化表达库 - metadata_ends = ["课程资料齐全，我们马上开讲！", "文本就位，接下来直入主题！"] - slide_intros = ["我们先从这里入手，聊聊", "接下来带大家走进", "现在一起探讨", "那就让我们开始"] - section_ends = ["这部分先告一段落，接下来有新亮点", "聊到这里，我们转向新内容", "先到这儿，后续更精彩"] - final_closings = ["今天的课程到此结束，大家收获如何？下次见！", - "好了，这节内容就先画个句号，咱们下次再续！"] - - for i, line in enumerate(lines): - # 元数据处理 - if current_section == "metadata" and ':' in line: - key, value = line.split(':', 1) - optimized_lines.append(f"{key.strip()}: {value.strip()}") - if "Revision" in key: - optimized_lines.append(f"\n{random.choice(metadata_ends)}") - current_section = "slides" - continue - - # 幻灯片分隔符 - slide_match = re.match(r'(?:接下来聊聊|让我们进入|现在我们来看)第 (\d+) 张幻灯片：', line) - if slide_match or re.match(r'@@@Slide_\d+@@@', line): - slide_count += 1 - if slide_buffer: - optimized_lines.append(_process_slide(slide_buffer, prev_context, slide_count - 1)) - slide_buffer = [] - if slide_count > 1: - optimized_lines.append(f"\n{random.choice(section_ends)}") - optimized_lines.append(f"\n{random.choice(slide_intros)}第 {slide_count} 张幻灯片：") - continue - - # 清理无用内容 - cleaned_line = re.sub(r'[▪•\-\t]|stablediffusionweb\.com|WHATEV-VEER\.|\s{2,}', ' ', line).strip() - if not cleaned_line: - continue - - # 添加到幻灯片缓冲区 - slide_buffer.append(cleaned_line) - - # 处理最后一个幻灯片 - if slide_buffer: - final_text = _process_final_slide(slide_buffer, prev_context, slide_count) - optimized_lines.append(final_text) - optimized_lines.append(f"\n{random.choice(final_closings)}") - - optimized_text = "\n".join(optimized_lines) - logger.info("Text optimization completed successfully.") - return optimized_text - - except Exception as e: - logger.error(f"Error optimizing text: {e}", exc_info=True) - return text - -def _process_slide(slide_lines, prev_context, slide_num): - """处理单个幻灯片的文本""" - doc = nlp(" ".join(slide_lines)) - sentences = [sent.text.strip() for sent in doc.sents] - narrative = [] - list_items = [] - - for i, sent in enumerate(sentences): - # 处理公式 - if any(symbol in sent for symbol in ['=', '∑', 'σ']): - purpose = "揭示模型计算的关键" if '∑' in sent else "提升模型的效率" - narrative.append(f"这里有个关键点，我们来看公式：{sent}，它{purpose}。") - continue - - # 处理图片 - if 'Image' in sent: - content = sent.split('Text:')[-1].strip() if 'Text:' in sent else "" - narrative.append(f"幻灯片上有个直观的展示{'，内容是：' + content if content else '，具体请看幻灯片'}。") - continue - - # 处理列表或正文 - if len(sent.split()) < 10 and (':' in sent or sent.endswith('，')): - list_items.append(sent.strip('：，')) - else: - if list_items: - narrative.append(_format_list(list_items, prev_context)) - list_items = [] - transition = _generate_transition(prev_context, sent) - narrative.append(f"{transition}{sent}。") - prev_context = sent - - if list_items: - narrative.append(_format_list(list_items, prev_context)) - - return " ".join(narrative) - -def _process_final_slide(slide_lines, prev_context, slide_num): - """将最后一个幻灯片处理为完整段落""" - doc = nlp(" ".join(slide_lines)) - sentences = [sent.text.strip() for sent in doc.sents] - narrative = [f"\n在第 {slide_num} 张幻灯片中，我们深入剖析了知识点的细节。"] - - for i, sent in enumerate(sentences): - if any(symbol in sent for symbol in ['=', '∑', 'σ']): - purpose = "揭示了计算的核心逻辑" if '∑' in sent else "让模型运行更高效" - narrative.append(f"其中一个关键点是公式：{sent}，它{purpose}，") - elif 'Image' in sent: - content = sent.split('Text:')[-1].strip() if 'Text:' in sent else "" - narrative.append(f"幻灯片上通过图示{'展示了' + content if content else '清晰呈现了相关内容'}，") - else: - if i == 0: - narrative.append(f"首先是{sent}，这为我们理解整体框架奠定了基础，") - elif i == len(sentences) - 1: - narrative.append(f"最后谈到{sent}，它不仅总结了前面的内容，也为后续学习提供了启发。") - else: - narrative.append(f"接着是{sent}，进一步丰富了我们的视角，") - - return " ".join(narrative).rstrip('，') + "。" - -def _generate_transition(prev_context, current_line): - """使用 Transformers 生成自然过渡语""" - if not prev_context: - return "我们先来看看，" - prompt = f"{prev_context}，接下来是{current_line}" - try: - generated = generator(prompt, num_return_sequences=1, max_new_tokens=10)[0]['generated_text'] - transition = generated.split('，')[-2] + "，" if len(generated.split('，')) > 1 else "接着是，" - except Exception: - transition = random.choice(["顺着这个思路，", "再来看看，", "基于此，"]) - return transition - -def _format_list(items, prev_context): - """格式化列表为自然叙述""" - if not items: - return "" - intro = "具体来说，" - connectors = ["接着是", "然后聊到", "再看看", "另外还有"] - sentences = [intro] - for i, item in enumerate(items): - connector = "" if i == 0 else random.choice(connectors) - sentences.append(f"{connector}{item}{'，这点很关键' if i % 2 == 0 else '，也很重要'}") - return " ".join(sentences) + "。" diff --git a/modules/config.py b/modules/config.py deleted file mode 100644 index d1657e0..0000000 --- a/modules/config.py +++ /dev/null @@ -1,5 +0,0 @@ - -PPTX_FILE = r"G:\\我的云端硬盘\\PPTX\\PPT_Text_Extractor\\ppt.pptx" -PPTX_FILE_2 = r"G:\\我的云端硬盘\\PPTX\\PPT_Text_Extractor\\DeepLearning.pptx" -OUTPUT_DIR = r"G:\\我的云端硬盘\\PPTX\\PPT_TEXT_EXTRACTOR\\output" -OUTPUT_DIR_2 = r"G:\\我的云端硬盘\\PPTX\\PPT_TEXT_EXTRACTOR\\output2" \ No newline at end of file diff --git a/modules/image_extraction_p.py b/modules/image_extraction_p.py deleted file mode 100644 index 41dc6ee..0000000 --- a/modules/image_extraction_p.py +++ /dev/null @@ -1,223 +0,0 @@ -from pptx import Presentation -from pptx.enum.shapes import MSO_SHAPE_TYPE -import os -import logging -from paddleocr import PaddleOCR -from modules.config import OUTPUT_DIR -import win32com.client -import pythoncom - -# 配置日志 -logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s") -logger = logging.getLogger(__name__) - -def ensure_dir(directory): - """确保目录存在，若不存在则创建""" - if not os.path.exists(directory): - os.makedirs(directory) - logger.info(f"Created directory: {directory}") - -def extract_images_from_ppt_paddleocr(file_path, output_dir=OUTPUT_DIR, output_format="text", use_gpu=False): - """ - 从 PPT 文件中提取图片，并使用 PaddleOCR 识别图片中的文本。 - - Args: - file_path (str): PPT 文件路径。 - output_dir (str): 输出目录路径，默认为 OUTPUT_DIR。 - output_format (str): 输出格式，可选 "text"（默认）或 "json"。 - use_gpu (bool): 是否使用 GPU 加速 OCR，默认 False。 - - Returns: - list: 包含每张图片识别文本的列表。 - """ - try: - # 判断文件格式 - ext = os.path.splitext(file_path.lower())[1] - if ext != '.pptx': - return extract_images_from_ppt_legacy(file_path, output_dir, output_format, use_gpu) - - image_texts = [] - presentation = Presentation(file_path) - logger.info(f"Processing PPT file: {file_path}") - - # 初始化 PaddleOCR - ocr = PaddleOCR(use_angle_cls=True, lang='ch', use_gpu=use_gpu) - - for slide_number, slide in enumerate(presentation.slides, start=1): - slide_folder = os.path.join(output_dir, f"slide_{slide_number}", "image") - ensure_dir(slide_folder) - image_index = 1 - - for shape in slide.shapes: - if shape.shape_type == MSO_SHAPE_TYPE.PICTURE: - image = shape.image - image_bytes = image.blob - image_name = f"image_{image_index}.jpg" - image_path = os.path.join(slide_folder, image_name) - - with open(image_path, "wb") as f: - f.write(image_bytes) - logger.debug(f"Saved image: {image_path}") - - if contains_text(image_path, ocr): - text = process_image_for_ocr(image_path, ocr) - if text: - text_entry = { - "slide": slide_number, - "image": image_index, - "text": text - } - if output_format == "json": - image_texts.append(text_entry) - else: - image_texts.append(f"\nSlide {slide_number}, Image {image_index} Text:\n{text}") - - text_file_path = os.path.join(slide_folder, f"slide_{slide_number}_image_{image_index}_text.txt") - with open(text_file_path, "w", encoding="utf-8") as f: - f.write(text) - logger.info(f"Extracted text from {image_name}: {text[:50]}...") - else: - logger.debug(f"No readable text in {image_name}") - else: - logger.debug(f"No text detected in {image_name}") - image_index += 1 - - logger.info(f"Completed processing {file_path}. Extracted {len(image_texts)} image texts.") - return image_texts - - except Exception as e: - logger.error(f"Failed to process PPT file {file_path}: {e}") - return [] - -def process_image_for_ocr(image_path, ocr): - """使用指定的 PaddleOCR 实例处理图片并提取文本""" - try: - result = ocr.ocr(image_path, cls=True) - if not result or not result[0]: - return "" - - texts = [] - for line in result[0]: - if line and len(line[1][0]) >= 2 and line[1][1] > 0.5: - texts.append(line[1][0]) - full_text = '\n'.join(texts).strip() - return full_text if full_text else "" - - except Exception as e: - logger.error(f"OCR processing failed for {image_path}: {e}") - return "" - -def contains_text(image_path, ocr): - """使用指定的 PaddleOCR 实例判断图片是否含有文本""" - try: - result = ocr.ocr(image_path, cls=True) - if result and result[0]: - valid_texts = [ - line for line in result[0] - if line and len(line[1][0]) >= 2 and line[1][1] > 0.5 - ] - return len(valid_texts) > 0 - return False - - except Exception as e: - logger.error(f"Text detection failed for {image_path}: {e}") - return False - -def extract_images_from_ppt_legacy(file_path, output_dir=OUTPUT_DIR, output_format="text", use_gpu=False): - """ - 从非 PPTX 格式的文件（如 .ppt, .pot, .pps）中提取图片并识别文本。 - - Args: - file_path (str): PPT 文件路径。 - output_dir (str): 输出目录路径，默认为 OUTPUT_DIR。 - output_format (str): 输出格式，可选 "text"（默认）或 "json"。 - use_gpu (bool): 是否使用 GPU 加速 OCR，默认 False。 - - Returns: - list: 包含每张图片识别文本的列表，与 extract_images_from_ppt_paddleocr 输出格式一致。 - """ - try: - pythoncom.CoInitialize() - image_texts = [] - app = win32com.client.Dispatch("PowerPoint.Application") - prs = app.Presentations.Open(file_path, WithWindow=False) - logger.info(f"Processing legacy PPT file: {file_path}") - - # 初始化 PaddleOCR - ocr = PaddleOCR(use_angle_cls=True, lang='ch', use_gpu=use_gpu) - - for slide_number, slide in enumerate(prs.Slides, start=1): - slide_folder = os.path.join(output_dir, f"slide_{slide_number}", "image") - ensure_dir(slide_folder) - image_index = 1 - - for shape in slide.Shapes: - if shape.Type == 13: - image_name = f"image_{image_index}.jpg" - image_path = os.path.join(slide_folder, image_name) - shape.Export(image_path, 2) - logger.debug(f"Saved image: {image_path}") - - if contains_text(image_path, ocr): - text = process_image_for_ocr(image_path, ocr) - if text: - text_entry = { - "slide": slide_number, - "image": image_index, - "text": text - } - if output_format == "json": - image_texts.append(text_entry) - else: - image_texts.append(f"\nSlide {slide_number}, Image {image_index} Text:\n{text}") - - text_file_path = os.path.join(slide_folder, f"slide_{slide_number}_image_{image_index}_text.txt") - with open(text_file_path, "w", encoding="utf-8") as f: - f.write(text) - logger.info(f"Extracted text from {image_name}: {text[:50]}...") - else: - logger.debug(f"No readable text in {image_name}") - else: - logger.debug(f"No text detected in {image_name}") - image_index += 1 - - if image_index == 1: - image_name = f"image_1.jpg" - image_path = os.path.join(slide_folder, image_name) - slide.Export(image_path, "JPG") - logger.debug(f"Saved slide image: {image_path}") - - if contains_text(image_path, ocr): - text = process_image_for_ocr(image_path, ocr) - if text: - text_entry = { - "slide": slide_number, - "image": 1, - "text": text - } - if output_format == "json": - image_texts.append(text_entry) - else: - image_texts.append(f"\nSlide {slide_number}, Image 1 Text:\n{text}") - - text_file_path = os.path.join(slide_folder, f"slide_{slide_number}_image_1_text.txt") - with open(text_file_path, "w", encoding="utf-8") as f: - f.write(text) - logger.info(f"Extracted text from {image_name}: {text[:50]}...") - else: - logger.debug(f"No readable text in slide {slide_number}") - else: - logger.debug(f"No text detected in slide {slide_number}") - - prs.Close() - app.Quit() - pythoncom.CoUninitialize() - logger.info(f"Completed processing {file_path}. Extracted {len(image_texts)} image texts.") - return image_texts - - except Exception as e: - logger.error(f"Failed to process legacy PPT file {file_path}: {e}") - if 'app' in locals(): - app.Quit() - pythoncom.CoUninitialize() - return [] \ No newline at end of file diff --git a/modules/image_extraction_t.py b/modules/image_extraction_t.py deleted file mode 100644 index 10e9ee0..0000000 --- a/modules/image_extraction_t.py +++ /dev/null @@ -1,78 +0,0 @@ -from pptx import Presentation -from pptx.enum.shapes import MSO_SHAPE_TYPE -from PIL import Image, ImageEnhance, ImageFilter -import pytesseract -import cv2 -import numpy as np -import io -import os - -pytesseract.pytesseract.tesseract_cmd = r'C:\\Program Files\\Tesseract-OCR\\tesseract.exe' - -def extract_images_from_ppt_tesseract(file_path, output_dir): - os.makedirs(output_dir, exist_ok=True) - image_texts = [] - presentation = Presentation(file_path) - - for slide_number, slide in enumerate(presentation.slides, start=1): - slide_folder = os.path.join(output_dir, f"slide_{slide_number}", "image") - os.makedirs(slide_folder, exist_ok=True) - image_index = 1 - - for shape in slide.shapes: - if shape.shape_type == MSO_SHAPE_TYPE.PICTURE: - image = shape.image - image_bytes = image.blob - image_name = f"image_{image_index}.jpg" - image_path = os.path.join(slide_folder, image_name) - - with open(image_path, "wb") as f: - f.write(image_bytes) - - text = process_image_for_ocr(image_bytes) - if text: - image_texts.append(f"Slide {slide_number}, Image {image_index} Text:\n{text}") - image_index += 1 - return image_texts - -def process_image_for_ocr(image_bytes): - try: - if not contains_text(image_bytes): - return "" # 直接跳过 OCR，避免乱码 - # 读取图片 - with Image.open(io.BytesIO(image_bytes)) as img: - img = img.convert('L') # 转换为灰度 - img = ImageEnhance.Contrast(img).enhance(2.0) # 增强对比度 - img_cv = np.array(img) # 将 PIL 图片转换为 OpenCV 格式 - img_cv = cv2.medianBlur(img_cv, 3) # 应用中值滤波（减少噪声） - _, img_cv = cv2.threshold(img_cv, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU) # Otsu 二值化（自动调整阈值） - edges = cv2.Canny(img_cv, 30, 150) # 边缘检测，检查是否含有文字 - edge_ratio = np.sum(edges > 0) / edges.size - if edge_ratio < 0.005: # 设置一个经验阈值（如果边缘像素太少，说明可能没有文字） - return "" # 直接跳过 OCR 处理 - text = pytesseract.image_to_string(img_cv, lang='chi_sim+eng', config='--psm 6') # OCR 识别（使用合适的 PSM 模式） - # 过滤无效文本（防止全是符号、噪声） - if len(text.strip()) < 5: # 设定最小长度阈值 - return "" - return text.strip() - except Exception as e: - return f"OCR processing failed: {e}" - -def contains_text(image_bytes): - """判断图片是否含有文本""" - with Image.open(io.BytesIO(image_bytes)) as img: - img_cv = np.array(img.convert('L')) # 转换为灰度 - img_blur = cv2.GaussianBlur(img_cv, (5, 5), 0) # 应用高斯模糊，减少噪声 - _, img_bin = cv2.threshold(img_blur, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU) # 使用 Otsu 二值化 - edges = cv2.Canny(img_bin, 30, 150) # 计算边缘像素比例 - edge_ratio = np.sum(edges > 0) / edges.size - contours, _ = cv2.findContours(img_bin, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) # 轮廓检测 - text_area = sum(cv2.contourArea(cnt) for cnt in contours) - text_ratio = text_area / (img_cv.shape[0] * img_cv.shape[1]) - # **文本判断标准**： - # - **边缘像素比例 > 0.005** (说明有文字结构) - # - **文本区域面积比 > 0.01** (大于1%图像面积) - if edge_ratio > 0.005 and text_ratio > 0.01: - print('word') - return True # 说明图像含有文字 - return False # 图像无明显文字 \ No newline at end of file diff --git a/modules/ppt_text_extraction.py b/modules/ppt_text_extraction.py deleted file mode 100644 index e990d03..0000000 --- a/modules/ppt_text_extraction.py +++ /dev/null @@ -1,131 +0,0 @@ -# ppt_text_extraction.py -from pptx import Presentation -from modules.config import OUTPUT_DIR_2 -import os -import logging -import win32com.client -import pythoncom - -logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s") -logger = logging.getLogger(__name__) - -def extract_text_from_ppt(file_path): - os.makedirs(OUTPUT_DIR_2, exist_ok=True) - ext = os.path.splitext(file_path.lower())[1] - if ext != '.pptx': - return extract_text_from_ppt_legacy(file_path) - - text_output = [] - presentation = Presentation(file_path) - logger.info(f"Processing PPTX file: {file_path}") - - for slide_number, slide in enumerate(presentation.slides, start=1): - slide_text = [] - for shape in slide.shapes: - if shape.has_text_frame: - for paragraph in shape.text_frame.paragraphs: - for run in paragraph.runs: - slide_text.append(run.text) - if slide_text: - text_output.append(f"\n\n@@@Slide_{slide_number}@@@\n" + "\n".join(slide_text)) - logger.info(f"Completed processing {file_path}. Extracted text from {len(text_output)} slides.") - return text_output - -def extract_metadata(file_path): - try: - presentation = Presentation(file_path) - props = presentation.core_properties - logger.info(f"Extracting metadata from PPTX: {file_path}") - metadata = { - "Title": props.title or "N/A", - "Author": props.author or "N/A", - "Subject": props.subject or "N/A", - "Keywords": props.keywords or "N/A", - "Comments": props.comments or "N/A", - "Last Modified By": props.last_modified_by or "N/A", - "Created": props.created.strftime('%Y-%m-%d %H:%M:%S') if props.created else "N/A", - "Modified": props.modified.strftime('%Y-%m-%d %H:%M:%S') if props.modified else "N/A", - "Category": props.category or "N/A", - "Content Status": props.content_status or "N/A", - "Identifier": props.identifier or "N/A", - "Language": props.language or "N/A", - "Revision": props.revision or "N/A" - } - logger.info(f"Metadata extracted successfully from {file_path}") - return metadata - except Exception as e: - logger.error(f"Error extracting metadata from PPTX {file_path}: {str(e)}") - return {"Error": str(e)} - -def extract_text_from_ppt_legacy(file_path, output_dir=OUTPUT_DIR_2): - try: - if not os.path.exists(file_path): - logger.error(f"File does not exist: {file_path}") - return [] - pythoncom.CoInitialize() - os.makedirs(output_dir, exist_ok=True) - text_output = [] - app = win32com.client.Dispatch("PowerPoint.Application") - prs = app.Presentations.Open(file_path, WithWindow=False) - logger.info(f"Processing legacy PPT format: {file_path}") - - for slide_number, slide in enumerate(prs.Slides, start=1): - slide_text = [] - for shape in slide.Shapes: - if shape.HasTextFrame and shape.TextFrame.HasText: - text = shape.TextFrame.TextRange.Text.strip() - if text: - slide_text.append(text) - if slide_text: - text_output.append(f"\n\n@@@Slide_{slide_number}@@@\n" + "\n".join(slide_text)) - - prs.Close() - app.Quit() - pythoncom.CoUninitialize() - logger.info(f"Completed processing {file_path}. Extracted text from {len(text_output)} slides.") - return text_output - except Exception as e: - logger.error(f"Error processing legacy PPT format {file_path}: {str(e)}") - if 'app' in locals(): - app.Quit() - pythoncom.CoUninitialize() - return [] - -def extract_metadata_from_ppt_legacy(file_path): - try: - if not os.path.exists(file_path): - logger.error(f"File does not exist: {file_path}") - return {"Error": f"File not found: {file_path}"} - pythoncom.CoInitialize() - app = win32com.client.Dispatch("PowerPoint.Application") - logger.info(f"Opening PowerPoint application for {file_path}") - prs = app.Presentations.Open(file_path, WithWindow=False) - props = prs.BuiltInDocumentProperties - logger.info(f"Extracting metadata from legacy PPT format: {file_path}") - - metadata = { - "Title": props.Item("Title").Value if props("Title") else "N/A", - "Author": props.Item("Author").Value if props("Author") else "N/A", - "Subject": props.Item("Subject").Value if props("Subject") else "N/A", - "Keywords": props.Item("Keywords").Value if props("Keywords") else "N/A", - "Comments": props.Item("Comments").Value if props("Comments") else "N/A", - "Last Modified By": props.Item("Last Saved By").Value if props("Last Saved By") else "N/A", - "Created": props.Item("Creation Date").Value.strftime('%Y-%m-%d %H:%M:%S') if props("Creation Date") else "N/A", - "Modified": props.Item("Last Save Time").Value.strftime('%Y-%m-%d %H:%M:%S') if props("Last Save Time") else "N/A", - "Category": props.Item("Category").Value if props("Category") else "N/A", - "Content Status": "N/A", - "Identifier": "N/A", - "Language": "N/A", - "Revision": str(props.Item("Revision Number").Value) if props("Revision Number") else "N/A" - } - prs.Close() - app.Quit() - pythoncom.CoUninitialize() - logger.info(f"Metadata extracted successfully from {file_path}") - return metadata - except Exception as e: - logger.error(f"Error extracting metadata from legacy PPT format {file_path}: {str(e)}") - if 'app' in locals(): - app.Quit() - pythoncom.CoUninitialize() - return {"Error": str(e)} \ No newline at end of file diff --git a/modules/text_extraction.py b/modules/text_extraction.py deleted file mode 100644 index 4a10414..0000000 --- a/modules/text_extraction.py +++ /dev/null @@ -1,129 +0,0 @@ -from pptx import Presentation -from pptx.enum.shapes import MSO_SHAPE_TYPE -import logging -from datetime import datetime - -# 配置日志 -logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s") -logger = logging.getLogger(__name__) - -def extract_text_from_ppt(file_path): - """ - 从 PPT 文件中提取文本，保留层次结构并支持多种形状类型。 - - Args: - file_path (str): PPT 文件路径。 - - Returns: - list: 包含每张幻灯片文本的列表，格式为分隔符标记的字符串。 - """ - try: - text_output = [] - presentation = Presentation(file_path) - logger.info(f"Extracting text from PPT: {file_path}") - - for slide_number, slide in enumerate(presentation.slides, start=1): - slide_text = [] - logger.debug(f"Processing Slide {slide_number}") - - # 提取标题（如果有） - if slide.shapes.title: - slide_text.append(f"Title: {slide.shapes.title.text.strip()}") - - # 遍历所有形状 - for shape in slide.shapes: - if shape.has_text_frame: - # 处理文本框 - for paragraph in shape.text_frame.paragraphs: - paragraph_text = "".join(run.text for run in paragraph.runs if run.text.strip()) - if paragraph_text: - # 根据层级添加前缀（粗略判断） - level = paragraph.level if hasattr(paragraph, 'level') else 0 - prefix = " " * level + ("▪ " if level > 0 else "") - slide_text.append(f"{prefix}{paragraph_text}") - - elif shape.shape_type == MSO_SHAPE_TYPE.TABLE: - # 处理表格 - table_text = [] - for row in shape.table.rows: - row_text = [cell.text.strip() for cell in row.cells if cell.text.strip()] - if row_text: - table_text.append(" | ".join(row_text)) - if table_text: - slide_text.append("Table:\n" + "\n".join(table_text)) - - elif shape.has_chart: - # 处理图表（简单提取标题） - if shape.chart.chart_title.has_text_frame: - slide_text.append(f"Chart Title: {shape.chart.chart_title.text_frame.text.strip()}") - - # 处理注释（如果有） - if slide.has_notes_slide and slide.notes_slide.notes_text_frame: - notes_text = slide.notes_slide.notes_text_frame.text.strip() - if notes_text: - slide_text.append(f"Notes:\n{notes_text}") - - # 清理并格式化输出 - if slide_text: - cleaned_text = "\n".join(line for line in slide_text if line.strip()) - text_output.append(f"@@@Slide_{slide_number}@@@\n{cleaned_text}") - else: - text_output.append(f"@@@Slide_{slide_number}@@@\n[No text content]") - - logger.info(f"Extracted text from {len(text_output)} slides.") - return text_output - - except Exception as e: - logger.error(f"Failed to extract text from {file_path}: {e}") - return [f"Error: Unable to process {file_path}"] - -def extract_metadata(file_path): - """ - 从 PPT 文件中提取元数据，包含更多核心属性。 - - Args: - file_path (str): PPT 文件路径。 - - Returns: - dict: 包含元数据的字典。 - """ - try: - presentation = Presentation(file_path) - props = presentation.core_properties - logger.info(f"Extracting metadata from PPT: {file_path}") - - metadata = { - "Title": props.title or "N/A", - "Author": props.author or "N/A", - "Subject": props.subject or "N/A", - "Keywords": props.keywords or "N/A", - "Comments": props.comments or "N/A", - "Last Modified By": props.last_modified_by or "N/A", - "Created": props.created.strftime('%Y-%m-%d %H:%M:%S') if props.created else "N/A", - "Modified": props.modified.strftime('%Y-%m-%d %H:%M:%S') if props.modified else "N/A", - "Category": props.category or "N/A", - "Content Status": props.content_status or "N/A", - "Identifier": props.identifier or "N/A", - "Language": props.language or "N/A", - "Revision": str(props.revision) if props.revision else "N/A" - } - - logger.info("Metadata extraction completed.") - return metadata - - except Exception as e: - logger.error(f"Failed to extract metadata from {file_path}: {e}") - return {"Error": f"Unable to process {file_path}"} - -# 测试代码 -if __name__ == "__main__": - sample_ppt = "path/to/your/sample.pptx" - text_result = extract_text_from_ppt(sample_ppt) - metadata_result = extract_metadata(sample_ppt) - - print("Metadata:") - for key, value in metadata_result.items(): - print(f"{key}: {value}") - print("\nText:") - for slide in text_result: - print(slide) \ No newline at end of file diff --git a/modules/utils.py b/modules/utils.py deleted file mode 100644 index 8759b58..0000000 --- a/modules/utils.py +++ /dev/null @@ -1,121 +0,0 @@ -import logging -import os -from typing import List, Optional - -# 默认日志格式 -DEFAULT_LOG_FORMAT = "%(asctime)s - %(levelname)s - %(message)s" - - -def validate_file_type(file_path: str, valid_extensions: List[str], raise_exception: bool = False) -> bool: - """ - 验证文件路径是否有效且扩展名在允许的列表中。 - - Args: - file_path (str): 文件路径。 - valid_extensions (List[str]): 允许的扩展名列表（如 ['.pptx', '.pdf']）。 - raise_exception (bool): 是否在验证失败时抛出异常，默认为 False。 - - Returns: - bool: 如果文件有效且扩展名匹配，返回 True；否则返回 False。 - - Raises: - FileNotFoundError: 如果 raise_exception=True 且文件不存在。 - ValueError: 如果 raise_exception=True 且扩展名无效。 - """ - logger = logging.getLogger(__name__) - - # 检查文件是否存在 - if not os.path.isfile(file_path): - error_msg = f"The file {file_path} does not exist." - logger.error(error_msg) - if raise_exception: - raise FileNotFoundError(error_msg) - return False - - # 获取文件扩展名 - _, ext = os.path.splitext(file_path) - ext = ext.lower() - - # 规范化扩展名列表（去除多余的点号） - valid_extensions = [e.lower().lstrip('.') for e in valid_extensions] - - # 检查扩展名是否匹配 - is_valid = ext[1:] in valid_extensions # 去掉点号比较 - if not is_valid: - error_msg = f"Invalid file type: {ext}. Expected one of {valid_extensions}." - logger.warning(error_msg) - if raise_exception: - raise ValueError(error_msg) - - logger.debug(f"Validated file: {file_path} with extension {ext}") - return is_valid - - -def setup_logger( - name: str = __name__, - level: int = logging.INFO, - log_file: Optional[str] = None, - log_format: str = DEFAULT_LOG_FORMAT -) -> logging.Logger: - """ - 配置并返回一个日志器，支持控制台和文件输出。 - - Args: - name (str): 日志器名称，默认为当前模块名。 - level (int): 日志级别，默认为 logging.INFO。 - log_file (Optional[str]): 日志文件路径，若提供则记录到文件。 - log_format (str): 日志输出格式，默认为时间-级别-消息。 - - Returns: - logging.Logger: 配置好的日志器实例。 - """ - # 获取或创建日志器 - logger = logging.getLogger(name) - - # 避免重复配置 - if logger.handlers: - logger.debug("Logger already configured, skipping setup.") - return logger - - # 设置日志级别 - logger.setLevel(level) - - # 创建格式器 - formatter = logging.Formatter(log_format) - - # 添加控制台处理器 - console_handler = logging.StreamHandler() - console_handler.setFormatter(formatter) - logger.addHandler(console_handler) - - # 如果指定了日志文件，添加文件处理器 - if log_file: - try: - os.makedirs(os.path.dirname(log_file), exist_ok=True) - file_handler = logging.FileHandler(log_file, encoding="utf-8") - file_handler.setFormatter(formatter) - logger.addHandler(file_handler) - logger.info(f"Logging to file: {log_file}") - except Exception as e: - logger.error(f"Failed to set up file logging: {e}") - - logger.info(f"Logger initialized with level {logging.getLevelName(level)}") - return logger - - -# 测试代码 -if __name__ == "__main__": - # 设置日志 - logger = setup_logger(level=logging.DEBUG, log_file="logs/test.log") - - # 测试文件验证 - valid_extensions = ['ppt', 'pptx', 'pdf'] - test_files = [ - "sample.pptx", - "sample.doc", - "nonexistent.pptx" - ] - - for file in test_files: - result = validate_file_type(file, valid_extensions, raise_exception=False) - logger.info(f"File {file} is valid: {result}") \ No newline at end of file diff --git a/output.srt b/output.srt deleted file mode 100644 index ba509a5..0000000 --- a/output.srt +++ /dev/null @@ -1,60 +0,0 @@ -1 -00:00:00,000 --> 00:00:03,299 -我永远没法改变别人怎么评判我的容貌 - -2 -00:00:03,299 --> 00:00:05,480 -啊 我只能自信起来 - -3 -00:00:05,480 --> 00:00:06,960 -从那天开始我接受了 - -4 -00:00:06,960 --> 00:00:08,099 -我就长这样 - -5 -00:00:08,099 --> 00:00:09,339 -你觉得我长得帅 - -6 -00:00:09,339 --> 00:00:11,400 -就是我给你的福利 - -7 -00:00:12,300 --> 00:00:13,500 -你觉得我长得吵 - -8 -00:00:13,500 --> 00:00:15,400 -就是我给你的宠吧 - -9 -00:00:16,439 --> 00:00:17,760 -我就是我 - -10 -00:00:17,760 --> 00:00:18,719 -笨蛋 - -11 -00:00:18,719 --> 00:00:19,920 -要有自信啊 - -12 -00:00:19,920 --> 00:00:21,500 -你是最好的 你知道吗 - -13 -00:00:22,219 --> 00:00:24,519 -你这命运是掌握在自己手中的 - -14 -00:00:25,579 --> 00:00:27,000 -给你自己的信心 - -15 -00:00:27,980 --> 00:00:28,839 -你行的 - diff --git a/output2/optimized_output.txt b/output2/optimized_output.txt deleted file mode 100644 index c9f0618..0000000 --- a/output2/optimized_output.txt +++ /dev/null @@ -1,121 +0,0 @@ -Title: N/A -Author: huawei -Subject: N/A -Keywords: N/A -Comments: generated using python-pptx -Last Modified By: 厚冰 张 -Created: 2013-01-27 09:14:16 -Modified: 2025-02-11 12:31:16 -Category: N/A -Content Status: N/A -Identifier: N/A -Language: N/A -Revision: 5 - -现在我们来看看第 1 张幻灯片： -那么我们说说 深度学习基础。 -值得一提的是 人工智能学院核心课程。 -那么我们说说 2024年春季学期。 -这里有个重点 本课程将深入讲解深度学习的基本原理、核心算法及典型应用，涵盖神经网络、卷积神经网络、生成对抗网络等内容，帮助学生掌握深度学习的前沿技术。。 - -那我们就先讲到这里，接下来看看新的内容。 - -这一节我们讲讲第 2 张幻灯片： -那么我们说说 课程内容概览。 -那么我们说说 1. 神经网络基本原理。 -这里有个重点 神经元模型/激活函数/前向传播。 -那么我们说说 2. 卷积神经网络。 -这里有个重点 局部感受野/池化操作/经典架构。 -那么我们说说 3. 生成对抗网络。 -首先是 博弈论框架/生成器判别器。 -这里有个重点 4. 注意力机制。 -值得一提的是 Self-Attention/Transformer架构。 -这里有个重点 5. 强化学习基础。 -值得一提的是 马尔可夫决策过程/Q-Learning。 - -那我们就先讲到这里，接下来看看新的内容。 - -现在我们来看看第 3 张幻灯片： -这里有个重点 神经网络的基本概念。 -首先是 神经网络的三大要素：。 -比如说： - 加权求和，挺关键的吧？ -这些要点就先讲到这里。 -简单解释一下：z = ∑w_i x_i + b，它描述了模型的计算过程。 -这里包括： - w_i: 权重，挺关键的吧？ -接着是 x_i: 输入，挺关键的吧？ -然后是 b: 偏置，挺关键的吧？ -另外一点 非线性激活，挺关键的吧？ -这些要点就先讲到这里。 -简单解释一下：σ(z) = max(0,z) (ReLU)，它让模型更有效。 -这里包括： - 引入非线性，挺关键的吧？ -还有呢 解决梯度消失问题，挺关键的吧？ -然后是 损失函数，挺关键的吧？ -这些要点就先讲到这里。 -简单解释一下：L = ½(y - ŷ)^2 (均方误差)，它让模型更有效。 -这里包括： - y: 真实值，挺关键的吧？ -另外一点 ŷ: 预测值，挺关键的吧？ - -那我们就先讲到这里，接下来看看新的内容。 - -接下来聊聊第 4 张幻灯片： -那么我们说说 卷积运算可视化。 -这里有个重点 卷积运算的核心思想：。 -先来看看： - 卷积核（Kernel）：一个小的权重矩阵，用于提取局部特征。，挺关键的吧？ -然后是 步长（Stride）：卷积核在输入上滑动的步幅，影响输出特征图的大小。，挺关键的吧？ -然后是 填充（Padding）：在输入边缘添加额外的像素，控制输出特征图的尺寸。，挺关键的吧？ -然后是 特征图（Feature Map）：卷积运算的输出，反映了输入中某种特征的分布。，挺关键的吧？ - -那我们就先讲到这里，接下来看看新的内容。 - -让我们进入第 5 张幻灯片： -那么我们说说 典型应用场景。 -这里有个重点 医疗影像分析。 -那么我们说说 CT图像分割。 -先来看看： - 肿瘤检测，挺关键的吧？ -接着是 器官定位，挺关键的吧？ -这些要点就先讲到这里。 -这里有个重点 深度学习在医疗影像中的应用显著提高了诊断的准确性和效率。。 -那么我们说说 自动驾驶。 -那么我们说说 实时目标检测。 -比如说： - 行人识别，挺关键的吧？ -另外一点 车道线检测，挺关键的吧？ -这些要点就先讲到这里。 -这里有个重点 自动驾驶技术依赖于深度学习模型对复杂环境。 -首先是 的实时感知和决策。。 -这里有个重点 艺术创作。 -首先是 风格迁移示例。 -我们聊聊： - 图像风格化，挺关键的吧？ -另外一点 视频风格化，挺关键的吧？ -这些要点就先讲到这里。 -首先是 生成对抗网络（GAN）为艺术创作提供了全新的可能性。。 - -那我们就先讲到这里，接下来看看新的内容。 - -让我们进入第 6 张幻灯片： -这里有个重点 知识点详解。 -值得一提的是 1. 神经网络的基本结构：。 -这里包括： - 神经网络由输入层、隐藏层和输出层组成。，挺关键的吧？ -另外一点 每一层包含多个神经元，神经元之间通过权重连接。，挺关键的吧？ -然后是 输入层接收原始数据，隐藏层提取特征，输出层生成最终结果。，挺关键的吧？ -这些要点就先讲到这里。 -值得一提的是 2. 激活函数的作用：。 -这里包括： - 激活函数引入非线性，使神经网络能够学习复杂的模式。，挺关键的吧？ -还有呢 常用的激活函数包括ReLU、Sigmoid和Tanh。，挺关键的吧？ -接着是 ReLU（Rectified Linear Unit）是目前最常用的激活函数，因其简单且有效。，挺关键的吧？ -这里有张图，具体内容可以参考幻灯片。 -这有个图片说明，具体内容可以参考幻灯片。 -我们看个示例，具体内容可以参考幻灯片。 -这里有张图，具体内容可以参考幻灯片。 -这些要点就先讲到这里。 - -好了，今天的内容就到这儿，希望大家收获满满，下次再聊！ \ No newline at end of file diff --git a/output2/slide_1/image/image_1.jpg b/output2/slide_1/image/image_1.jpg deleted file mode 100644 index 913cb17..0000000 Binary files a/output2/slide_1/image/image_1.jpg and /dev/null differ diff --git a/output2/slide_1/slide_1_texts.txt b/output2/slide_1/slide_1_texts.txt deleted file mode 100644 index dca8398..0000000 --- a/output2/slide_1/slide_1_texts.txt +++ /dev/null @@ -1,3 +0,0 @@ -@@@content@@@ -深度学习基础人工智能学院核心课程2024年春季学期本课程将深入讲解深度学习的基本原理、核心算法及典型应用，涵盖神经网络、卷积神经网络、生成对抗网络等内容，帮助学生掌握深度学习的前沿技术。 - diff --git a/output2/slide_2/slide_2_texts.txt b/output2/slide_2/slide_2_texts.txt deleted file mode 100644 index 3b62ee3..0000000 --- a/output2/slide_2/slide_2_texts.txt +++ /dev/null @@ -1,3 +0,0 @@ -@@@content@@@ -课程内容概览1. 神经网络基本原理神经元模型/激活函数/前向传播2. 卷积神经网络局部感受野/池化操作/经典架构3. 生成对抗网络博弈论框架/生成器判别器4. 注意力机制Self-Attention/Transformer架构5. 强化学习基础马尔可夫决策过程/Q-Learning - diff --git a/output2/slide_3/image/image_1.jpg b/output2/slide_3/image/image_1.jpg deleted file mode 100644 index 01dab72..0000000 Binary files a/output2/slide_3/image/image_1.jpg and /dev/null differ diff --git a/output2/slide_3/slide_3_texts.txt b/output2/slide_3/slide_3_texts.txt deleted file mode 100644 index 7b0bdf6..0000000 --- a/output2/slide_3/slide_3_texts.txt +++ /dev/null @@ -1,3 +0,0 @@ -@@@content@@@ -神经网络的基本概念神经网络的三大要素：▪ 加权求和 z = ∑w_i x_i + b - w_i: 权重 - x_i: 输入 - b: 偏置▪ 非线性激活 σ(z) = max(0,z) (ReLU) - 引入非线性 - 解决梯度消失问题▪ 损失函数 L = ½(y - ŷ)^2 (均方误差) - y: 真实值 - ŷ: 预测值 - diff --git a/output2/slide_4/image/image_1.jpg b/output2/slide_4/image/image_1.jpg deleted file mode 100644 index d9509c9..0000000 Binary files a/output2/slide_4/image/image_1.jpg and /dev/null differ diff --git a/output2/slide_4/image/slide_4_image_1_text.txt b/output2/slide_4/image/slide_4_image_1_text.txt deleted file mode 100644 index 27a2912..0000000 --- a/output2/slide_4/image/slide_4_image_1_text.txt +++ /dev/null @@ -1,2 +0,0 @@ -WHATEV-VEER. -stablediffusionweb.com \ No newline at end of file diff --git a/output2/slide_4/image/texts.txt b/output2/slide_4/image/texts.txt deleted file mode 100644 index cc1786b..0000000 --- a/output2/slide_4/image/texts.txt +++ /dev/null @@ -1,24 +0,0 @@ -@@@image1@@@ -WHATEV-VEER. -stablediffusionweb.com - -@@@image1@@@ -WHATEV-VEER. -stablediffusionweb.com - -@@@image1@@@ -WHATEV-VEER. -stablediffusionweb.com - -@@@image1@@@ -WHATEV-VEER. -stablediffusionweb.com - -@@@image1@@@ -WHATEV-VEER. -stablediffusionweb.com - -@@@image1@@@ -WHATEV-VEER. -stablediffusionweb.com - diff --git a/output2/slide_4/slide_4_texts.txt b/output2/slide_4/slide_4_texts.txt deleted file mode 100644 index 3fd86cd..0000000 --- a/output2/slide_4/slide_4_texts.txt +++ /dev/null @@ -1,3 +0,0 @@ -@@@content@@@ -卷积运算可视化卷积运算的核心思想：▪ 卷积核（Kernel）：一个小的权重矩阵，用于提取局部特征。▪ 步长（Stride）：卷积核在输入上滑动的步幅，影响输出特征图的大小。▪ 填充（Padding）：在输入边缘添加额外的像素，控制输出特征图的尺寸。▪ 特征图（Feature Map）：卷积运算的输出，反映了输入中某种特征的分布。 - diff --git a/output2/slide_5/image/image_1.jpg b/output2/slide_5/image/image_1.jpg deleted file mode 100644 index fc4c75c..0000000 Binary files a/output2/slide_5/image/image_1.jpg and /dev/null differ diff --git a/output2/slide_5/image/image_2.jpg b/output2/slide_5/image/image_2.jpg deleted file mode 100644 index fc4c75c..0000000 Binary files a/output2/slide_5/image/image_2.jpg and /dev/null differ diff --git a/output2/slide_5/image/image_3.jpg b/output2/slide_5/image/image_3.jpg deleted file mode 100644 index fc4c75c..0000000 Binary files a/output2/slide_5/image/image_3.jpg and /dev/null differ diff --git a/output2/slide_5/image/slide_5_image_1_text.txt b/output2/slide_5/image/slide_5_image_1_text.txt deleted file mode 100644 index d80a243..0000000 --- a/output2/slide_5/image/slide_5_image_1_text.txt +++ /dev/null @@ -1 +0,0 @@ -stablediffusionweb.com \ No newline at end of file diff --git a/output2/slide_5/image/slide_5_image_2_text.txt b/output2/slide_5/image/slide_5_image_2_text.txt deleted file mode 100644 index d80a243..0000000 --- a/output2/slide_5/image/slide_5_image_2_text.txt +++ /dev/null @@ -1 +0,0 @@ -stablediffusionweb.com \ No newline at end of file diff --git a/output2/slide_5/image/slide_5_image_3_text.txt b/output2/slide_5/image/slide_5_image_3_text.txt deleted file mode 100644 index d80a243..0000000 --- a/output2/slide_5/image/slide_5_image_3_text.txt +++ /dev/null @@ -1 +0,0 @@ -stablediffusionweb.com \ No newline at end of file diff --git a/output2/slide_5/image/texts.txt b/output2/slide_5/image/texts.txt deleted file mode 100644 index df4f34f..0000000 --- a/output2/slide_5/image/texts.txt +++ /dev/null @@ -1,54 +0,0 @@ -@@@image1@@@ -stablediffusionweb.com - -@@@image2@@@ -stablediffusionweb.com - -@@@image3@@@ -stablediffusionweb.com - -@@@image1@@@ -stablediffusionweb.com - -@@@image2@@@ -stablediffusionweb.com - -@@@image3@@@ -stablediffusionweb.com - -@@@image1@@@ -stablediffusionweb.com - -@@@image2@@@ -stablediffusionweb.com - -@@@image3@@@ -stablediffusionweb.com - -@@@image1@@@ -stablediffusionweb.com - -@@@image2@@@ -stablediffusionweb.com - -@@@image3@@@ -stablediffusionweb.com - -@@@image1@@@ -stablediffusionweb.com - -@@@image2@@@ -stablediffusionweb.com - -@@@image3@@@ -stablediffusionweb.com - -@@@image1@@@ -stablediffusionweb.com - -@@@image2@@@ -stablediffusionweb.com - -@@@image3@@@ -stablediffusionweb.com - diff --git a/output2/slide_5/slide_5_texts.txt b/output2/slide_5/slide_5_texts.txt deleted file mode 100644 index f092f27..0000000 --- a/output2/slide_5/slide_5_texts.txt +++ /dev/null @@ -1,3 +0,0 @@ -@@@content@@@ -典型应用场景医疗影像分析CT图像分割- 肿瘤检测- 器官定位深度学习在医疗影像中的应用显著提高了诊断的准确性和效率。自动驾驶实时目标检测- 行人识别- 车道线检测自动驾驶技术依赖于深度学习模型对复杂环境的实时感知和决策。艺术创作风格迁移示例- 图像风格化- 视频风格化生成对抗网络（GAN）为艺术创作提供了全新的可能性。 - diff --git a/output2/slide_6/slide_6_texts.txt b/output2/slide_6/slide_6_texts.txt deleted file mode 100644 index 187e962..0000000 --- a/output2/slide_6/slide_6_texts.txt +++ /dev/null @@ -1,3 +0,0 @@ -@@@content@@@ -知识点详解1. 神经网络的基本结构：▪ 神经网络由输入层、隐藏层和输出层组成。▪ 每一层包含多个神经元，神经元之间通过权重连接。▪ 输入层接收原始数据，隐藏层提取特征，输出层生成最终结果。2. 激活函数的作用：▪ 激活函数引入非线性，使神经网络能够学习复杂的模式。▪ 常用的激活函数包括ReLU、Sigmoid和Tanh。▪ ReLU（Rectified Linear Unit）是目前最常用的激活函数，因其简单且有效。 - diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..b845da0 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,74 @@ +[build-system] +requires = ["setuptools>=69", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "pptx-extraction" +version = "2.0.0" +description = "Extract PowerPoint decks into traceable, AI-ready structured data." +readme = "README.md" +requires-python = ">=3.10" +license = "MIT" +license-files = ["LICENSE"] +authors = [{ name = "pptx_extraction contributors" }] +keywords = ["powerpoint", "pptx", "extraction", "ocr", "rag", "agents"] +classifiers = [ + "Development Status :: 4 - Beta", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Topic :: Office/Business :: Office Suites", +] +dependencies = [ + "python-pptx>=1.0.2,<2", + "Pillow>=10.3,<13", +] + +[project.optional-dependencies] +ocr = ["pytesseract>=0.3.13,<1"] +api = [ + "fastapi>=0.115,<1", + "python-multipart>=0.0.18,<1", + "uvicorn[standard]>=0.32,<1", +] +dev = [ + "build>=1.2,<2", + "httpx2>=2.10,<3", + "mypy>=1.13,<2", + "pytest>=8.3,<9", + "pytest-cov>=6,<7", + "ruff>=0.8,<1", +] + +[project.scripts] +pptx-extraction = "pptx_extraction.cli:main" + +[project.urls] +Homepage = "https://github.com/BlairCode/pptx_extraction" +Issues = "https://github.com/BlairCode/pptx_extraction/issues" + +[tool.setuptools] +package-dir = { "" = "src" } +include-package-data = true + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.ruff] +target-version = "py310" +line-length = 100 + +[tool.ruff.lint] +select = ["E", "F", "I", "B", "UP", "SIM", "RUF"] + +[tool.mypy] +python_version = "3.10" +warn_return_any = true +warn_unused_configs = true +disallow_untyped_defs = true +no_implicit_optional = true + +[tool.pytest.ini_options] +addopts = "-ra --strict-markers --cov=pptx_extraction --cov-report=term-missing" +testpaths = ["tests"] diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index e69de29..0000000 diff --git a/sample/sample.mp3 b/sample/sample.mp3 deleted file mode 100644 index 20cb170..0000000 Binary files a/sample/sample.mp3 and /dev/null differ diff --git a/sample/sample1.mp3 b/sample/sample1.mp3 deleted file mode 100644 index 0b685d1..0000000 Binary files a/sample/sample1.mp3 and /dev/null differ diff --git a/schemas/pptx-extraction.presentation.v1.schema.json b/schemas/pptx-extraction.presentation.v1.schema.json new file mode 100644 index 0000000..337a62c --- /dev/null +++ b/schemas/pptx-extraction.presentation.v1.schema.json @@ -0,0 +1,54 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://example.invalid/pptx-extraction.presentation.v1.schema.json", + "title": "pptx_extraction Presentation v1", + "type": "object", + "required": [ + "schema_version", + "source_name", + "source_sha256", + "source_size_bytes", + "slide_width_pt", + "slide_height_pt", + "metadata", + "slides", + "warnings" + ], + "properties": { + "schema_version": { "const": "1.0" }, + "source_name": { "type": "string" }, + "source_sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" }, + "source_size_bytes": { "type": "integer", "minimum": 1 }, + "slide_width_pt": { "type": "number", "minimum": 0 }, + "slide_height_pt": { "type": "number", "minimum": 0 }, + "metadata": { "type": "object" }, + "slides": { + "type": "array", + "items": { + "type": "object", + "required": [ + "number", + "hidden", + "text_blocks", + "tables", + "charts", + "images" + ], + "properties": { + "number": { "type": "integer", "minimum": 1 }, + "title": { "type": ["string", "null"] }, + "hidden": { "type": "boolean" }, + "layout_name": { "type": ["string", "null"] }, + "text_blocks": { "type": "array" }, + "tables": { "type": "array" }, + "charts": { "type": "array" }, + "images": { "type": "array" }, + "notes": { "type": ["string", "null"] } + }, + "additionalProperties": false + } + }, + "warnings": { "type": "array" } + }, + "additionalProperties": false +} diff --git a/scripts/build_release.py b/scripts/build_release.py new file mode 100644 index 0000000..44befde --- /dev/null +++ b/scripts/build_release.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +"""Create deterministic, privacy-scanned project and Agent Skill release archives.""" + +from __future__ import annotations + +import argparse +import zipfile +from pathlib import Path + +from privacy_scan import publishable_files, scan + +VERSION = "2.0.0" +ZIP_TIMESTAMP = (2026, 8, 11, 0, 0, 0) + + +def write_zip(destination: Path, entries: list[tuple[Path, str]]) -> None: + destination.parent.mkdir(parents=True, exist_ok=True) + if destination.exists(): + destination.unlink() + with zipfile.ZipFile( + destination, "w", compression=zipfile.ZIP_DEFLATED, compresslevel=9 + ) as archive: + for source, archive_name in sorted(entries, key=lambda item: item[1]): + info = zipfile.ZipInfo(archive_name, ZIP_TIMESTAMP) + info.compress_type = zipfile.ZIP_DEFLATED + info.external_attr = 0o644 << 16 + archive.writestr(info, source.read_bytes()) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--output", type=Path, default=Path("release")) + args = parser.parse_args() + root = Path(__file__).resolve().parents[1] + findings = scan(root) + if findings: + raise SystemExit("Privacy scan failed:\n" + "\n".join(findings)) + + output = (root / args.output).resolve() if not args.output.is_absolute() else args.output + project_entries = [ + (path, f"pptx_extraction-{VERSION}/{path.relative_to(root).as_posix()}") + for path in publishable_files(root) + if "agent-skill" not in path.relative_to(root).parts + ] + project_archive = output / f"pptx_extraction-v{VERSION}.zip" + write_zip(project_archive, project_entries) + + skill_root = root / "agent-skill" / "pptx-extraction" + skill_entries = [ + (path, f"pptx-extraction/{path.relative_to(skill_root).as_posix()}") + for path in skill_root.rglob("*") + if path.is_file() and "__pycache__" not in path.parts + ] + skill_archive = output / f"pptx_extraction-skill-v{VERSION}.zip" + write_zip(skill_archive, skill_entries) + print(project_archive) + print(skill_archive) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/privacy_scan.py b/scripts/privacy_scan.py new file mode 100644 index 0000000..9c699a9 --- /dev/null +++ b/scripts/privacy_scan.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +"""Fail when publishable sources contain common private artifacts or secrets.""" + +from __future__ import annotations + +import argparse +import re +import sys +from pathlib import Path + +PUBLISH_ROOTS = ( + ".github", + "agent-skill", + "docs", + "schemas", + "scripts", + "src", + "tests", +) +ROOT_FILES = ( + ".env.example", + ".gitignore", + "CHANGELOG.md", + "CONTRIBUTING.md", + "LICENSE", + "MANIFEST.in", + "README.md", + "SECURITY.md", + "pyproject.toml", +) +TEXT_SUFFIXES = {".md", ".py", ".toml", ".yaml", ".yml", ".json", ".txt", ".example"} +BANNED_ARTIFACT_SUFFIXES = {".ppt", ".pptx", ".pptm", ".potx", ".ppsx", ".mp3", ".wav", ".srt"} +EXCLUDED_PARTS = {"__pycache__", ".pytest_cache", ".mypy_cache", ".ruff_cache"} +EXCLUDED_SUFFIXES = {".pyc", ".pyo"} +PATTERNS = { + "Windows absolute path": re.compile(r"(? list[Path]: + files = [root / name for name in ROOT_FILES if (root / name).is_file()] + for relative in PUBLISH_ROOTS: + base = root / relative + if base.is_dir(): + files.extend( + path + for path in base.rglob("*") + if path.is_file() + and not EXCLUDED_PARTS.intersection(path.parts) + and path.suffix.lower() not in EXCLUDED_SUFFIXES + ) + return sorted(set(files)) + + +def scan(root: Path) -> list[str]: + findings: list[str] = [] + scanner = (root / "scripts" / "privacy_scan.py").resolve() + for path in publishable_files(root): + relative = path.relative_to(root).as_posix() + if path.suffix.lower() in BANNED_ARTIFACT_SUFFIXES: + findings.append(f"{relative}: publishable binary/private artifact") + continue + if path.resolve() == scanner or path.suffix.lower() not in TEXT_SUFFIXES: + continue + try: + text = path.read_text(encoding="utf-8") + except UnicodeDecodeError: + findings.append(f"{relative}: non-UTF-8 text file") + continue + for line_number, line in enumerate(text.splitlines(), start=1): + for label, pattern in PATTERNS.items(): + if pattern.search(line): + findings.append(f"{relative}:{line_number}: {label}") + return findings + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--root", type=Path, default=Path(__file__).resolve().parents[1]) + args = parser.parse_args() + findings = scan(args.root.resolve()) + if findings: + print("Privacy scan failed:", file=sys.stderr) + print("\n".join(f"- {item}" for item in findings), file=sys.stderr) + return 1 + print("Privacy scan passed: no common private artifacts or secrets in publishable files.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/pptx_extraction/__init__.py b/src/pptx_extraction/__init__.py new file mode 100644 index 0000000..7d5cb73 --- /dev/null +++ b/src/pptx_extraction/__init__.py @@ -0,0 +1,7 @@ +"""pptx_extraction public Python API.""" + +from .models import ExtractionOptions, PresentationRecord +from .pipeline import extract_file, inspect_file + +__all__ = ["ExtractionOptions", "PresentationRecord", "extract_file", "inspect_file"] +__version__ = "2.0.0" diff --git a/src/pptx_extraction/__main__.py b/src/pptx_extraction/__main__.py new file mode 100644 index 0000000..bfdcd0c --- /dev/null +++ b/src/pptx_extraction/__main__.py @@ -0,0 +1,4 @@ +from .cli import main + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/pptx_extraction/api.py b/src/pptx_extraction/api.py new file mode 100644 index 0000000..a0c7369 --- /dev/null +++ b/src/pptx_extraction/api.py @@ -0,0 +1,152 @@ +"""Optional single-node job API for pptx_extraction.""" + +from __future__ import annotations + +import json +import os +import shutil +import threading +import uuid +from concurrent.futures import ThreadPoolExecutor +from contextlib import asynccontextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import Annotated, Any + +from .exceptions import OptionalDependencyError +from .models import ExtractionOptions +from .pipeline import extract_file +from .security import SUPPORTED_OOXML_EXTENSIONS + + +@dataclass(frozen=True, slots=True) +class ServiceSettings: + work_dir: Path + max_upload_bytes: int = 50 * 1024 * 1024 + workers: int = 2 + + @classmethod + def from_env(cls) -> ServiceSettings: + return cls( + work_dir=Path(os.getenv("PPTX_EXTRACTION_WORK_DIR", "./work")).expanduser().resolve(), + max_upload_bytes=int(os.getenv("PPTX_EXTRACTION_MAX_UPLOAD_MB", "50")) * 1024 * 1024, + workers=max(1, min(int(os.getenv("PPTX_EXTRACTION_WORKERS", "2")), 16)), + ) + + +class JobStore: + def __init__(self) -> None: + self._jobs: dict[str, dict[str, Any]] = {} + self._lock = threading.Lock() + + def create(self, job_id: str, source_name: str) -> None: + with self._lock: + self._jobs[job_id] = {"id": job_id, "status": "queued", "source_name": source_name} + + def update(self, job_id: str, **values: Any) -> None: + with self._lock: + self._jobs[job_id].update(values) + + def get(self, job_id: str) -> dict[str, Any] | None: + with self._lock: + value = self._jobs.get(job_id) + return dict(value) if value else None + + +def create_app(settings: ServiceSettings | None = None) -> Any: + try: + from fastapi import ( # type: ignore[import-not-found] + FastAPI, + File, + HTTPException, + UploadFile, + ) + from fastapi.responses import JSONResponse # type: ignore[import-not-found] + except ImportError as exc: + raise OptionalDependencyError( + "HTTP API dependencies are not installed.", + hint="Install `pptx-extraction[api]`.", + ) from exc + + # FastAPI resolves postponed endpoint annotations from module globals. + globals().update({"UploadFile": UploadFile, "File": File}) + + active = settings or ServiceSettings.from_env() + active.work_dir.mkdir(parents=True, exist_ok=True) + store = JobStore() + executor = ThreadPoolExecutor( + max_workers=active.workers, thread_name_prefix="pptx-extraction-api" + ) + + @asynccontextmanager + async def lifespan(_: Any) -> Any: + yield + executor.shutdown(wait=True, cancel_futures=True) + + app = FastAPI(title="pptx_extraction API", version="2.0.0", docs_url="/docs", lifespan=lifespan) + + @app.get("/healthz") + def health() -> dict[str, str]: + return {"status": "ok"} + + @app.post("/v1/jobs", status_code=202) + async def create_job(file: Annotated[UploadFile, File(...)]) -> dict[str, str]: + original_name = Path(file.filename or "upload.pptx").name + suffix = Path(original_name).suffix.lower() + if suffix not in SUPPORTED_OOXML_EXTENSIONS: + raise HTTPException(status_code=415, detail="Unsupported PowerPoint format.") + job_id = uuid.uuid4().hex + job_dir = active.work_dir / job_id + job_dir.mkdir(parents=False, exist_ok=False) + source_path = job_dir / f"source{suffix}" + total = 0 + try: + with source_path.open("wb") as stream: + while chunk := await file.read(1024 * 1024): + total += len(chunk) + if total > active.max_upload_bytes: + raise HTTPException( + status_code=413, detail="Upload exceeds configured limit." + ) + stream.write(chunk) + except Exception: + shutil.rmtree(job_dir, ignore_errors=True) + raise + finally: + await file.close() + store.create(job_id, original_name) + executor.submit(_run_job, store, job_id, source_path, job_dir / "result") + return {"id": job_id, "status": "queued"} + + @app.get("/v1/jobs/{job_id}") + def get_job(job_id: str) -> dict[str, Any]: + job = store.get(job_id) + if job is None: + raise HTTPException(status_code=404, detail="Job not found.") + return job + + @app.get("/v1/jobs/{job_id}/result") + def get_result(job_id: str) -> Any: + job = store.get(job_id) + if job is None: + raise HTTPException(status_code=404, detail="Job not found.") + if job["status"] != "succeeded": + raise HTTPException(status_code=409, detail=f"Job is {job['status']}.") + result_path = active.work_dir / job_id / "result" / "presentation.json" + return JSONResponse(json.loads(result_path.read_text(encoding="utf-8"))) + + return app + + +def _run_job(store: JobStore, job_id: str, source_path: Path, output_dir: Path) -> None: + store.update(job_id, status="running") + try: + result = extract_file( + source_path, + output_dir, + options=ExtractionOptions(redact_metadata=True), + formats=("json", "markdown"), + ) + store.update(job_id, status="succeeded", summary=result.record.summary) + except Exception as exc: + store.update(job_id, status="failed", error=str(exc)) diff --git a/src/pptx_extraction/cli.py b/src/pptx_extraction/cli.py new file mode 100644 index 0000000..d4407aa --- /dev/null +++ b/src/pptx_extraction/cli.py @@ -0,0 +1,205 @@ +"""Dependency-light pptx_extraction command-line interface.""" + +from __future__ import annotations + +import argparse +import json +import logging +import sys +from collections.abc import Sequence +from pathlib import Path +from typing import TextIO + +from . import __version__ +from .converter import convert_legacy +from .exceptions import ( + InputValidationError, + OptionalDependencyError, + OutputExistsError, + PptxExtractionError, +) +from .models import ExtractionOptions +from .pipeline import batch_extract, extract_file, inspect_file +from .security import validate_package + +EXIT_SUCCESS = 0 +EXIT_USAGE = 2 +EXIT_EXTRACTION = 3 +EXIT_PARTIAL = 4 +EXIT_DEPENDENCY = 5 + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="pptx-extraction", + description="Extract PowerPoint decks into traceable, AI-ready data.", + ) + parser.add_argument("--version", action="version", version=f"pptx_extraction {__version__}") + parser.add_argument("--verbose", action="store_true", help="Enable diagnostic logs.") + commands = parser.add_subparsers(dest="command", required=True) + + extract = commands.add_parser("extract", help="Extract one OOXML presentation.") + extract.add_argument("source", type=Path) + extract.add_argument("--output", "-o", type=Path, required=True) + extract.add_argument( + "--format", + dest="formats", + action="append", + choices=("json", "markdown", "text"), + help="Repeat for multiple formats; defaults to json and markdown.", + ) + _add_extraction_options(extract) + extract.add_argument("--overwrite", action="store_true") + extract.add_argument("--tesseract-command") + + inspect = commands.add_parser("inspect", help="Print a deck summary without writing assets.") + inspect.add_argument("source", type=Path) + inspect.add_argument("--full", action="store_true", help="Print the complete JSON record.") + inspect.add_argument("--redact-metadata", action="store_true") + + validate = commands.add_parser("validate", help="Validate OOXML structure and safety limits.") + validate.add_argument("source", type=Path) + + batch = commands.add_parser("batch", help="Extract files/directories concurrently.") + batch.add_argument("sources", type=Path, nargs="+") + batch.add_argument("--output", "-o", type=Path, required=True) + batch.add_argument( + "--format", dest="formats", action="append", choices=("json", "markdown", "text") + ) + batch.add_argument("--workers", type=int, default=2) + batch.add_argument("--no-recursive", action="store_true") + batch.add_argument("--overwrite", action="store_true") + _add_extraction_options(batch) + + convert = commands.add_parser("convert", help="Convert one legacy deck with LibreOffice.") + convert.add_argument("source", type=Path) + convert.add_argument("--output", "-o", type=Path, required=True) + convert.add_argument("--soffice", default="soffice") + convert.add_argument("--timeout", type=int, default=120) + return parser + + +def _add_extraction_options(parser: argparse.ArgumentParser) -> None: + parser.add_argument("--no-assets", action="store_true") + parser.add_argument("--no-notes", action="store_true") + parser.add_argument("--no-metadata", action="store_true") + parser.add_argument("--redact-metadata", action="store_true") + parser.add_argument("--ocr", choices=("none", "tesseract"), default="none") + parser.add_argument("--ocr-language", default="eng") + + +def _options(args: argparse.Namespace) -> ExtractionOptions: + return ExtractionOptions( + include_assets=not getattr(args, "no_assets", False), + include_notes=not getattr(args, "no_notes", False), + include_metadata=not getattr(args, "no_metadata", False), + redact_metadata=getattr(args, "redact_metadata", False), + ocr_backend=getattr(args, "ocr", "none"), + ocr_language=getattr(args, "ocr_language", "eng"), + ) + + +def main(argv: Sequence[str] | None = None) -> int: + parser = build_parser() + args = parser.parse_args(argv) + logging.basicConfig( + level=logging.DEBUG if args.verbose else logging.WARNING, + format="%(levelname)s %(name)s: %(message)s", + ) + try: + if args.command == "extract": + result = extract_file( + args.source, + args.output, + options=_options(args), + formats=tuple(args.formats or ("json", "markdown")), + overwrite=args.overwrite, + tesseract_command=args.tesseract_command, + ) + _print_json( + { + "status": "ok", + "output_dir": str(result.output_dir), + "files": {key: str(value) for key, value in result.files.items()}, + "summary": result.record.summary, + } + ) + return EXIT_SUCCESS + if args.command == "inspect": + record = inspect_file( + args.source, + options=ExtractionOptions(redact_metadata=args.redact_metadata), + ) + _print_json(record.to_dict() if args.full else record.summary) + return EXIT_SUCCESS + if args.command == "validate": + report = validate_package(args.source) + _print_json( + { + "status": "valid", + "entries": report.entries, + "expanded_bytes": report.expanded_bytes, + "has_macros": report.has_macros, + "warnings": report.warnings, + } + ) + return EXIT_SUCCESS + if args.command == "batch": + items = batch_extract( + list(args.sources), + args.output, + options=_options(args), + formats=tuple(args.formats or ("json", "markdown")), + workers=args.workers, + overwrite=args.overwrite, + recursive=not args.no_recursive, + ) + payload = [ + { + "source": str(item.source), + "success": item.success, + "output_dir": str(item.output_dir) if item.output_dir else None, + "error": item.error, + } + for item in items + ] + _print_json({"results": payload}) + return EXIT_SUCCESS if all(item.success for item in items) else EXIT_PARTIAL + if args.command == "convert": + destination = convert_legacy( + args.source, + args.output, + soffice_command=args.soffice, + timeout_seconds=args.timeout, + ) + _print_json({"status": "ok", "output": str(destination)}) + return EXIT_SUCCESS + raise InputValidationError(f"Unknown command: {args.command}") + except OptionalDependencyError as exc: + _print_error(exc) + return EXIT_DEPENDENCY + except (InputValidationError, OutputExistsError) as exc: + _print_error(exc) + return EXIT_USAGE + except PptxExtractionError as exc: + _print_error(exc) + return EXIT_EXTRACTION + except Exception as exc: # final CLI boundary; library calls retain typed exceptions + logging.getLogger(__name__).exception("Unexpected failure") + _print_json({"error": {"code": "unexpected_error", "message": str(exc)}}, sys.stderr) + return EXIT_EXTRACTION + + +def _print_error(error: PptxExtractionError) -> None: + payload = {"error": {"code": error.code, "message": error.message}} + if error.hint: + payload["error"]["hint"] = error.hint + _print_json(payload, sys.stderr) + + +def _print_json(payload: object, stream: TextIO | None = None) -> None: + print(json.dumps(payload, ensure_ascii=False, indent=2), file=stream or sys.stdout) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/pptx_extraction/converter.py b/src/pptx_extraction/converter.py new file mode 100644 index 0000000..d5115a6 --- /dev/null +++ b/src/pptx_extraction/converter.py @@ -0,0 +1,53 @@ +"""Explicit legacy PowerPoint conversion through LibreOffice.""" + +from __future__ import annotations + +import shutil +import subprocess +from pathlib import Path + +from .exceptions import ExtractionError, InputValidationError, OptionalDependencyError +from .security import LEGACY_EXTENSIONS + + +def convert_legacy( + source: str | Path, + output_dir: str | Path, + *, + soffice_command: str = "soffice", + timeout_seconds: int = 120, +) -> Path: + input_path = Path(source).expanduser().resolve() + target_dir = Path(output_dir).expanduser().resolve() + if not input_path.is_file(): + raise InputValidationError(f"Presentation does not exist: {input_path}") + if input_path.suffix.lower() not in LEGACY_EXTENSIONS: + raise InputValidationError("convert accepts only .ppt, .pot or .pps input.") + executable = shutil.which(soffice_command) + if not executable: + raise OptionalDependencyError( + "LibreOffice executable was not found.", + hint="Install LibreOffice or pass --soffice with its executable path.", + ) + target_dir.mkdir(parents=True, exist_ok=True) + completed = subprocess.run( + [ + executable, + "--headless", + "--convert-to", + "pptx", + "--outdir", + str(target_dir), + str(input_path), + ], + capture_output=True, + text=True, + timeout=timeout_seconds, + check=False, + shell=False, + ) + destination = target_dir / f"{input_path.stem}.pptx" + if completed.returncode != 0 or not destination.is_file(): + detail = (completed.stderr or completed.stdout or "unknown conversion error").strip() + raise ExtractionError(f"LibreOffice conversion failed: {detail}") + return destination diff --git a/src/pptx_extraction/exceptions.py b/src/pptx_extraction/exceptions.py new file mode 100644 index 0000000..01ea728 --- /dev/null +++ b/src/pptx_extraction/exceptions.py @@ -0,0 +1,34 @@ +"""Stable exception hierarchy used by the library, CLI and API.""" + +from __future__ import annotations + + +class PptxExtractionError(Exception): + """Base error with a machine-readable code.""" + + code = "pptx_extraction_error" + + def __init__(self, message: str, *, hint: str | None = None) -> None: + super().__init__(message) + self.message = message + self.hint = hint + + +class InputValidationError(PptxExtractionError): + code = "invalid_input" + + +class UnsafePackageError(InputValidationError): + code = "unsafe_package" + + +class ExtractionError(PptxExtractionError): + code = "extraction_failed" + + +class OptionalDependencyError(PptxExtractionError): + code = "optional_dependency_missing" + + +class OutputExistsError(PptxExtractionError): + code = "output_exists" diff --git a/src/pptx_extraction/exporters.py b/src/pptx_extraction/exporters.py new file mode 100644 index 0000000..86fe632 --- /dev/null +++ b/src/pptx_extraction/exporters.py @@ -0,0 +1,146 @@ +"""Deterministic serializers for pptx_extraction records.""" + +from __future__ import annotations + +import json +from pathlib import Path + +from .exceptions import InputValidationError +from .models import ChartRecord, ImageRecord, PresentationRecord, TableRecord, TextBlock + +SUPPORTED_FORMATS = frozenset({"json", "markdown", "text"}) + + +def export_record( + record: PresentationRecord, + output_dir: Path, + formats: tuple[str, ...] = ("json", "markdown"), +) -> dict[str, Path]: + normalized = tuple(dict.fromkeys(item.lower().strip() for item in formats if item.strip())) + invalid = sorted(set(normalized) - SUPPORTED_FORMATS) + if invalid: + raise InputValidationError(f"Unsupported output format(s): {', '.join(invalid)}") + if not normalized: + raise InputValidationError("At least one output format is required.") + output_dir.mkdir(parents=True, exist_ok=True) + written: dict[str, Path] = {} + if "json" in normalized: + destination = output_dir / "presentation.json" + destination.write_text( + json.dumps(record.to_dict(), ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + written["json"] = destination + if "markdown" in normalized: + destination = output_dir / "presentation.md" + destination.write_text(to_markdown(record), encoding="utf-8") + written["markdown"] = destination + if "text" in normalized: + destination = output_dir / "presentation.txt" + destination.write_text(to_text(record), encoding="utf-8") + written["text"] = destination + return written + + +def to_markdown(record: PresentationRecord) -> str: + lines = [ + f"# {record.metadata.get('title') or Path(record.source_name).stem}", + "", + f"> Source: `{record.source_name}` · SHA-256: `{record.source_sha256}` · " + f"Slides: {len(record.slides)} · Schema: {record.schema_version}", + "", + ] + for slide in record.slides: + suffix = " _(hidden)_" if slide.hidden else "" + lines.extend([f"## Slide {slide.number}: {slide.title or 'Untitled'}{suffix}", ""]) + ordered: list[tuple[int, int, object]] = [] + ordered.extend((item.order, 0, item) for item in slide.text_blocks) + ordered.extend((item.order, 1, item) for item in slide.tables) + ordered.extend((item.order, 2, item) for item in slide.charts) + ordered.extend((item.order, 3, item) for item in slide.images) + for _, _, item in sorted(ordered, key=lambda value: (value[0], value[1])): + if isinstance(item, TextBlock): + prefix = " " * item.level + ("- " if item.level else "") + lines.append(f"{prefix}{item.text}") + for link in item.hyperlinks: + lines.append(f" - Link: <{link}>") + lines.append("") + elif isinstance(item, TableRecord): + lines.extend(_markdown_table(item.rows)) + elif isinstance(item, ChartRecord): + lines.append(f"### Chart: {item.title or item.chart_type}") + lines.append("") + if item.categories: + lines.append("Categories: " + ", ".join(item.categories)) + for series in item.series: + values = ", ".join( + "" if value is None else str(value) for value in series.values + ) + lines.append(f"- {series.name or 'Series'}: {values}") + lines.append("") + elif isinstance(item, ImageRecord): + description = item.alt_text or "Embedded image" + if item.asset_path: + lines.append(f"![{_escape_alt(description)}]({item.asset_path})") + else: + lines.append(f"_[{description}; asset export disabled]_ ") + if item.ocr_text: + lines.extend(["", "OCR:", "", item.ocr_text]) + lines.append("") + if slide.notes: + lines.extend(["### Speaker notes", "", slide.notes, ""]) + if record.warnings: + lines.extend(["## Extraction warnings", ""]) + for issue in record.warnings: + location = f" (slide {issue.slide_number})" if issue.slide_number else "" + lines.append(f"- `{issue.code}`{location}: {issue.message}") + lines.append("") + return "\n".join(lines).rstrip() + "\n" + + +def to_text(record: PresentationRecord) -> str: + lines = [ + f"SOURCE: {record.source_name}", + f"SHA256: {record.source_sha256}", + f"SLIDES: {len(record.slides)}", + "", + ] + for slide in record.slides: + lines.append(f"=== SLIDE {slide.number}: {slide.title or 'Untitled'} ===") + for block in sorted(slide.text_blocks, key=lambda item: (item.order, item.z_order)): + lines.append(f"{' ' * block.level}{block.text}") + for table in sorted(slide.tables, key=lambda item: (item.order, item.z_order)): + lines.append("[TABLE]") + lines.extend(" | ".join(row) for row in table.rows) + for chart in sorted(slide.charts, key=lambda item: (item.order, item.z_order)): + lines.append(f"[CHART] {chart.title or chart.chart_type}") + for series in chart.series: + lines.append(f"{series.name}: {', '.join(map(str, series.values))}") + for image in sorted(slide.images, key=lambda item: (item.order, item.z_order)): + lines.append(f"[IMAGE] {image.alt_text or image.sha256[:16]}") + if image.ocr_text: + lines.append(image.ocr_text) + if slide.notes: + lines.extend(["[NOTES]", slide.notes]) + lines.append("") + return "\n".join(lines).rstrip() + "\n" + + +def _markdown_table(rows: tuple[tuple[str, ...], ...]) -> list[str]: + if not rows: + return ["_[Empty table]_", ""] + width = max(len(row) for row in rows) + normalized = [list(row) + [""] * (width - len(row)) for row in rows] + + def escape(value: str) -> str: + return value.replace("|", "\\|").replace("\n", "
") + + lines = ["| " + " | ".join(escape(cell) for cell in normalized[0]) + " |"] + lines.append("| " + " | ".join("---" for _ in range(width)) + " |") + lines.extend("| " + " | ".join(escape(cell) for cell in row) + " |" for row in normalized[1:]) + lines.append("") + return lines + + +def _escape_alt(value: str) -> str: + return value.replace("[", "\\[").replace("]", "\\]").replace("\n", " ") diff --git a/src/pptx_extraction/extractors/__init__.py b/src/pptx_extraction/extractors/__init__.py new file mode 100644 index 0000000..2960eeb --- /dev/null +++ b/src/pptx_extraction/extractors/__init__.py @@ -0,0 +1,3 @@ +from .pptx import PptxExtractor + +__all__ = ["PptxExtractor"] diff --git a/src/pptx_extraction/extractors/pptx.py b/src/pptx_extraction/extractors/pptx.py new file mode 100644 index 0000000..8d87ade --- /dev/null +++ b/src/pptx_extraction/extractors/pptx.py @@ -0,0 +1,419 @@ +"""PowerPoint OOXML extraction into pptx_extraction domain records.""" + +from __future__ import annotations + +import hashlib +import logging +from collections.abc import Iterable +from datetime import date, datetime +from pathlib import Path +from typing import Any + +from pptx import Presentation +from pptx.enum.shapes import MSO_SHAPE_TYPE + +from ..exceptions import ExtractionError, OptionalDependencyError +from ..models import ( + BoundingBox, + ChartRecord, + ChartSeries, + ExtractionOptions, + ImageRecord, + Issue, + PresentationRecord, + SlideRecord, + TableRecord, + TextBlock, +) +from ..ocr import OCRBackend, create_ocr_backend +from ..security import PackageLimits, sha256_file, validate_package + +logger = logging.getLogger(__name__) + + +class PptxExtractor: + """Extract one validated OOXML presentation without network access.""" + + def __init__( + self, + options: ExtractionOptions | None = None, + *, + package_limits: PackageLimits | None = None, + ocr_backend: OCRBackend | None = None, + tesseract_command: str | None = None, + ) -> None: + self.options = options or ExtractionOptions() + self.package_limits = package_limits or PackageLimits() + self.ocr = ocr_backend or create_ocr_backend(self.options.ocr_backend, tesseract_command) + self._ocr_cache: dict[str, str] = {} + self._asset_cache: dict[str, str] = {} + + def extract(self, source: str | Path, asset_dir: Path | None = None) -> PresentationRecord: + path = Path(source).expanduser().resolve() + report = validate_package(path, self.package_limits) + warnings = [Issue("macro_present", item) for item in report.warnings] + self._ocr_cache.clear() + self._asset_cache.clear() + try: + presentation = Presentation(str(path)) + slide_width = int(presentation.slide_width or 0) + slide_height = int(presentation.slide_height or 0) + if asset_dir and self.options.include_assets: + asset_dir.mkdir(parents=True, exist_ok=True) + slides = [ + self._extract_slide( + slide, + number, + slide_width, + slide_height, + asset_dir if self.options.include_assets else None, + warnings, + ) + for number, slide in enumerate(presentation.slides, start=1) + ] + metadata = self._extract_metadata(presentation) if self.options.include_metadata else {} + if self.options.redact_metadata: + metadata = self._redact_metadata(metadata) + return PresentationRecord( + source_name=path.name, + source_sha256=sha256_file(path), + source_size_bytes=path.stat().st_size, + slide_width_pt=round(slide_width / 12_700, 3), + slide_height_pt=round(slide_height / 12_700, 3), + metadata=metadata, + slides=slides, + warnings=warnings, + ) + except OptionalDependencyError: + raise + except Exception as exc: + raise ExtractionError(f"Unable to extract {path.name}: {exc}") from exc + + def _extract_slide( + self, + slide: Any, + number: int, + slide_width: int, + slide_height: int, + asset_dir: Path | None, + warnings: list[Issue], + ) -> SlideRecord: + title_shape = slide.shapes.title + title = title_shape.text.strip() if title_shape and title_shape.text.strip() else None + record = SlideRecord( + number=number, + title=title, + hidden=slide._element.get("show") == "0", + layout_name=getattr(slide.slide_layout, "name", None), + notes=self._extract_notes(slide, warnings, number) + if self.options.include_notes + else None, + ) + flattened = list(self._walk_shapes(slide.shapes)) + row_band = max(int(slide_height * 0.015), 1) + ordered = sorted( + flattened, + key=lambda item: ( + int(getattr(item[1], "top", 0)) // row_band, + int(getattr(item[1], "left", 0)), + item[0], + ), + ) + for reading_order, (z_order, shape) in enumerate(ordered, start=1): + bbox = self._bbox(shape, slide_width, slide_height) + shape_id = int(getattr(shape, "shape_id", 0)) + shape_name = str(getattr(shape, "name", f"shape-{shape_id}")) + try: + if getattr(shape, "has_table", False): + rows = tuple( + tuple(cell.text.strip() for cell in row.cells) for row in shape.table.rows + ) + record.tables.append( + TableRecord(rows, reading_order, z_order, shape_id, shape_name, bbox) + ) + elif getattr(shape, "has_chart", False): + record.charts.append( + self._extract_chart( + shape, reading_order, z_order, shape_id, shape_name, bbox + ) + ) + elif shape.shape_type == MSO_SHAPE_TYPE.PICTURE: + record.images.append( + self._extract_image( + shape, + reading_order, + z_order, + shape_id, + shape_name, + bbox, + asset_dir, + number, + warnings, + ) + ) + elif getattr(shape, "has_text_frame", False): + is_title = bool( + title_shape is not None and shape._element is title_shape._element + ) + record.text_blocks.extend( + self._extract_text_blocks( + shape, + "title" if is_title else "body", + reading_order, + z_order, + shape_id, + shape_name, + bbox, + ) + ) + elif self._is_material_unsupported_shape(shape): + warnings.append( + Issue( + "unsupported_shape", + f"Shape type {shape.shape_type} was not extracted.", + slide_number=number, + shape_id=shape_id, + ) + ) + except OptionalDependencyError: + raise + except Exception as exc: + warnings.append( + Issue( + "shape_extraction_failed", + f"{shape_name}: {exc}", + slide_number=number, + shape_id=shape_id, + ) + ) + return record + + def _walk_shapes(self, shapes: Iterable[Any]) -> Iterable[tuple[int, Any]]: + z_order = 0 + for shape in shapes: + if shape.shape_type == MSO_SHAPE_TYPE.GROUP: + for _, child in self._walk_shapes(shape.shapes): + yield z_order, child + z_order += 1 + else: + yield z_order, shape + z_order += 1 + + @staticmethod + def _bbox(shape: Any, slide_width: int, slide_height: int) -> BoundingBox: + return BoundingBox.from_emu( + int(getattr(shape, "left", 0)), + int(getattr(shape, "top", 0)), + int(getattr(shape, "width", 0)), + int(getattr(shape, "height", 0)), + slide_width, + slide_height, + ) + + @staticmethod + def _extract_text_blocks( + shape: Any, + kind: str, + order: int, + z_order: int, + shape_id: int, + shape_name: str, + bbox: BoundingBox, + ) -> list[TextBlock]: + blocks: list[TextBlock] = [] + for paragraph in shape.text_frame.paragraphs: + text = paragraph.text.strip() + if not text: + continue + links: list[str] = [] + for run in paragraph.runs: + try: + address = run.hyperlink.address + except (KeyError, ValueError): + address = None + if address and address not in links: + links.append(address) + blocks.append( + TextBlock( + text=text, + kind=kind, + level=int(getattr(paragraph, "level", 0)), + order=order, + z_order=z_order, + shape_id=shape_id, + shape_name=shape_name, + bbox=bbox, + hyperlinks=tuple(links), + ) + ) + return blocks + + @staticmethod + def _extract_chart( + shape: Any, + order: int, + z_order: int, + shape_id: int, + shape_name: str, + bbox: BoundingBox, + ) -> ChartRecord: + chart = shape.chart + title: str | None = None + if getattr(chart, "has_title", False): + candidate = chart.chart_title.text_frame.text.strip() + title = candidate or None + categories: list[str] = [] + try: + if chart.plots: + for category in chart.plots[0].categories: + label = getattr(category, "label", category) + categories.append(str(label)) + except (AttributeError, TypeError, ValueError): + categories = [] + series = tuple( + ChartSeries( + name=str(item.name or ""), + values=tuple(PptxExtractor._json_scalar(value) for value in item.values), + ) + for item in chart.series + ) + chart_type = getattr(getattr(chart, "chart_type", None), "name", None) + return ChartRecord( + chart_type=chart_type or str(chart.chart_type), + title=title, + categories=tuple(categories), + series=series, + order=order, + z_order=z_order, + shape_id=shape_id, + shape_name=shape_name, + bbox=bbox, + ) + + def _extract_image( + self, + shape: Any, + order: int, + z_order: int, + shape_id: int, + shape_name: str, + bbox: BoundingBox, + asset_dir: Path | None, + slide_number: int, + warnings: list[Issue], + ) -> ImageRecord: + image = shape.image + blob = image.blob + digest = hashlib.sha256(blob).hexdigest() + extension = (image.ext or "bin").lower().lstrip(".") + media_extension = "jpeg" if extension in {"jpg", "jpeg"} else extension + media_type = f"image/{media_extension}" + asset_path: str | None = None + if asset_dir is not None: + asset_path = self._asset_cache.get(digest) + if asset_path is None: + filename = f"{digest[:16]}.{extension}" + destination = asset_dir / filename + if not destination.exists(): + destination.write_bytes(blob) + asset_path = f"assets/{filename}" + self._asset_cache[digest] = asset_path + alt_text = self._alt_text(shape) + if not alt_text: + warnings.append( + Issue( + "image_missing_alt_text", + f"Image {shape_name} has no alternative text.", + slide_number=slide_number, + shape_id=shape_id, + ) + ) + ocr_text = self._ocr_cache.get(digest) + if ocr_text is None: + ocr_text = self.ocr.recognize(blob, self.options.ocr_language).strip() + self._ocr_cache[digest] = ocr_text + return ImageRecord( + sha256=digest, + media_type=media_type, + asset_path=asset_path, + alt_text=alt_text, + ocr_text=ocr_text or None, + order=order, + z_order=z_order, + shape_id=shape_id, + shape_name=shape_name, + bbox=bbox, + ) + + @staticmethod + def _alt_text(shape: Any) -> str | None: + try: + nodes = shape._element.xpath(".//p:cNvPr") + if nodes: + value = nodes[0].get("descr") or nodes[0].get("title") + return value.strip() if value and value.strip() else None + except (AttributeError, IndexError, TypeError): + return None + return None + + @staticmethod + def _extract_notes(slide: Any, warnings: list[Issue], slide_number: int) -> str | None: + try: + if not slide.has_notes_slide: + return None + text_frame = slide.notes_slide.notes_text_frame + text = text_frame.text.strip() if text_frame is not None else "" + return text or None + except Exception as exc: + warnings.append( + Issue( + "notes_extraction_failed", + str(exc), + slide_number=slide_number, + ) + ) + return None + + @staticmethod + def _extract_metadata(presentation: Any) -> dict[str, Any]: + props = presentation.core_properties + values = { + "title": props.title, + "author": props.author, + "subject": props.subject, + "keywords": props.keywords, + "comments": props.comments, + "last_modified_by": props.last_modified_by, + "created": props.created, + "modified": props.modified, + "category": props.category, + "content_status": props.content_status, + "identifier": props.identifier, + "language": props.language, + "revision": props.revision, + "version": props.version, + } + return { + key: PptxExtractor._json_scalar(value) + for key, value in values.items() + if value not in {None, ""} + } + + @staticmethod + def _redact_metadata(metadata: dict[str, Any]) -> dict[str, Any]: + sensitive = {"author", "last_modified_by", "comments", "identifier"} + return {key: "[redacted]" if key in sensitive else value for key, value in metadata.items()} + + @staticmethod + def _json_scalar(value: Any) -> float | int | str | None: + if value is None or isinstance(value, (float, int, str)): + return value + if isinstance(value, (date, datetime)): + return value.isoformat() + return str(value) + + @staticmethod + def _is_material_unsupported_shape(shape: Any) -> bool: + names = {"MEDIA", "OLE_OBJECT", "LINKED_OLE_OBJECT", "WEB_VIDEO"} + shape_type = getattr(shape, "shape_type", None) + return getattr(shape_type, "name", "") in names diff --git a/src/pptx_extraction/models.py b/src/pptx_extraction/models.py new file mode 100644 index 0000000..0f39db7 --- /dev/null +++ b/src/pptx_extraction/models.py @@ -0,0 +1,158 @@ +"""Serializable records forming the pptx_extraction schema v1 contract.""" + +from __future__ import annotations + +from dataclasses import asdict, dataclass, field +from typing import Any + +SCHEMA_VERSION = "1.0" + + +@dataclass(frozen=True, slots=True) +class ExtractionOptions: + include_assets: bool = True + include_notes: bool = True + include_metadata: bool = True + redact_metadata: bool = False + ocr_backend: str = "none" + ocr_language: str = "eng" + + +@dataclass(frozen=True, slots=True) +class BoundingBox: + left_pt: float + top_pt: float + width_pt: float + height_pt: float + left_ratio: float + top_ratio: float + width_ratio: float + height_ratio: float + + @classmethod + def from_emu( + cls, + left: int, + top: int, + width: int, + height: int, + slide_width: int, + slide_height: int, + ) -> BoundingBox: + emu_per_point = 12_700 + return cls( + left_pt=round(left / emu_per_point, 3), + top_pt=round(top / emu_per_point, 3), + width_pt=round(width / emu_per_point, 3), + height_pt=round(height / emu_per_point, 3), + left_ratio=round(left / slide_width, 6) if slide_width else 0.0, + top_ratio=round(top / slide_height, 6) if slide_height else 0.0, + width_ratio=round(width / slide_width, 6) if slide_width else 0.0, + height_ratio=round(height / slide_height, 6) if slide_height else 0.0, + ) + + +@dataclass(frozen=True, slots=True) +class Issue: + code: str + message: str + severity: str = "warning" + slide_number: int | None = None + shape_id: int | None = None + + +@dataclass(frozen=True, slots=True) +class TextBlock: + text: str + kind: str + level: int + order: int + z_order: int + shape_id: int + shape_name: str + bbox: BoundingBox + hyperlinks: tuple[str, ...] = () + + +@dataclass(frozen=True, slots=True) +class TableRecord: + rows: tuple[tuple[str, ...], ...] + order: int + z_order: int + shape_id: int + shape_name: str + bbox: BoundingBox + + +@dataclass(frozen=True, slots=True) +class ChartSeries: + name: str + values: tuple[float | int | str | None, ...] + + +@dataclass(frozen=True, slots=True) +class ChartRecord: + chart_type: str + title: str | None + categories: tuple[str, ...] + series: tuple[ChartSeries, ...] + order: int + z_order: int + shape_id: int + shape_name: str + bbox: BoundingBox + + +@dataclass(frozen=True, slots=True) +class ImageRecord: + sha256: str + media_type: str + asset_path: str | None + alt_text: str | None + ocr_text: str | None + order: int + z_order: int + shape_id: int + shape_name: str + bbox: BoundingBox + + +@dataclass(slots=True) +class SlideRecord: + number: int + title: str | None + hidden: bool + layout_name: str | None + text_blocks: list[TextBlock] = field(default_factory=list) + tables: list[TableRecord] = field(default_factory=list) + charts: list[ChartRecord] = field(default_factory=list) + images: list[ImageRecord] = field(default_factory=list) + notes: str | None = None + + +@dataclass(slots=True) +class PresentationRecord: + source_name: str + source_sha256: str + source_size_bytes: int + slide_width_pt: float + slide_height_pt: float + metadata: dict[str, Any] + slides: list[SlideRecord] + warnings: list[Issue] + schema_version: str = SCHEMA_VERSION + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + @property + def summary(self) -> dict[str, int | str]: + return { + "schema_version": self.schema_version, + "slides": len(self.slides), + "text_blocks": sum(len(slide.text_blocks) for slide in self.slides), + "tables": sum(len(slide.tables) for slide in self.slides), + "charts": sum(len(slide.charts) for slide in self.slides), + "images": sum(len(slide.images) for slide in self.slides), + "warnings": len(self.warnings), + } diff --git a/src/pptx_extraction/ocr.py b/src/pptx_extraction/ocr.py new file mode 100644 index 0000000..6d8beb5 --- /dev/null +++ b/src/pptx_extraction/ocr.py @@ -0,0 +1,59 @@ +"""Lazy OCR adapters. Core extraction never imports an OCR engine.""" + +from __future__ import annotations + +import io +from typing import Protocol + +from PIL import Image + +from .exceptions import OptionalDependencyError + + +class OCRBackend(Protocol): + name: str + + def recognize(self, image_bytes: bytes, language: str) -> str: + """Return recognized text without mutating the source image.""" + + +class NoOCR: + name = "none" + + def recognize(self, image_bytes: bytes, language: str) -> str: + return "" + + +class TesseractOCR: + name = "tesseract" + + def __init__(self, executable: str | None = None) -> None: + try: + import pytesseract # type: ignore[import-not-found] + except ImportError as exc: + raise OptionalDependencyError( + "Tesseract OCR adapter is not installed.", + hint="Install `pptx-extraction[ocr]` and the Tesseract executable.", + ) from exc + self._module = pytesseract + if executable: + self._module.pytesseract.tesseract_cmd = executable + + def recognize(self, image_bytes: bytes, language: str) -> str: + try: + with Image.open(io.BytesIO(image_bytes)) as image: + return str(self._module.image_to_string(image, lang=language)).strip() + except self._module.TesseractNotFoundError as exc: + raise OptionalDependencyError( + "Tesseract executable was not found.", + hint="Install Tesseract or pass --tesseract-command.", + ) from exc + + +def create_ocr_backend(name: str, executable: str | None = None) -> OCRBackend: + normalized = name.strip().lower() + if normalized in {"", "none", "off"}: + return NoOCR() + if normalized == "tesseract": + return TesseractOCR(executable) + raise OptionalDependencyError(f"Unknown OCR backend: {name}") diff --git a/src/pptx_extraction/pipeline.py b/src/pptx_extraction/pipeline.py new file mode 100644 index 0000000..c33bb06 --- /dev/null +++ b/src/pptx_extraction/pipeline.py @@ -0,0 +1,149 @@ +"""High-level extraction workflows with atomic output directories.""" + +from __future__ import annotations + +import os +import shutil +import tempfile +from concurrent.futures import ThreadPoolExecutor, as_completed +from dataclasses import dataclass, replace +from pathlib import Path + +from .exceptions import InputValidationError, OutputExistsError +from .exporters import export_record +from .extractors import PptxExtractor +from .models import ExtractionOptions, PresentationRecord +from .security import SUPPORTED_OOXML_EXTENSIONS, sha256_file + + +@dataclass(frozen=True, slots=True) +class ExtractionResult: + output_dir: Path + files: dict[str, Path] + record: PresentationRecord + + +@dataclass(frozen=True, slots=True) +class BatchItem: + source: Path + output_dir: Path | None + success: bool + error: str | None = None + + +def extract_file( + source: str | Path, + output_dir: str | Path, + *, + options: ExtractionOptions | None = None, + formats: tuple[str, ...] = ("json", "markdown"), + overwrite: bool = False, + tesseract_command: str | None = None, +) -> ExtractionResult: + source_path = Path(source).expanduser().resolve() + destination = Path(output_dir).expanduser().resolve() + destination.parent.mkdir(parents=True, exist_ok=True) + if destination.exists(): + if not destination.is_dir(): + raise InputValidationError(f"Output path is not a directory: {destination}") + if any(destination.iterdir()): + if not overwrite: + raise OutputExistsError( + f"Output directory is not empty: {destination}", + hint="Choose another directory or pass --overwrite.", + ) + _remove_exact_output(destination) + else: + destination.rmdir() + + temporary = Path(tempfile.mkdtemp(prefix=f".{destination.name}-", dir=str(destination.parent))) + try: + extractor = PptxExtractor( + options=options or ExtractionOptions(), + tesseract_command=tesseract_command, + ) + record = extractor.extract(source_path, temporary / "assets") + files = export_record(record, temporary, formats) + os.replace(temporary, destination) + resolved_files = { + name: destination / path.relative_to(temporary) for name, path in files.items() + } + return ExtractionResult(destination, resolved_files, record) + except Exception: + shutil.rmtree(temporary, ignore_errors=True) + raise + + +def inspect_file( + source: str | Path, + *, + options: ExtractionOptions | None = None, +) -> PresentationRecord: + base = options or ExtractionOptions() + inspect_options = replace(base, include_assets=False, ocr_backend="none") + return PptxExtractor(options=inspect_options).extract(source) + + +def discover_sources(inputs: list[str | Path], recursive: bool = True) -> list[Path]: + discovered: dict[Path, None] = {} + for value in inputs: + candidate = Path(value).expanduser().resolve() + if candidate.is_file(): + if candidate.suffix.lower() in SUPPORTED_OOXML_EXTENSIONS: + discovered[candidate] = None + continue + if candidate.is_dir(): + iterator = candidate.rglob("*") if recursive else candidate.glob("*") + for path in iterator: + if path.is_file() and path.suffix.lower() in SUPPORTED_OOXML_EXTENSIONS: + discovered[path.resolve()] = None + continue + raise InputValidationError(f"Input does not exist: {candidate}") + return sorted(discovered) + + +def batch_extract( + sources: list[str | Path], + output_root: str | Path, + *, + options: ExtractionOptions | None = None, + formats: tuple[str, ...] = ("json", "markdown"), + workers: int = 2, + overwrite: bool = False, + recursive: bool = True, +) -> list[BatchItem]: + if workers < 1 or workers > 32: + raise InputValidationError("workers must be between 1 and 32.") + inputs = discover_sources(sources, recursive=recursive) + if not inputs: + raise InputValidationError("No supported presentations were found.") + root = Path(output_root).expanduser().resolve() + root.mkdir(parents=True, exist_ok=True) + + def run(path: Path) -> BatchItem: + target = root / f"{path.stem}-{sha256_file(path)[:8]}" + try: + result = extract_file( + path, + target, + options=options, + formats=formats, + overwrite=overwrite, + ) + return BatchItem(path, result.output_dir, True) + except Exception as exc: + return BatchItem(path, None, False, str(exc)) + + by_source: dict[Path, BatchItem] = {} + with ThreadPoolExecutor(max_workers=workers, thread_name_prefix="pptx-extraction") as pool: + futures = {pool.submit(run, path): path for path in inputs} + for future in as_completed(futures): + by_source[futures[future]] = future.result() + return [by_source[path] for path in inputs] + + +def _remove_exact_output(path: Path) -> None: + resolved = path.resolve() + if resolved == Path(resolved.anchor) or resolved == resolved.parent: + raise InputValidationError(f"Refusing to overwrite broad path: {resolved}") + shutil.rmtree(resolved) diff --git a/src/pptx_extraction/security.py b/src/pptx_extraction/security.py new file mode 100644 index 0000000..09ec697 --- /dev/null +++ b/src/pptx_extraction/security.py @@ -0,0 +1,103 @@ +"""Pre-parse validation for untrusted OOXML packages.""" + +from __future__ import annotations + +import hashlib +import zipfile +from dataclasses import dataclass, field +from pathlib import Path, PurePosixPath + +from .exceptions import InputValidationError, UnsafePackageError + +SUPPORTED_OOXML_EXTENSIONS = frozenset({".pptx", ".pptm", ".potx", ".ppsx"}) +LEGACY_EXTENSIONS = frozenset({".ppt", ".pot", ".pps"}) + + +@dataclass(frozen=True, slots=True) +class PackageLimits: + max_source_bytes: int = 250 * 1024 * 1024 + max_entries: int = 10_000 + max_expanded_bytes: int = 1_000 * 1024 * 1024 + max_compression_ratio: float = 250.0 + + +@dataclass(slots=True) +class PackageReport: + entries: int + expanded_bytes: int + has_macros: bool + warnings: list[str] = field(default_factory=list) + + +def sha256_file(path: Path, chunk_size: int = 1024 * 1024) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(chunk_size), b""): + digest.update(chunk) + return digest.hexdigest() + + +def validate_package(path: str | Path, limits: PackageLimits | None = None) -> PackageReport: + source = Path(path).expanduser().resolve() + active_limits = limits or PackageLimits() + if not source.is_file(): + raise InputValidationError(f"Presentation does not exist: {source}") + if source.suffix.lower() not in SUPPORTED_OOXML_EXTENSIONS: + if source.suffix.lower() in LEGACY_EXTENSIONS: + raise InputValidationError( + f"Legacy format requires conversion: {source.suffix}", + hint="Run `pptx-extraction convert` with LibreOffice installed.", + ) + raise InputValidationError( + f"Unsupported presentation format: {source.suffix or '[no extension]'}" + ) + size = source.stat().st_size + if size == 0: + raise InputValidationError("Presentation is empty.") + if size > active_limits.max_source_bytes: + raise UnsafePackageError( + f"Source size {size} exceeds limit {active_limits.max_source_bytes} bytes." + ) + if not zipfile.is_zipfile(source): + raise InputValidationError("File is not a valid OOXML ZIP package.") + + expanded = 0 + has_macros = False + warnings: list[str] = [] + with zipfile.ZipFile(source) as archive: + entries = archive.infolist() + if len(entries) > active_limits.max_entries: + raise UnsafePackageError( + f"Archive has {len(entries)} entries; limit is {active_limits.max_entries}." + ) + names: set[str] = set() + for entry in entries: + normalized_name = entry.filename.replace("\\", "/") + normalized = PurePosixPath(normalized_name) + if normalized.is_absolute() or ".." in normalized.parts: + raise UnsafePackageError(f"Unsafe archive path: {entry.filename}") + if entry.flag_bits & 0x1: + raise UnsafePackageError( + f"Encrypted archive entry is unsupported: {entry.filename}" + ) + expanded += entry.file_size + if expanded > active_limits.max_expanded_bytes: + raise UnsafePackageError( + f"Expanded archive exceeds {active_limits.max_expanded_bytes} bytes." + ) + if entry.file_size: + ratio = entry.file_size / max(entry.compress_size, 1) + if ratio > active_limits.max_compression_ratio: + raise UnsafePackageError( + f"Suspicious compression ratio {ratio:.1f} for {entry.filename}." + ) + names.add(normalized_name) + has_macros = has_macros or normalized_name.lower() == "ppt/vbaproject.bin" + + required = {"[Content_Types].xml", "ppt/presentation.xml"} + missing = sorted(required - names) + if missing: + raise InputValidationError(f"OOXML package is missing: {', '.join(missing)}") + if has_macros: + warnings.append("The package contains VBA macros; pptx_extraction does not execute them.") + return PackageReport(len(entries), expanded, has_macros, warnings) diff --git a/static/index.html b/static/index.html deleted file mode 100644 index d67cf6e..0000000 --- a/static/index.html +++ /dev/null @@ -1,139 +0,0 @@ - - - - - - PPT处理测试 - - - -

PPT文件处理测试

- - -

支持的文件格式：PPT, PPTX, POT, POTX, PPS, PPSX, PPTM, PDF

-
-
- - - - - \ No newline at end of file diff --git a/test.py b/test.py deleted file mode 100644 index 23365ab..0000000 --- a/test.py +++ /dev/null @@ -1,19 +0,0 @@ -import numpy -import paddleocr -import skimage -import flask -import flask_cors -import dotenv -import spacy -import transformers -import google.protobuf - -print(f"NumPy: {numpy.__version__}") -print(f"PaddleOCR: {paddleocr.__version__}") -print(f"Scikit-image: {skimage.__version__}") -print(f"Flask: {flask.__version__}") -print(f"Flask-CORS: {flask_cors.__version__}") -print(f"python-dotenv: {dotenv.__version__}") -print(f"Spacy: {spacy.__version__}") -print(f"Transformers: {transformers.__version__}") -print(f"Protobuf: {google.protobuf.__version__}") \ No newline at end of file diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1 @@ + diff --git a/tests/helpers.py b/tests/helpers.py new file mode 100644 index 0000000..4d7cb6a --- /dev/null +++ b/tests/helpers.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +from pathlib import Path + +from PIL import Image +from pptx import Presentation +from pptx.chart.data import ChartData +from pptx.enum.chart import XL_CHART_TYPE +from pptx.util import Inches + + +def build_sample_deck(path: Path) -> Path: + presentation = Presentation() + presentation.core_properties.title = "Synthetic quarterly review" + presentation.core_properties.author = "Private Author" + + slide = presentation.slides.add_slide(presentation.slide_layouts[5]) + slide.shapes.title.text = "Quarterly review" + text_box = slide.shapes.add_textbox(Inches(0.7), Inches(1.3), Inches(4.0), Inches(0.8)) + paragraph = text_box.text_frame.paragraphs[0] + paragraph.text = "Revenue increased" + linked = text_box.text_frame.add_paragraph() + linked.level = 1 + run = linked.add_run() + run.text = "Evidence" + run.hyperlink.address = "https://example.com/evidence" + + table_shape = slide.shapes.add_table(2, 2, Inches(0.7), Inches(2.2), Inches(4), Inches(1.1)) + table_shape.table.cell(0, 0).text = "Metric" + table_shape.table.cell(0, 1).text = "Value" + table_shape.table.cell(1, 0).text = "ARR" + table_shape.table.cell(1, 1).text = "42" + + chart_data = ChartData() + chart_data.categories = ["Q1", "Q2"] + chart_data.add_series("Revenue", (30, 42)) + slide.shapes.add_chart( + XL_CHART_TYPE.COLUMN_CLUSTERED, + Inches(5.0), + Inches(1.3), + Inches(4.0), + Inches(2.4), + chart_data, + ) + + image_path = path.with_suffix(".png") + Image.new("RGB", (64, 48), color=(32, 92, 160)).save(image_path) + picture = slide.shapes.add_picture( + str(image_path), Inches(0.7), Inches(4.0), Inches(1.6), Inches(1.2) + ) + picture._element.nvPicPr.cNvPr.set("descr", "Blue test image") + slide.notes_slide.notes_text_frame.text = "Confidential speaker note" + + second = presentation.slides.add_slide(presentation.slide_layouts[5]) + second.shapes.title.text = "Appendix" + second._element.set("show", "0") + second.shapes.add_picture(str(image_path), Inches(1.0), Inches(1.5), Inches(1.6), Inches(1.2)) + presentation.save(path) + image_path.unlink() + return path diff --git a/tests/test_ai.py b/tests/test_ai.py deleted file mode 100644 index 6b07a29..0000000 --- a/tests/test_ai.py +++ /dev/null @@ -1,9 +0,0 @@ -import os - -# 测试 main.py 路径 -main_path = r"G:\我的云端硬盘\PPTX\PPT_Text_Extractor\main.py" -print(f"main.py 路径是否存在: {os.path.exists(main_path)}") - -# 测试 text_extraction.py 路径 -text_extraction_path = r"G:\我的云端硬盘\PPTX\PPT_Text_Extractor\modules\text_extraction.py" -print(f"text_extraction.py 路径是否存在: {os.path.exists(text_extraction_path)}") diff --git a/tests/test_api.py b/tests/test_api.py new file mode 100644 index 0000000..b1d8163 --- /dev/null +++ b/tests/test_api.py @@ -0,0 +1,59 @@ +from __future__ import annotations + +import tempfile +import time +import unittest +from pathlib import Path + +try: + from fastapi.testclient import TestClient +except ImportError: # optional dependency in core-only environments + TestClient = None # type: ignore[misc,assignment] + +from pptx_extraction.api import ServiceSettings, create_app +from tests.helpers import build_sample_deck + + +@unittest.skipIf(TestClient is None, "API extras are not installed") +class ApiTests(unittest.TestCase): + def test_job_upload_poll_and_result(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source = build_sample_deck(root / "sample.pptx") + app = create_app(ServiceSettings(root / "jobs", workers=1)) + with TestClient(app) as client: # type: ignore[operator] + response = client.post( + "/v1/jobs", + files={ + "file": ( + source.name, + source.read_bytes(), + "application/vnd.openxmlformats-officedocument.presentationml.presentation", + ) + }, + ) + self.assertEqual(response.status_code, 202) + job_id = response.json()["id"] + for _ in range(100): + status = client.get(f"/v1/jobs/{job_id}").json() + if status["status"] in {"succeeded", "failed"}: + break + time.sleep(0.01) + self.assertEqual(status["status"], "succeeded") + result = client.get(f"/v1/jobs/{job_id}/result") + self.assertEqual(result.status_code, 200) + self.assertEqual(result.json()["schema_version"], "1.0") + self.assertEqual(result.json()["metadata"]["author"], "[redacted]") + + def test_rejects_unsupported_upload(self) -> None: + with tempfile.TemporaryDirectory() as directory: + app = create_app(ServiceSettings(Path(directory) / "jobs", workers=1)) + with TestClient(app) as client: # type: ignore[operator] + response = client.post( + "/v1/jobs", files={"file": ("notes.txt", b"hello", "text/plain")} + ) + self.assertEqual(response.status_code, 415) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..0682b88 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +import contextlib +import io +import json +import tempfile +import unittest +from pathlib import Path + +from pptx_extraction.cli import EXIT_SUCCESS, EXIT_USAGE, main +from tests.helpers import build_sample_deck + + +class CliTests(unittest.TestCase): + def test_validate_and_inspect_emit_json(self) -> None: + with tempfile.TemporaryDirectory() as directory: + source = build_sample_deck(Path(directory) / "sample.pptx") + output = io.StringIO() + with contextlib.redirect_stdout(output): + code = main(["validate", str(source)]) + self.assertEqual(code, EXIT_SUCCESS) + self.assertEqual(json.loads(output.getvalue())["status"], "valid") + + output = io.StringIO() + with contextlib.redirect_stdout(output): + code = main(["inspect", str(source)]) + self.assertEqual(code, EXIT_SUCCESS) + self.assertEqual(json.loads(output.getvalue())["slides"], 2) + + def test_missing_input_uses_usage_exit(self) -> None: + errors = io.StringIO() + with contextlib.redirect_stderr(errors): + code = main(["validate", "missing.pptx"]) + self.assertEqual(code, EXIT_USAGE) + self.assertEqual(json.loads(errors.getvalue())["error"]["code"], "invalid_input") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_extraction.py b/tests/test_extraction.py new file mode 100644 index 0000000..547bb43 --- /dev/null +++ b/tests/test_extraction.py @@ -0,0 +1,77 @@ +from __future__ import annotations + +import json +import tempfile +import unittest +from pathlib import Path + +from pptx_extraction.exceptions import InputValidationError, OutputExistsError +from pptx_extraction.models import ExtractionOptions +from pptx_extraction.pipeline import extract_file, inspect_file +from tests.helpers import build_sample_deck + + +class ExtractionTests(unittest.TestCase): + def test_extracts_structured_content_and_deduplicates_assets(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source = build_sample_deck(root / "sample.pptx") + result = extract_file( + source, + root / "output", + formats=("json", "markdown", "text"), + ) + + self.assertEqual(result.record.summary["slides"], 2) + first = result.record.slides[0] + self.assertEqual(first.title, "Quarterly review") + self.assertTrue(any(block.text == "Revenue increased" for block in first.text_blocks)) + self.assertEqual(first.tables[0].rows[1], ("ARR", "42")) + self.assertEqual(first.charts[0].series[0].values, (30.0, 42.0)) + self.assertIn("Confidential speaker note", first.notes or "") + self.assertTrue(result.record.slides[1].hidden) + self.assertEqual(first.images[0].sha256, result.record.slides[1].images[0].sha256) + self.assertEqual(len(list((root / "output" / "assets").iterdir())), 1) + + payload = json.loads((root / "output" / "presentation.json").read_text("utf-8")) + self.assertEqual(payload["schema_version"], "1.0") + self.assertEqual(payload["slides"][0]["images"][0]["alt_text"], "Blue test image") + markdown = (root / "output" / "presentation.md").read_text("utf-8") + self.assertIn("## Slide 1: Quarterly review", markdown) + self.assertIn("| ARR | 42 |", markdown) + + def test_metadata_can_be_redacted_and_assets_disabled(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source = build_sample_deck(root / "sample.pptx") + record = inspect_file( + source, + options=ExtractionOptions(redact_metadata=True), + ) + self.assertEqual(record.metadata["author"], "[redacted]") + self.assertIsNone(record.slides[0].images[0].asset_path) + + def test_nonempty_output_requires_explicit_overwrite(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source = build_sample_deck(root / "sample.pptx") + output = root / "output" + output.mkdir() + (output / "keep.txt").write_text("keep", encoding="utf-8") + with self.assertRaises(OutputExistsError): + extract_file(source, output) + self.assertEqual((output / "keep.txt").read_text("utf-8"), "keep") + + def test_output_file_is_rejected_without_mutation(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source = build_sample_deck(root / "sample.pptx") + output = root / "output" + output.write_text("keep", encoding="utf-8") + with self.assertRaises(InputValidationError): + extract_file(source, output) + self.assertEqual(output.read_text("utf-8"), "keep") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_image.py b/tests/test_image.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/test_security.py b/tests/test_security.py new file mode 100644 index 0000000..acd65c7 --- /dev/null +++ b/tests/test_security.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +import tempfile +import unittest +import zipfile +from pathlib import Path + +from pptx_extraction.exceptions import InputValidationError, UnsafePackageError +from pptx_extraction.security import PackageLimits, validate_package +from tests.helpers import build_sample_deck + + +class SecurityTests(unittest.TestCase): + def test_rejects_non_zip_input(self) -> None: + with tempfile.TemporaryDirectory() as directory: + source = Path(directory) / "fake.pptx" + source.write_text("not a package", encoding="utf-8") + with self.assertRaises(InputValidationError): + validate_package(source) + + def test_rejects_traversal_entry(self) -> None: + with tempfile.TemporaryDirectory() as directory: + source = Path(directory) / "traversal.pptx" + with zipfile.ZipFile(source, "w") as archive: + archive.writestr("[Content_Types].xml", "") + archive.writestr("ppt/presentation.xml", "") + archive.writestr("../outside.txt", "bad") + with self.assertRaises(UnsafePackageError): + validate_package(source) + + def test_applies_configured_source_limit(self) -> None: + with tempfile.TemporaryDirectory() as directory: + source = build_sample_deck(Path(directory) / "sample.pptx") + with self.assertRaises(UnsafePackageError): + validate_package(source, PackageLimits(max_source_bytes=1)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_text.py b/tests/test_text.py deleted file mode 100644 index 2aa682b..0000000 --- a/tests/test_text.py +++ /dev/null @@ -1,9 +0,0 @@ -import unittest -from modules.ppt_text_extraction import extract_text_from_ppt - -class TestTextExtraction(unittest.TestCase): - def test_extract_text(self): - file_path = "tests/sample.pptx" - result = extract_text_from_ppt(file_path) - self.assertTrue(len(result) > 0) - self.assertIn("Sample Slide Text", result[0]) diff --git a/tests/test_workflows.py b/tests/test_workflows.py new file mode 100644 index 0000000..f8b7766 --- /dev/null +++ b/tests/test_workflows.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +import tempfile +import unittest +from pathlib import Path + +from pptx_extraction.converter import convert_legacy +from pptx_extraction.exceptions import InputValidationError, OptionalDependencyError +from pptx_extraction.ocr import NoOCR, create_ocr_backend +from pptx_extraction.pipeline import batch_extract, discover_sources +from tests.helpers import build_sample_deck + + +class WorkflowTests(unittest.TestCase): + def test_batch_reports_success_and_failure_without_stopping(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + build_sample_deck(root / "valid.pptx") + (root / "broken.pptx").write_text("not a zip", encoding="utf-8") + items = batch_extract([root], root / "outputs", workers=2) + self.assertEqual(len(items), 2) + self.assertEqual(sum(item.success for item in items), 1) + self.assertEqual(sum(not item.success for item in items), 1) + + def test_discovery_rejects_missing_input(self) -> None: + with self.assertRaises(InputValidationError): + discover_sources(["definitely-missing-directory"]) + + def test_legacy_conversion_errors_are_actionable(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source = root / "legacy.ppt" + source.write_bytes(b"legacy") + with self.assertRaises(OptionalDependencyError): + convert_legacy(source, root / "out", soffice_command="missing-soffice-command") + wrong = root / "modern.pptx" + wrong.write_bytes(b"modern") + with self.assertRaises(InputValidationError): + convert_legacy(wrong, root / "out") + + def test_no_ocr_is_deterministic_and_unknown_backend_fails(self) -> None: + self.assertEqual(NoOCR().recognize(b"anything", "eng"), "") + self.assertIsInstance(create_ocr_backend("none"), NoOCR) + with self.assertRaises(OptionalDependencyError): + create_ocr_backend("unknown") + + +if __name__ == "__main__": + unittest.main()