From 9c4edc814edd4dde37828f4effe89c02a8bfe37f Mon Sep 17 00:00:00 2001 From: wiseaidev Date: Sat, 22 Aug 2026 19:41:34 +0300 Subject: [PATCH] feat: expand watermark detection & synonym engine && impl image scrubbing. Signed-off-by: wiseaidev --- Cargo.lock | 114 +++ Cargo.toml | 2 + README.md | 76 +- RUST.md | 4 +- examples/unmark/Cargo.lock | 131 +++ examples/unmark/Cargo.toml | 4 +- examples/unmark/src/app.rs | 96 ++- .../unmark/src/components/controls_panel.rs | 226 +++-- examples/unmark/src/components/header.rs | 31 +- examples/unmark/src/components/input_panel.rs | 117 ++- .../unmark/src/components/output_panel.rs | 82 +- examples/unmark/src/components/stats_badge.rs | 33 +- examples/unmark/src/types.rs | 57 ++ examples/unmark/test_files/pixel_scrub.png | Bin 0 -> 67 bytes examples/unmark/test_files/synonyms.md | 3 + examples/unmark/test_files/unicode.txt | 4 + src/{container_meta.rs => container.rs} | 0 src/image.rs | 19 + src/{image_meta.rs => image/meta.rs} | 0 src/image/pixel_scrub.rs | 214 +++++ src/lib.rs | 18 +- src/stochastic.rs | 796 ------------------ src/text.rs | 16 + src/text/stochastic.rs | 766 +++++++++++++++++ src/text/stochastic/arabic.rs | 52 ++ src/text/stochastic/detect.rs | 175 ++++ src/text/stochastic/english.rs | 488 +++++++++++ src/text/stochastic/french.rs | 53 ++ src/text/stochastic/german.rs | 53 ++ src/text/stochastic/spanish.rs | 57 ++ src/{ => text}/unicode.rs | 479 ++++++++++- src/types.rs | 10 + tests/multilingual.rs | 94 +++ tests/pixel_scrub.rs | 99 +++ tests/punctuation.rs | 68 ++ tests/stochastic.rs | 145 +++- tests/unicode.rs | 96 ++- 37 files changed, 3658 insertions(+), 1020 deletions(-) create mode 100644 examples/unmark/test_files/pixel_scrub.png create mode 100644 examples/unmark/test_files/synonyms.md create mode 100644 examples/unmark/test_files/unicode.txt rename src/{container_meta.rs => container.rs} (100%) create mode 100644 src/image.rs rename src/{image_meta.rs => image/meta.rs} (100%) create mode 100644 src/image/pixel_scrub.rs delete mode 100644 src/stochastic.rs create mode 100644 src/text.rs create mode 100644 src/text/stochastic.rs create mode 100644 src/text/stochastic/arabic.rs create mode 100644 src/text/stochastic/detect.rs create mode 100644 src/text/stochastic/english.rs create mode 100644 src/text/stochastic/french.rs create mode 100644 src/text/stochastic/german.rs create mode 100644 src/text/stochastic/spanish.rs rename src/{ => text}/unicode.rs (64%) create mode 100644 tests/multilingual.rs create mode 100644 tests/pixel_scrub.rs create mode 100644 tests/punctuation.rs diff --git a/Cargo.lock b/Cargo.lock index dbc5fe9..e10ac0c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -73,6 +73,12 @@ version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + [[package]] name = "bitflags" version = "2.13.1" @@ -85,6 +91,18 @@ version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" +[[package]] +name = "bytemuck" +version = "1.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" + +[[package]] +name = "byteorder-lite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" + [[package]] name = "bytes" version = "1.12.1" @@ -205,6 +223,7 @@ dependencies = [ "anyhow", "clap", "console_error_panic_hook", + "image", "miniz_oxide 0.9.1", "napi", "napi-build", @@ -247,12 +266,22 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" +[[package]] +name = "fdeflate" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e6853b52649d4ac5c0bd02320cddc5ba956bdb407c4b75a2c6b75bf51500f8c" +dependencies = [ + "simd-adler32", +] + [[package]] name = "flate2" version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" dependencies = [ + "crc32fast", "miniz_oxide 0.8.9", "zlib-rs", ] @@ -380,6 +409,32 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "image" +version = "0.25.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85ab80394333c02fe689eaf900ab500fbd0c2213da414687ebf995a65d5a6104" +dependencies = [ + "bytemuck", + "byteorder-lite", + "image-webp", + "moxcms", + "num-traits", + "png", + "zune-core", + "zune-jpeg", +] + +[[package]] +name = "image-webp" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "525e9ff3e1a4be2fbea1fdf0e98686a6d98b4d8f937e1bf7402245af1909e8c3" +dependencies = [ + "byteorder-lite", + "quick-error", +] + [[package]] name = "indexmap" version = "2.14.0" @@ -486,6 +541,16 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "moxcms" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb85c154ba489f01b25c0d36ae69a87e4a1c73a72631fc6c0eb6dde34a73e44b" +dependencies = [ + "num-traits", + "pxfm", +] + [[package]] name = "napi" version = "3.12.1" @@ -550,6 +615,15 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2bf50223579dc7cdcfb3bfcacf7069ff68243f8c363f62ffa99cf000a6b9c451" +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -672,6 +746,19 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" +[[package]] +name = "png" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60769b8b31b2a9f263dae2776c37b1b28ae246943cf719eb6946a1db05128a61" +dependencies = [ + "bitflags", + "crc32fast", + "fdeflate", + "flate2", + "miniz_oxide 0.8.9", +] + [[package]] name = "portable-atomic" version = "1.15.0" @@ -696,6 +783,12 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "pxfm" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d55d956fa96f5ec02be2e13af0e20391a5aa83d6a074e3ad368959d0fab299ea" + [[package]] name = "pyo3" version = "0.29.2" @@ -753,6 +846,12 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "quick-error" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" + [[package]] name = "quote" version = "1.0.47" @@ -1277,3 +1376,18 @@ dependencies = [ "log", "simd-adler32", ] + +[[package]] +name = "zune-core" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d56377fd46368984a170bc5aac5567e52ca5da874caa60bea39fcbca78fb658b" + +[[package]] +name = "zune-jpeg" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27bc9d5b815bc103f142aa054f561d9187d191692ec7c2d1e2b4737f8dbd7296" +dependencies = [ + "zune-core", +] diff --git a/Cargo.toml b/Cargo.toml index 9605909..6df915a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -53,6 +53,7 @@ regex = "1.13.1" serde = { version = "1.0.229", features = ["derive"] } serde_json = "1.0.151" zip = { version = "8.6.0", default-features = false, features = ["deflate"] } +image = { version = "0.25", default-features = false, features = ["png", "jpeg", "webp"], optional = true } pyo3 = { version = "0.29.2", features = ["extension-module"], optional = true } napi = { version = "3.12.1", default-features = false, features = ["napi4"], optional = true } napi-derive = { version = "3.6.3", optional = true } @@ -77,6 +78,7 @@ wasm = ["wasm-bindgen", "serde-wasm-bindgen", "console_error_panic_hook"] python = ["pyo3"] node = ["napi", "napi-derive", "napi-build"] rust-binary = ["cli"] +pixel-scrub = ["image"] [profile.release] opt-level = "z" diff --git a/README.md b/README.md index d7f80d9..177d17f 100644 --- a/README.md +++ b/README.md @@ -34,9 +34,10 @@ The good news: we have the specialized equipment. And it is written in Rust, so | Layer | What lurks in the shadows | What we do about it | | ------------------ | ------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------- | -| **A: Unicode** | ZWSP, bidi controls, tag chars, variation selectors, private-use codepoints, basically a Unicode horror movie | Deterministic, lossless exorcism 🧹 | +| **A: Unicode** | ZWSP, bidi controls, tag chars, variation selectors, private-use codepoints, **dash homoglyphs** (U+2011 non-breaking hyphen, en-dash, em-dash, etc.), **punctuation homoglyphs** (curly quotes, ellipsis U+2026, etc.), **mathematical alphanumerics** (𝑨→A), Braille blank U+2800 | Deterministic, lossless exorcism 🧹 | | **File: Metadata** | C2PA manifests, EXIF, XMP, document properties, the digital equivalent of a tracking ankle bracelet | Stripped from PNG, JPEG, WebP, SVG, PDF, DOCX, ODT, HTML, Markdown | -| **B: Statistical** | Token-sampling watermarks (SynthID-Text, KGW), watermarks baked into the actual word choices | Best-effort via stochastic synonym replacement: `enhance` command / `enhance_text` API | +| **B: Statistical** | Token-sampling watermarks (SynthID-Text, KGW), watermarks baked into the actual word choices | Best-effort via stochastic synonym replacement (400+ English entry table + ES/FR/DE/AR multilingual support) | +| **Pixel** | SynthID-Image, StegaStamp, Tree-Ring, StableSignature: pixel-domain perturbations invisible to the eye | Decode→raw RGBA→lossless PNG re-encode via `pixel-scrub` feature | > **Fun fact:** some of those invisible characters are technically in the Unicode "Tag" block, which was originally designed for plane tickets in 1997 and then deprecated. AI providers found a new use for them. The Unicode Consortium is presumably very proud. @@ -73,14 +74,15 @@ See **[CLI.md](CLI.md)** for the full command reference. It has tables and every Available on [crates.io](https://crates.io/crates/cum-rs). Because of course it is. Full API docs: **[RUST.md](RUST.md)**. -| Feature | Description | -| ------------- | ----------------------------------------------------------------------------------------------- | -| _(default)_ | Pure-Rust core: `clean`, `inspect`, all media formats. Zero drama. | -| `cli` | Clap CLI module required for the binary. Comes with an ASCII banner, because we have standards. | -| `rust-binary` | Enables the `cum` binary. Ship it. | -| `python` | Python extension via PyO3. For the snake people. 🐍 | -| `node` | Node.js native add-on via napi-rs. For the `node_modules` enjoyers. 🟩 | -| `wasm` | WASM bindings. Run watermark removal in the browser. Why? Because we can. | +| Feature | Description | +| -------------- | ----------------------------------------------------------------------------------------------- | +| _(default)_ | Pure-Rust core: `clean`, `inspect`, all media formats. Zero drama. | +| `cli` | Clap CLI module required for the binary. Comes with an ASCII banner, because we have standards. | +| `rust-binary` | Enables the `cum` binary. Ship it. | +| `pixel-scrub` | Adds `pixel_scrub::scrub_pixels()`: decode-then-re-encode any raster image to strip pixel-domain watermarks. Uses the `image` crate. Not available on wasm32. | +| `python` | Python extension via PyO3. For the snake people. 🐍 | +| `node` | Node.js native add-on via napi-rs. For the `node_modules` enjoyers. 🟩 | +| `wasm` | WASM bindings. Run watermark removal in the browser. Why? Because we can. | ## ⚡ Quick Start @@ -140,7 +142,59 @@ println!("Substituted {} words", output.words_substituted); println!("{}", output.text); ``` -The synonyms table relies on a curated perfect-hash map combined with the host's `/usr/share/dict/` system wordlist. +The English table has **400+ curated entries** covering common verbs, nouns, and adjectives. A two-tier fallback uses `/usr/share/dict/` system wordlists for same-length substitution when no curated synonym exists. + +### 🌍 Multilingual Support + +Pass a `LanguageHint` or let `detect_language()` auto-detect: + +```rust +use cum_rs::stochastic::{StochasticEnhancer, LanguageHint, detect_language}; + +// Explicit language +let es = StochasticEnhancer::with_language(LanguageHint::Spanish); +let out = es.enhance("El texto contiene marcas invisibles"); +println!("{}", out.text); +println!("Language: {}", out.language.as_bcp47()); // "es" + +// Auto-detect +let lang = detect_language("يحتوي النص على علامات مائية"); +assert_eq!(lang, LanguageHint::Arabic); +``` + +| Language | Identifier | Entries | +| -------- | --------------------- | ------- | +| English | `LanguageHint::English` | 400+ | +| Spanish | `LanguageHint::Spanish` | 35 | +| French | `LanguageHint::French` | 32 | +| German | `LanguageHint::German` | 32 | +| Arabic | `LanguageHint::Arabic` | 31 | + +## 🖼️ Pixel-Domain Scrubbing + +Some AI image generators embed invisible watermarks by perturbing pixel values at a level imperceptible to humans but detectable by a matched neural decoder (SynthID-Image, StegaStamp, Tree-Ring, StableSignature). + +Enable the `pixel-scrub` feature to strip them: + +```toml +cum-rs = { version = "0.1.2", features = ["pixel-scrub"] } +``` + +```rust,no_run +#[cfg(feature = "pixel-scrub")] +{ + use cum_rs::pixel_scrub::scrub_pixels; + + let png_bytes = std::fs::read("watermarked.png").unwrap(); + let clean = scrub_pixels(&png_bytes).unwrap(); + std::fs::write("clean.png", &clean).unwrap(); + // Output is always lossless PNG regardless of input format +} +``` + +Supported input: **PNG, JPEG, WebP**. Output is always **PNG** (lossless). Not available on `wasm32`. + +> **How it works:** Decode any supported raster image to raw RGBA pixels, then re-encode from scratch as PNG. The watermark signal lives in the original compression stream's state; a fresh encode from raw pixels cannot carry it. ## 🚨 Disclaimer _(the responsible adult part)_ diff --git a/RUST.md b/RUST.md index ab35af7..f050dc0 100644 --- a/RUST.md +++ b/RUST.md @@ -45,7 +45,7 @@ println!("Words substituted: {}", output.words_substituted); ## Image Metadata Stripping -```rust +```rust,no_run use cum_rs::image_meta::clean_image; use std::fs; @@ -56,7 +56,7 @@ fs::write("output.png", &cleaned).unwrap(); ## Unified API (auto-detect format) -```rust +```rust,no_run use cum_rs::cleaner::{clean, inspect}; let bytes = std::fs::read("draft.docx").unwrap(); diff --git a/examples/unmark/Cargo.lock b/examples/unmark/Cargo.lock index 0ed4133..cb516ad 100644 --- a/examples/unmark/Cargo.lock +++ b/examples/unmark/Cargo.lock @@ -17,6 +17,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + [[package]] name = "base64" version = "0.23.1" @@ -32,12 +38,30 @@ dependencies = [ "serde", ] +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + [[package]] name = "bumpalo" version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" +[[package]] +name = "bytemuck" +version = "1.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" + +[[package]] +name = "byteorder-lite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" + [[package]] name = "bytes" version = "1.12.1" @@ -93,6 +117,7 @@ dependencies = [ name = "cum-rs" version = "0.1.1" dependencies = [ + "image", "miniz_oxide 0.9.1", "phf 0.14.0", "rand 0.10.2", @@ -118,12 +143,22 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" +[[package]] +name = "fdeflate" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e6853b52649d4ac5c0bd02320cddc5ba956bdb407c4b75a2c6b75bf51500f8c" +dependencies = [ + "simd-adler32", +] + [[package]] name = "flate2" version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" dependencies = [ + "crc32fast", "miniz_oxide 0.8.9", "zlib-rs", ] @@ -495,6 +530,32 @@ dependencies = [ "itoa", ] +[[package]] +name = "image" +version = "0.25.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85ab80394333c02fe689eaf900ab500fbd0c2213da414687ebf995a65d5a6104" +dependencies = [ + "bytemuck", + "byteorder-lite", + "image-webp", + "moxcms", + "num-traits", + "png", + "zune-core", + "zune-jpeg", +] + +[[package]] +name = "image-webp" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "525e9ff3e1a4be2fbea1fdf0e98686a6d98b4d8f937e1bf7402245af1909e8c3" +dependencies = [ + "byteorder-lite", + "quick-error", +] + [[package]] name = "implicit-clone" version = "0.6.0" @@ -525,6 +586,16 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "input-rs" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26af89ee2120d44804c8fd9aee22efcce9e43eadcc0766ac0d1b1eea3868f602" +dependencies = [ + "web-sys", + "yew", +] + [[package]] name = "itoa" version = "1.0.18" @@ -579,6 +650,25 @@ dependencies = [ "adler2", ] +[[package]] +name = "moxcms" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb85c154ba489f01b25c0d36ae69a87e4a1c73a72631fc6c0eb6dde34a73e44b" +dependencies = [ + "num-traits", + "pxfm", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + [[package]] name = "num_cpus" version = "1.17.0" @@ -719,6 +809,19 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "png" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60769b8b31b2a9f263dae2776c37b1b28ae246943cf719eb6946a1db05128a61" +dependencies = [ + "bitflags", + "crc32fast", + "fdeflate", + "flate2", + "miniz_oxide 0.8.9", +] + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -781,6 +884,18 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "pxfm" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d55d956fa96f5ec02be2e13af0e20391a5aa83d6a074e3ad368959d0fab299ea" + +[[package]] +name = "quick-error" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" + [[package]] name = "quote" version = "1.0.47" @@ -1210,6 +1325,7 @@ dependencies = [ "gloo-file 0.4.0", "gloo-timers 0.4.0", "gloo-utils 0.3.0", + "input-rs", "js-sys", "serde", "serde_json", @@ -1402,3 +1518,18 @@ dependencies = [ "log", "simd-adler32", ] + +[[package]] +name = "zune-core" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d56377fd46368984a170bc5aac5567e52ca5da874caa60bea39fcbca78fb658b" + +[[package]] +name = "zune-jpeg" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27bc9d5b815bc103f142aa054f561d9187d191692ec7c2d1e2b4737f8dbd7296" +dependencies = [ + "zune-core", +] diff --git a/examples/unmark/Cargo.toml b/examples/unmark/Cargo.toml index 2ca5176..c787a20 100644 --- a/examples/unmark/Cargo.toml +++ b/examples/unmark/Cargo.toml @@ -10,8 +10,9 @@ name = "unmark" path = "src/main.rs" [dependencies] -cum-rs = { path = "../..", default-features = false } +cum-rs = { path = "../..", default-features = false, features = ["pixel-scrub"] } yew = { version = "0.22", features = ["csr"] } +input-rs = { version = "0.2.6", features = ["yew"] } wasm-bindgen = "0.2.127" wasm-bindgen-futures = "0.4.77" web-sys = { version = "0.3.104", features = [ @@ -20,6 +21,7 @@ web-sys = { version = "0.3.104", features = [ "FileList", "FileReader", "HtmlInputElement", + "HtmlSelectElement", "HtmlTextAreaElement", "HtmlButtonElement", "Blob", diff --git a/examples/unmark/src/app.rs b/examples/unmark/src/app.rs index a139152..2ffae80 100644 --- a/examples/unmark/src/app.rs +++ b/examples/unmark/src/app.rs @@ -11,19 +11,34 @@ use crate::components::controls_panel::ControlsPanel; use crate::components::header::Header; use crate::components::input_panel::InputPanel; use crate::components::output_panel::OutputPanel; -use crate::types::{CleanResult, CleanStats, MediaKind, StochasticConfig}; +use crate::types::{AppLanguage, CleanResult, CleanStats, MediaKind, StochasticConfig}; use base64::{Engine, engine::general_purpose::STANDARD as B64}; use cum_rs::cleaner::clean; -use cum_rs::stochastic::StochasticEnhancer; +use cum_rs::stochastic::{LanguageHint, StochasticEnhancer}; use cum_rs::types::MediaHint; use cum_rs::unicode::{CleanOpts, clean_text}; +use web_sys::MouseEvent; use yew::prelude::*; -/// Returns the [`StochasticConfig`] default: disabled, probability 50 %. +/// Returns the [`StochasticConfig`] default: disabled, 50 %, Auto language, Layer A normalize ON. fn default_stochastic_config() -> StochasticConfig { StochasticConfig { enabled: false, probability_pct: 50, + language: AppLanguage::Auto, + normalize_punctuation: true, + } +} + +/// Maps the UI [`AppLanguage`] to the crate [`LanguageHint`]. +fn to_lang_hint(lang: &AppLanguage) -> LanguageHint { + match lang { + AppLanguage::Auto => LanguageHint::Auto, + AppLanguage::English => LanguageHint::English, + AppLanguage::Spanish => LanguageHint::Spanish, + AppLanguage::French => LanguageHint::French, + AppLanguage::German => LanguageHint::German, + AppLanguage::Arabic => LanguageHint::Arabic, } } @@ -38,18 +53,6 @@ pub fn app() -> Html { let debounce = use_mut_ref(|| Option::::None); - let _recompute_text = { - let result_state = result.clone(); - let stochastic = stochastic.clone(); - move |s: &str| { - if s.trim().is_empty() { - result_state.set(None); - return; - } - result_state.set(Some(run_clean_text(s, &stochastic))); - } - }; - let on_text_change = { let text_state = text.clone(); let result_state = result.clone(); @@ -74,6 +77,15 @@ pub fn app() -> Html { }) }; + let on_clear: Callback = { + let text_state = text.clone(); + let result_state = result.clone(); + Callback::from(move |_: MouseEvent| { + text_state.set(String::new()); + result_state.set(None); + }) + }; + let on_stochastic_change = { let stochastic_state = stochastic.clone(); let text_val = text.clone(); @@ -106,27 +118,38 @@ pub fn app() -> Html { }; html! { -
+
-
-
- - +
+
+ +
+
+ +
+
+
+
-
} @@ -149,6 +172,7 @@ pub fn app() -> Html { fn run_clean_text(text: &str, stochastic: &StochasticConfig) -> CleanResult { let opts = CleanOpts { aggressive_confusables: true, + normalize_punctuation: stochastic.normalize_punctuation, ..CleanOpts::safe() }; match clean_text(text, &opts) { @@ -158,12 +182,14 @@ fn run_clean_text(text: &str, stochastic: &StochasticConfig) -> CleanResult { let mut replaced_count = raw.replaced_count; if stochastic.enabled { - let enhancer = StochasticEnhancer::new(stochastic.probability_pct as f64 / 100.0); + let lang_hint = to_lang_hint(&stochastic.language); + let prob = stochastic.probability_pct as f64 / 100.0; + let enhancer = StochasticEnhancer::with_language_and_probability(lang_hint, prob); let out = enhancer.enhance(&final_text); final_text = out.text; if out.words_substituted > 0 { replaced_count += out.words_substituted; - summary.push(format!("layer_b_syonyms: {}", out.words_substituted)); + summary.push(format!("layer_b_synonyms: {}", out.words_substituted)); } } diff --git a/examples/unmark/src/components/controls_panel.rs b/examples/unmark/src/components/controls_panel.rs index b7ef3ac..1bc690f 100644 --- a/examples/unmark/src/components/controls_panel.rs +++ b/examples/unmark/src/components/controls_panel.rs @@ -7,21 +7,24 @@ //! # Controls Panel //! -//! A sidebar panel that exposes the stochastic synonym-replacement settings. +//! A sidebar panel that exposes the stochastic synonym-replacement settings +//! and the Layer A `normalize_punctuation` flag. //! //! ## Controls //! //! | Control | ID | Description | //! |---------|----|-------------| -//! | Toggle | `stochastic-toggle` | Enable / disable synonym replacement. | -//! | Slider | `prob-slider` | Per-word substitution probability (1 – 100 %). | +//! | Toggle | `stochastic-toggle` | Enable / disable synonym replacement. | +//! | Slider | `prob-slider` | Per-word substitution probability (1 - 100 %). | +//! | Select | `language-select` | Target language for the curated synonym table. | +//! | Toggle | `normalize-punct-toggle` | Enable / disable punctuation normalisation (Layer A). | //! //! Every change fires the parent callback immediately (no debounce) so the //! output panel reflects the new setting on each interaction. -use crate::types::StochasticConfig; +use crate::types::{AppLanguage, StochasticConfig}; use wasm_bindgen::JsCast; -use web_sys::{HtmlInputElement, MouseEvent}; +use web_sys::{HtmlInputElement, HtmlSelectElement, MouseEvent}; use yew::prelude::*; /// Props for [`ControlsPanel`]. @@ -41,7 +44,18 @@ pub fn controls_panel(props: &ControlsPanelProps) -> Html { Callback::from(move |_: MouseEvent| { cb.emit(StochasticConfig { enabled: !config.enabled, - probability_pct: config.probability_pct, + ..config.clone() + }); + }) + }; + + let on_norm_punct_toggle = { + let config = props.config.clone(); + let cb = props.on_change.clone(); + Callback::from(move |_: MouseEvent| { + cb.emit(StochasticConfig { + normalize_punctuation: !config.normalize_punctuation, + ..config.clone() }); }) }; @@ -56,91 +70,187 @@ pub fn controls_panel(props: &ControlsPanelProps) -> Html { { if let Ok(v) = el.value().parse::() { cb.emit(StochasticConfig { - enabled: config.enabled, probability_pct: v.clamp(1, 100), + ..config.clone() }); } } }) }; - let track_class = if props.config.enabled { + let on_language_change = { + let config = props.config.clone(); + let cb = props.on_change.clone(); + Callback::from(move |e: Event| { + if let Some(el) = e + .target() + .and_then(|t| t.dyn_into::().ok()) + { + cb.emit(StochasticConfig { + language: AppLanguage::from_value(&el.value()), + ..config.clone() + }); + } + }) + }; + + let stoch_track_class = if props.config.enabled { "um-toggle-track bg-um-accent" } else { "um-toggle-track bg-um-border" }; + let stoch_thumb_class = if props.config.enabled { + "um-toggle-thumb translate-x-4" + } else { + "um-toggle-thumb translate-x-0" + }; - let thumb_class = if props.config.enabled { + let norm_track_class = if props.config.normalize_punctuation { + "um-toggle-track bg-um-accent" + } else { + "um-toggle-track bg-um-border" + }; + let norm_thumb_class = if props.config.normalize_punctuation { "um-toggle-thumb translate-x-4" } else { "um-toggle-thumb translate-x-0" }; - html! { -