From 672ddaa2c6ceb0b0b2f8ad546e5229a9933dd7bb Mon Sep 17 00:00:00 2001 From: vovarbv Date: Thu, 19 Jun 2025 17:20:04 +0200 Subject: [PATCH 1/2] feat(performance): Implement lazy loading for large repositories This commit introduces a lazy loading architecture to significantly improve performance and UI responsiveness when handling large codebases. The previous implementation loaded all file contents and tokenized them upfront, causing severe slowdowns and freezing on large folders. This new architecture addresses these issues by: - Performing a lightweight initial scan that only gathers file metadata and provides token estimations based on file type and size. - Deferring the expensive work of reading file content and performing accurate tokenization until a file is explicitly selected by the user. - Introducing UI components to handle large folder warnings (LargeFolderModal, LargeSubfolderModal) and provide clear user feedback during on-demand processing (ProcessingOverlay). - Refactoring App.tsx by extracting workspace logic into a dedicated useWorkspaces hook to improve state management and readability. - Updating documentation to reflect the new architecture (ARCHITECTURE.md, lazy-loading.md). --- CHANGELOG.md | 51 + CONTRIBUTING.md | 84 +- README.md | 120 +- TODO.md | 10 + docs/ARCHITECTURE.md | 64 + docs/features/lazy-loading.md | 110 ++ electron/backup/OldMain.js | 1316 ------------------- electron/build.js | 2 +- electron/dev.js | 34 +- electron/main.js | 675 +++++++++- electron/preload.js | 8 +- index.html | 4 +- package-lock.json | 7 - package.json | 4 +- scripts/README.md | 47 +- scripts/fix-dependencies.js | 5 +- src/App.tsx | 844 +++++++----- src/components/CopyButton.tsx | 20 +- src/components/FileCard.tsx | 16 +- src/components/FileList.tsx | 29 +- src/components/LargeFolderModal.tsx | 57 + src/components/LargeSubfolderModal.tsx | 77 ++ src/components/ProcessingOverlay.tsx | 24 + src/components/Sidebar.tsx | 13 +- src/components/TreeItem.tsx | 101 +- src/global.d.ts | 1 - src/hooks/useWorkspaces.ts | 356 +++++ src/main.tsx | 3 + src/styles/components/ProcessingOverlay.css | 72 + src/styles/contentarea/FileCard.css | 11 + src/styles/modals/LargeFolderModal.css | 150 +++ src/styles/modals/LargeSubfolderModal.css | 161 +++ src/styles/sidebar/TreeItem.css | 43 +- src/types/FileTypes.ts | 5 + src/utils/contentFormatUtils.ts | 12 +- 35 files changed, 2645 insertions(+), 1891 deletions(-) create mode 100644 docs/ARCHITECTURE.md create mode 100644 docs/features/lazy-loading.md delete mode 100644 electron/backup/OldMain.js create mode 100644 src/components/LargeFolderModal.tsx create mode 100644 src/components/LargeSubfolderModal.tsx create mode 100644 src/components/ProcessingOverlay.tsx create mode 100644 src/hooks/useWorkspaces.ts create mode 100644 src/styles/components/ProcessingOverlay.css create mode 100644 src/styles/modals/LargeFolderModal.css create mode 100644 src/styles/modals/LargeSubfolderModal.css diff --git a/CHANGELOG.md b/CHANGELOG.md index 541245e5..10bc9c8c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,54 @@ +## [1.2.0] - 2025-01-10 + +### Added + +- **Lazy Loading Architecture for Improved Performance:** + - Implemented lightweight metadata-only initial file scanning that defers expensive file I/O and tokenization + - Added token estimation function that calculates approximate token counts based on file extension and size without reading file content + - Files now display estimated token counts with a visual indicator (~tokens and "est" badge) until actual tokenization occurs + - Individual files are processed for real tokens only when selected, dramatically improving initial load times + +- **Visual Processing Indicators:** + - Added loading spinner next to individual files being processed for token counting + - Implemented full-page processing overlay for batch operations (reserved for future use) + - Clear visual distinction between estimated and actual token counts in both TreeItem and FileCard components + +### Improved + +- **Large Repository Handling:** + - All folders now use lightweight scanning by default, making the app responsive even with massive codebases + - Large folder warning now triggers instantly based on file count estimates rather than after full processing + - Token threshold warning uses more realistic estimates (~2000 tokens per file instead of 100) + - Removed automatic batch processing from folder selection to maintain instant UI response + +- **Performance Optimizations:** + - Optimized file path comparison using Set-based lookups (O(1)) instead of array iterations (O(n²)) + - Deferred content reading and tokenization until files are actually needed + - Significantly reduced memory usage by not loading file contents until selected + +### Fixed + +- **UI Responsiveness:** + - Fixed issue where selecting folders would freeze the UI while processing all files + - Fixed double-click requirement for first-time folder expansion in tree view + - Token counts now properly update when files are selected and processed + +## [1.1.2] - 2025-06-18 + +### Added + +- **Large Folder Warning System:** + - Added a warning modal that appears when selecting folders with more than 500,000 tokens, alerting users about potential performance impacts. + - Users can choose to proceed with full selection, load files but keep them deselected, or cancel the folder load operation. + - Prevents application freezing when loading extremely large repositories. + +### Improved + +- **Default Folder Tree State:** + - Folder tree now defaults to collapsed state instead of expanded, improving initial load experience for large projects. + - Only the root folder is expanded by default when a new folder is selected. + - This change significantly improves the user experience when working with projects containing many nested folders. + ## [1.1.1] - 2025-05-23 ### Added diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e1789c2d..17d3af5d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,36 +1,76 @@ # Contributing to PasteMax -Thank you for considering contributing to PasteMax! This document outlines the process for contributing to the project. +Thank you for considering contributing to PasteMax! We welcome all contributions, from bug reports to feature requests and code changes. ## Getting Started -1. Fork the repository -2. Clone your fork: `git clone https://github.com/yourusername/pastemax.git` -3. Create a branch for your changes: `git checkout -b feature/your-feature-name` -4. Install dependencies: `npm install` -5. Make your changes -6. Run tests and linting: `npm run lint` -7. Commit your changes with a descriptive message +1. **Fork the repository** on GitHub. +2. **Clone your fork** locally: + ```bash + git clone https://github.com/your-username/pastemax.git + cd pastemax + ``` +3. **Install dependencies**: + ```bash + npm install + ``` +4. **Create a new branch** for your feature or bugfix: + ```bash + git checkout -b feature/your-amazing-feature + ``` -## Pull Request Process +## Development Workflow -1. Update the README.md or documentation with details of your changes if appropriate -2. Make sure your code passes all linting checks -3. Submit a pull request to the main repository -4. The maintainers will review your PR as soon as possible +To run the application in development mode with hot-reloading for both the frontend (React) and backend (Electron), use the following command: -## Code Style +```bash +npm run dev:all +``` -- Follow the existing code style in the project -- Run `npm run lint` to ensure your code meets the project's style guidelines -- Comment your code where appropriate +This will: +- Start the Vite dev server for the React app. +- Start the Electron main process, which will load the app from the Vite server. +- Automatically restart the Electron process when you make changes to files in the `electron/` directory. -## Issues +### Scripts -- Use GitHub Issues to report bugs or suggest new features -- Check existing issues before creating a new one -- When reporting bugs, include steps to reproduce, expected behavior, and actual behavior +- `npm run dev`: Starts only the Vite dev server. +- `npm run dev:electron`: Starts only the Electron process (expects the Vite server to be running). +- `npm run lint`: Runs ESLint to check for code quality issues. +- `npm run lint:fix`: Attempts to automatically fix linting issues. +- `npm run format:all`: Formats all code using Prettier. + +## Making Changes + +1. Make your code changes in your feature branch. +2. Ensure your code follows the existing style and conventions. +3. Add or update documentation in the `docs/` directory if you are changing functionality. +4. Run the linter to ensure your code is clean: + ```bash + npm run lint + ``` +5. Commit your changes with a clear and descriptive commit message. + +## Submitting a Pull Request + +1. Push your feature branch to your fork on GitHub: + ```bash + git push origin feature/your-amazing-feature + ``` +2. Open a **Pull Request** from your feature branch to the `main` branch of the `kleneway/pastemax` repository. +3. Provide a clear title and description for your pull request, explaining the changes you've made and why. +4. The maintainers will review your PR as soon as possible. Thank you for your contribution! + +## Reporting Issues + +- Use the [GitHub Issues](https://github.com/kleneway/pastemax/issues) page to report bugs or suggest new features. +- Before creating a new issue, please check if a similar one already exists. +- When reporting bugs, please include: + - Steps to reproduce the issue. + - The expected behavior. + - The actual behavior. + - Your operating system and app version. ## License -By contributing to PasteMax, you agree that your contributions will be licensed under the project's MIT License. +By contributing to PasteMax, you agree that your contributions will be licensed under the project's [MIT License](LICENSE). diff --git a/README.md b/README.md index 284b94bd..53b3b05e 100644 --- a/README.md +++ b/README.md @@ -82,54 +82,39 @@ npm install ``` npm run build:electron -npm run package ``` -**Note**: If you encounter issues with `npm run package`, you can try the platform-specific command: - +4. Package the app for your platform: ``` -npm run package:win +# For macOS npm run package:mac -npm run package:linux -``` - -After successful build, you'll find the executable files inside the `release-builds` directory: - -**Windows:** - -- `PasteMax Setup 1.0.0.exe` - Installer version -- `PasteMax 1.0.0.exe` - Portable version - -**Mac:** -- `PasteMax 1.0.0.dmg` - Installer version -- `PasteMax 1.0.0.zip` - Portable version +# For Windows +npm run package:win -**Linx:** +# For Linux +npm run package:linux +``` -- `PasteMax 1.0.0.deb` - Installer version (Deb package) -- `PasteMax 1.0.0.rpm` - Installer version (RPM package) -- `PasteMax 1.0.0.AppImage` - Portable version +After successful packaging, you'll find the executable files inside the `release-builds` directory. ## Development ### Prerequisites -- Node.js (v14 or higher) -- npm or yarn +- Node.js (v18 or higher) +- npm ### Running in Development Mode -To run the application in development mode: +To run the application in development mode with hot-reloading for both the frontend and backend: ``` -# Start the Vite dev server -npm run dev - -# In a separate terminal, start Electron -npm run dev:electron +npm run dev:all ``` +This will start the Vite dev server for the React app and the Electron main process concurrently. + ### Building for Production To build the application for production: @@ -139,60 +124,51 @@ To build the application for production: npm run build:electron # Create platform-specific distributables -npm run package +npm run package:mac # macOS +npm run package:win # Windows +npm run package:linux # Linux ``` ## Project Structure -- `src/` - React application source code - - `components/` - React components - - `context/` - React context providers - - `hooks/` - Custom React hooks +- **`src/`** - React application source code (Renderer Process) + - `components/` - Reusable React components + - `context/` - React context providers (e.g., ThemeContext) + - `hooks/` - Custom React hooks for stateful logic - `types/` - TypeScript type definitions - - `utils/` - Utility functions - - `styles/` - CSS styles - - `assets/` - Static assets like images -- `electron/` - Electron-Backend related files - - `main.js` - Electron main process - - `preload.js` - Preload script for secure IPC - - `renderer.js` - Renderer process utilities - - `build.js` - Build script for production - - `dev.js` - Development script - - `excluded-files.js` - Configuration for files to exclude by default - - `file-processor.js` - File processing utilities - - `ignore-manager.js` - Ignore pattern management - - `update-checker.js` - Update checking functionality - - `update-manager.js` - Update management - - `utils.js` - Utility functions - - `watcher.js` - File change watcher -- `public/` - Public assets (favicon, etc.) -- `scripts/` - Utility scripts for building and testing -- `docs/` - Documentation + - `utils/` - Utility functions for the frontend + - `styles/` - Modularized CSS stylesheets +- **`electron/`** - Electron-Backend related files (Main Process) + - `main.js` - Main process entry point, window management, and IPC handling + - `preload.js` - Preload script for secure IPC communication + - `file-processor.js` - Logic for reading files, counting tokens, etc. + - `ignore-manager.js` - Logic for handling `.gitignore` and other ignore patterns + - `update-manager.js` - Logic for managing update checks + - `watcher.js` - File system watcher logic using Chokidar +- **`public/`** - Static assets (e.g., icons) +- **`scripts/`** - Utility scripts for building, testing, and debugging ## Libraries Used -- Electron - Desktop application framework -- React - UI library -- TypeScript - Type safety -- Vite - Build tool and development server -- tiktoken - Token counting for LLM context estimation -- ignore - .gitignore-style pattern matching for file exclusions -- chokidar - File Watcher +- **Electron** - Desktop application framework +- **React** & **TypeScript** - For building the user interface +- **Vite** - Build tool and development server +- **Tiktoken** - Fast BPE tokenization for LLM context estimation +- **ignore** - For `.gitignore`-style pattern matching +- **Chokidar** - Advanced file system watcher ## Troubleshooting ### Getting "Warning: Not trusted" on Windows -If you see a warning about the app not being trusted, you can bypass this by clicking "run anyways". This is a common issue with Electron apps, especially since PasteMax is not signed. - -### Getting "App not responding" on Mac +If you see a warning about the app not being trusted, you can bypass this by clicking "More info" -> "Run anyway". This is a common issue with unsigned Electron apps. -If you encounter an "App not responding" message on Mac, it may be due to macOS security settings. You can try the following: +### Getting "App can't be opened" on Mac -1. Open System Preferences. -2. Go to Security & Privacy. -3. Under the General tab, look for the "Allow apps downloaded from" section. -4. Look for "PasteMax" and click "Open Anyway". +If you encounter this message on macOS, it may be due to security settings. +1. Right-click the `PasteMax.app` file and select "Open". +2. You may see a warning dialog. Click "Open" again to confirm. +You should only need to do this the first time you run the app. ### Other Issues @@ -204,13 +180,7 @@ MIT License - see the [LICENSE](LICENSE) file for details. ## Contributing -Contributions are welcome! Please feel free to submit a Pull Request. - -1. Fork the repository -2. Create your feature branch (`git checkout -b feature/amazing-feature`) -3. Commit your changes (`git commit -m 'Add some amazing feature'`) -4. Push to the branch (`git push origin feature/amazing-feature`) -5. Open a Pull Request +Contributions are welcome! Please see the [CONTRIBUTING.md](CONTRIBUTING.md) file for details. ## Star History ⭐ diff --git a/TODO.md b/TODO.md index 538d27d3..bf8af2e6 100644 --- a/TODO.md +++ b/TODO.md @@ -25,3 +25,13 @@ - [ ] Consider using a different library for the file tree navigation - [ ] Chokidar for file watching - [ ] Zustand for state management + +!!! +!!! +!!! +!!! +!!! +!!! +!!! +!!! +!!! \ No newline at end of file diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md new file mode 100644 index 00000000..f1e58945 --- /dev/null +++ b/docs/ARCHITECTURE.md @@ -0,0 +1,64 @@ +# PasteMax Architecture + +This document provides a high-level overview of the PasteMax application's architecture, data flow, and core components. + +## Core Technologies + +- **Framework:** [Electron](https://www.electronjs.org/) for cross-platform desktop application development. +- **Frontend:** [React](https://reactjs.org/) with [TypeScript](https://www.typescriptlang.org/) for building the user interface. +- **Build Tool:** [Vite](https://vitejs.dev/) for fast development and optimized production builds. +- **Tokenizer:** [Tiktoken](https://github.com/openai/tiktoken) for accurate language model token counting. +- **File Watching:** [Chokidar](https://github.com/paulmillr/chokidar) for efficient file system monitoring. + +## Application Structure + +The application is divided into three main parts: + +1. **Electron Main Process (`electron/`)**: The backend of the application, running in a Node.js environment. It manages the application lifecycle, windows, and native operating system interactions (like file dialogs). It's also responsible for all heavy lifting, such as file system scanning and processing. +2. **Preload Script (`electron/preload.js`)**: A secure bridge that exposes a limited, controlled API from the Main Process to the Renderer Process via `contextBridge`. This is crucial for maintaining Electron's security model. +3. **Renderer Process (`src/`)**: The frontend of the application, running in a browser-like environment (Chromium). This is where the React application lives, handling all UI rendering and user interactions. + +## Key Architectural Concepts + +### 1. Lazy Loading & On-Demand Processing + +To handle large repositories efficiently, PasteMax uses a lazy-loading approach: + +- **Initial Scan:** When a folder is selected, the Main Process performs a **lightweight scan**. It only gathers file metadata (path, name, size) and calculates an *estimated* token count based on file extension and size, without reading file content. This makes the initial load extremely fast. +- **On-Demand Processing:** The actual file content is only read and tokenized when a file is selected by the user in the UI. The Renderer sends an IPC request (`process-selected-files`) to the Main Process, which then reads the file, calculates the precise token count, and sends the updated data back. +- **User Feedback:** The UI clearly indicates when a file has an estimated token count (`~123 est`) and shows a loading spinner next to files that are being processed on-demand. + +### 2. IPC (Inter-Process Communication) + +Communication between the Main and Renderer processes is handled exclusively through IPC channels defined in `electron/main.js` and exposed via `electron/preload.js`. + +- **Renderer to Main (`invoke`):** For actions that require a response, like fetching data (`get-ignore-patterns`, `process-selected-files`). +- **Main to Renderer (`send`):** For pushing updates to the UI, such as file system changes from the watcher (`file-added`, `file-updated`, `file-removed`) or processing status updates. + +### 3. Ignore Logic (`electron/ignore-manager.js`) + +PasteMax supports two modes for ignoring files: + +- **Automatic Mode:** Respects `.gitignore` files found within the project directory. It traverses the directory structure, applying rules hierarchically. +- **Global Mode:** Uses a predefined set of global ignore patterns (e.g., `node_modules`, build artifacts) and any custom patterns defined by the user. + +This logic is centralized in `ignore-manager.js` to ensure consistent filtering during both initial scans and file watching. + +### 4. File System Watcher (`electron/watcher.js`) + +- A single, persistent `chokidar` instance monitors the selected folder for changes. +- It is initialized *after* the initial file scan is complete. +- It uses the same ignore logic as the initial scan to avoid processing ignored files. +- Events (`add`, `change`, `unlink`) are debounced and trigger IPC messages to the Renderer, which then updates the UI reactively. +- The watcher's lifecycle is carefully managed to prevent resource leaks, shutting down and restarting when the user selects a new folder or changes ignore settings. + +### 5. State Management (Frontend) + +- The primary application state is managed within the `App.tsx` component using React hooks (`useState`, `useEffect`, `useCallback`). +- Stateful logic is further modularized into custom hooks: + - `useWorkspaces`: Manages workspace creation, selection, and persistence. + - `useIgnorePatterns`: Manages ignore mode and custom ignore patterns. + - `useModels`: Manages fetching and selecting LLM models. +- State related to the user's session (e.g., selected folder, files, UI settings) is persisted to `localStorage`. + +This architecture ensures a responsive user experience by deferring heavy work, provides a secure communication channel between the frontend and backend, and organizes code into maintainable, single-responsibility modules. \ No newline at end of file diff --git a/docs/features/lazy-loading.md b/docs/features/lazy-loading.md new file mode 100644 index 00000000..92688238 --- /dev/null +++ b/docs/features/lazy-loading.md @@ -0,0 +1,110 @@ +# Lazy Loading Architecture + +## Overview + +PasteMax uses a lazy loading architecture to efficiently handle large repositories. Instead of reading and tokenizing all files upfront (which can take minutes for large codebases), the app performs a lightweight initial scan that only gathers file metadata and estimates token counts. Actual file content and accurate token counts are loaded on-demand when files are selected. + +## How It Works + +### 1. Initial Folder Load - Lightweight Scan + +When a folder is selected: +- The app performs a fast metadata-only scan using `scanDirectoryLightweight()` +- For each file, it collects: + - File path and name + - File size from `fs.stat()` + - Estimated token count based on file extension and size +- Files are marked with `isTokenEstimate: true` flag +- File content is left empty (`content: ''`) + +### 2. Token Estimation Algorithm + +The `estimateTokens()` function in `electron/main.js` estimates tokens without reading files: + +- **Binary files** (images, videos, executables): 0 tokens +- **Code files** (.js, .py, .ts, etc.): ~3 characters per token +- **Text files** (.md, .txt, .json, etc.): ~4 characters per token +- **Unknown files**: Default to ~4 characters per token + +This provides reasonably accurate estimates for the file tree display. + +### 3. Visual Indicators + +Files with estimated tokens are displayed with visual cues: +- Token count prefixed with `~` (e.g., "~1,234 tokens") +- Small "est" badge next to the count +- Loading spinner when processing begins + +### 4. On-Demand Processing + +When a file is selected: +1. The UI immediately adds it to the selection +2. If `isTokenEstimate === true`, it triggers `processFileForRealTokens()` +3. A loading spinner appears next to the file +4. The backend reads the file content and calculates actual tokens +5. The file data is updated with real values and `isTokenEstimate: false` +6. The spinner disappears and real token count is shown + +### 5. Performance Benefits + +- **Initial load time**: Reduced from minutes to seconds for large repos +- **Memory usage**: Minimal until files are actually needed +- **UI responsiveness**: No freezing during folder selection +- **Scalability**: Can handle repositories with 10,000+ files + +## Implementation Details + +### Frontend Components + +- **App.tsx**: Manages `processingFiles` state and triggers on-demand processing +- **TreeItem.tsx**: Shows loading spinners and estimate badges +- **FileCard.tsx**: Displays estimate badges for selected files + +### Backend Services + +- **main.js**: + - `scanDirectoryLightweight()`: Performs metadata-only scanning + - `estimateTokens()`: Calculates token estimates + - `process-selected-files` IPC handler: Processes files on-demand + +- **file-processor.js**: + - `processSingleFile()`: Reads and tokenizes individual files + +### IPC Communication + +```typescript +// Frontend requests processing +await window.electron.ipcRenderer.invoke('process-selected-files', [filePath]); + +// Backend response +{ + success: true, + processedFiles: [{ + path: string, + content: string, + tokenCount: number, + isTokenEstimate: false, + // ... other metadata + }] +} +``` + +## Configuration + +Currently, lazy loading is always enabled. Future versions may add: +- Threshold settings for automatic vs on-demand processing +- Batch processing options +- Preloading for commonly selected files + +## Known Limitations + +1. **Estimates may be inaccurate** for files with unusual formatting +2. **Binary detection** is extension-based until file is processed +3. **Large files** still take time to process when selected + +## Future Improvements + +- Background pre-processing of likely selections +- Smart caching of processed files between sessions +- Progressive loading for very large files +- Parallel processing of multiple selections \ No newline at end of file diff --git a/electron/backup/OldMain.js b/electron/backup/OldMain.js deleted file mode 100644 index 073fced6..00000000 --- a/electron/backup/OldMain.js +++ /dev/null @@ -1,1316 +0,0 @@ -// ====================== -// IMPORTS AND CONSTANTS -// ====================== -const { app, BrowserWindow, ipcMain, dialog, session } = require('electron'); -const fs = require('fs'); -const path = require('path'); -const os = require('os'); -const { default: PQueue } = require('p-queue'); // Added for controlled concurrency -const watcher = require('./watcher.js'); // New watcher module -const { excludedFiles, binaryExtensions } = require('./excluded-files'); // Import the excluded files list - -// Configuration constants -const MAX_DIRECTORY_LOAD_TIME = 300000; // 5 minutes timeout for large repositories -const MAX_FILE_SIZE = 5 * 1024 * 1024; // 5MB max file size -const CONCURRENT_DIRS = os.cpus().length * 2; // Increase based on CPU count for better parallelism -// const CHUNK_SIZE = 30; // Number of files to process in one chunk (no longer used, might bring back) - -// Default ignore patterns that should always be applied -const DEFAULT_PATTERNS = [ - '.git', - '.svn', - '.hg', - 'node_modules', - 'bower_components', - 'vendor', - 'dist', - 'build', - 'out', - '.next', - 'target', - 'bin', - 'Debug', - 'Release', - 'x64', - 'x86', - '.output', - '*.min.js', - '*.min.css', - '*.bundle.js', - '*.compiled.*', - '*.generated.*', - '.cache', - '.parcel-cache', - '.webpack', - '.turbo', - '.idea', - '.vscode', - '.vs', - '.DS_Store', - 'Thumbs.db', - 'desktop.ini', - '*.asar', - 'release-builds', -]; - -// ====================== -// GLOBAL STATE -// ====================== -/** runtime ignore-mode */ -/** @type {'automatic' | 'global'} */ -let currentIgnoreMode = 'automatic'; -let isLoadingDirectory = false; -let loadingTimeoutId = null; - -/** - * @typedef {Object} DirectoryLoadingProgress - * @property {number} directories - Number of directories processed - * @property {number} files - Number of files processed - */ -let currentProgress = { directories: 0, files: 0 }; - -// Throttling for status updates -let lastStatusUpdateTime = 0; -const STATUS_UPDATE_INTERVAL = 200; // ms - -// Global caches -const ignoreCache = new Map(); // Cache for ignore filters keyed by normalized root directory -const fileCache = new Map(); // Cache for file metadata keyed by normalized file path -const fileTypeCache = new Map(); // Cache for binary file type detection results -const gitIgnoreFound = new Map(); // Cache for already found/processed gitignore files -let defaultExcludeFilter = null; // Cache for default exclude ignore filter - -// ====================== -// PATH UTILITIES -// ====================== -const { - normalizePath, - ensureAbsolutePath, - safePathJoin, - safeRelativePath, - isValidPath, -} = require('./utils.js'); - -// ====================== -// MODULE INITIALIZATION -// ====================== -let ignore; -try { - ignore = require('ignore'); - console.log('Successfully loaded ignore module'); -} catch (err) { - console.error('Failed to load ignore module:', err); - // Simple fallback implementation - ignore = { - createFilter: () => (path) => !excludedFiles.includes(path), - }; - console.log('Using fallback for ignore module'); -} - -let tiktoken; -try { - tiktoken = require('tiktoken'); - console.log('Successfully loaded tiktoken module'); -} catch (err) { - console.error('Failed to load tiktoken module:', err); - tiktoken = null; -} - -let encoder; -try { - if (tiktoken) { - encoder = tiktoken.get_encoding('o200k_base'); // gpt-4o encoding - console.log('Tiktoken encoder initialized successfully'); - } else { - throw new Error('Tiktoken module not available'); - } -} catch (err) { - console.error('Failed to initialize tiktoken encoder:', err); - console.log('Using fallback token counter'); - encoder = null; -} - -function shouldExcludeByDefault(filePath, rootDir) { - filePath = ensureAbsolutePath(filePath); - rootDir = ensureAbsolutePath(rootDir); - - const relativePath = safeRelativePath(rootDir, filePath); - - if (!isValidPath(relativePath) || relativePath.startsWith('..')) { - return true; - } - - if (process.platform === 'win32') { - if (/^(CON|PRN|AUX|NUL|COM[1-9]|LPT[1-9])$/i.test(path.basename(filePath))) { - console.log(`Excluding reserved Windows name: ${path.basename(filePath)}`); - return true; - } - - if ( - filePath.toLowerCase().includes('\\windows\\') || - filePath.toLowerCase().includes('\\system32\\') - ) { - console.log(`Excluding system path: ${filePath}`); - return true; - } - } - - if (process.platform === 'darwin') { - if ( - filePath.includes('/.Spotlight-') || - filePath.includes('/.Trashes') || - filePath.includes('/.fseventsd') - ) { - console.log(`Excluding macOS system path: ${filePath}`); - return true; - } - } - - if (process.platform === 'linux') { - if ( - filePath.startsWith('/proc/') || - filePath.startsWith('/sys/') || - filePath.startsWith('/dev/') - ) { - console.log(`Excluding Linux system path: ${filePath}`); - return true; - } - } - - // Create the filter only once and reuse it - if (!defaultExcludeFilter) { - defaultExcludeFilter = ignore().add(excludedFiles); - console.log(`[Default Exclude] Initialized filter with ${excludedFiles.length} excluded files`); - } - - const isExcluded = defaultExcludeFilter.ignores(relativePath); - - // Only log exclusions periodically to reduce spam - if (isExcluded && Math.random() < 0.05) { - // Log ~5% of exclusions as samples - console.log(`[Default Exclude] Excluded file: ${relativePath}`); - } - - return isExcluded; -} - -// ====================== -// IGNORE CACHE LOGIC -// ====================== -async function collectGitignoreMapRecursive(startDir, rootDir, currentMap = new Map()) { - const normalizedStartDir = normalizePath(startDir); - const normalizedRootDir = normalizePath(rootDir); - - try { - await fs.promises.access(normalizedStartDir, fs.constants.R_OK); - } catch (err) { - console.warn(`Cannot access directory: ${normalizedStartDir}`, err); - return currentMap; - } - - // Read .gitignore in current directory - const gitignorePath = safePathJoin(normalizedStartDir, '.gitignore'); - try { - const content = await fs.promises.readFile(gitignorePath, 'utf8'); - const patterns = content - .split(/\r?\n/) - .map((line) => line.trim()) - .filter((line) => line && !line.startsWith('#')); - - if (patterns.length > 0) { - const relativeDirPath = safeRelativePath(normalizedRootDir, normalizedStartDir) || '.'; - currentMap.set(relativeDirPath, patterns); - console.log(`Found .gitignore in ${relativeDirPath} with ${patterns.length} patterns`); - } - } catch (err) { - if (err.code !== 'ENOENT') { - console.error(`Error reading ${gitignorePath}:`, err); - } - } - - // Recursively scan subdirectories in parallel - try { - const dirents = await fs.promises.readdir(normalizedStartDir, { withFileTypes: true }); - const subdirs = dirents.filter((dirent) => dirent.isDirectory()); - - // Process subdirectories in parallel - await Promise.all( - subdirs.map(async (dirent) => { - const subDir = safePathJoin(normalizedStartDir, dirent.name); - await collectGitignoreMapRecursive(subDir, normalizedRootDir, currentMap); - }) - ); - } catch (err) { - console.error(`Error reading directory ${normalizedStartDir} for recursion:`, err); - } - - return currentMap; -} - -// Pre-compiled default ignore filter for early checks -const defaultIgnoreFilter = ignore().add(DEFAULT_PATTERNS); - -function shouldIgnorePath(filePath, rootDir, currentDir, ignoreFilter, ignoreMode = 'automatic') { - // Validate paths to prevent empty path errors - if (!filePath || filePath.trim() === '') { - console.warn('Ignoring empty path in shouldIgnorePath'); - return true; // Treat empty paths as "should ignore" - } - - const relativeToRoot = safeRelativePath(rootDir, filePath); - const relativeToCurrent = safeRelativePath(currentDir, filePath); - - // Validate that the relative paths are not empty - if (!relativeToRoot || relativeToRoot.trim() === '') { - console.warn(`Skipping empty relativeToRoot path for: ${filePath}`); - return true; - } - - // First check against default patterns (fast path) - if (defaultIgnoreFilter.ignores(relativeToRoot)) { - console.log('Skipped by default ignore patterns:', relativeToRoot); - return true; - } - - // Then check against root-relative patterns (global/default) - if (ignoreFilter.ignores(relativeToRoot)) { - return true; - } - - // In global mode, we don't need contextual checks - if (ignoreMode === 'global') { - return false; - } - - // Then check against current directory context (automatic mode only) - const currentIgnoreFilter = createContextualIgnoreFilter(rootDir, currentDir, ignoreFilter); - - // Ensure relativeToCurrent is not empty before calling ignores - if (!relativeToCurrent || relativeToCurrent.trim() === '') { - console.warn(`Skipping empty relativeToCurrent path for: ${filePath}`); - return false; // Don't ignore if we can't determine the relative path - } - - return currentIgnoreFilter.ignores(relativeToCurrent); -} - -function createGlobalIgnoreFilter(customIgnores = []) { - const normalizedCustomIgnores = (customIgnores || []).map((p) => p.trim()).sort(); - const ig = ignore(); - const globalPatterns = [...DEFAULT_PATTERNS, ...excludedFiles, ...normalizedCustomIgnores].map( - (pattern) => normalizePath(pattern) - ); - ig.add(globalPatterns); - console.log( - `[Global Mode] Added ${DEFAULT_PATTERNS.length} default patterns, ${excludedFiles.length} excluded files, and ${normalizedCustomIgnores.length} custom ignores` - ); - - console.log( - `[Global Mode] Added ${globalPatterns.length} global patterns (${excludedFiles.length} excluded + ${normalizedCustomIgnores.length} custom)` - ); - console.log(`[Global Mode] Custom ignores added:`, normalizedCustomIgnores); - - return ig; -} - -function createContextualIgnoreFilter( - rootDir, - currentDir, - parentIgnoreFilter, - ignoreMode = 'automatic' -) { - const ig = ignore(); - - // 1. Add all patterns from parent filter (global/default patterns) - if (parentIgnoreFilter && parentIgnoreFilter.rules) { - const parentRules = parentIgnoreFilter.rules; - // Extract pattern strings from parent rules - const parentPatterns = Object.values(parentRules).map((rule) => rule.pattern); - // Filter out any undefined/empty patterns - const validPatterns = parentPatterns.filter((p) => p && typeof p === 'string'); - ig.add(validPatterns); - } - - // 2. Only add patterns from .gitignore if in automatic mode - if (ignoreMode === 'automatic') { - const gitignorePath = safePathJoin(currentDir, '.gitignore'); - - // Create a cache key for this .gitignore file - const cacheKey = normalizePath(gitignorePath); - - let patterns = []; - let needToProcessFile = true; - - // Check if we've already processed this .gitignore file - if (gitIgnoreFound.has(cacheKey)) { - patterns = gitIgnoreFound.get(cacheKey); - needToProcessFile = false; - } - - if (needToProcessFile) { - try { - const content = fs.readFileSync(gitignorePath, 'utf8'); - patterns = content - .split(/\r?\n/) - .map((line) => line.trim()) - .filter((line) => line && !line.startsWith('#')); - - // Cache the patterns for future use - if (patterns.length > 0) { - gitIgnoreFound.set(cacheKey, patterns); - - // Get a more concise path for display - const relativePath = safeRelativePath(rootDir, currentDir); - console.log( - `[Contextual Filter] Added ${patterns.length} patterns from ${relativePath === '.' ? 'root' : relativePath} .gitignore` - ); - } - } catch (err) { - if (err.code !== 'ENOENT') { - console.error(`Error reading ${gitignorePath}:`, err); - } - } - } - - if (patterns.length > 0) { - // Adjust patterns to be relative to current directory - const adjustedPatterns = patterns.map((pattern) => { - if (pattern.startsWith('/')) { - return pattern.substring(1); // Make root-relative - } - if (!pattern.includes('**')) { - // Make relative to current directory - const relPath = safeRelativePath(rootDir, currentDir); - return safePathJoin(relPath, pattern); - } - return pattern; - }); - - ig.add(adjustedPatterns); - } - } - - return ig; -} - -async function loadGitignore(rootDir, window) { - rootDir = ensureAbsolutePath(rootDir); - const cacheKey = `${rootDir}:automatic`; - - if (ignoreCache.has(cacheKey)) { - console.log(`Using cached ignore filter for automatic mode in:`, rootDir); - const cached = ignoreCache.get(cacheKey); - console.log('Cache entry details:', { - patternCount: Object.keys(cached.patterns.gitignoreMap || {}).length, - }); - return cached.ig; - } - console.log(`Cache miss for key: ${cacheKey}`); - - const ig = ignore(); - - try { - // Combine default patterns with excludedFiles - const defaultPatterns = [...DEFAULT_PATTERNS, ...excludedFiles]; - - ig.add(defaultPatterns); - console.log( - `[Automatic Mode] Added ${DEFAULT_PATTERNS.length} default patterns and ${excludedFiles.length} excluded files` - ); - - const gitignoreMap = await collectGitignoreMapRecursive(rootDir, rootDir); - let totalGitignorePatterns = 0; - - // Store raw patterns with their origin directory - const patternOrigins = new Map(); - for (const [relativeDirPath, patterns] of gitignoreMap) { - patternOrigins.set(relativeDirPath, patterns); - - // Add patterns to root filter (for backward compatibility) - const patternsToAdd = patterns.map((pattern) => { - if (!pattern.startsWith('/') && !pattern.includes('**')) { - const joinedPath = normalizePath( - path.join(relativeDirPath === '.' ? '' : relativeDirPath, pattern) - ); - return joinedPath.replace(/^\.\//, ''); - } else if (pattern.startsWith('/')) { - return pattern.substring(1); - } - return pattern; - }); - - if (patternsToAdd.length > 0) { - ig.add(patternsToAdd); - totalGitignorePatterns += patternsToAdd.length; - console.log( - `[Automatic Mode] Added ${patternsToAdd.length} repository patterns from ${relativeDirPath}/.gitignore` - ); - } - } - - if (totalGitignorePatterns > 0) { - console.log( - `[Automatic Mode] Added ${totalGitignorePatterns} repository-specific patterns (combined with ${defaultPatterns.length} default patterns) for:`, - rootDir - ); - } - - ignoreCache.set(cacheKey, { - ig, - patterns: { - gitignoreMap: Object.fromEntries(gitignoreMap), - patternOrigins: Object.fromEntries(patternOrigins), - }, - }); - - return ig; - } catch (err) { - console.error(`Error in loadGitignore for ${rootDir}:`, err); - return ig; - } -} - -// ====================== -// FILE PROCESSING -// ====================== -function isBinaryFile(filePath) { - const ext = path.extname(filePath).toLowerCase(); - - if (fileTypeCache.has(ext)) { - return fileTypeCache.get(ext); - } - - const isBinary = binaryExtensions.includes(ext); - fileTypeCache.set(ext, isBinary); - return isBinary; -} - -function countTokens(text) { - if (!encoder) { - return Math.ceil(text.length / 4); - } - - try { - const cleanText = text.replace(/<\|endoftext\|>/g, ''); - const tokens = encoder.encode(cleanText); - return tokens.length; - } catch (err) { - console.error('Error counting tokens:', err); - return Math.ceil(text.length / 4); - } -} - -// Process a single file for the file watcher -async function processSingleFile(fullPath, rootDir, ignoreFilter) { - try { - fullPath = ensureAbsolutePath(fullPath); - rootDir = ensureAbsolutePath(rootDir); - const relativePath = safeRelativePath(rootDir, fullPath); - - if (!isValidPath(relativePath) || relativePath.startsWith('..')) { - return null; - } - - if (ignoreFilter.ignores(relativePath)) { - return null; - } - - const stats = await fs.promises.stat(fullPath); - const fileData = { - name: path.basename(fullPath), - path: normalizePath(fullPath), - relativePath: relativePath, - size: stats.size, - isBinary: false, - isSkipped: false, - content: '', - tokenCount: 0, - excludedByDefault: shouldExcludeByDefault(fullPath, rootDir), - }; - - if (stats.size > MAX_FILE_SIZE) { - fileData.isSkipped = true; - fileData.error = 'File too large to process'; - return fileData; - } - - const ext = path.extname(fullPath).toLowerCase(); - if (binaryExtensions.includes(ext)) { - fileData.isBinary = true; - fileData.fileType = ext.toUpperCase(); - return fileData; - } - - const content = await fs.promises.readFile(fullPath, 'utf8'); - fileData.content = content; - fileData.tokenCount = countTokens(content); - - return fileData; - } catch (err) { - console.error(`Error processing single file ${fullPath}:`, err); - return { - name: path.basename(fullPath), - path: normalizePath(fullPath), - relativePath: safeRelativePath(rootDir, fullPath), - size: 0, - isBinary: false, - isSkipped: true, - error: `Error: ${err.message}`, - content: '', - tokenCount: 0, - excludedByDefault: shouldExcludeByDefault(fullPath, rootDir), - }; - } -} - -async function processDirectory({ - dirent, - dir, - rootDir, - ignoreFilter, - window, - progress, - currentDir = dir, - ignoreMode = 'automatic', - fileQueue = null, -}) { - await watcher.shutdownWatcher(); - const fullPath = safePathJoin(dir, dirent.name); - const relativePath = safeRelativePath(rootDir, fullPath); - - // Early check against default ignore patterns - if (defaultIgnoreFilter.ignores(relativePath)) { - console.log('Skipped by default ignore patterns:', relativePath); - return { results: [], progress }; - } - - if ( - fullPath.includes('.app') || - fullPath === app.getAppPath() || - !isValidPath(relativePath) || - relativePath.startsWith('..') - ) { - console.log('Skipping directory:', fullPath); - return { results: [], progress }; - } - - // In global mode, use the passed ignoreFilter directly - const filterToUse = - ignoreMode === 'global' - ? ignoreFilter - : createContextualIgnoreFilter(rootDir, currentDir, ignoreFilter, ignoreMode); - - if (!shouldIgnorePath(fullPath, rootDir, currentDir, filterToUse, ignoreMode)) { - progress.directories++; - await watcher.initializeWatcher(dir, window, ignoreFilter, defaultIgnoreFilter); - window.webContents.send('file-processing-status', { - status: 'processing', - message: `Scanning directories (${progress.directories} processed)... (Press ESC to cancel)`, - }); - return readFilesRecursively( - fullPath, - rootDir, - filterToUse, - window, - progress, - fullPath, - ignoreMode, - fileQueue - ); - } - return { results: [], progress }; -} - -async function readFilesRecursively( - dir, - rootDir, - ignoreFilter, - window, - progress = { directories: 0, files: 0 }, - currentDir = dir, - ignoreMode = 'automatic', - fileQueue = null -) { - await watcher.shutdownWatcher(); - if (!ignoreFilter) { - throw new Error('readFilesRecursively requires an ignoreFilter parameter'); - } - if (!isLoadingDirectory) return { results: [], progress }; - - dir = ensureAbsolutePath(dir); - rootDir = ensureAbsolutePath(rootDir || dir); - - // Initialize queue only once at the top level call - let shouldCleanupQueue = false; - let queueToUse = fileQueue; - if (!queueToUse) { - // Determine concurrency based on CPU cores, with a reasonable minimum and maximum - const cpuCount = os.cpus().length; - const fileQueueConcurrency = Math.max(2, Math.min(cpuCount, 8)); // e.g., Use between 2 and 8 concurrent file operations - queueToUse = new PQueue({ concurrency: fileQueueConcurrency }); - shouldCleanupQueue = true; - - // Only log the initialization message for the root directory to reduce spam - if (dir === rootDir) { - console.log(`Initializing file processing queue with concurrency: ${fileQueueConcurrency}`); - } - } - - let results = []; - let fileProcessingErrors = []; // To collect errors without stopping - - try { - const dirents = await fs.promises.readdir(dir, { withFileTypes: true }); - if (!isLoadingDirectory) return { results: [], progress }; - - const directories = dirents.filter((dirent) => dirent.isDirectory()); - const files = dirents.filter((dirent) => dirent.isFile()); - - for (let i = 0; i < directories.length; i += CONCURRENT_DIRS) { - if (!isLoadingDirectory) return { results: [], progress }; - - const batch = directories.slice(i, Math.min(i + CONCURRENT_DIRS, directories.length)); - - const batchPromises = batch.map((dirent) => - processDirectory({ - dirent, - dir, - rootDir, - ignoreFilter, - window, - progress, - currentDir, - ignoreMode, - fileQueue, - }) - ); - - const batchResults = await Promise.all(batchPromises); - - const combinedResults = batchResults.reduce( - (acc, curr) => { - acc.results = acc.results.concat(curr.results); - return acc; - }, - { results: [], progress } - ); - - results = results.concat(combinedResults.results); - if (!isLoadingDirectory) return { results: [], progress }; - } - - // Process files using the controlled concurrency queue - for (const dirent of files) { - if (!isLoadingDirectory) break; // Check cancellation before adding to queue - - queueToUse.add(async () => { - if (!isLoadingDirectory) return; // Check cancellation again inside the task - - const fullPath = safePathJoin(dir, dirent.name); - const relativePath = safeRelativePath(rootDir, fullPath); - const fullPathNormalized = normalizePath(fullPath); - - try { - // Wrap file processing in try/catch to handle errors within the queue task - if (!isValidPath(relativePath) || relativePath.startsWith('..')) { - console.log('Invalid path, skipping:', fullPath); - return; - } - - if (fullPath.includes('.app') || fullPath === app.getAppPath()) { - console.log('System path, skipping:', fullPath); - return; - } - - // Early check against default ignore patterns - if (defaultIgnoreFilter.ignores(relativePath)) { - console.log('Skipped by default ignore patterns:', relativePath); - return; - } - - if (shouldIgnorePath(fullPath, rootDir, currentDir, ignoreFilter, ignoreMode)) { - // console.log('Ignored by filter, skipping:', relativePath); // Can be noisy - return; - } - - if (fileCache.has(fullPathNormalized)) { - // console.log('Using cached file data for:', fullPathNormalized); // Can be noisy - results.push(fileCache.get(fullPathNormalized)); - progress.files++; - return; - } - - if (isBinaryFile(fullPath)) { - // console.log('Binary file by extension, skipping content read:', fullPath); // Can be noisy - const fileData = { - name: dirent.name, - path: fullPathNormalized, - relativePath: relativePath, - tokenCount: 0, - size: 0, - content: '', - isBinary: true, - isSkipped: false, - fileType: path.extname(fullPath).substring(1).toUpperCase(), - }; - - try { - const stats = await fs.promises.stat(fullPath); - if (!isLoadingDirectory) return; - fileData.size = stats.size; - } catch (statErr) { - console.log('Could not get size for binary file:', fullPath, statErr.code); - // Still add the file entry, just with size 0 - } - - fileCache.set(fullPathNormalized, fileData); - results.push(fileData); - progress.files++; - return; - } - - // Process non-binary files - const stats = await fs.promises.stat(fullPath); - if (!isLoadingDirectory) return; - - if (stats.size > MAX_FILE_SIZE) { - const fileData = { - name: dirent.name, - path: fullPathNormalized, - relativePath: relativePath, - tokenCount: 0, - size: stats.size, - content: '', - isBinary: false, - isSkipped: true, - error: 'File too large to process', - }; - fileCache.set(fullPathNormalized, fileData); - results.push(fileData); - progress.files++; - return; - } - - const fileContent = await fs.promises.readFile(fullPath, 'utf8'); - if (!isLoadingDirectory) return; - - const fileData = { - name: dirent.name, - path: fullPathNormalized, - relativePath: relativePath, - content: fileContent, // Still loading full content for token counting - tokenCount: countTokens(fileContent), - size: stats.size, - isBinary: false, - isSkipped: false, - }; - fileCache.set(fullPathNormalized, fileData); - results.push(fileData); - progress.files++; - } catch (err) { - console.error(`Error processing file ${fullPath}:`, err.code || err.message); - const errorData = { - name: dirent.name, - path: fullPathNormalized, - relativePath: relativePath, - tokenCount: 0, - size: 0, // Attempt to get size if possible, otherwise 0 - isBinary: false, - isSkipped: true, - error: - err.code === 'EPERM' - ? 'Permission denied' - : err.code === 'ENOENT' - ? 'File not found' - : err.code === 'EBUSY' - ? 'File busy' - : err.code === 'EMFILE' - ? 'Too many open files' - : 'Could not read file', - }; - // Try to get stats even if read failed - try { - const errorStats = await fs.promises.stat(fullPath); - errorData.size = errorStats.size; - } catch (statErr) { - /* ignore */ - } - - fileCache.set(fullPathNormalized, errorData); - results.push(errorData); // Add error entry to results - progress.files++; // Count errors as processed files for progress - fileProcessingErrors.push({ path: fullPathNormalized, error: err.message }); - } - - // Throttle status updates (moved outside finally) - const now = Date.now(); - if (now - lastStatusUpdateTime > STATUS_UPDATE_INTERVAL) { - if (!isLoadingDirectory) return; // Check cancellation before sending IPC - window.webContents.send('file-processing-status', { - status: 'processing', - message: `Processing files (${progress.directories} dirs, ${progress.files} files)... (Press ESC to cancel)`, - }); - lastStatusUpdateTime = now; - if (progress.files % 500 === 0) { - // Log less frequently - console.log( - `Progress update - Dirs: ${progress.directories}, Files: ${progress.files}, Queue Size: ${queueToUse.size}, Pending: ${queueToUse.pending}` - ); - } - } - }); - } - - // Wait for all queued file processing tasks to complete - await queueToUse.onIdle(); - - if (fileProcessingErrors.length > 0) { - console.warn(`Encountered ${fileProcessingErrors.length} errors during file processing.`); - // Optionally send a summary of errors to the renderer - // window.webContents.send("file-processing-errors", fileProcessingErrors); - } - } catch (err) { - console.error(`Error reading directory ${dir}:`, err); - if (err.code === 'EPERM' || err.code === 'EACCES') { - console.log(`Skipping inaccessible directory: ${dir}`); - return { results: [], progress }; - } - } - - // Cleanup queue if it was initialized in this call - if (shouldCleanupQueue) { - await queueToUse.onIdle(); - queueToUse.clear(); - } - - return { results, progress }; -} - -// ====================== -// DIRECTORY LOADING MANAGEMENT -// ====================== -function setupDirectoryLoadingTimeout(window, folderPath) { - if (loadingTimeoutId) { - clearTimeout(loadingTimeoutId); - } - - loadingTimeoutId = setTimeout(() => { - console.log( - `Directory loading timed out after ${MAX_DIRECTORY_LOAD_TIME / 1000} seconds: ${folderPath}` - ); - console.log( - `Stats at timeout: Processed ${currentProgress.directories} directories and ${currentProgress.files} files` - ); - cancelDirectoryLoading(window, 'timeout'); - }, MAX_DIRECTORY_LOAD_TIME); - - currentProgress = { directories: 0, files: 0 }; -} - -async function cancelDirectoryLoading(window, reason = 'user') { - await watcher.shutdownWatcher(); - if (!isLoadingDirectory) return; - - console.log(`Cancelling directory loading process (Reason: ${reason})`); - console.log( - `Stats at cancellation: Processed ${currentProgress.directories} directories and ${currentProgress.files} files` - ); - - isLoadingDirectory = false; - - if (loadingTimeoutId) { - clearTimeout(loadingTimeoutId); - loadingTimeoutId = null; - } - - currentProgress = { directories: 0, files: 0 }; - - if (window && window.webContents && !window.webContents.isDestroyed()) { - const message = - reason === 'timeout' - ? 'Directory loading timed out after 5 minutes. Try clearing data and retrying.' - : 'Directory loading cancelled'; - - window.webContents.send('file-processing-status', { - status: 'cancelled', - message: message, - }); - } else { - console.log('Window not available to send cancellation status.'); - } -} - -// ====================== -// IPC HANDLERS -// ====================== -ipcMain.on('clear-main-cache', () => { - console.log('Clearing main process caches'); - ignoreCache.clear(); - fileCache.clear(); - fileTypeCache.clear(); - console.log('Main process caches cleared (including ignoreCache)'); -}); - -ipcMain.on('clear-ignore-cache', () => { - console.log('Clearing ignore cache due to ignore settings change'); - ignoreCache.clear(); - console.log('Ignore cache cleared'); -}); - -ipcMain.on('open-folder', async (event) => { - const result = await dialog.showOpenDialog({ - properties: ['openDirectory'], - }); - - if (!result.canceled && result.filePaths && result.filePaths.length > 0) { - const rawPath = result.filePaths[0]; - const normalizedPath = normalizePath(rawPath); - try { - console.log('Sending folder-selected event with normalized path:', normalizedPath); - event.sender.send('folder-selected', normalizedPath); - } catch (err) { - console.error('Error sending folder-selected event:', err); - event.sender.send('folder-selected', normalizedPath); - } - } -}); - -if (!ipcMain.eventNames().includes('get-ignore-patterns')) { - ipcMain.handle( - 'get-ignore-patterns', - async (event, { folderPath, mode = 'automatic', customIgnores = [] } = {}) => { - if (!folderPath) { - console.log('get-ignore-patterns called without folderPath - returning default patterns'); - return { - patterns: { - global: [...DEFAULT_PATTERNS, ...excludedFiles, ...(customIgnores || [])], - }, - }; - } - - try { - let patterns; - const normalizedPath = ensureAbsolutePath(folderPath); - - if (mode === 'global') { - patterns = { global: [...excludedFiles, ...(customIgnores || [])] }; - const cacheKey = `${normalizedPath}:global:${JSON.stringify(customIgnores?.sort() || [])}`; - ignoreCache.set(cacheKey, { - ig: createGlobalIgnoreFilter(customIgnores), - patterns, - }); - } else { - await loadGitignore(normalizedPath); - const cacheKey = `${normalizedPath}:automatic`; - patterns = ignoreCache.get(cacheKey)?.patterns || { gitignoreMap: {} }; - } - - return { patterns }; - } catch (err) { - console.error(`Error getting ignore patterns for ${folderPath}:`, err); - return { error: err.message }; - } - } - ); -} - -ipcMain.on('cancel-directory-loading', (event) => { - cancelDirectoryLoading(BrowserWindow.fromWebContents(event.sender)); -}); - -ipcMain.on('debug-file-selection', (event, data) => { - console.log('DEBUG - File Selection:', data); -}); - -if (!ipcMain.eventNames().includes('set-ignore-mode')) { - /** - * Handles ignore mode changes. Validates the mode, clears caches, - * resets the watcher, and notifies renderer windows of the change. - * @param {string} mode - The new ignore mode ('automatic' or 'global') - */ - ipcMain.on('set-ignore-mode', async (_event, mode) => { - if (mode !== 'automatic' && mode !== 'global') { - console.warn(`[IgnoreMode] Received invalid mode: ${mode}`); - return; - } - - currentIgnoreMode = mode; - console.log(`[IgnoreMode] switched -> ${mode}`); - console.log('[IgnoreMode] DEBUG - Current mode set to:', currentIgnoreMode); - - ignoreCache.clear(); - fileCache.clear(); - fileTypeCache.clear(); - - // Watcher cleanup is now handled by the watcher module itself - - BrowserWindow.getAllWindows().forEach((win) => { - if (win && win.webContents) { - win.webContents.send('ignore-mode-updated', mode); - } - }); - }); -} - -ipcMain.on('request-file-list', async (event, folderPath) => { - console.log('Received request-file-list payload:', folderPath); // Log the entire payload - - if (isLoadingDirectory) { - console.log('Already processing a directory, ignoring new request for:', folderPath); - const window = BrowserWindow.fromWebContents(event.sender); - if (window && window.webContents && !window.webContents.isDestroyed()) { - window.webContents.send('file-processing-status', { - status: 'busy', - message: 'Already processing another directory. Please wait.', - }); - } - return; - } - - try { - isLoadingDirectory = true; - setupDirectoryLoadingTimeout(BrowserWindow.fromWebContents(event.sender), folderPath); - - event.sender.send('file-processing-status', { - status: 'processing', - message: 'Scanning directory structure... (Press ESC to cancel)', - }); - - currentProgress = { directories: 0, files: 0 }; - - // Clear ignore cache if ignore settings were modified - if (folderPath.ignoreSettingsModified) { - console.log('Clearing ignore cache due to modified ignore settings'); - ignoreCache.clear(); - } - - console.log( - `Loading ignore patterns for: ${folderPath.folderPath} in mode: ${folderPath.ignoreMode}` - ); - let ignoreFilter; - if (folderPath.ignoreMode === 'global') { - console.log('Using global ignore filter with custom ignores:', folderPath.customIgnores); - ignoreFilter = createGlobalIgnoreFilter(folderPath.customIgnores); - } else { - // Default to automatic - console.log('Using automatic ignore filter (loading .gitignore)'); - ignoreFilter = await loadGitignore( - folderPath.folderPath, - BrowserWindow.fromWebContents(event.sender) - ); - } - if (!ignoreFilter) { - throw new Error('Failed to load ignore patterns'); - } - console.log('Ignore patterns loaded successfully'); - - const { results: files } = await readFilesRecursively( - folderPath.folderPath, - folderPath.folderPath, - ignoreFilter, - BrowserWindow.fromWebContents(event.sender), - currentProgress, - folderPath.folderPath, - folderPath?.ignoreMode ?? currentIgnoreMode - ); - - if (!isLoadingDirectory) { - return; - } - - if (loadingTimeoutId) { - clearTimeout(loadingTimeoutId); - loadingTimeoutId = null; - } - isLoadingDirectory = false; - - event.sender.send('file-processing-status', { - status: 'complete', - message: `Found ${files.length} files`, - }); - - const serializedFiles = files - .filter((file) => { - if (typeof file?.path !== 'string') { - console.warn('Invalid file object in files array:', file); - return false; - } - return true; - }) - .map((file) => { - return { - path: file.path, - relativePath: file.relativePath, - name: file.name, - size: file.size, - isDirectory: file.isDirectory, - extension: path.extname(file.name).toLowerCase(), - excluded: shouldExcludeByDefault(file.path, folderPath.folderPath), - content: file.content, - tokenCount: file.tokenCount, - isBinary: file.isBinary, - isSkipped: file.isSkipped, - error: file.error, - }; - }); - - event.sender.send('file-list-data', serializedFiles); - } catch (err) { - console.error('Error processing file list:', err); - isLoadingDirectory = false; - - if (loadingTimeoutId) { - clearTimeout(loadingTimeoutId); - loadingTimeoutId = null; - } - - event.sender.send('file-processing-status', { - status: 'error', - message: `Error: ${err.message}`, - }); - } finally { - isLoadingDirectory = false; - if (loadingTimeoutId) { - clearTimeout(loadingTimeoutId); - loadingTimeoutId = null; - } - } -}); - -// ====================== -// ELECTRON WINDOW SETUP -// ====================== -console.log('--- createWindow() ENTERED ---'); -let mainWindow; -function createWindow() { - const isSafeMode = process.argv.includes('--safe-mode'); - - // Set CSP header for all environments - session.defaultSession.webRequest.onHeadersReceived((details, callback) => { - callback({ - responseHeaders: { - ...details.responseHeaders, - 'Content-Security-Policy': [ - "default-src 'self'; script-src 'self' 'unsafe-inline'; style-src 'self' 'unsafe-inline' https://fonts.googleapis.com; img-src 'self' data:; font-src 'self' https://fonts.gstatic.com; connect-src 'self' http://localhost:* ws://localhost:*; object-src 'none';", - ], - }, - }); - }); - mainWindow = new BrowserWindow({ - width: 1200, - height: 800, - webPreferences: { - nodeIntegration: false, - contextIsolation: true, - preload: path.join(__dirname, 'preload.js'), - devTools: { - isDevToolsExtension: false, - htmlFullscreen: false, - }, - // Always enable security - // Disable manually for testing - webSecurity: true, - allowRunningInsecureContent: false, - }, - }); - - // Set up window event handlers - mainWindow.on('closed', async () => { - await watcher.shutdownWatcher(); - mainWindow = null; // Now allowed since mainWindow is let - }); - - app.on('before-quit', async (event) => { - await watcher.shutdownWatcher(); - }); - - app.on('window-all-closed', async () => { - if (process.platform !== 'darwin') { - await watcher.shutdownWatcher(); - app.quit(); - } - }); - - // handle Escape locally (only when focused), not globally - mainWindow.webContents.on('before-input-event', (event, input) => { - // only intercept Esc when our window is focused and a load is in progress - if (input.key === 'Escape' && isLoadingDirectory) { - cancelDirectoryLoading(mainWindow); - event.preventDefault(); // stop further in-app handling - } - }); - - // Only verify file existence in production mode - if (process.env.NODE_ENV !== 'development') { - // Verify file exists before loading - const prodPath = path.join(__dirname, 'dist', 'index.html'); - console.log('Production path:', prodPath); - try { - fs.accessSync(prodPath, fs.constants.R_OK); - console.log('File exists and is readable'); - } catch (err) { - console.error('File access error:', err); - } - } - - // Clean up watcher when window is closed - mainWindow.on('closed', () => { - // Watcher cleanup is now handled by the watcher module itself - }); - - mainWindow.webContents.once('did-finish-load', () => { - mainWindow.webContents.send('startup-mode', { - safeMode: isSafeMode, - }); - }); - - app.on('window-all-closed', () => { - if (process.platform !== 'darwin') { - app.quit(); - } - }); - - if (process.env.NODE_ENV === 'development') { - mainWindow.loadURL('http://localhost:5173'); - mainWindow.webContents.openDevTools(); - } else { - const prodPath = app.isPackaged - ? path.join(process.resourcesPath, 'app.asar', 'dist', 'index.html') - : path.join(__dirname, 'dist', 'index.html'); - - console.log('--- PRODUCTION LOAD ---'); - console.log('NODE_ENV:', process.env.NODE_ENV); - console.log('app.isPackaged:', app.isPackaged); - console.log('__dirname:', __dirname); - console.log('Resources Path:', process.resourcesPath); - console.log('Attempting to load file:', prodPath); - console.log('File exists:', fs.existsSync(prodPath)); - - mainWindow - .loadFile(prodPath) - .then(() => { - console.log('Successfully loaded index.html'); - mainWindow.webContents.on('did-finish-load', () => { - console.log('Finished loading all page resources'); - }); - }) - .catch((err) => { - console.error('Failed to load index.html:', err); - // Fallback to showing error page - mainWindow.loadURL( - `data:text/html,

Loading Error

${encodeURIComponent(err.message)}

` - ); - }); - } -} - -// ====================== -// APP LIFECYCLE -// ====================== -app.whenReady().then(() => { - createWindow(); - - app.on('activate', () => { - if (BrowserWindow.getAllWindows().length === 0) createWindow(); - }); -}); diff --git a/electron/build.js b/electron/build.js index ccf268f9..43ab3080 100644 --- a/electron/build.js +++ b/electron/build.js @@ -9,7 +9,7 @@ async function main() { console.log('✅ React build completed successfully!'); // Fix the paths in index.html for Electron compatibility - const indexHtmlPath = path.join(__dirname, 'dist', 'index.html'); + const indexHtmlPath = path.join(__dirname, '..', 'dist', 'index.html'); if (fs.existsSync(indexHtmlPath)) { let content = fs.readFileSync(indexHtmlPath, 'utf8'); diff --git a/electron/dev.js b/electron/dev.js index 6aa21190..10cba6d8 100644 --- a/electron/dev.js +++ b/electron/dev.js @@ -3,7 +3,6 @@ try { // Test loading key dependencies require('ignore'); require('tiktoken'); - require('gpt-3-encoder'); } catch (err) { console.error(`\n❌ Missing dependency: ${err.message}`); console.error('Please run: npm install\n'); @@ -18,8 +17,8 @@ console.log('🚀 Starting development environment...'); // Set environment variable for development mode process.env.NODE_ENV = 'development'; -// Default port -let vitePort = 3000; +// Default port (Vite's default) +let vitePort = 5173; // Start Vite dev server console.log('📦 Starting Vite dev server...'); @@ -36,15 +35,26 @@ viteProcess.stdout?.on('data', (data) => { const output = data.toString(); console.log(output); // Echo output to console - // Extract port from Vite output (supports both formats) - const portMatch = output.match(/(?:Local|➜\s+Local):\s+http:\/\/localhost:(\d+)/); + // Extract port from Vite output - improved regex to handle Unicode characters + const portMatch = output.match(/Local.*?http:\/\/localhost:(\d+)/); if (portMatch && portMatch[1]) { vitePort = parseInt(portMatch[1], 10); console.log(`🔍 Detected Vite server running on port ${vitePort}`); } + + // Alternative extraction method if main regex fails + if (output.includes('localhost:') && !portMatch) { + const simpleMatch = output.match(/localhost:(\d+)/); + if (simpleMatch && simpleMatch[1]) { + vitePort = parseInt(simpleMatch[1], 10); + console.log(`🔍 Detected Vite server running on port ${vitePort} (alternative method)`); + } + } - if (output.includes('Local:') && !viteStarted) { + // Check for Vite ready indicators (more robust) + if ((output.includes('Local:') || output.includes('ready in') || output.includes('localhost:')) && !viteStarted) { viteStarted = true; + console.log('🎯 Vite server detected as ready, starting Electron...'); startElectron(vitePort); } }); @@ -54,10 +64,10 @@ viteProcess.stderr?.on('data', (data) => { const output = data.toString(); console.error(output); // Echo error output to console - if (output.includes('Port 3000 is already in use')) { - console.error('\n❌ Port 3000 is already in use. Try one of the following:'); + if (output.includes('Port 5173 is already in use') || output.includes('already in use')) { + console.error('\n❌ Vite port is already in use. Try one of the following:'); console.error( - " 1. Kill the process using port 3000: 'lsof -i :3000 | grep LISTEN' then 'kill -9 [PID]'" + " 1. Kill the process using the port: 'lsof -i :5173 | grep LISTEN' then 'kill -9 [PID]'" ); console.error(' 2. Change the Vite port in vite.config.ts'); console.error(' 3. Restart your computer if the issue persists\n'); @@ -68,12 +78,12 @@ viteProcess.stderr?.on('data', (data) => { setTimeout(() => { if (!viteStarted) { console.log('⚠️ Vite server might not be ready yet, but starting Electron anyway...'); - startElectron(); + startElectron(vitePort); } }, 5000); // Wait 5 seconds before attempting to start Electron -function startElectron(port) { - console.log(`🔌 Starting Electron app with Vite server at port ${vitePort}...`); +function startElectron(port = vitePort) { + console.log(`🔌 Starting Electron app with Vite server at port ${port}...`); const electronProcess = spawn('npm', ['start'], { stdio: 'inherit', shell: platform() === 'win32', // Use shell on Windows diff --git a/electron/main.js b/electron/main.js index 527209b9..426289d8 100644 --- a/electron/main.js +++ b/electron/main.js @@ -11,6 +11,217 @@ const { getUpdateStatus, resetUpdateSessionState } = require('./update-manager') // Configuration constants const MAX_DIRECTORY_LOAD_TIME = 300000; // 5 minutes timeout for large repositories +// Token estimation based on file extension and size +function estimateTokens(fileName, fileSize) { + const path = require('path'); + const ext = path.extname(fileName).toLowerCase(); + + // Binary/media files have 0 tokens + const binaryExtensions = [ + // Images + '.png', '.jpg', '.jpeg', '.gif', '.bmp', '.ico', '.svg', '.webp', '.tiff', '.tif', + // Videos + '.mp4', '.avi', '.mov', '.wmv', '.flv', '.mkv', '.webm', '.m4v', + // Audio + '.mp3', '.wav', '.flac', '.aac', '.ogg', '.wma', '.m4a', + // Archives + '.zip', '.rar', '.7z', '.tar', '.gz', '.bz2', '.xz', + // Executables + '.exe', '.dll', '.so', '.dylib', '.bin', '.app', '.deb', '.rpm', + // Fonts + '.woff', '.woff2', '.ttf', '.otf', '.eot', + // Documents (binary format) + '.pdf', '.doc', '.docx', '.xls', '.xlsx', '.ppt', '.pptx', + // Other binary + '.sqlite', '.db', '.lock' + ]; + + if (binaryExtensions.includes(ext)) { + return 0; + } + + // Code files are denser (more tokens per character) + const codeExtensions = [ + // Web technologies + '.js', '.ts', '.jsx', '.tsx', '.html', '.css', '.scss', '.sass', '.less', + '.vue', '.svelte', '.astro', + // Programming languages + '.py', '.java', '.cpp', '.c', '.h', '.hpp', '.cs', '.php', '.rb', '.go', + '.rs', '.swift', '.kt', '.scala', '.clj', '.cljs', '.r', '.m', '.mm', + // Shell scripts + '.sh', '.bash', '.zsh', '.fish', '.ps1', '.bat', '.cmd', + // SQL and databases + '.sql', '.mysql', '.postgres', '.sqlite', + // Other code + '.dart', '.lua', '.perl', '.pl', '.haskell', '.hs', '.elm', '.nim' + ]; + + if (codeExtensions.includes(ext)) { + return Math.ceil(fileSize / 3); // ~3 chars per token for code + } + + // Text/config files + const textExtensions = [ + // Documentation + '.txt', '.md', '.rst', '.adoc', '.tex', + // Config files + '.json', '.xml', '.yml', '.yaml', '.toml', '.ini', '.cfg', '.conf', '.config', + // Environment and build + '.env', '.gitignore', '.gitattributes', '.dockerfile', '.dockerignore', + '.makefile', '.cmake', '.gradle', '.maven', + // Data files + '.csv', '.tsv', '.log', '.logs', + // License and readme files + 'license', 'readme', 'changelog', 'contributing' + ]; + + if (textExtensions.includes(ext) || !ext) { + return Math.ceil(fileSize / 4); // ~4 chars per token for text + } + + // Files without extensions - check filename + if (!ext) { + const lowerName = fileName.toLowerCase(); + if (['readme', 'license', 'changelog', 'contributing', 'dockerfile', 'makefile', 'gemfile', 'procfile'].includes(lowerName)) { + return Math.ceil(fileSize / 4); + } + } + + // Default estimation for unknown file types + return Math.ceil(fileSize / 4); +} + +// Enhanced lightweight directory scanning - gets metadata + token estimates +async function scanDirectoryLightweight(folderPath, ignoreFilter) { + const fs = require('fs').promises; + const path = require('path'); + const files = []; + const maxFiles = 10000; // Limit to prevent infinite loops + + async function scanDir(dir, relativeTo = folderPath) { + if (files.length > maxFiles) return; + + try { + const items = await fs.readdir(dir, { withFileTypes: true }); + + for (const item of items) { + if (files.length > maxFiles) break; + + const fullPath = path.join(dir, item.name); + const relativePath = path.relative(relativeTo, fullPath).replace(/\\/g, '/'); + + // Check if path should be ignored + if (ignoreFilter && ignoreFilter.ignores(relativePath)) { + continue; + } + + if (item.isDirectory()) { + // Add directory to list + files.push({ + path: fullPath, + relativePath: relativePath, + name: item.name, + isDirectory: true, + size: 0, + estimatedTokens: 0 // Will be calculated as sum of children + }); + + // Recurse into directory + await scanDir(fullPath, relativeTo); + } else { + // Add file to list with basic metadata and token estimate + try { + const stats = await fs.stat(fullPath); + const estimatedTokens = estimateTokens(item.name, stats.size); + + if (files.length < 5) { // Debug first few files + console.log(`[TOKEN ESTIMATE DEBUG] ${item.name}: size=${stats.size}, estimated=${estimatedTokens}`); + } + + files.push({ + path: fullPath, + relativePath: relativePath, + name: item.name, + isDirectory: false, + size: stats.size, + estimatedTokens: estimatedTokens + }); + } catch (error) { + // Skip files we can't stat + console.warn(`Cannot stat file ${fullPath}:`, error.message); + } + } + } + } catch (error) { + console.warn(`Cannot read directory ${dir}:`, error.message); + } + } + + await scanDir(folderPath); + + // The frontend will calculate directory token totals from the file list + // No need to calculate them here since we're filtering out directories + + const totalEstimatedTokens = files.reduce((sum, file) => sum + (file.estimatedTokens || 0), 0); + console.log(`[MAIN] Lightweight scan found ${files.length} items with ~${totalEstimatedTokens} estimated tokens`); + return files; +} + +// Helper function for recursive file counting +async function countFilesRecursive(dir, depth, maxDepth, maxFiles, counters, fs, path) { + if (depth > maxDepth || counters.fileCount > maxFiles) return; + + try { + const items = await fs.readdir(dir, { withFileTypes: true }); + + for (const item of items) { + if (counters.fileCount > maxFiles) break; + + // Skip common ignore patterns quickly + if (item.name.startsWith('.git') || + item.name === 'node_modules' || + item.name === 'dist' || + item.name === '__pycache__' || + item.name === '.venv' || + item.name === 'venv') { + continue; + } + + if (item.isDirectory()) { + counters.dirCount++; + await countFilesRecursive(path.join(dir, item.name), depth + 1, maxDepth, maxFiles, counters, fs, path); + } else { + counters.fileCount++; + } + } + } catch (error) { + // Skip directories we can't read + console.warn(`Cannot read directory ${dir}:`, error.message); + } +} + +// Quick folder size estimation function +async function getEstimatedFileCount(folderPath) { + const fs = require('fs').promises; + const path = require('path'); + + try { + const counters = { fileCount: 0, dirCount: 0 }; + const maxDepth = 5; // Limit depth to avoid deep recursion + const maxFiles = 50000; // Stop counting after this many files + + await countFilesRecursive(folderPath, 0, maxDepth, maxFiles, counters, fs, path); + console.log(`[MAIN] Quick scan found ~${counters.fileCount} files in ${counters.dirCount} directories`); + return { + fileCount: counters.fileCount > maxFiles ? -1 : counters.fileCount, // -1 indicates too many files to count + dirCount: counters.dirCount + }; + } catch (error) { + console.error('Error estimating folder size:', error); + return { fileCount: 0, dirCount: 0 }; // Return 0s on error to allow normal processing + } +} + // ====================== // GLOBAL STATE // ====================== @@ -26,6 +237,9 @@ let loadingTimeoutId = null; */ let currentProgress = { directories: 0, files: 0 }; +// State to hold large folder data while waiting for user confirmation +let pendingLargeFolderData = null; + // ====================== // PATH UTILITIES // ====================== @@ -313,6 +527,25 @@ ipcMain.handle('get-token-count', async (event, textToTokenize) => { ipcMain.on('request-file-list', async (event, payload) => { console.log('Received request-file-list payload:', payload); // Log the entire payload + // Validate payload structure + if (!payload || typeof payload !== 'object') { + console.error('Invalid payload received in request-file-list:', payload); + event.sender.send('file-processing-status', { + status: 'error', + message: 'Invalid request format. Please try again.', + }); + return; + } + + if (!payload.folderPath || typeof payload.folderPath !== 'string') { + console.error('Invalid or missing folderPath in payload:', payload); + event.sender.send('file-processing-status', { + status: 'error', + message: 'Invalid folder path. Please select a folder again.', + }); + return; + } + // Always clear file caches before scanning clearFileCaches(); @@ -366,18 +599,79 @@ ipcMain.on('request-file-list', async (event, payload) => { } console.log('Ignore patterns loaded successfully'); - const { results: files } = await readFilesRecursively( + // Quick folder size check before full processing + console.log(`[MAIN] Performing quick folder size check for ${payload.folderPath}`); + try { + const { fileCount: estimatedFileCount, dirCount } = await getEstimatedFileCount(payload.folderPath); + console.log(`[MAIN] Estimated file count: ${estimatedFileCount}, directories: ${dirCount}`); + + // If estimated file count is high OR we found many directories (indicating complex repo structure), show modal immediately + const shouldShowModal = estimatedFileCount > 1000 || + estimatedFileCount === -1 || + dirCount > 200; // Many directories indicate complex structure + + if (shouldShowModal) { + console.log(`[MAIN] Large/complex directory detected (${estimatedFileCount} files, ${dirCount} dirs), showing modal immediately`); + + // Store minimal data for the modal + pendingLargeFolderData = { + folderPath: payload.folderPath, + files: [], // Empty for now, will be populated if user chooses to proceed + ignoreFilter: ignoreFilter, + payload: payload + }; + + event.sender.send('large-folder-warning', { + totalTokens: estimatedFileCount === -1 ? 50000000 : estimatedFileCount * 2000, // More realistic estimate: ~2000 tokens per file + folderPath: payload.folderPath, + isEstimate: true + }); + + // Stop further execution until user responds + isLoadingDirectory = false; + stopFileProcessing(); + if (loadingTimeoutId) { + clearTimeout(loadingTimeoutId); + loadingTimeoutId = null; + } + return; // IMPORTANT: Stop here. + } + } catch (error) { + console.error('Error during folder size estimation:', error); + // Continue with normal processing if estimation fails + } + + // Always use lightweight scan for initial load (per TASKS_2.md) + console.log(`[MAIN] Performing lightweight scan for ${payload.folderPath}`); + const lightweightFiles = await scanDirectoryLightweight( payload.folderPath, - payload.folderPath, // rootDir is the same as the initial dir for top-level call - ignoreFilter, - BrowserWindow.fromWebContents(event.sender), - currentProgress, - payload.folderPath, // currentDir is also the same for top-level - payload?.ignoreMode ?? currentIgnoreMode, - null, // fileQueue - watcher.shutdownWatcher, - watcher.initializeWatcher + ignoreFilter ); + + // Convert lightweight files to the expected format + const files = lightweightFiles + .filter(file => !file.isDirectory) // Only include actual files + .map((file) => ({ + path: file.path, + relativePath: file.relativePath, + name: file.name, + size: file.size || 0, + isDirectory: false, + extension: path.extname(file.name).toLowerCase(), + excluded: isPathExcludedByDefaults( + file.path, + payload.folderPath, + payload.ignoreMode ?? currentIgnoreMode + ), + content: '', // Empty content for lightweight mode + tokenCount: file.estimatedTokens || 0, + isTokenEstimate: true, // Flag to indicate this is an estimate + isBinary: false, // Will be determined later if file is selected + isSkipped: false, + error: null, + })); + + console.log(`[MAIN] Lightweight scan completed: ${files.length} files found`); if (!isLoadingDirectory) { return; @@ -395,6 +689,8 @@ ipcMain.on('request-file-list', async (event, payload) => { message: `Found ${files.length} files`, }); + console.log(`[MAIN] Starting serialization of ${files.length} files for ${payload.folderPath}`); + const serializedFiles = files .filter((file) => { if (typeof file?.path !== 'string') { @@ -424,26 +720,51 @@ ipcMain.on('request-file-list', async (event, payload) => { }; }); - event.sender.send('file-list-data', serializedFiles); - - // After sending file-list-data, start watcher for the root folder - // Use the same ignoreFilter as used for the scan - // Pass rootDir as payload.folderPath - watcher.initializeWatcher( - payload.folderPath, // rootDir - BrowserWindow.fromWebContents(event.sender), - ignoreFilter, - // For defaultIgnoreFilterInstance, use the system default filter - require('./ignore-manager.js').systemDefaultFilter, - // processSingleFileCallback - (filePath) => - require('./file-processor.js').processSingleFile( - filePath, - payload.folderPath, - ignoreFilter, - payload?.ignoreMode ?? currentIgnoreMode - ) - ); + // Calculate total token count + const totalTokens = serializedFiles.reduce((sum, file) => sum + (file.tokenCount || 0), 0); + const TOKEN_THRESHOLD = 500000; + + console.log(`[MAIN] Token count check: ${totalTokens} tokens, threshold: ${TOKEN_THRESHOLD}`); + + // Check if folder exceeds token threshold + if (totalTokens > TOKEN_THRESHOLD) { + // Store the large folder data temporarily + pendingLargeFolderData = { + folderPath: payload.folderPath, + files: serializedFiles, + ignoreFilter: ignoreFilter, + payload: payload + }; + + console.log(`[MAIN] Sending large-folder-warning for ${payload.folderPath} with ${totalTokens} tokens`); + // Send warning to renderer + event.sender.send('large-folder-warning', { totalTokens, folderPath: payload.folderPath }); + + // Do NOT send file-list-data yet - wait for user's choice + } else { + console.log(`[MAIN] Sending file-list-data for ${payload.folderPath} with ${totalTokens} tokens`); + // Send data as an object with a 'selectAll' flag + event.sender.send('file-list-data', { files: serializedFiles, selectAll: true }); + + // After sending file-list-data, start watcher for the root folder + // Use the same ignoreFilter as used for the scan + // Pass rootDir as payload.folderPath + watcher.initializeWatcher( + payload.folderPath, // rootDir + BrowserWindow.fromWebContents(event.sender), + ignoreFilter, + // For defaultIgnoreFilterInstance, use the system default filter + require('./ignore-manager.js').systemDefaultFilter, + // processSingleFileCallback + (filePath) => + require('./file-processor.js').processSingleFile( + filePath, + payload.folderPath, + ignoreFilter, + payload?.ignoreMode ?? currentIgnoreMode + ) + ); + } } catch (err) { console.error('Error processing file list:', err); stopFileProcessing(); // Stop file processor state @@ -468,6 +789,296 @@ ipcMain.on('request-file-list', async (event, payload) => { } }); +// Handler to proceed with full selection for large folders +ipcMain.on('proceed-with-large-folder', async (event, folderPath) => { + if (!pendingLargeFolderData) { + console.error('No pending large folder data available'); + event.sender.send('file-processing-status', { + status: 'error', + message: 'Large folder data is no longer available. Please try selecting the folder again.' + }); + return; + } + + if (pendingLargeFolderData.folderPath === folderPath) { + console.log('[MAIN] User chose to proceed with large folder, performing FULL scan with actual token counts:', folderPath); + + // User explicitly chose "Proceed Anyway" - perform full processing with actual token counts + isLoadingDirectory = true; + startFileProcessing(); + setupDirectoryLoadingTimeout(BrowserWindow.fromWebContents(event.sender), folderPath); + + event.sender.send('file-processing-status', { + status: 'processing', + message: 'Reading files and calculating exact token counts...', + }); + + try { + // Use full file processing to get actual token counts (not estimates) + const result = await readFilesRecursively( + pendingLargeFolderData.folderPath, // dir + pendingLargeFolderData.folderPath, // rootDir + pendingLargeFolderData.ignoreFilter, // ignoreFilter + BrowserWindow.fromWebContents(event.sender), // window + currentProgress, // progress + pendingLargeFolderData.folderPath, // currentDir + pendingLargeFolderData.payload?.ignoreMode ?? currentIgnoreMode, // ignoreMode + null // fileQueue + ); + + const files = result.results; + + pendingLargeFolderData.files = files; + + } catch (error) { + console.error('Error during confirmed large folder scan:', error); + event.sender.send('file-processing-status', { + status: 'error', + message: `Error scanning folder: ${error.message}` + }); + + // Clean up loading state on error + isLoadingDirectory = false; + stopFileProcessing(); + if (loadingTimeoutId) { + clearTimeout(loadingTimeoutId); + loadingTimeoutId = null; + } + pendingLargeFolderData = null; + return; + } finally { + isLoadingDirectory = false; + stopFileProcessing(); + if (loadingTimeoutId) { + clearTimeout(loadingTimeoutId); + loadingTimeoutId = null; + } + } + + // Check if pendingLargeFolderData still exists + if (!pendingLargeFolderData) { + console.error('pendingLargeFolderData is null when trying to proceed with large folder'); + event.sender.send('file-processing-status', { + status: 'error', + message: 'Large folder data is no longer available. Please try selecting the folder again.' + }); + return; + } + + // Send the data with selectAll: true + event.sender.send('file-list-data', { files: pendingLargeFolderData.files, selectAll: true }); + + // Initialize watcher after sending data + console.log('Large folder data sent with selectAll: true'); + + // Initialize watcher for the large folder + try { + // Store references before clearing pendingLargeFolderData + const folderPath = pendingLargeFolderData.folderPath; + const ignoreFilter = pendingLargeFolderData.ignoreFilter; + + await watcher.initializeWatcher( + folderPath, // rootDir + BrowserWindow.fromWebContents(event.sender), + ignoreFilter, + // For defaultIgnoreFilterInstance, use the system default filter + require('./ignore-manager.js').systemDefaultFilter, + // processSingleFileCallback + (filePath) => + require('./file-processor.js').processSingleFile( + filePath, + folderPath, // Use stored folderPath, not pendingLargeFolderData.folderPath + ignoreFilter // Use stored ignoreFilter, not pendingLargeFolderData.ignoreFilter + ) + ); + console.log('[MAIN] Watcher initialized for large folder (selectAll: true)'); + } catch (error) { + console.error('[MAIN] Failed to initialize watcher for large folder:', error); + } + + // Clear the pending data + pendingLargeFolderData = null; + } +}); + +// Handler to load files but keep them deselected +ipcMain.on('load-large-folder-deselected', async (event, folderPath) => { + if (!pendingLargeFolderData) { + console.error('No pending large folder data available'); + event.sender.send('file-processing-status', { + status: 'error', + message: 'Large folder data is no longer available. Please try selecting the folder again.' + }); + return; + } + + if (pendingLargeFolderData.folderPath === folderPath) { + console.log('[MAIN] User chose to load large folder with files deselected, starting lightweight scan:', folderPath); + + // Always perform lightweight scan for large folders (per TASKS_4.md) + isLoadingDirectory = true; + startFileProcessing(); + setupDirectoryLoadingTimeout(BrowserWindow.fromWebContents(event.sender), folderPath); + + event.sender.send('file-processing-status', { + status: 'processing', + message: 'Scanning file structure (lightweight mode)...', + }); + + try { + const lightweightFiles = await scanDirectoryLightweight( + pendingLargeFolderData.folderPath, + pendingLargeFolderData.ignoreFilter + ); + + const files = lightweightFiles.filter(file => !file.isDirectory).map(file => ({ + path: file.path, + relativePath: file.relativePath, + name: file.name, + size: file.size || 0, + isDirectory: false, + extension: path.extname(file.name).toLowerCase(), + excluded: isPathExcludedByDefaults( + file.path, + pendingLargeFolderData.payload.folderPath, + pendingLargeFolderData.payload.ignoreMode ?? currentIgnoreMode + ), + content: '', // Empty content for lightweight mode + tokenCount: file.estimatedTokens || 0, + isTokenEstimate: true, // Flag to indicate this is an estimate + isBinary: false, // Will be determined later if file is selected + isSkipped: false, + error: null, + })); + + pendingLargeFolderData.files = files; + + const fileCount = files.length; + const totalItems = lightweightFiles.length; + const dirCount = totalItems - fileCount; + console.log(`[MAIN] Lightweight scan completed: ${fileCount} files found (${dirCount} directories excluded from file list)`); + + } catch (error) { + console.error('Error during confirmed large folder scan:', error); + event.sender.send('file-processing-status', { + status: 'error', + message: `Error scanning folder: ${error.message}` + }); + + // Clean up loading state on error + isLoadingDirectory = false; + stopFileProcessing(); + if (loadingTimeoutId) { + clearTimeout(loadingTimeoutId); + loadingTimeoutId = null; + } + pendingLargeFolderData = null; + return; + } finally { + isLoadingDirectory = false; + stopFileProcessing(); + if (loadingTimeoutId) { + clearTimeout(loadingTimeoutId); + loadingTimeoutId = null; + } + } + + // Check if pendingLargeFolderData still exists + if (!pendingLargeFolderData) { + console.error('pendingLargeFolderData is null when trying to proceed with large folder'); + event.sender.send('file-processing-status', { + status: 'error', + message: 'Large folder data is no longer available. Please try selecting the folder again.' + }); + return; + } + + // Send the data with selectAll: false + event.sender.send('file-list-data', { files: pendingLargeFolderData.files, selectAll: false }); + + // Initialize watcher after sending data + console.log('Large folder data sent with selectAll: false'); + + // Initialize watcher for the large folder + try { + // Store references before clearing pendingLargeFolderData + const folderPath = pendingLargeFolderData.folderPath; + const ignoreFilter = pendingLargeFolderData.ignoreFilter; + + await watcher.initializeWatcher( + folderPath, // rootDir + BrowserWindow.fromWebContents(event.sender), + ignoreFilter, + // For defaultIgnoreFilterInstance, use the system default filter + require('./ignore-manager.js').systemDefaultFilter, + // processSingleFileCallback + (filePath) => + require('./file-processor.js').processSingleFile( + filePath, + folderPath, // Use stored folderPath, not pendingLargeFolderData.folderPath + ignoreFilter // Use stored ignoreFilter, not pendingLargeFolderData.ignoreFilter + ) + ); + console.log('[MAIN] Watcher initialized for large folder (selectAll: false)'); + } catch (error) { + console.error('[MAIN] Failed to initialize watcher for large folder:', error); + } + + // Clear the pending data + pendingLargeFolderData = null; + } +}); + +// Handler to cancel the large folder load +ipcMain.on('cancel-large-folder-load', () => { + // Clear the pending data + pendingLargeFolderData = null; + console.log('Large folder load cancelled by user'); +}); + +// Handler for on-demand file processing when files are selected +ipcMain.handle('process-selected-files', async (event, filePaths) => { + console.log(`[MAIN] Processing ${filePaths.length} selected files for real tokenization...`); + + const fileProcessor = require('./file-processor'); + const processedFiles = []; + + try { + for (const filePath of filePaths) { + try { + // Process each file individually to get real content and tokens + // Create a minimal ignore filter for individual file processing + const { createGlobalIgnoreFilter } = require('./ignore-manager'); + const minimalIgnoreFilter = createGlobalIgnoreFilter([]); + + const processedFile = await fileProcessor.processSingleFile( + filePath, + require('path').dirname(filePath), // Use file's directory as rootDir + minimalIgnoreFilter, // Use minimal ignore filter + 'automatic' // Default ignore mode + ); + + if (processedFile) { + processedFiles.push({ + path: filePath, + ...processedFile, + isTokenEstimate: false // Mark as real tokenization + }); + } + } catch (error) { + console.warn(`Error processing file ${filePath}:`, error.message); + // Keep the original estimated data if processing fails + } + } + + console.log(`[MAIN] Successfully processed ${processedFiles.length}/${filePaths.length} files`); + return { success: true, processedFiles }; + } catch (error) { + console.error('Error in process-selected-files:', error); + return { success: false, error: error.message }; + } +}); + // Handle fetch-models request from renderer ipcMain.handle('fetch-models', async () => { try { @@ -642,7 +1253,9 @@ function createWindow() { }); if (process.env.NODE_ENV === 'development') { - mainWindow.loadURL('http://localhost:5173'); + const devURL = process.env.ELECTRON_START_URL || 'http://localhost:5173'; + console.log('Loading development URL:', devURL); + mainWindow.loadURL(devURL); mainWindow.webContents.openDevTools(); } else { const prodPath = app.isPackaged diff --git a/electron/preload.js b/electron/preload.js index 1bbc6b45..40d24963 100644 --- a/electron/preload.js +++ b/electron/preload.js @@ -48,6 +48,9 @@ contextBridge.exposeInMainWorld('electron', { 'request-file-list', 'debug-file-selection', 'cancel-directory-loading', + 'proceed-with-large-folder', + 'load-large-folder-deselected', + 'cancel-large-folder-load', ]; if (validChannels.includes(channel)) { // Ensure data is serializable before sending @@ -64,6 +67,7 @@ contextBridge.exposeInMainWorld('electron', { 'file-added', 'file-updated', 'file-removed', + 'large-folder-warning', ]; if (validChannels.includes(channel)) { // Remove any existing listeners to avoid duplicates @@ -101,6 +105,7 @@ contextBridge.exposeInMainWorld('electron', { 'file-added', 'file-updated', 'file-removed', + 'large-folder-warning', ]; if (validChannels.includes(channel)) { ipcRenderer.removeListener(channel, (event, ...args) => func(...args)); @@ -113,7 +118,8 @@ contextBridge.exposeInMainWorld('electron', { 'check-for-updates', 'get-token-count', 'fetch-models', - ]; // Added 'fetch-models' + 'process-selected-files', + ]; // Added 'fetch-models' and 'process-selected-files' if (validChannels.includes(channel)) { return ipcRenderer.invoke(channel, data); } diff --git a/index.html b/index.html index 02219ea8..d0e9896c 100644 --- a/index.html +++ b/index.html @@ -2,7 +2,7 @@ - + - + \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index c704e33e..85e5a837 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,7 +10,6 @@ "license": "MIT", "dependencies": { "chokidar": "^3.6.0", - "gpt-3-encoder": "^1.1.4", "ignore": "^7.0.3", "lodash": "^4.17.21", "lucide-react": "^0.477.0", @@ -4679,12 +4678,6 @@ "url": "https://github.com/sindresorhus/got?sponsor=1" } }, - "node_modules/gpt-3-encoder": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/gpt-3-encoder/-/gpt-3-encoder-1.1.4.tgz", - "integrity": "sha512-fSQRePV+HUAhCn7+7HL7lNIXNm6eaFWFbNLOOGtmSJ0qJycyQvj60OvRlH7mee8xAMjBDNRdMXlMwjAbMTDjkg==", - "license": "MIT" - }, "node_modules/graceful-fs": { "version": "4.2.11", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", diff --git a/package.json b/package.json index cef65017..b1571207 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "pastemax", - "version": "v1.1.0-stable", + "version": "1.1.0-stable", "main": "electron/main.js", "scripts": { "dev:all": "concurrently \"npm:dev\" \"npm:dev:electron\"", @@ -103,7 +103,6 @@ "asarUnpack": [ "node_modules/ignore/**", "node_modules/tiktoken/**", - "node_modules/gpt-3-encoder/**", "node_modules/chokidar/**" ], "asar": true, @@ -136,7 +135,6 @@ }, "dependencies": { "chokidar": "^3.6.0", - "gpt-3-encoder": "^1.1.4", "ignore": "^7.0.3", "lodash": "^4.17.21", "lucide-react": "^0.477.0", diff --git a/scripts/README.md b/scripts/README.md index 0cdebaf0..e308a9da 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -1,15 +1,14 @@ # Build Scripts -This directory contains various utility scripts for building, packaging, and debugging the Electron application. +This directory contains various utility scripts for building, packaging, debugging, and testing the Electron application. ## Available Scripts ### `verify-build.js` -Verifies that your package.json build configuration is correct for Electron builds. It checks that all required fields are present and that the main file exists. - -Usage: +Verifies that your `package.json` build configuration is correct for Electron builds. It checks that all required fields are present and that the main file exists. +**Usage:** ```bash npm run verify-build ``` @@ -18,8 +17,7 @@ npm run verify-build Tests the complete build and packaging process for Electron locally. This is useful for debugging issues with the build process before pushing to GitHub. -Usage: - +**Usage:** ```bash # Test the build for the current platform npm run test-build @@ -30,19 +28,44 @@ npm run test-build:win npm run test-build:linux ``` -## Debugging GitHub Actions +### `notarize.js` -If you're having issues with GitHub Actions not building the binaries correctly, use the debug workflow: +This script is called by `electron-builder` after signing a macOS app. It handles the notarization process with Apple, which is required for distribution on macOS. It uses environment variables (`APPLE_ID`, `APPLE_APP_SPECIFIC_PASSWORD`, `TEAM_ID`) for authentication. + +**Usage:** This script is typically run automatically by the build process and does not need to be called manually. + +### `fix-dependencies.js` + +A utility script to help ensure that critical native dependencies (like `tiktoken` and `ignore`) are correctly unpacked from the ASAR archive in the final packaged application. It modifies the `asarUnpack` configuration in `package.json`. -1. Run the debug-gh-release script to create a debug tag: +**Usage:** Run this script if you encounter "module not found" errors in the packaged application. +```bash +node scripts/fix-dependencies.js +``` + +### `test-file-watcher.js` + +A script to test the file watcher functionality. Run this script in a directory that PasteMax is currently watching to simulate file additions, modifications, and deletions. +**Usage:** +1. Open PasteMax and select a folder. +2. In your terminal, `cd` into that same folder. +3. Run the script: ```bash - npm run debug-gh-release + node /path/to/pastemax/scripts/test-file-watcher.js ``` +4. Observe the output in the terminal and see the file list update in the PasteMax UI. -2. This will trigger the `.github/workflows/debug-build.yml` workflow, which includes extensive logging. +## Debugging GitHub Actions + +If you're having issues with GitHub Actions not building the binaries correctly, use the debug workflow: -3. Check the GitHub Actions logs for detailed information about the build process. +1. Run the `debug-gh-release` script to create a debug tag: + ```bash + npm run debug-gh-release + ``` +2. This will trigger the `.github/workflows/debug-build.yml` workflow, which includes extensive logging. +3. Check the GitHub Actions logs for detailed information about the build process. ## Troubleshooting Common Issues diff --git a/scripts/fix-dependencies.js b/scripts/fix-dependencies.js index 5a5ba7ad..52533aad 100644 --- a/scripts/fix-dependencies.js +++ b/scripts/fix-dependencies.js @@ -13,7 +13,7 @@ console.log('🔧 PasteMax Dependency Fixer'); console.log('============================'); // Define the dependencies we need to ensure are installed -const criticalDependencies = ['ignore', 'tiktoken', 'gpt-3-encoder']; +const criticalDependencies = ['ignore', 'tiktoken']; // Get the application path (platform-dependent) function getAppResourcesPath() { @@ -61,7 +61,7 @@ function fixDependencies() { // Install required dependencies console.log('📦 Installing dependencies locally...'); - execSync('npm install ignore tiktoken gpt-3-encoder --no-save', { + execSync('npm install ignore tiktoken --no-save', { stdio: 'inherit', }); @@ -79,7 +79,6 @@ function fixDependencies() { packageJson.build.asarUnpack = [ 'node_modules/ignore/**', 'node_modules/tiktoken/**', - 'node_modules/gpt-3-encoder/**', ]; // Write updated package.json diff --git a/src/App.tsx b/src/App.tsx index 82e89c85..6b9931cf 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -9,6 +9,7 @@ import IgnoreListModal from './components/IgnoreListModal'; import ThemeToggle from './components/ThemeToggle'; import UpdateModal from './components/UpdateModal'; import { useIgnorePatterns } from './hooks/useIgnorePatterns'; +import { useWorkspaces } from './hooks/useWorkspaces'; import UserInstructions from './components/UserInstructions'; import { STORAGE_KEY_TASK_TYPE } from './types/TaskTypes'; import { @@ -28,6 +29,9 @@ import CopyHistoryModal, { CopyHistoryItem } from './components/CopyHistoryModal import CopyHistoryButton from './components/CopyHistoryButton'; import ModelDropdown from './components/ModelDropdown'; import ToggleSwitch from './components/base/ToggleSwitch'; +import LargeFolderModal from './components/LargeFolderModal'; +import LargeSubfolderModal from './components/LargeSubfolderModal'; +import ProcessingOverlay from './components/ProcessingOverlay'; /** * Import path utilities for handling file paths across different operating systems. @@ -91,45 +95,7 @@ const App = (): JSX.Element => { const [allFiles, setAllFiles] = useState([] as FileData[]); /* ============================== STATE: Workspace Management ============================== */ - const [isWorkspaceManagerOpen, setIsWorkspaceManagerOpen] = useState(false); - const [currentWorkspaceId, setCurrentWorkspaceId] = useState(() => { - return localStorage.getItem(STORAGE_KEYS.CURRENT_WORKSPACE) || null; - }); - // State for confirm folder modal - const [isConfirmUseFolderModalOpen, setIsConfirmUseFolderModalOpen] = useState(false); - const [confirmFolderModalDetails, setConfirmFolderModalDetails] = useState<{ - workspaceId: string | null; - workspaceName: string; - folderPath: string; - }>({ workspaceId: null, workspaceName: '', folderPath: '' }); - - const [workspaces, setWorkspaces] = useState(() => { - const savedWorkspaces = localStorage.getItem(STORAGE_KEYS.WORKSPACES); - if (savedWorkspaces) { - try { - const parsed = JSON.parse(savedWorkspaces); - if (Array.isArray(parsed)) { - console.log(`Initialized workspaces state with ${parsed.length} workspaces`); - return parsed as Workspace[]; - } else { - console.warn( - 'Invalid workspaces data in localStorage (not an array), resetting to empty array' - ); - localStorage.setItem(STORAGE_KEYS.WORKSPACES, JSON.stringify([])); - return [] as Workspace[]; - } - } catch (error) { - console.error('Failed to parse workspaces from localStorage during initialization:', error); - // Reset localStorage to prevent further errors - localStorage.setItem(STORAGE_KEYS.WORKSPACES, JSON.stringify([])); - return [] as Workspace[]; - } - } - // Initialize with empty array and ensure localStorage has a valid value - console.log('No workspaces found in localStorage, initializing with empty array'); - localStorage.setItem(STORAGE_KEYS.WORKSPACES, JSON.stringify([])); - return [] as Workspace[]; - }); + // Workspace management is now handled by the useWorkspaces hook /* ============================== STATE: Ignore Patterns ============================== */ const { @@ -148,6 +114,17 @@ const App = (): JSX.Element => { const [selectedFiles, setSelectedFiles] = useState( (savedFiles ? JSON.parse(savedFiles).map(normalizePath) : []) as string[] ); + + // Debug logging for selectedFiles changes + useEffect(() => { + console.log(`[DEBUG] selectedFiles changed: ${selectedFiles.length} files selected`); + if (selectedFiles.length > 0 && selectedFiles.length < 10) { + console.log(`[DEBUG] Selected files:`, selectedFiles); + } else if (selectedFiles.length >= 10) { + console.log(`[DEBUG] Too many files to log individually (${selectedFiles.length})`); + } + }, [selectedFiles]); + const [sortOrder, setSortOrder] = useState(savedSortOrder || 'tokens-desc'); const [searchTerm, setSearchTerm] = useState(savedSearchTerm || ''); const [expandedNodes, setExpandedNodes] = useState({} as Record); @@ -160,6 +137,10 @@ const App = (): JSX.Element => { const [includeBinaryPaths, setIncludeBinaryPaths] = useState( localStorage.getItem(STORAGE_KEYS.INCLUDE_BINARY_PATHS) === 'true' ); + const [processingFiles, setProcessingFiles] = useState(new Set()); + const [isBatchProcessing, setIsBatchProcessing] = useState(false); + const [isFolderProcessing, setIsFolderProcessing] = useState(false); + const [processingFolderName, setProcessingFolderName] = useState(''); /* ============================== STATE: UI Controls ============================== */ const [sortDropdownOpen, setSortDropdownOpen] = useState(false); @@ -195,6 +176,15 @@ const App = (): JSX.Element => { }); const [isCopyHistoryModalOpen, setIsCopyHistoryModalOpen] = useState(false); + /* ============================== STATE: Large Folder Modal ============================== */ + const [isLargeFolderModalOpen, setIsLargeFolderModalOpen] = useState(false); + const [largeFolderDetails, setLargeFolderDetails] = useState({ totalTokens: 0, folderPath: '' }); + + /* ============================== STATE: Large Subfolder Modal ============================== */ + const [isLargeSubfolderModalOpen, setIsLargeSubfolderModalOpen] = useState(false); + const [largeSubfolderDetails, setLargeSubfolderDetails] = useState({ totalTokens: 0, folderPath: '', hasEstimates: false }); + const [pendingFolderSelection, setPendingFolderSelection] = useState<{ folderPath: string; isSelected: boolean } | null>(null); + const [selectedModelId, setSelectedModelId] = useState(() => { const savedModelId = localStorage.getItem('pastemax-selected-model'); return savedModelId || ''; @@ -481,23 +471,15 @@ const App = (): JSX.Element => { // Clear selections if folder changed if (!arePathsEqual(normalizedFolderPath, selectedFolder)) { setSelectedFiles([]); + // Reset expanded nodes to only show the root folder expanded + const newExpandedState = { [`node-${normalizedFolderPath}`]: true }; + setExpandedNodes(newExpandedState); + localStorage.setItem(STORAGE_KEYS.EXPANDED_NODES, JSON.stringify(newExpandedState)); } - // Update current workspace's folder path if a workspace is active - if (currentWorkspaceId) { - setWorkspaces((prevWorkspaces: Workspace[]) => { - const updatedWorkspaces = prevWorkspaces.map((workspace: Workspace) => - workspace.id === currentWorkspaceId - ? { ...workspace, folderPath: normalizedFolderPath, lastUsed: Date.now() } - : workspace - ); - // Save to localStorage - localStorage.setItem(STORAGE_KEYS.WORKSPACES, JSON.stringify(updatedWorkspaces)); - return updatedWorkspaces; - }); - } + // Workspace update is handled by the useWorkspaces hook }, - [selectedFolder, allFiles, processingStatus, currentWorkspaceId] + [selectedFolder, allFiles, processingStatus] ); // The handleFileListData function is implemented as stableHandleFileListData below @@ -522,7 +504,22 @@ const App = (): JSX.Element => { ); const stableHandleFileListData = useCallback( - (files: FileData[]) => { + (payload: { files: FileData[], selectAll: boolean }) => { + const { files, selectAll } = payload; + + console.log(`[handleFileListData] Called with ${files.length} files, selectAll: ${selectAll}`); + + // Debug: Check token counts in loaded files + const filesWithTokens = files.filter(f => f.tokenCount > 0); + console.log(`[TOKEN DEBUG] Loaded ${filesWithTokens.length} files with tokens out of ${files.length}`); + if (filesWithTokens.length > 0) { + console.log(`[TOKEN DEBUG] Sample files:`, filesWithTokens.slice(0, 3).map(f => ({ + name: f.name, + tokens: f.tokenCount, + isEstimate: f.isTokenEstimate + }))); + } + setAllFiles((prevFiles: FileData[]) => { if (files.length !== prevFiles.length) { console.debug( @@ -540,22 +537,34 @@ const App = (): JSX.Element => { message: `Loaded ${files.length} files`, }); - setSelectedFiles((prevSelected: string[]) => { - // If we have previous selections, preserve all existing selections + // Force clear selections if selectAll is false - do this immediately and separately + if (!selectAll) { + console.log('[handleFileListData] selectAll is false, FORCE clearing all selections'); + setSelectedFiles([]); + return; // Exit early to prevent any other selection logic + } + + setSelectedFiles((prevSelected: string[]) => { + // If we have previous selections, preserve all existing selections that still exist if (prevSelected.length > 0) { // Only filter out files that no longer exist in the new list - return prevSelected.filter((selectedPath: string) => + const preservedSelections = prevSelected.filter((selectedPath: string) => files.some((file) => arePathsEqual(file.path, selectedPath)) ); + console.log(`[handleFileListData] Preserving ${preservedSelections.length} existing selections`); + return preservedSelections; } - // No previous selections - select all eligible files - return files + // No previous selections - select all eligible files if selectAll is true + console.log(`[handleFileListData] selectAll is true, auto-selecting eligible files`); + const eligibleFiles = files .filter( (file: FileData) => !file.isSkipped && !file.excludedByDefault && (includeBinaryPaths || !file.isBinary) ) .map((file: FileData) => file.path); + console.log(`[handleFileListData] Auto-selecting ${eligibleFiles.length} files`); + return eligibleFiles; }); }, [includeBinaryPaths] @@ -595,9 +604,17 @@ const App = (): JSX.Element => { stableHandleFolderSelectedRef.current(folderPath); }; - const handleFileListDataIPC = (files: FileData[]) => { - console.log('[IPC] Received file-list-data:', files.length, 'files'); - stableHandleFileListDataRef.current(files); + const handleFileListDataIPC = (payload: FileData[] | { files: FileData[], selectAll: boolean }) => { + // Handle both old and new payload formats + if (Array.isArray(payload)) { + // Old format - backward compatibility + console.log('[IPC] Received file-list-data (legacy format):', payload.length, 'files'); + stableHandleFileListDataRef.current({ files: payload, selectAll: true }); + } else { + // New format + console.log('[IPC] Received file-list-data:', payload.files.length, 'files, selectAll:', payload.selectAll); + stableHandleFileListDataRef.current(payload); + } }; type ProcessingStatusIPCHandler = (payload: FileProcessingStatusIPCPayload) => void; @@ -647,6 +664,22 @@ const App = (): JSX.Element => { }; }, [isElectron]); + // Listen for large folder warning from main process + useEffect(() => { + if (!isElectron) return; + const handleLargeFolderWarning = (details: { totalTokens: number; folderPath: string }) => { + console.log(`[APP] Received large-folder-warning:`, details); + setLargeFolderDetails(details); + setIsLargeFolderModalOpen(true); + setProcessingStatus({ status: 'idle', message: '' }); // Stop the "loading" indicator + console.log(`[APP] Modal should now be open: isLargeFolderModalOpen set to true`); + }; + window.electron.ipcRenderer.on('large-folder-warning', handleLargeFolderWarning); + return () => { + window.electron.ipcRenderer.removeListener('large-folder-warning', handleLargeFolderWarning); + }; + }, [isElectron]); + /* ============================== HANDLERS & UTILITIES ============================== */ /** @@ -703,6 +736,56 @@ const App = (): JSX.Element => { } }; + // Initialize workspace management hook + const { + workspaces, + setWorkspaces, + currentWorkspaceId, + setCurrentWorkspaceId, + isWorkspaceManagerOpen, + setIsWorkspaceManagerOpen, + isConfirmUseFolderModalOpen, + setIsConfirmUseFolderModalOpen, + confirmFolderModalDetails, + setConfirmFolderModalDetails, + currentWorkspaceName, + handleOpenWorkspaceManager, + handleSelectWorkspace, + handleCreateWorkspace, + handleDeleteWorkspace, + handleUpdateWorkspaceFolder, + handleConfirmUseCurrentFolder, + handleDeclineUseCurrentFolder, + } = useWorkspaces({ + selectedFolder, + setSelectedFolder, + setSelectedFiles, + setAllFiles, + setProcessingStatus, + openFolder, + handleFolderSelected, + isElectron, + }); + + // Large folder modal handlers + const handleProceedWithLargeFolder = () => { + window.electron.ipcRenderer.send('proceed-with-large-folder', largeFolderDetails.folderPath); + setIsLargeFolderModalOpen(false); + setProcessingStatus({ status: 'processing', message: 'Loading large folder...' }); + }; + + const handleLoadLargeFolderDeselected = () => { + window.electron.ipcRenderer.send('load-large-folder-deselected', largeFolderDetails.folderPath); + setIsLargeFolderModalOpen(false); + setProcessingStatus({ status: 'processing', message: 'Loading folder with files deselected...' }); + }; + + const handleCancelLargeFolder = () => { + window.electron.ipcRenderer.send('cancel-large-folder-load'); + clearSavedState(); + setIsLargeFolderModalOpen(false); + }; + // Apply filters and sorting to files const applyFiltersAndSort = useCallback( (files: FileData[], sort: string, filter: string) => { @@ -808,8 +891,54 @@ const App = (): JSX.Element => { }; }, [isElectron, handleFileAdded, handleFileUpdated, handleFileRemoved]); + // Process files for real tokenization when selected + const processFileForRealTokens = async (filePath: string) => { + if (!isElectron) return; + + // Add file to processing set + setProcessingFiles(prev => new Set([...prev, filePath])); + + try { + console.log(`[APP] Processing file for real tokens: ${filePath}`); + const result = await window.electron.ipcRenderer.invoke('process-selected-files', [filePath]); + + if (result.success && result.processedFiles.length > 0) { + const processedFile = result.processedFiles[0]; + + // Update the file in allFiles with real token data + setAllFiles((prevFiles: FileData[]) => + prevFiles.map((file: FileData) => { + if (arePathsEqual(file.path, filePath)) { + console.log(`[TOKEN UPDATE] Updating ${file.name}: ${file.tokenCount} -> ${processedFile.tokenCount}`); + return { + ...file, + content: processedFile.content, + tokenCount: processedFile.tokenCount, + isTokenEstimate: false, + isBinary: processedFile.isBinary, + error: processedFile.error + }; + } + return file; + }) + ); + + console.log(`[APP] Updated file ${filePath} with real tokens: ${processedFile.tokenCount}`); + } + } catch (error) { + console.error(`[APP] Error processing file ${filePath}:`, error); + } finally { + // Remove file from processing set + setProcessingFiles(prev => { + const newSet = new Set(prev); + newSet.delete(filePath); + return newSet; + }); + } + }; + // Toggle file selection - const toggleFileSelection = (filePath: string) => { + const toggleFileSelection = async (filePath: string) => { // Normalize the incoming file path const normalizedPath = normalizePath(filePath); @@ -827,19 +956,29 @@ const App = (): JSX.Element => { return prev.filter((path: string) => !arePathsEqual(path, normalizedPath)); } else { // Add the file to selected files - return [...prev, normalizedPath]; + const newSelection = [...prev, normalizedPath]; + + // If this file has estimated tokens, process it for real tokenization + if (f && f.isTokenEstimate && !f.isDirectory) { + console.log(`[toggleFileSelection] File ${normalizedPath} has estimated tokens, processing for real tokens`); + processFileForRealTokens(normalizedPath); + } else if (f) { + console.log(`[toggleFileSelection] File ${normalizedPath} - isTokenEstimate: ${f.isTokenEstimate}, isDirectory: ${f.isDirectory}`); + } + + return newSelection; } }); }; - // Toggle folder selection (select/deselect all files in folder) - const toggleFolderSelection = (folderPath: string, isSelected: boolean) => { - // Normalize the folder path for cross-platform compatibility + // Calculate folder tokens and detect if they're estimated + const calculateFolderTokensWithEstimateInfo = (folderPath: string): { totalTokens: number; hasEstimates: boolean } => { const normalizedFolderPath = normalizePath(folderPath); + let totalTokens = 0; + let hasEstimates = false; // Function to check if a file is in the given folder or its subfolders const isFileInFolder = (filePath: string, folderPath: string): boolean => { - // Ensure paths are normalized with consistent slashes let normalizedFilePath = normalizePath(filePath); let normalizedFolderPath = normalizePath(folderPath); @@ -852,61 +991,200 @@ const App = (): JSX.Element => { normalizedFolderPath = '/' + normalizedFolderPath; } - // A file is in the folder if: - // 1. The paths are equal (exact match) - // 2. The file path is a subpath of the folder - const isMatch = - arePathsEqual(normalizedFilePath, normalizedFolderPath) || - isSubPath(normalizedFolderPath, normalizedFilePath); + return arePathsEqual(normalizedFilePath, normalizedFolderPath) || isSubPath(normalizedFolderPath, normalizedFilePath); + }; + + // Find all files in this folder + const filesInFolder = allFiles.filter((file: FileData) => { + const inFolder = isFileInFolder(file.path, normalizedFolderPath); + const selectable = !file.isSkipped && !file.excludedByDefault && (includeBinaryPaths || !file.isBinary); + return selectable && inFolder && !file.isDirectory; + }); + + // Calculate total tokens and check for estimates + filesInFolder.forEach((file: FileData) => { + if (file.tokenCount > 0) { + totalTokens += file.tokenCount; + if (file.isTokenEstimate) { + hasEstimates = true; + } + } + }); + + return { totalTokens, hasEstimates }; + }; + + // Toggle folder selection (select/deselect all files in folder) + const toggleFolderSelection = (folderPath: string, isSelected: boolean) => { + // Normalize the folder path for cross-platform compatibility + const normalizedFolderPath = normalizePath(folderPath); + + // Check for large folder selection (500k+ tokens) before proceeding + if (isSelected) { + const { totalTokens, hasEstimates } = calculateFolderTokensWithEstimateInfo(normalizedFolderPath); + const TOKEN_THRESHOLD = 500000; // 500k tokens + + if (totalTokens >= TOKEN_THRESHOLD) { + console.log(`[toggleFolderSelection] Large folder detected: ${totalTokens.toLocaleString()} tokens (estimated: ${hasEstimates})`); + + // Store the pending selection and show modal + setPendingFolderSelection({ folderPath: normalizedFolderPath, isSelected }); + setLargeSubfolderDetails({ + totalTokens, + folderPath: normalizedFolderPath, + hasEstimates + }); + setIsLargeSubfolderModalOpen(true); + return; // Don't proceed with selection until user confirms + } + } + + // For non-large folders, proceed with normal selection + performFolderSelection(normalizedFolderPath, isSelected); + }; + + // Perform the actual folder selection (extracted from toggleFolderSelection) + const performFolderSelection = async (folderPath: string, isSelected: boolean) => { + const normalizedFolderPath = normalizePath(folderPath); + const folderName = normalizedFolderPath.split(/[/\\]/).pop() || normalizedFolderPath; + + // Function to check if a file is in the given folder or its subfolders (same as in toggleFolderSelection) + const isFileInFolder = (filePath: string, folderPath: string): boolean => { + let normalizedFilePath = normalizePath(filePath); + let normalizedFolderPath = normalizePath(folderPath); + + if (!normalizedFilePath.startsWith('/') && !normalizedFilePath.match(/^[a-z]:/i)) { + normalizedFilePath = '/' + normalizedFilePath; + } - if (isMatch) { - // File is in folder + if (!normalizedFolderPath.startsWith('/') && !normalizedFolderPath.match(/^[a-z]:/i)) { + normalizedFolderPath = '/' + normalizedFolderPath; } - return isMatch; + return arePathsEqual(normalizedFilePath, normalizedFolderPath) || isSubPath(normalizedFolderPath, normalizedFilePath); }; // Filter all files to get only those in this folder (and subfolders) that are selectable const filesInFolder = allFiles.filter((file: FileData) => { const inFolder = isFileInFolder(file.path, normalizedFolderPath); - const selectable = - !file.isSkipped && !file.excludedByDefault && (includeBinaryPaths || !file.isBinary); + const selectable = !file.isSkipped && !file.excludedByDefault && (includeBinaryPaths || !file.isBinary); return selectable && inFolder; }); console.log('Found', filesInFolder.length, 'selectable files in folder'); - // If no selectable files were found, do nothing if (filesInFolder.length === 0) { console.warn('No selectable files found in folder, nothing to do'); return; } - // Extract just the paths from the files and normalize them const folderFilePaths = filesInFolder.map((file: FileData) => normalizePath(file.path)); if (isSelected) { - // Adding files - create a new Set with all existing + new files - setSelectedFiles((prev: string[]) => { - const existingSelection = new Set(prev.map(normalizePath)); - folderFilePaths.forEach((pathToAdd: string) => existingSelection.add(pathToAdd)); - const newSelection = Array.from(existingSelection); - console.log( - `Added ${folderFilePaths.length} files to selection, total now: ${newSelection.length}` - ); - return newSelection; - }); + // Check if significant processing is needed (threshold: 25+ files with estimates OR 100+ total files) + const filesToProcess = filesInFolder.filter((file: FileData) => + file.isTokenEstimate && !file.isDirectory + ); + + const needsSignificantProcessing = filesToProcess.length >= 25 || filesInFolder.length >= 100; + + if (needsSignificantProcessing) { + // Show processing overlay before starting + setIsFolderProcessing(true); + setProcessingFolderName(folderName); + console.log(`[performFolderSelection] Starting processing for folder "${folderName}" with ${filesToProcess.length} files to process`); + + try { + // Process files in a single batch call if needed + if (filesToProcess.length > 0) { + console.log(`[performFolderSelection] Starting batch processing of ${filesToProcess.length} files`); + const filePaths = filesToProcess.map(file => file.path); + const result = await window.electron.ipcRenderer.invoke('process-selected-files', filePaths); + + if (result.success && result.processedFiles.length > 0) { + // Update all processed files in state + setAllFiles((prevFiles: FileData[]) => + prevFiles.map((file: FileData) => { + const processedFile = result.processedFiles.find((pf: FileData) => + arePathsEqual(pf.path, file.path) + ); + + if (processedFile) { + console.log(`[performFolderSelection] Updated ${file.name}: ${file.tokenCount} -> ${processedFile.tokenCount}`); + return { + ...file, + content: processedFile.content, + tokenCount: processedFile.tokenCount, + isTokenEstimate: false, + isBinary: processedFile.isBinary, + error: processedFile.error + }; + } + return file; + }) + ); + } + console.log(`[performFolderSelection] Completed batch processing of ${filesToProcess.length} files`); + } + + // Only update selection after processing completes + setSelectedFiles((prev: string[]) => { + const existingSelection = new Set(prev.map(normalizePath)); + folderFilePaths.forEach((pathToAdd: string) => existingSelection.add(pathToAdd)); + const newSelection = Array.from(existingSelection); + console.log(`Added ${folderFilePaths.length} files to selection, total now: ${newSelection.length}`); + return newSelection; + }); + + } catch (error) { + console.error(`[performFolderSelection] Error processing folder "${folderName}":`, error); + } finally { + setIsFolderProcessing(false); + setProcessingFolderName(''); + } + } else { + // For small folders, proceed with immediate selection as before + setSelectedFiles((prev: string[]) => { + const existingSelection = new Set(prev.map(normalizePath)); + folderFilePaths.forEach((pathToAdd: string) => existingSelection.add(pathToAdd)); + const newSelection = Array.from(existingSelection); + console.log(`Added ${folderFilePaths.length} files to selection, total now: ${newSelection.length}`); + return newSelection; + }); + + // Process any remaining files in background + if (filesToProcess.length > 0) { + setIsBatchProcessing(true); + Promise.all( + filesToProcess.map((file: FileData) => processFileForRealTokens(file.path)) + ).finally(() => { + setIsBatchProcessing(false); + }); + } + } } else { - // Removing files - filter out any file that's in our folder + // For deselection, proceed immediately setSelectedFiles((prev: string[]) => { - const newSelection = prev.filter( - (path: string) => !isFileInFolder(path, normalizedFolderPath) - ); + const newSelection = prev.filter((path: string) => !isFileInFolder(path, normalizedFolderPath)); return newSelection; }); } }; + // Handle large subfolder modal responses + const handleLargeSubfolderConfirm = () => { + if (pendingFolderSelection) { + performFolderSelection(pendingFolderSelection.folderPath, pendingFolderSelection.isSelected); + setPendingFolderSelection(null); + } + setIsLargeSubfolderModalOpen(false); + }; + + const handleLargeSubfolderCancel = () => { + setPendingFolderSelection(null); + setIsLargeSubfolderModalOpen(false); + }; + // Handle sort change const handleSortChange = (newSort: string) => { setSortOrder(newSort); @@ -943,6 +1221,37 @@ const App = (): JSX.Element => { ); }; + // Refresh only the currently selected files without reloading the entire folder + const refreshSelectedFiles = async () => { + if (!isElectron || selectedFiles.length === 0) { + console.log('[refreshSelectedFiles] No files selected or not in Electron environment'); + return; + } + + console.log(`[refreshSelectedFiles] Refreshing ${selectedFiles.length} selected files`); + setIsBatchProcessing(true); + + try { + // Process all selected files that need updating + const filesToRefresh = selectedFiles.filter(filePath => { + const file = allFiles.find(f => arePathsEqual(f.path, filePath)); + return file && !file.isDirectory; + }); + + if (filesToRefresh.length > 0) { + console.log(`[refreshSelectedFiles] Processing ${filesToRefresh.length} files`); + await Promise.all( + filesToRefresh.map(filePath => processFileForRealTokens(filePath)) + ); + console.log(`[refreshSelectedFiles] Completed refreshing ${filesToRefresh.length} files`); + } + } catch (error) { + console.error('[refreshSelectedFiles] Error refreshing files:', error); + } finally { + setIsBatchProcessing(false); + } + }; + // Handle select all files const selectAllFiles = () => { console.time('selectAllFiles'); @@ -962,6 +1271,26 @@ const App = (): JSX.Element => { }); return newSelection; }); + + // Process files with estimated tokens immediately for instant copy + const filesToProcess = displayedFiles.filter((file: FileData) => + !file.isSkipped && + (includeBinaryPaths || !file.isBinary) && + file.isTokenEstimate && + !file.isDirectory + ); + + if (filesToProcess.length > 0) { + setIsBatchProcessing(true); + console.log(`[selectAllFiles] Processing ${filesToProcess.length} files immediately for instant copy`); + + // Process all files and wait for completion + Promise.all( + filesToProcess.map((file: FileData) => processFileForRealTokens(file.path)) + ).finally(() => { + setIsBatchProcessing(false); + }); + } } finally { console.timeEnd('selectAllFiles'); } @@ -997,7 +1326,7 @@ const App = (): JSX.Element => { setExpandedNodes((prev: Record) => { const newState = { ...prev, - [nodeId]: prev[nodeId] === undefined ? false : !prev[nodeId], + [nodeId]: prev[nodeId] === undefined ? true : !prev[nodeId], }; // Save to localStorage @@ -1067,19 +1396,32 @@ const App = (): JSX.Element => { setCachedBaseContentString(baseContent); - if (isElectron && baseContent) { - try { - const result = await window.electron.ipcRenderer.invoke('get-token-count', baseContent); - if (result?.tokenCount !== undefined) { - setCachedBaseContentTokens(result.tokenCount); - } - } catch (error) { - console.error('Error getting base content token count:', error); - setCachedBaseContentTokens(0); + // Calculate tokens by summing individual file tokens instead of tokenizing concatenated content + // This works correctly with lazy loading where some files may not have content loaded yet + const normalizedSelectedPaths = new Set(selectedFiles.map(path => normalizePath(path))); + const selectedFileData = allFiles.filter(file => + normalizedSelectedPaths.has(normalizePath(file.path)) && !file.isBinary + ); + + const baseContentTokens = selectedFileData.reduce((total, file) => { + const tokens = file.tokenCount || 0; + console.log(`[TOKEN DEBUG] File: ${file.name}, tokens: ${tokens}, isEstimate: ${file.isTokenEstimate}`); + return total + tokens; + }, 0); + + console.log(`[TOKEN DEBUG] Selected ${selectedFileData.length} files, total tokens: ${baseContentTokens}`); + + // Add estimated tokens for file tree if enabled + let fileTreeTokens = 0; + if (includeFileTree && baseContent.includes('')) { + // Estimate tokens for file tree (approximately 1 token per 4 characters) + const fileMapMatch = baseContent.match(/([\s\S]*?)<\/file_map>/); + if (fileMapMatch) { + fileTreeTokens = Math.ceil(fileMapMatch[1].length / 4); } - } else { - setCachedBaseContentTokens(0); } + + setCachedBaseContentTokens(baseContentTokens + fileTreeTokens); }; const debounceTimer = setTimeout(updateBaseContent, 300); @@ -1187,262 +1529,15 @@ const App = (): JSX.Element => { setSelectedTaskType(taskTypeId); }; - // Workspace functions - const handleOpenWorkspaceManager = () => { - // Force reload workspaces from localStorage before opening - const storedWorkspaces = localStorage.getItem(STORAGE_KEYS.WORKSPACES); - if (storedWorkspaces) { - try { - const parsed = JSON.parse(storedWorkspaces); - if (Array.isArray(parsed)) { - // Update state with a fresh copy from localStorage - setWorkspaces(parsed); - console.log('Workspaces refreshed from localStorage before opening manager'); - } - } catch (error) { - console.error('Failed to parse workspaces from localStorage:', error); - } - } - - // Open the workspace manager - setIsWorkspaceManagerOpen(true); - }; - - const handleSelectWorkspace = (workspaceId: string) => { - // Find the workspace - const workspace = workspaces.find((w: Workspace) => w.id === workspaceId); - if (!workspace) return; - - // Save current workspace id - localStorage.setItem(STORAGE_KEYS.CURRENT_WORKSPACE, workspaceId); - setCurrentWorkspaceId(workspaceId); - - // Update last used timestamp using functional state update - setWorkspaces((currentWorkspaces: Workspace[]) => { - const updatedWorkspaces = currentWorkspaces.map((w: Workspace) => - w.id === workspaceId ? { ...w, lastUsed: Date.now() } : w - ); - - // Save to localStorage - localStorage.setItem(STORAGE_KEYS.WORKSPACES, JSON.stringify(updatedWorkspaces)); - - return updatedWorkspaces; - }); - - // If the workspace has a folder associated with it - if (workspace.folderPath) { - // Only reload if it's different from the current folder - if (!arePathsEqual(workspace.folderPath, selectedFolder)) { - console.log(`Switching to workspace folder: ${workspace.folderPath}`); - - // First set the selected folder - setSelectedFolder(workspace.folderPath); - localStorage.setItem(STORAGE_KEYS.SELECTED_FOLDER, workspace.folderPath); - - // Request file data from the main process (if in Electron) - if (isElectron && !isSafeMode) { - setProcessingStatus({ - status: 'processing', - message: 'Loading files...', - }); - - // Ensure we're sending the updated folder path to the main process - window.electron.ipcRenderer.send('request-file-list', { - folderPath: workspace.folderPath, - ignoreMode, - customIgnores, - }); - } - } - } else { - // Clear current selection if workspace has no folder - setSelectedFolder(null); - localStorage.removeItem(STORAGE_KEYS.SELECTED_FOLDER); - setSelectedFiles([]); - setAllFiles([]); - setProcessingStatus({ - status: 'idle', - message: '', - }); - } - - setIsWorkspaceManagerOpen(false); - }; - - const handleCreateWorkspace = (name: string) => { - console.log('App: Creating new workspace with name:', name); - - // Create a new workspace with a unique id - const newWorkspace = { - id: `workspace-${Date.now()}-${Math.random().toString(36).substring(2, 9)}`, - name, - folderPath: null, - createdAt: Date.now(), - lastUsed: Date.now(), - }; - - // Add to workspaces list - setWorkspaces((currentWorkspaces: Workspace[]) => { - console.log('Updating workspaces state, current count:', currentWorkspaces.length); - const updatedWorkspaces = [...currentWorkspaces, newWorkspace]; - localStorage.setItem(STORAGE_KEYS.WORKSPACES, JSON.stringify(updatedWorkspaces)); - console.log('Saved updated workspaces to localStorage, new count:', updatedWorkspaces.length); - return updatedWorkspaces; - }); + // Workspace functions are now handled by the useWorkspaces hook - // Set as current workspace - localStorage.setItem(STORAGE_KEYS.CURRENT_WORKSPACE, newWorkspace.id); - setCurrentWorkspaceId(newWorkspace.id); - console.log('Set current workspace ID to:', newWorkspace.id); - - if (selectedFolder) { - // Show confirmation modal to use current folder - setConfirmFolderModalDetails({ - workspaceId: newWorkspace.id, - workspaceName: name, - folderPath: selectedFolder, - }); - setIsConfirmUseFolderModalOpen(true); - } else { - // No folder selected - proceed with folder selection - setSelectedFolder(null); - localStorage.removeItem(STORAGE_KEYS.SELECTED_FOLDER); - localStorage.removeItem(STORAGE_KEYS.SELECTED_FILES); - setSelectedFiles([]); - setAllFiles([]); - setProcessingStatus({ - status: 'idle', - message: '', - }); - openFolder(); - } - - // Close the workspace manager - setIsWorkspaceManagerOpen(false); - console.log('Workspace creation complete, manager closed'); - }; - - const handleConfirmUseCurrentFolder = () => { - if (!confirmFolderModalDetails.workspaceId) return; - - // Update workspace with current folder path - handleUpdateWorkspaceFolder( - confirmFolderModalDetails.workspaceId, - confirmFolderModalDetails.folderPath - ); - setIsConfirmUseFolderModalOpen(false); - }; - - const handleDeclineUseCurrentFolder = () => { - setIsConfirmUseFolderModalOpen(false); - // Clear state and open folder selector - setSelectedFolder(null); - localStorage.removeItem(STORAGE_KEYS.SELECTED_FOLDER); - localStorage.removeItem(STORAGE_KEYS.SELECTED_FILES); - setSelectedFiles([]); - setAllFiles([]); - setProcessingStatus({ - status: 'idle', - message: '', - }); - openFolder(); - }; - - const handleDeleteWorkspace = (workspaceId: string) => { - console.log('App: Deleting workspace with ID:', workspaceId); - // Ensure any open modal is closed first - setIsConfirmUseFolderModalOpen(false); - - const workspaceBeingDeleted = workspaces.find((w: Workspace) => w.id === workspaceId); - console.log('Deleting workspace:', workspaceBeingDeleted?.name); - - // Filter out the deleted workspace, using functional update to prevent stale state - setWorkspaces((currentWorkspaces: Workspace[]) => { - const filteredWorkspaces = currentWorkspaces.filter((w: Workspace) => w.id !== workspaceId); - console.log( - `Filtered workspaces: ${currentWorkspaces.length} -> ${filteredWorkspaces.length}` - ); - - // Save the updated workspaces list to localStorage - localStorage.setItem(STORAGE_KEYS.WORKSPACES, JSON.stringify(filteredWorkspaces)); - console.log('Saved filtered workspaces to localStorage'); - - // Ensure empty array is properly saved when deleting the last workspace - if (filteredWorkspaces.length === 0) { - console.log('No workspaces left, ensuring empty array is saved'); - localStorage.setItem(STORAGE_KEYS.WORKSPACES, JSON.stringify([])); - } - - return filteredWorkspaces; - }); - - // (Removed workspaceManagerVersion increment) - - // If deleting current workspace, clear current selection - if (currentWorkspaceId === workspaceId) { - console.log('Deleted the current workspace, clearing workspace state'); - localStorage.removeItem(STORAGE_KEYS.CURRENT_WORKSPACE); - setCurrentWorkspaceId(null); - - // Also clear folder selection when current workspace is deleted - setSelectedFolder(null); - localStorage.removeItem(STORAGE_KEYS.SELECTED_FOLDER); - setSelectedFiles([]); - setAllFiles([]); - setProcessingStatus({ - status: 'idle', - message: '', - }); - } - - console.log('Workspace deletion complete'); - - // Important: Keep the workspace manager open so user can create a new workspace immediately - // The visual update with the deleted workspace removed will happen thanks to our useEffect in WorkspaceManager - }; - - // Handler to update a workspace's folder path - const handleUpdateWorkspaceFolder = (workspaceId: string, folderPath: string | null) => { - setWorkspaces((prevWorkspaces: Workspace[]) => { - const updatedWorkspaces = prevWorkspaces.map((workspace: Workspace) => - workspace.id === workspaceId - ? { ...workspace, folderPath, lastUsed: Date.now() } - : workspace - ); - localStorage.setItem(STORAGE_KEYS.WORKSPACES, JSON.stringify(updatedWorkspaces)); - return updatedWorkspaces; - }); - - // If updating the current workspace, also update the selected folder - if (currentWorkspaceId === workspaceId) { - if (folderPath) { - // Update local storage and request file list - localStorage.setItem(STORAGE_KEYS.SELECTED_FOLDER, folderPath); - handleFolderSelected(folderPath); - } else { - // Clear folder selection in localStorage and state - localStorage.removeItem(STORAGE_KEYS.SELECTED_FOLDER); - setSelectedFolder(null); - setSelectedFiles([]); - setAllFiles([]); - setProcessingStatus({ - status: 'idle', - message: '', - }); - } - } - }; - - // Get current workspace name for display - const currentWorkspaceName = currentWorkspaceId - ? workspaces.find((w: Workspace) => w.id === currentWorkspaceId)?.name || 'Untitled' - : null; // Handle copying content to clipboard const handleCopy = async () => { if (selectedFiles.length === 0) return; try { + // Files should already be processed when selected, so copy should be instant const content = getSelectedFilesContent(); await navigator.clipboard.writeText(content); setProcessingStatus({ status: 'complete', message: 'Copied to clipboard!' }); @@ -1562,12 +1657,14 @@ const App = (): JSX.Element => { @@ -1663,6 +1760,7 @@ const App = (): JSX.Element => { currentWorkspaceName={currentWorkspaceName} collapseAllFolders={collapseAllFolders} expandAllFolders={expandAllFolders} + processingFiles={processingFiles} /> ) : (
@@ -1702,7 +1800,7 @@ const App = (): JSX.Element => {
{selectedFolder - ? `${displayedFiles.length} files | ~${totalFormattedContentTokens.toLocaleString()} tokens` + ? `${selectedFiles.length} files | ~${totalFormattedContentTokens.toLocaleString()} tokens` : '0 files | ~0 tokens'}
{selectedFolder && ( @@ -1764,9 +1862,10 @@ const App = (): JSX.Element => {
{selectedFolder ? ( ) : (
@@ -1894,6 +1993,29 @@ const App = (): JSX.Element => { workspaceName={confirmFolderModalDetails.workspaceName} folderPath={confirmFolderModalDetails.folderPath} /> + setIsLargeFolderModalOpen(false)} + details={largeFolderDetails} + onProceed={handleProceedWithLargeFolder} + onLoadDeselected={handleLoadLargeFolderDeselected} + onCancel={handleCancelLargeFolder} + /> + + {/* ProcessingOverlay for folder and batch file processing */} +
); diff --git a/src/components/CopyButton.tsx b/src/components/CopyButton.tsx index 50c3597a..a75cd395 100644 --- a/src/components/CopyButton.tsx +++ b/src/components/CopyButton.tsx @@ -2,29 +2,28 @@ import { useState } from 'react'; import { Copy, Check } from 'lucide-react'; interface CopyButtonProps { - text: string; + onClick: () => Promise; + disabled?: boolean; className?: string; - children?: JSX.Element | string; + children?: React.ReactNode; } -const CopyButton = ({ text, className = '', children }: CopyButtonProps) => { +const CopyButton = ({ onClick, disabled = false, className = '', children }: CopyButtonProps) => { const [copied, setCopied] = useState(false); - const handleCopy = async () => { + const handleCopyClick = async () => { + if (disabled) return; try { - await navigator.clipboard.writeText(text); + await onClick(); setCopied(true); - - // Reset the copied state after 2 seconds setTimeout(() => { setCopied(false); }, 2000); } catch (err) { - console.error('Failed to copy:', err); + console.error('Copy operation failed:', err); } }; - // Add inline styles to ensure no focus outline appears const buttonStyle = { outline: 'none', }; @@ -33,7 +32,8 @@ const CopyButton = ({ text, className = '', children }: CopyButtonProps) => { - + navigator.clipboard.writeText(textToCopy)} className="file-card-action"> {''} @@ -83,4 +93,4 @@ const FileCard = ({ file, isSelected, toggleSelection, onPreview }: FileCardComp }; // Wrap component with React.memo to prevent unnecessary re-renders -export default memo(FileCard); +export default memo(FileCard); \ No newline at end of file diff --git a/src/components/FileList.tsx b/src/components/FileList.tsx index 1dc39f33..c4b1844a 100644 --- a/src/components/FileList.tsx +++ b/src/components/FileList.tsx @@ -6,17 +6,36 @@ import FilePreviewModal from './FilePreviewModal'; import { arePathsEqual } from '../utils/pathUtils'; // Add proper memoization to avoid unnecessary re-renders -const FileList = ({ files, selectedFiles, toggleFileSelection }: FileListProps) => { +const FileList = ({ files, selectedFiles, toggleFileSelection, sortOrder = 'tokens-desc' }: FileListProps) => { // Only show files that are in the selectedFiles array and not binary/skipped const displayableFiles = useMemo( - () => - files.filter( + () => { + const filtered = files.filter( (file: FileData) => selectedFiles.some((selectedPath) => arePathsEqual(selectedPath, file.path)) && !file.isSkipped && !file.excludedByDefault - ), - [files, selectedFiles] + ); + + // Apply sorting to selected files + const [sortKey, sortDir] = sortOrder.split('-'); + const sorted = [...filtered].sort((a, b) => { + let comparison = 0; + + if (sortKey === 'name') { + comparison = a.name.localeCompare(b.name); + } else if (sortKey === 'tokens') { + comparison = a.tokenCount - b.tokenCount; + } else if (sortKey === 'size') { + comparison = a.size - b.size; + } + + return sortDir === 'asc' ? comparison : -comparison; + }); + + return sorted; + }, + [files, selectedFiles, sortOrder] ); const [previewModalOpen, setPreviewModalOpen] = useState(false); diff --git a/src/components/LargeFolderModal.tsx b/src/components/LargeFolderModal.tsx new file mode 100644 index 00000000..fdef97e3 --- /dev/null +++ b/src/components/LargeFolderModal.tsx @@ -0,0 +1,57 @@ +import React from 'react'; +import '../styles/modals/LargeFolderModal.css'; + +interface LargeFolderModalProps { + isOpen: boolean; + onClose: () => void; + details: { + totalTokens: number; + folderPath: string; + }; + onProceed: () => void; + onLoadDeselected: () => void; + onCancel: () => void; +} + +const LargeFolderModal: React.FC = ({ + isOpen, + onClose, + details, + onProceed, + onLoadDeselected, + onCancel, +}) => { + if (!isOpen) return null; + + return ( +
+
+
+

Large Folder Detected

+ +
+
+

+ The selected folder contains approximately {details.totalTokens.toLocaleString()} tokens, + which may impact application performance. How would you like to proceed? +

+
+
+ + + +
+
+
+ ); +}; + +export default LargeFolderModal; \ No newline at end of file diff --git a/src/components/LargeSubfolderModal.tsx b/src/components/LargeSubfolderModal.tsx new file mode 100644 index 00000000..a9f32e9b --- /dev/null +++ b/src/components/LargeSubfolderModal.tsx @@ -0,0 +1,77 @@ +import React from 'react'; +import '../styles/modals/LargeSubfolderModal.css'; + +interface LargeSubfolderModalProps { + isOpen: boolean; + onClose: () => void; + onConfirm: () => void; + details: { + totalTokens: number; + folderPath: string; + hasEstimates: boolean; + }; +} + +const LargeSubfolderModal = ({ isOpen, onClose, onConfirm, details }: LargeSubfolderModalProps) => { + if (!isOpen) return null; + + const { totalTokens, folderPath, hasEstimates } = details; + const folderName = folderPath.split(/[/\\]/).pop() || folderPath; + + return ( +
+
+
+

Large Folder Selection

+ +
+ +
+
+ ⚠️ Large Folder Detected +
+ +
+

+ The folder "{folderName}" contains approximately{' '} + + {hasEstimates ? '~' : ''}{totalTokens.toLocaleString()} + {hasEstimates && (estimated)} + {' '} + tokens. +

+ + {hasEstimates && ( +

+ Some token counts are estimated. Actual counts may differ when files are processed. +

+ )} + +

+ Selecting this folder may impact application performance. Do you want to proceed? +

+
+
+ +
+ + +
+
+
+ ); +}; + +export default LargeSubfolderModal; \ No newline at end of file diff --git a/src/components/ProcessingOverlay.tsx b/src/components/ProcessingOverlay.tsx new file mode 100644 index 00000000..a21589bd --- /dev/null +++ b/src/components/ProcessingOverlay.tsx @@ -0,0 +1,24 @@ +import { Loader } from 'lucide-react'; +import '../styles/components/ProcessingOverlay.css'; + +interface ProcessingOverlayProps { + isVisible: boolean; + title?: string; + message?: string; +} + +const ProcessingOverlay = ({ isVisible, title = "Processing Files", message = "Calculating precise tokens for copying..." }: ProcessingOverlayProps) => { + if (!isVisible) return null; + + return ( +
+
+ +

{title}

+

{message}

+
+
+ ); +}; + +export default ProcessingOverlay; \ No newline at end of file diff --git a/src/components/Sidebar.tsx b/src/components/Sidebar.tsx index ab0e67be..4329b1bb 100644 --- a/src/components/Sidebar.tsx +++ b/src/components/Sidebar.tsx @@ -36,6 +36,7 @@ const Sidebar = ({ onManageCustomTypes, collapseAllFolders, expandAllFolders, + processingFiles, }: Omit) => { // State for managing the file tree and UI const [fileTree, setFileTree] = useState(() => [] as TreeNode[]); @@ -153,6 +154,10 @@ const Sidebar = ({ children: {}, }; } + // Safety check - ensure children exists + if (!current[part].children) { + current[part].children = {}; + } current = current[part].children; } } @@ -176,8 +181,7 @@ const Sidebar = ({ return item as TreeNode; } else { const children = convertToTreeNodes(item.children, level + 1); - const isExpanded = - expandedNodes[item.id] !== undefined ? expandedNodes[item.id] : true; + const isExpanded = !!expandedNodes[item.id]; // Check if this directory contains any binary files const hasBinaries = hasBinaryFiles(children); @@ -239,7 +243,7 @@ const Sidebar = ({ const applyExpandedState = (nodes: TreeNode[]): TreeNode[] => { return nodes.map((node: TreeNode): TreeNode => { if (node.type === 'directory') { - const isExpanded = expandedNodes[node.id] !== undefined ? expandedNodes[node.id] : true; // Default to expanded if not in state + const isExpanded = !!expandedNodes[node.id]; return { ...node, @@ -331,9 +335,10 @@ const Sidebar = ({ toggleFolderSelection={toggleFolderSelection} toggleExpanded={toggleExpanded} includeBinaryPaths={includeBinaryPaths} + processingFiles={processingFiles} /> )); - }, [visibleTree, selectedFiles, toggleFileSelection, toggleFolderSelection, toggleExpanded]); + }, [visibleTree, selectedFiles, toggleFileSelection, toggleFolderSelection, toggleExpanded, processingFiles]); return (
diff --git a/src/components/TreeItem.tsx b/src/components/TreeItem.tsx index 5c976e62..952d1b7d 100644 --- a/src/components/TreeItem.tsx +++ b/src/components/TreeItem.tsx @@ -1,6 +1,6 @@ import { useRef, useEffect, useMemo, useCallback, memo } from 'react'; -import { TreeItemProps, TreeNode } from '../types/FileTypes'; -import { ChevronRight, File, Folder } from 'lucide-react'; +import { TreeItemProps, TreeNode, FileData } from '../types/FileTypes'; +import { ChevronRight, File, Folder, Loader } from 'lucide-react'; import { arePathsEqual } from '../utils/pathUtils'; /** @@ -30,10 +30,55 @@ const TreeItem = ({ toggleFolderSelection, toggleExpanded, includeBinaryPaths, + processingFiles, }: TreeItemProps) => { const { id, name, path, type, level, isExpanded, fileData } = node; const checkboxRef = useRef(null); + // Calculate total tokens for a folder by summing all child files + const calculateFolderTokens = useCallback((folderNode: TreeNode): number => { + if (!folderNode.children) return 0; + + let totalTokens = 0; + + const traverseChildren = (children: TreeNode[]) => { + children.forEach(child => { + if (child.type === 'file' && child.fileData && child.fileData.tokenCount > 0) { + totalTokens += child.fileData.tokenCount; + } else if (child.type === 'directory' && child.children) { + traverseChildren(child.children); + } + }); + }; + + traverseChildren(folderNode.children); + return totalTokens; + }, []); + + // Calculate folder tokens with estimate information + const calculateFolderTokensWithEstimates = useCallback((folderNode: TreeNode): { totalTokens: number; hasEstimates: boolean } => { + if (!folderNode.children) return { totalTokens: 0, hasEstimates: false }; + + let totalTokens = 0; + let hasEstimates = false; + + const traverseChildren = (children: TreeNode[]) => { + children.forEach(child => { + if (child.type === 'file' && child.fileData && child.fileData.tokenCount > 0) { + totalTokens += child.fileData.tokenCount; + if (child.fileData.isTokenEstimate) { + hasEstimates = true; + } + } else if (child.type === 'directory' && child.children) { + traverseChildren(child.children); + } + }); + }; + + traverseChildren(folderNode.children); + return { totalTokens, hasEstimates }; + }, []); + // Check if this file is in the selected files list - memoize this calculation const isSelected = useMemo( () => @@ -172,22 +217,27 @@ const TreeItem = ({ return fileData ? isFileExcluded(fileData, includeBinaryPaths) : false; }, [fileData, includeBinaryPaths]); + // Check if this file is currently being processed for tokens + const isProcessing = useMemo(() => { + return processingFiles ? processingFiles.has(path) : false; + }, [processingFiles, path]); + // Event Handlers - memoize them to prevent recreating on each render const handleToggle = useCallback( (e: any) => { e.stopPropagation(); + e.preventDefault(); toggleExpanded(id); }, [toggleExpanded, id] ); const handleItemClick = useCallback(() => { - if (type === 'directory') { - toggleExpanded(id); - } else if (type === 'file' && !isCheckboxDisabled) { + // Only handle file clicks, directories should only be expanded via the arrow + if (type === 'file' && !isCheckboxDisabled) { toggleFileSelection(path); } - }, [type, id, path, toggleExpanded, toggleFileSelection, isCheckboxDisabled]); + }, [type, path, toggleFileSelection, isCheckboxDisabled]); const handleCheckboxChange = useCallback( (e: any) => { @@ -201,18 +251,9 @@ const TreeItem = ({ const isChecked = e.target.checked; - console.log('Checkbox clicked:', { - type, - path, - isChecked, - isDirectory: type === 'directory', - isFile: type === 'file', - }); - if (type === 'file') { toggleFileSelection(path); } else if (type === 'directory') { - console.log('Calling toggleFolderSelection with:', path, isChecked); toggleFolderSelection(path, isChecked); } }, @@ -258,10 +299,34 @@ const TreeItem = ({
{name}
+ {/* Show loading indicator for files being processed */} + {isProcessing && type === 'file' && ( + + + + )} + {/* Show token count for files that have it */} - {fileData && fileData.tokenCount > 0 && ( - (~{fileData.tokenCount.toLocaleString()}) + {!isProcessing && fileData && fileData.tokenCount > 0 && ( + + ({fileData.isTokenEstimate ? '~' : ''}{fileData.tokenCount.toLocaleString()} + {fileData.isTokenEstimate && est}) + )} + + {/* Show folder token totals */} + {type === 'directory' && node.children && (() => { + const { totalTokens, hasEstimates } = calculateFolderTokensWithEstimates(node); + if (totalTokens > 0) { + return ( + + ({hasEstimates ? '~' : ''}{totalTokens.toLocaleString()} + {hasEstimates && est} tokens) + + ); + } + return null; + })()} {/* Show badges for files and folders */} {type === 'file' && fileData && ( @@ -288,4 +353,4 @@ const TreeItem = ({ }; // Wrap the component with React.memo to prevent unnecessary re-renders -export default memo(TreeItem); +export default memo(TreeItem); \ No newline at end of file diff --git a/src/global.d.ts b/src/global.d.ts index 8ef6652e..cb260cc4 100644 --- a/src/global.d.ts +++ b/src/global.d.ts @@ -35,7 +35,6 @@ declare module 'react/jsx-runtime'; declare module 'electron'; declare module 'tiktoken'; declare module 'ignore'; -declare module 'gpt-3-encoder'; // asset imports declare module '*.css' { diff --git a/src/hooks/useWorkspaces.ts b/src/hooks/useWorkspaces.ts new file mode 100644 index 00000000..a32451f3 --- /dev/null +++ b/src/hooks/useWorkspaces.ts @@ -0,0 +1,356 @@ +import { useState, useEffect, useCallback } from 'react'; +import { Workspace } from '../types/WorkspaceTypes'; +import { normalizePath } from '../utils/pathUtils'; + +// Storage keys - replicating from App.tsx +const STORAGE_KEYS = { + WORKSPACES: 'pastemax-workspaces', + CURRENT_WORKSPACE: 'pastemax-current-workspace', + SELECTED_FOLDER: 'pastemax-selected-folder', + SELECTED_FILES: 'pastemax-selected-files', +}; + +interface UseWorkspacesProps { + selectedFolder: string | null; + setSelectedFolder: (folder: string | null) => void; + setSelectedFiles: (files: string[]) => void; + setAllFiles: (files: any[]) => void; + setProcessingStatus: (status: { status: string; message: string }) => void; + openFolder: () => void; + handleFolderSelected: (folderPath: string) => void; + isElectron: boolean; +} + +interface UseWorkspacesReturn { + workspaces: Workspace[]; + setWorkspaces: React.Dispatch>; + currentWorkspaceId: string | null; + setCurrentWorkspaceId: React.Dispatch>; + isWorkspaceManagerOpen: boolean; + setIsWorkspaceManagerOpen: React.Dispatch>; + isConfirmUseFolderModalOpen: boolean; + setIsConfirmUseFolderModalOpen: React.Dispatch>; + confirmFolderModalDetails: { + workspaceId: string | null; + workspaceName: string; + folderPath: string; + }; + setConfirmFolderModalDetails: React.Dispatch>; + currentWorkspaceName: string | null; + handleOpenWorkspaceManager: () => void; + handleSelectWorkspace: (workspaceId: string) => void; + handleCreateWorkspace: (name: string) => void; + handleDeleteWorkspace: (workspaceId: string) => void; + handleUpdateWorkspaceFolder: (workspaceId: string, folderPath: string | null) => void; + handleConfirmUseCurrentFolder: () => void; + handleDeclineUseCurrentFolder: () => void; +} + +export const useWorkspaces = ({ + selectedFolder, + setSelectedFolder, + setSelectedFiles, + setAllFiles, + setProcessingStatus, + openFolder, + handleFolderSelected, + isElectron, +}: UseWorkspacesProps): UseWorkspacesReturn => { + // Initialize workspaces from localStorage + const [workspaces, setWorkspaces] = useState(() => { + const savedWorkspaces = localStorage.getItem(STORAGE_KEYS.WORKSPACES); + if (savedWorkspaces) { + try { + const parsed = JSON.parse(savedWorkspaces); + if (Array.isArray(parsed)) { + console.log(`Loaded ${parsed.length} workspaces from localStorage`); + return parsed as Workspace[]; + } else { + console.warn('Invalid workspaces format in localStorage, resetting to empty array'); + // Reset localStorage to prevent further errors + localStorage.setItem(STORAGE_KEYS.WORKSPACES, JSON.stringify([])); + return [] as Workspace[]; + } + } catch (error) { + console.error('Error parsing workspaces from localStorage:', error); + // Reset localStorage to prevent further errors + localStorage.setItem(STORAGE_KEYS.WORKSPACES, JSON.stringify([])); + return [] as Workspace[]; + } + } + // Initialize with empty array and ensure localStorage has a valid value + console.log('No workspaces found in localStorage, initializing with empty array'); + localStorage.setItem(STORAGE_KEYS.WORKSPACES, JSON.stringify([])); + return [] as Workspace[]; + }); + + const [currentWorkspaceId, setCurrentWorkspaceId] = useState(() => { + return localStorage.getItem(STORAGE_KEYS.CURRENT_WORKSPACE) || null; + }); + + const [isWorkspaceManagerOpen, setIsWorkspaceManagerOpen] = useState(false); + const [isConfirmUseFolderModalOpen, setIsConfirmUseFolderModalOpen] = useState(false); + const [confirmFolderModalDetails, setConfirmFolderModalDetails] = useState<{ + workspaceId: string | null; + workspaceName: string; + folderPath: string; + }>({ + workspaceId: null, + workspaceName: '', + folderPath: '', + }); + + // Sync workspaces to localStorage whenever they change + useEffect(() => { + if (workspaces.length > 0) { + localStorage.setItem(STORAGE_KEYS.WORKSPACES, JSON.stringify(workspaces)); + console.log(`Workspaces updated: ${workspaces.length} workspaces saved to localStorage`); + + // If we have a current workspace, ensure it still exists in the workspaces array + if (currentWorkspaceId && !workspaces.some((w: Workspace) => w.id === currentWorkspaceId)) { + console.log('Current workspace no longer exists, clearing currentWorkspaceId'); + localStorage.removeItem(STORAGE_KEYS.CURRENT_WORKSPACE); + setCurrentWorkspaceId(null); + } + } + }, [workspaces, currentWorkspaceId]); + + // Update current workspace's folder path when selectedFolder changes + useEffect(() => { + if (selectedFolder && currentWorkspaceId) { + setWorkspaces((prevWorkspaces: Workspace[]) => { + const updatedWorkspaces = prevWorkspaces.map((workspace: Workspace) => + workspace.id === currentWorkspaceId + ? { ...workspace, folderPath: normalizePath(selectedFolder), lastUsed: Date.now() } + : workspace + ); + localStorage.setItem(STORAGE_KEYS.WORKSPACES, JSON.stringify(updatedWorkspaces)); + return updatedWorkspaces; + }); + } + }, [selectedFolder, currentWorkspaceId]); + + // Get current workspace name for display + const currentWorkspaceName = currentWorkspaceId + ? workspaces.find((w: Workspace) => w.id === currentWorkspaceId)?.name || 'Untitled' + : null; + + // Workspace handler functions + const handleOpenWorkspaceManager = useCallback(() => { + setIsWorkspaceManagerOpen(true); + }, []); + + const handleSelectWorkspace = useCallback((workspaceId: string) => { + console.log('Selecting workspace with ID:', workspaceId); + const workspace = workspaces.find((w: Workspace) => w.id === workspaceId); + if (!workspace) { + console.error('Workspace not found:', workspaceId); + return; + } + + // Update timestamps and set as current + setWorkspaces((currentWorkspaces: Workspace[]) => { + const updatedWorkspaces = currentWorkspaces.map((w: Workspace) => + w.id === workspaceId ? { ...w, lastUsed: Date.now() } : w + ); + localStorage.setItem(STORAGE_KEYS.WORKSPACES, JSON.stringify(updatedWorkspaces)); + return updatedWorkspaces; + }); + + localStorage.setItem(STORAGE_KEYS.CURRENT_WORKSPACE, workspaceId); + setCurrentWorkspaceId(workspaceId); + console.log('Current workspace ID set to:', workspaceId); + + // Handle folder selection + if (workspace.folderPath) { + localStorage.setItem(STORAGE_KEYS.SELECTED_FOLDER, workspace.folderPath); + handleFolderSelected(workspace.folderPath); + } else { + localStorage.removeItem(STORAGE_KEYS.SELECTED_FOLDER); + localStorage.removeItem(STORAGE_KEYS.SELECTED_FILES); + setSelectedFolder(null); + setSelectedFiles([]); + setAllFiles([]); + setProcessingStatus({ + status: 'idle', + message: '', + }); + } + + setIsWorkspaceManagerOpen(false); + console.log('Workspace selection complete, manager closed'); + }, [workspaces, handleFolderSelected, setSelectedFolder, setSelectedFiles, setAllFiles, setProcessingStatus]); + + const handleCreateWorkspace = useCallback((name: string) => { + console.log('Creating workspace with name:', name); + const newWorkspace: Workspace = { + id: `workspace_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`, + name, + folderPath: null, + lastUsed: Date.now(), + createdAt: Date.now(), + }; + + setWorkspaces((currentWorkspaces: Workspace[]) => { + const updatedWorkspaces = [...currentWorkspaces, newWorkspace]; + localStorage.setItem(STORAGE_KEYS.WORKSPACES, JSON.stringify(updatedWorkspaces)); + return updatedWorkspaces; + }); + + // Set as current workspace + localStorage.setItem(STORAGE_KEYS.CURRENT_WORKSPACE, newWorkspace.id); + setCurrentWorkspaceId(newWorkspace.id); + console.log('Set current workspace ID to:', newWorkspace.id); + + if (selectedFolder) { + // Show confirmation modal to use current folder + setConfirmFolderModalDetails({ + workspaceId: newWorkspace.id, + workspaceName: name, + folderPath: selectedFolder, + }); + setIsConfirmUseFolderModalOpen(true); + } else { + // No folder selected - proceed with folder selection + setSelectedFolder(null); + localStorage.removeItem(STORAGE_KEYS.SELECTED_FOLDER); + localStorage.removeItem(STORAGE_KEYS.SELECTED_FILES); + setSelectedFiles([]); + setAllFiles([]); + setProcessingStatus({ + status: 'idle', + message: '', + }); + openFolder(); + } + + // Close the workspace manager + setIsWorkspaceManagerOpen(false); + console.log('Workspace creation complete, manager closed'); + }, [selectedFolder, setSelectedFolder, setSelectedFiles, setAllFiles, setProcessingStatus, openFolder]); + + const handleConfirmUseCurrentFolder = useCallback(() => { + if (!confirmFolderModalDetails.workspaceId) return; + + // Update workspace with current folder path + handleUpdateWorkspaceFolder( + confirmFolderModalDetails.workspaceId, + confirmFolderModalDetails.folderPath + ); + setIsConfirmUseFolderModalOpen(false); + }, [confirmFolderModalDetails]); + + const handleDeclineUseCurrentFolder = useCallback(() => { + setIsConfirmUseFolderModalOpen(false); + // Clear state and open folder selector + setSelectedFolder(null); + localStorage.removeItem(STORAGE_KEYS.SELECTED_FOLDER); + localStorage.removeItem(STORAGE_KEYS.SELECTED_FILES); + setSelectedFiles([]); + setAllFiles([]); + setProcessingStatus({ + status: 'idle', + message: '', + }); + openFolder(); + }, [setSelectedFolder, setSelectedFiles, setAllFiles, setProcessingStatus, openFolder]); + + const handleDeleteWorkspace = useCallback((workspaceId: string) => { + console.log('App: Deleting workspace with ID:', workspaceId); + // Ensure any open modal is closed first + setIsConfirmUseFolderModalOpen(false); + + const workspaceBeingDeleted = workspaces.find((w: Workspace) => w.id === workspaceId); + console.log('Deleting workspace:', workspaceBeingDeleted?.name); + + // Filter out the deleted workspace, using functional update to prevent stale state + setWorkspaces((currentWorkspaces: Workspace[]) => { + const filteredWorkspaces = currentWorkspaces.filter((w: Workspace) => w.id !== workspaceId); + console.log( + `Filtered workspaces: ${currentWorkspaces.length} -> ${filteredWorkspaces.length}` + ); + + // Save the updated workspaces list to localStorage + localStorage.setItem(STORAGE_KEYS.WORKSPACES, JSON.stringify(filteredWorkspaces)); + console.log('Saved filtered workspaces to localStorage'); + + return filteredWorkspaces; + }); + + // If deleting current workspace, clear current selection + if (currentWorkspaceId === workspaceId) { + console.log('Deleted the current workspace, clearing workspace state'); + localStorage.removeItem(STORAGE_KEYS.CURRENT_WORKSPACE); + setCurrentWorkspaceId(null); + + // Optionally clear folder selection when deleting current workspace + localStorage.removeItem(STORAGE_KEYS.SELECTED_FOLDER); + localStorage.removeItem(STORAGE_KEYS.SELECTED_FILES); + setSelectedFolder(null); + setSelectedFiles([]); + setAllFiles([]); + setProcessingStatus({ + status: 'idle', + message: '', + }); + } + + console.log('Workspace deletion complete'); + }, [workspaces, currentWorkspaceId, setSelectedFolder, setSelectedFiles, setAllFiles, setProcessingStatus]); + + const handleUpdateWorkspaceFolder = useCallback((workspaceId: string, folderPath: string | null) => { + setWorkspaces((prevWorkspaces: Workspace[]) => { + const updatedWorkspaces = prevWorkspaces.map((workspace: Workspace) => + workspace.id === workspaceId + ? { ...workspace, folderPath, lastUsed: Date.now() } + : workspace + ); + localStorage.setItem(STORAGE_KEYS.WORKSPACES, JSON.stringify(updatedWorkspaces)); + return updatedWorkspaces; + }); + + // If updating the current workspace, also update the selected folder + if (currentWorkspaceId === workspaceId) { + if (folderPath) { + // Update local storage and request file list + localStorage.setItem(STORAGE_KEYS.SELECTED_FOLDER, folderPath); + handleFolderSelected(folderPath); + } else { + // Clear folder selection in localStorage and state + localStorage.removeItem(STORAGE_KEYS.SELECTED_FOLDER); + setSelectedFolder(null); + setSelectedFiles([]); + setAllFiles([]); + setProcessingStatus({ + status: 'idle', + message: '', + }); + } + } + }, [currentWorkspaceId, handleFolderSelected, setSelectedFolder, setSelectedFiles, setAllFiles, setProcessingStatus]); + + return { + workspaces, + setWorkspaces, + currentWorkspaceId, + setCurrentWorkspaceId, + isWorkspaceManagerOpen, + setIsWorkspaceManagerOpen, + isConfirmUseFolderModalOpen, + setIsConfirmUseFolderModalOpen, + confirmFolderModalDetails, + setConfirmFolderModalDetails, + currentWorkspaceName, + handleOpenWorkspaceManager, + handleSelectWorkspace, + handleCreateWorkspace, + handleDeleteWorkspace, + handleUpdateWorkspaceFolder, + handleConfirmUseCurrentFolder, + handleDeclineUseCurrentFolder, + }; +}; \ No newline at end of file diff --git a/src/main.tsx b/src/main.tsx index 13f05fcf..d392bddd 100644 --- a/src/main.tsx +++ b/src/main.tsx @@ -12,6 +12,7 @@ import './styles/base/Buttons.css'; import './styles/base/Input.css'; import './styles/base/Utilities.css'; import './styles/base/ToggleSwitch.css'; +import './styles/components/ProcessingOverlay.css'; /* ============================== HEADER STYLES ============================ */ import './styles/header/Header.css'; @@ -41,6 +42,8 @@ import './styles/modals/UpdateModal.css'; import './styles/modals/CustomTaskTypeModal.css'; import './styles/modals/WorkspaceManager.css'; import './styles/modals/CopyHistoryModal.css'; +import './styles/modals/ConfirmUseFolderModal.css'; +import './styles/modals/LargeFolderModal.css'; /** * Add an event listener to ensure state is saved properly before a page refresh. diff --git a/src/styles/components/ProcessingOverlay.css b/src/styles/components/ProcessingOverlay.css new file mode 100644 index 00000000..85dcf348 --- /dev/null +++ b/src/styles/components/ProcessingOverlay.css @@ -0,0 +1,72 @@ +.processing-overlay { + position: fixed; + top: 0; + left: 0; + right: 0; + bottom: 0; + background-color: rgba(0, 0, 0, 0.7); + display: flex; + align-items: center; + justify-content: center; + z-index: 9999; + backdrop-filter: blur(4px); +} + +.processing-overlay-content { + background-color: var(--background-primary); + border: 1px solid var(--border-color); + border-radius: 12px; + padding: 40px; + text-align: center; + box-shadow: 0 8px 32px rgba(0, 0, 0, 0.2); + max-width: 400px; + width: 90%; +} + +.processing-overlay-spinner { + color: var(--color-primary); + margin-bottom: 20px; + animation: spin 1s linear infinite; +} + +.processing-overlay-content h2 { + margin: 0 0 10px 0; + color: var(--text-primary); + font-size: 24px; + font-weight: 600; +} + +.processing-overlay-content p { + margin: 0; + color: var(--text-secondary); + font-size: 16px; +} + +.processing-progress { + margin-top: 20px; + padding: 10px 20px; + background-color: var(--background-secondary); + border-radius: 6px; + color: var(--text-primary); + font-size: 14px; + font-weight: 500; +} + +@keyframes spin { + from { + transform: rotate(0deg); + } + to { + transform: rotate(360deg); + } +} + +/* Dark mode adjustments */ +.dark-mode .processing-overlay { + background-color: rgba(0, 0, 0, 0.8); +} + +.dark-mode .processing-overlay-content { + background-color: var(--background-primary); + box-shadow: 0 8px 32px rgba(0, 0, 0, 0.4); +} \ No newline at end of file diff --git a/src/styles/contentarea/FileCard.css b/src/styles/contentarea/FileCard.css index 4b9948b5..e5eff45c 100644 --- a/src/styles/contentarea/FileCard.css +++ b/src/styles/contentarea/FileCard.css @@ -110,6 +110,17 @@ padding: 0 var(--space-xs); /* Add horizontal padding */ } +.estimate-badge { + background: var(--color-primary); + color: var(--background-primary); + font-size: 10px; + padding: 2px 4px; + border-radius: 8px; + margin-left: 4px; + font-weight: 500; + opacity: 0.8; +} + .file-card-actions { position: absolute; top: 8px; diff --git a/src/styles/modals/LargeFolderModal.css b/src/styles/modals/LargeFolderModal.css new file mode 100644 index 00000000..a75f2223 --- /dev/null +++ b/src/styles/modals/LargeFolderModal.css @@ -0,0 +1,150 @@ +/* Large Folder Modal Styles */ +.large-folder-modal-overlay { + position: fixed; + top: 0; + left: 0; + width: 100%; + height: 100%; + display: flex; + justify-content: center; + align-items: center; + z-index: var(--z-modal, 1000); + animation: fadeIn 0.2s ease-out; /* Fade in animation */ + background-color: rgba(0, 0, 0, 0.5); /* Semi-transparent background */ + backdrop-filter: blur(2px); /* Slight blur effect for depth */ + transition: opacity 0.2s ease; /* Smooth transition when closing */ +} + +.large-folder-modal { + box-shadow: 0 4px 30px rgba(0, 0, 0, 0.2); + background-color: var(--background-secondary); + border-radius: var(--border-radius-lg); + width: 600px; + max-width: 90%; + padding: var(--space-lg) var(--space-lg); + border: var(--standard-border); + border-color: var(--warning-color); + color: var(--text-primary); +} + +.large-folder-modal .modal-header { + display: flex; + justify-content: space-between; + align-items: center; + margin-bottom: var(--space-md); +} + +.large-folder-modal .modal-header h3 { + margin: 0; + font-size: var(--font-size-lg); + font-weight: var(--font-weight-bold); + color: var(--text-primary); +} + +.large-folder-modal button.icon-button.close-button { + background: none; + border: none; + font-size: 1.5rem; + cursor: pointer; + color: var(--text-muted); + padding: 0; + line-height: 1; + display: inline-flex; + align-items: center; + justify-content: center; + transition: all 0.2s var(--animation-curve); + -webkit-transition: all 0.2s var(--animation-curve); + -moz-transition: all 0.2s var(--animation-curve); + -ms-transition: all 0.2s var(--animation-curve); + -o-transition: all 0.2s var(--animation-curve); +} + +.large-folder-modal button.icon-button.close-button:hover, +.large-folder-modal button.icon-button.close-button:focus-visible { + color: var(--text-primary); + background: none; + box-shadow: none; +} + +.large-folder-modal .modal-content { + margin-bottom: var(--space-lg); + color: var(--text-primary); +} + +.large-folder-modal .modal-content p { + margin-bottom: var(--space-md); + line-height: 1.5; +} + +.large-folder-modal .modal-content strong { + color: var(--warning-color); + font-weight: var(--font-weight-bold); +} + +.large-folder-modal .modal-actions { + display: flex; + justify-content: center; + gap: var(--space-md); + flex-wrap: wrap; +} + +.large-folder-modal button.primary.proceed-button { + background-color: var(--warning-color); + color: var(--text-on-primary); + border-color: var(--warning-color); +} + +.large-folder-modal button.primary.proceed-button:hover { + border-color: var(--warning-color); + opacity: 0.9; +} + +.large-folder-modal button.primary.load-deselected-button { + background-color: var(--color-primary); + color: var(--text-on-primary); + border-color: var(--color-primary); +} + +.large-folder-modal button.primary.load-deselected-button:hover { + border-color: var(--color-primary); + opacity: 0.9; +} + +.large-folder-modal button.secondary.cancel-button { + background-color: transparent; + color: var(--text-primary); + border-color: var(--text-muted); +} + +.large-folder-modal button.secondary.cancel-button:hover { + background-color: var(--background-tertiary); + border-color: var(--text-primary); +} + +/* Dark mode styles */ +.dark-mode .large-folder-modal { + background-color: var(--background-secondary); + border-color: var(--warning-color); + color: var(--text-primary); +} + +.dark-mode .large-folder-modal .modal-content strong { + color: var(--warning-color); +} + +/* Responsive styles */ +@media (max-width: 600px) { + .large-folder-modal { + width: 95%; + padding: var(--space-md); + } + + .large-folder-modal .modal-actions { + flex-direction: column; + gap: var(--space-sm); + } + + .large-folder-modal .modal-actions button { + width: 100%; + } +} \ No newline at end of file diff --git a/src/styles/modals/LargeSubfolderModal.css b/src/styles/modals/LargeSubfolderModal.css new file mode 100644 index 00000000..ae1471da --- /dev/null +++ b/src/styles/modals/LargeSubfolderModal.css @@ -0,0 +1,161 @@ +.large-subfolder-modal-overlay { + position: fixed; + top: 0; + left: 0; + right: 0; + bottom: 0; + background-color: rgba(0, 0, 0, 0.7); + display: flex; + align-items: center; + justify-content: center; + z-index: 9999; + backdrop-filter: blur(4px); +} + +.large-subfolder-modal { + background: var(--background-primary); + border: 1px solid var(--border-color); + border-radius: 12px; + box-shadow: 0 8px 32px rgba(0, 0, 0, 0.2); + max-width: 500px; + width: 90%; + max-height: 80vh; + overflow-y: auto; +} + +.large-subfolder-modal-header { + display: flex; + justify-content: space-between; + align-items: center; + padding: 20px 24px; + border-bottom: 1px solid var(--border-color); +} + +.large-subfolder-modal-header h2 { + margin: 0; + font-size: 20px; + font-weight: 600; + color: var(--text-primary); +} + +.large-subfolder-modal-close { + background: none; + border: none; + font-size: 24px; + color: var(--text-secondary); + cursor: pointer; + padding: 0; + width: 32px; + height: 32px; + display: flex; + align-items: center; + justify-content: center; + border-radius: 6px; + transition: background-color var(--transition-fast); +} + +.large-subfolder-modal-close:hover { + background-color: var(--hover-color); + color: var(--text-primary); +} + +.large-subfolder-modal-content { + padding: 24px; +} + +.large-subfolder-modal-warning { + display: flex; + align-items: center; + gap: 8px; + font-size: 16px; + font-weight: 600; + color: var(--warning-color); + margin-bottom: 16px; + padding: 12px 16px; + background-color: rgba(243, 156, 18, 0.1); + border: 1px solid rgba(243, 156, 18, 0.3); + border-radius: 8px; +} + +.large-subfolder-modal-details { + color: var(--text-primary); + line-height: 1.6; +} + +.large-subfolder-modal-details p { + margin: 0 0 16px 0; +} + +.large-subfolder-modal-details p:last-child { + margin-bottom: 0; +} + +.estimate-indicator { + color: var(--color-primary); + font-weight: 500; + font-style: italic; +} + +.estimate-notice { + font-size: 14px; + color: var(--text-secondary); + background-color: var(--background-secondary); + padding: 12px; + border-radius: 6px; + border-left: 3px solid var(--color-primary); +} + +.large-subfolder-modal-actions { + display: flex; + gap: 12px; + justify-content: flex-end; + padding: 20px 24px; + border-top: 1px solid var(--border-color); +} + +.large-subfolder-modal-btn { + padding: 10px 20px; + border-radius: 6px; + font-size: 14px; + font-weight: 500; + cursor: pointer; + transition: all var(--transition-fast); + border: 1px solid transparent; + min-width: 100px; +} + +.large-subfolder-modal-btn-secondary { + background-color: var(--background-secondary); + color: var(--text-primary); + border-color: var(--border-color); +} + +.large-subfolder-modal-btn-secondary:hover { + background-color: var(--hover-color); +} + +.large-subfolder-modal-btn-primary { + background-color: var(--color-primary); + color: var(--background-primary); + border-color: var(--color-primary); +} + +.large-subfolder-modal-btn-primary:hover { + background-color: var(--color-primary-dark); + border-color: var(--color-primary-dark); +} + +/* Dark mode adjustments */ +.dark-mode .large-subfolder-modal-overlay { + background-color: rgba(0, 0, 0, 0.8); +} + +.dark-mode .large-subfolder-modal { + background: var(--background-primary); + box-shadow: 0 8px 32px rgba(0, 0, 0, 0.4); +} + +.dark-mode .large-subfolder-modal-warning { + background-color: rgba(243, 156, 18, 0.15); + border-color: rgba(243, 156, 18, 0.4); +} \ No newline at end of file diff --git a/src/styles/sidebar/TreeItem.css b/src/styles/sidebar/TreeItem.css index 0b8f2d57..fdda7f8c 100644 --- a/src/styles/sidebar/TreeItem.css +++ b/src/styles/sidebar/TreeItem.css @@ -76,8 +76,14 @@ margin-right: 4px; cursor: pointer; color: var(--icon-color); - z-index: 2; /* Ensure toggle is clickable */ + z-index: 10; /* Higher z-index to ensure it's clickable */ flex-shrink: 0; + border-radius: 3px; + position: relative; +} + +.tree-item-toggle:hover { + background-color: var(--hover-color, rgba(0, 0, 0, 0.1)); } .tree-item-toggle svg { @@ -129,6 +135,28 @@ flex-shrink: 0; } +.tree-item-processing { + display: flex; + align-items: center; + font-size: 12px; + color: var(--color-primary); + flex-shrink: 0; + margin-left: 8px; +} + +.tree-item-spinner { + animation: spin 1s linear infinite; +} + +@keyframes spin { + from { + transform: rotate(0deg); + } + to { + transform: rotate(360deg); + } +} + /* -------------------- File Tree (in Sidebar - File badges (For Binary Files) -------------------- */ /* Base Badge Style */ @@ -175,3 +203,16 @@ .dark-mode .tree-item-badge-folder { font-style: italic; } + +/* Small estimate badge for tree items */ +.estimate-badge-small { + background: var(--color-primary); + color: var(--background-primary); + font-size: 8px; + padding: 1px 3px; + border-radius: 6px; + margin-left: 2px; + font-weight: 500; + opacity: 0.8; + vertical-align: super; +} diff --git a/src/types/FileTypes.ts b/src/types/FileTypes.ts index 1f614151..51c0a0a8 100644 --- a/src/types/FileTypes.ts +++ b/src/types/FileTypes.ts @@ -12,6 +12,8 @@ export interface FileData { error?: string; fileType?: string; excludedByDefault?: boolean; + isTokenEstimate?: boolean; // Flag to indicate if tokenCount is an estimate or real + isDirectory?: boolean; // Flag to indicate if this is a directory } export interface TreeNode { @@ -46,12 +48,14 @@ export interface SidebarProps { currentWorkspaceName?: string | null; collapseAllFolders: () => void; expandAllFolders: () => void; + processingFiles?: Set; } export interface FileListProps { files: FileData[]; selectedFiles: string[]; toggleFileSelection: (filePath: string) => void; + sortOrder?: string; } export interface FileCardProps { @@ -67,6 +71,7 @@ export interface TreeItemProps { toggleFolderSelection: (folderPath: string, isSelected: boolean) => void; toggleExpanded: (nodeId: string) => void; includeBinaryPaths: boolean; + processingFiles?: Set; } export interface SortOption { diff --git a/src/utils/contentFormatUtils.ts b/src/utils/contentFormatUtils.ts index a1ed2060..cd121db3 100644 --- a/src/utils/contentFormatUtils.ts +++ b/src/utils/contentFormatUtils.ts @@ -3,7 +3,7 @@ */ import { FileData } from '../types/FileTypes'; -import { generateAsciiFileTree, normalizePath } from './pathUtils'; +import { generateAsciiFileTree, normalizePath, arePathsEqual } from './pathUtils'; import { getLanguageFromFilename } from './languageUtils'; /** @@ -41,8 +41,11 @@ export const formatBaseFileContent = ({ selectedFolder, }: Omit): string => { // Sort files according to current sort settings + // Create a Set of normalized paths for O(1) lookup performance + const normalizedSelectedPaths = new Set(selectedFiles.map(path => normalizePath(path))); + const sortedSelected = files - .filter((file: FileData) => selectedFiles.includes(file.path)) + .filter((file: FileData) => normalizedSelectedPaths.has(normalizePath(file.path))) .sort((a: FileData, b: FileData) => { let comparison = 0; const [sortKey, sortDir] = sortOrder.split('-'); @@ -117,8 +120,11 @@ export const formatContentForCopying = ({ userInstructions, }: FormatContentParams): string => { // Sort files according to current sort settings + // Create a Set of normalized paths for O(1) lookup performance + const normalizedSelectedPaths = new Set(selectedFiles.map(path => normalizePath(path))); + const sortedSelected = files - .filter((file: FileData) => selectedFiles.includes(file.path)) + .filter((file: FileData) => normalizedSelectedPaths.has(normalizePath(file.path))) .sort((a: FileData, b: FileData) => { let comparison = 0; const [sortKey, sortDir] = sortOrder.split('-'); From 0e04757516d57b382e4fb0a123c84a22a676176b Mon Sep 17 00:00:00 2001 From: vovarbv Date: Thu, 19 Jun 2025 21:07:11 +0200 Subject: [PATCH 2/2] chore: remove extra lines from TODO.md --- TODO.md | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/TODO.md b/TODO.md index bf8af2e6..6df487ef 100644 --- a/TODO.md +++ b/TODO.md @@ -24,14 +24,4 @@ - [ ] Explore alternative libraries for file tree navigation - [ ] Consider using a different library for the file tree navigation - [ ] Chokidar for file watching -- [ ] Zustand for state management - -!!! -!!! -!!! -!!! -!!! -!!! -!!! -!!! -!!! \ No newline at end of file +- [ ] Zustand for state management \ No newline at end of file