From 4e34167fe3fd1b64f8bdd3cbd533ea60c641032d Mon Sep 17 00:00:00 2001 From: Saoud Rizwan <7799382+saoudrizwan@users.noreply.github.com> Date: Tue, 16 Jun 2026 08:17:24 -0700 Subject: [PATCH 1/3] feat: add pigment plugin --- README.md | 5 +- plugins/pigment/LICENSE.pigment-skills | 56 + plugins/pigment/README.md | 65 ++ plugins/pigment/index.ts | 63 ++ plugins/pigment/package.json | 21 + .../skills/analyzing-pigment-data/SKILL.md | 225 +++++ .../SKILL.md | 123 +++ .../view_aggregators.md | 408 ++++++++ .../view_components.md | 119 +++ .../view_design_process.md | 28 + .../view_display_modes.md | 9 + .../view_filtering.md | 131 +++ .../view_pivoting.md | 180 ++++ .../view_sorting.md | 133 +++ .../view_troubleshooting.md | 95 ++ .../skills/designing-pigment-boards/SKILL.md | 238 +++++ .../board_design_rules.md | 228 +++++ .../designing-pigment-boards/board_pages.md | 176 ++++ .../relevant_views.md | 206 ++++ .../scoring_relevant_views.md | 42 + .../designing-pigment-boards/view_widgets.md | 36 + .../skills/integrating-external-data/SKILL.md | 196 ++++ .../data_import_csv.md | 151 +++ .../integrating-external-data/excel_import.md | 414 ++++++++ .../integration_overview.md | 44 + .../modeling-pigment-applications/SKILL.md | 179 ++++ .../modeling_architecture_design.md | 307 ++++++ .../modeling_dimensions_and_hierarchies.md | 266 +++++ .../modeling_fundamentals.md | 248 +++++ .../modeling_naming_conventions.md | 430 ++++++++ .../modeling_performance_considerations.md | 52 + .../modeling_principles.md | 463 +++++++++ .../modeling_subsets.md | 364 +++++++ .../modeling_time_and_calendars.md | 465 +++++++++ .../modeling_working_with_folders.md | 137 +++ .../optimizing-pigment-performance/SKILL.md | 144 +++ .../performance_access_rights.md | 115 +++ .../performance_auditing_application.md | 218 ++++ .../performance_calendar_considerations.md | 50 + .../performance_cleaning_application.md | 189 ++++ .../performance_formula_optimization.md | 584 +++++++++++ .../performance_iterative_calculations.md | 477 +++++++++ .../performance_profiling.md | 114 +++ .../performance_scoping_patterns.md | 956 ++++++++++++++++++ .../performance_sparsity_deep_dive.md | 627 ++++++++++++ .../performance_troubleshooting_workflow.md | 85 ++ .../SKILL.md | 140 +++ .../planning_cycles_scenarios.md | 46 + .../planning_cycles_snapshots.md | 42 + .../planning_cycles_versions.md | 129 +++ .../securing-pigment-applications/SKILL.md | 105 ++ .../securing_access_rights.md | 270 +++++ .../solving-specific-use-cases/SKILL.md | 93 ++ .../finance_nexus_financial_statements.md | 242 +++++ .../fx_currency_conversion.md | 117 +++ ...pex_forecasting_planning_methods_engine.md | 237 +++++ .../opex_planning_application_architecture.md | 139 +++ ...orkforce_planning_architecture_patterns.md | 231 +++++ ...kforce_planning_cards_mapped_dimensions.md | 135 +++ .../workforce_planning_changelog_overrides.md | 164 +++ .../workforce_planning_snapshot_spread.md | 154 +++ .../skills/writing-pigment-formulas/SKILL.md | 331 ++++++ .../formula_by_mapping_arrow.md | 221 ++++ .../formula_conditionals_style.md | 279 +++++ .../formula_modifiers.md | 578 +++++++++++ .../formula_performance_patterns.md | 435 ++++++++ .../formula_segmentation_tiered_lookup.md | 115 +++ .../formula_writing_workflow.md | 262 +++++ .../functions_basic_aggregations.md | 221 ++++ .../functions_finance.md | 219 ++++ .../functions_forecasting.md | 220 ++++ .../functions_iterative_calculation.md | 335 ++++++ .../functions_logical.md | 660 ++++++++++++ .../functions_lookup.md | 108 ++ .../functions_numeric.md | 200 ++++ .../functions_security.md | 102 ++ .../functions_text.md | 387 +++++++ .../functions_time_and_date.md | 505 +++++++++ 78 files changed, 17283 insertions(+), 1 deletion(-) create mode 100644 plugins/pigment/LICENSE.pigment-skills create mode 100644 plugins/pigment/README.md create mode 100644 plugins/pigment/index.ts create mode 100644 plugins/pigment/package.json create mode 100644 plugins/pigment/skills/analyzing-pigment-data/SKILL.md create mode 100644 plugins/pigment/skills/creating-and-editing-pigment-views/SKILL.md create mode 100644 plugins/pigment/skills/creating-and-editing-pigment-views/view_aggregators.md create mode 100644 plugins/pigment/skills/creating-and-editing-pigment-views/view_components.md create mode 100644 plugins/pigment/skills/creating-and-editing-pigment-views/view_design_process.md create mode 100644 plugins/pigment/skills/creating-and-editing-pigment-views/view_display_modes.md create mode 100644 plugins/pigment/skills/creating-and-editing-pigment-views/view_filtering.md create mode 100644 plugins/pigment/skills/creating-and-editing-pigment-views/view_pivoting.md create mode 100644 plugins/pigment/skills/creating-and-editing-pigment-views/view_sorting.md create mode 100644 plugins/pigment/skills/creating-and-editing-pigment-views/view_troubleshooting.md create mode 100644 plugins/pigment/skills/designing-pigment-boards/SKILL.md create mode 100644 plugins/pigment/skills/designing-pigment-boards/board_design_rules.md create mode 100644 plugins/pigment/skills/designing-pigment-boards/board_pages.md create mode 100644 plugins/pigment/skills/designing-pigment-boards/relevant_views.md create mode 100644 plugins/pigment/skills/designing-pigment-boards/scoring_relevant_views.md create mode 100644 plugins/pigment/skills/designing-pigment-boards/view_widgets.md create mode 100644 plugins/pigment/skills/integrating-external-data/SKILL.md create mode 100644 plugins/pigment/skills/integrating-external-data/data_import_csv.md create mode 100644 plugins/pigment/skills/integrating-external-data/excel_import.md create mode 100644 plugins/pigment/skills/integrating-external-data/integration_overview.md create mode 100644 plugins/pigment/skills/modeling-pigment-applications/SKILL.md create mode 100644 plugins/pigment/skills/modeling-pigment-applications/modeling_architecture_design.md create mode 100644 plugins/pigment/skills/modeling-pigment-applications/modeling_dimensions_and_hierarchies.md create mode 100644 plugins/pigment/skills/modeling-pigment-applications/modeling_fundamentals.md create mode 100644 plugins/pigment/skills/modeling-pigment-applications/modeling_naming_conventions.md create mode 100644 plugins/pigment/skills/modeling-pigment-applications/modeling_performance_considerations.md create mode 100644 plugins/pigment/skills/modeling-pigment-applications/modeling_principles.md create mode 100644 plugins/pigment/skills/modeling-pigment-applications/modeling_subsets.md create mode 100644 plugins/pigment/skills/modeling-pigment-applications/modeling_time_and_calendars.md create mode 100644 plugins/pigment/skills/modeling-pigment-applications/modeling_working_with_folders.md create mode 100644 plugins/pigment/skills/optimizing-pigment-performance/SKILL.md create mode 100644 plugins/pigment/skills/optimizing-pigment-performance/performance_access_rights.md create mode 100644 plugins/pigment/skills/optimizing-pigment-performance/performance_auditing_application.md create mode 100644 plugins/pigment/skills/optimizing-pigment-performance/performance_calendar_considerations.md create mode 100644 plugins/pigment/skills/optimizing-pigment-performance/performance_cleaning_application.md create mode 100644 plugins/pigment/skills/optimizing-pigment-performance/performance_formula_optimization.md create mode 100644 plugins/pigment/skills/optimizing-pigment-performance/performance_iterative_calculations.md create mode 100644 plugins/pigment/skills/optimizing-pigment-performance/performance_profiling.md create mode 100644 plugins/pigment/skills/optimizing-pigment-performance/performance_scoping_patterns.md create mode 100644 plugins/pigment/skills/optimizing-pigment-performance/performance_sparsity_deep_dive.md create mode 100644 plugins/pigment/skills/optimizing-pigment-performance/performance_troubleshooting_workflow.md create mode 100644 plugins/pigment/skills/planning-cycles-pigment-applications/SKILL.md create mode 100644 plugins/pigment/skills/planning-cycles-pigment-applications/planning_cycles_scenarios.md create mode 100644 plugins/pigment/skills/planning-cycles-pigment-applications/planning_cycles_snapshots.md create mode 100644 plugins/pigment/skills/planning-cycles-pigment-applications/planning_cycles_versions.md create mode 100644 plugins/pigment/skills/securing-pigment-applications/SKILL.md create mode 100644 plugins/pigment/skills/securing-pigment-applications/securing_access_rights.md create mode 100644 plugins/pigment/skills/solving-specific-use-cases/SKILL.md create mode 100644 plugins/pigment/skills/solving-specific-use-cases/finance_nexus_financial_statements.md create mode 100644 plugins/pigment/skills/solving-specific-use-cases/fx_currency_conversion.md create mode 100644 plugins/pigment/skills/solving-specific-use-cases/opex_forecasting_planning_methods_engine.md create mode 100644 plugins/pigment/skills/solving-specific-use-cases/opex_planning_application_architecture.md create mode 100644 plugins/pigment/skills/solving-specific-use-cases/workforce_planning_architecture_patterns.md create mode 100644 plugins/pigment/skills/solving-specific-use-cases/workforce_planning_cards_mapped_dimensions.md create mode 100644 plugins/pigment/skills/solving-specific-use-cases/workforce_planning_changelog_overrides.md create mode 100644 plugins/pigment/skills/solving-specific-use-cases/workforce_planning_snapshot_spread.md create mode 100644 plugins/pigment/skills/writing-pigment-formulas/SKILL.md create mode 100644 plugins/pigment/skills/writing-pigment-formulas/formula_by_mapping_arrow.md create mode 100644 plugins/pigment/skills/writing-pigment-formulas/formula_conditionals_style.md create mode 100644 plugins/pigment/skills/writing-pigment-formulas/formula_modifiers.md create mode 100644 plugins/pigment/skills/writing-pigment-formulas/formula_performance_patterns.md create mode 100644 plugins/pigment/skills/writing-pigment-formulas/formula_segmentation_tiered_lookup.md create mode 100644 plugins/pigment/skills/writing-pigment-formulas/formula_writing_workflow.md create mode 100644 plugins/pigment/skills/writing-pigment-formulas/functions_basic_aggregations.md create mode 100644 plugins/pigment/skills/writing-pigment-formulas/functions_finance.md create mode 100644 plugins/pigment/skills/writing-pigment-formulas/functions_forecasting.md create mode 100644 plugins/pigment/skills/writing-pigment-formulas/functions_iterative_calculation.md create mode 100644 plugins/pigment/skills/writing-pigment-formulas/functions_logical.md create mode 100644 plugins/pigment/skills/writing-pigment-formulas/functions_lookup.md create mode 100644 plugins/pigment/skills/writing-pigment-formulas/functions_numeric.md create mode 100644 plugins/pigment/skills/writing-pigment-formulas/functions_security.md create mode 100644 plugins/pigment/skills/writing-pigment-formulas/functions_text.md create mode 100644 plugins/pigment/skills/writing-pigment-formulas/functions_time_and_date.md diff --git a/README.md b/README.md index a400e81c..4ffb083e 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,7 @@ Each plugin lives in `plugins/`. The directory name is the install keyword | `linear` | Linear SDK scripting skill for issue, project, team, cycle, and comment workflows. | | `mac-notify` | macOS notifications when a Cline run completes. | | `nanobanana` | Image generation through OpenRouter and Gemini image models. | +| `pigment` | Pigment planning, formula, modeling, board, view, import, and performance skills with optional Pigment MCP registration. | | `speak` | Speaks completed Cline replies with ElevenLabs text to speech. | | `typescript-lsp` | TypeScript language service `goto_definition` support. | | `weather-metrics` | Demo weather tool plus runtime metrics hooks. | @@ -53,4 +54,6 @@ npm run validate ## License -[Apache 2.0 © 2026 Cline Bot Inc.](./LICENSE) +Most plugins in this repository are distributed under [Apache 2.0 © 2026 Cline Bot Inc.](./LICENSE). + +Some bundled third-party skill content has additional license terms. Check each plugin directory for plugin-specific license files, such as `plugins/pigment/LICENSE.pigment-skills`. diff --git a/plugins/pigment/LICENSE.pigment-skills b/plugins/pigment/LICENSE.pigment-skills new file mode 100644 index 00000000..e53a7479 --- /dev/null +++ b/plugins/pigment/LICENSE.pigment-skills @@ -0,0 +1,56 @@ +Pigment Skills License v1 + +1. Definitions + +“Pigment” means Pigment SAS and its affiliates. +“Skills” means the text files, prompts, instructions, examples, and any accompanying materials provided by Pigment that are intended for use in connection with the Services. +“You” means the individual or entity using the Skills. +“Services” means Pigment’s products, APIs, MCP servers, and related services. + +2. License Grant + +Subject to this License, Pigment grants You a limited, non-exclusive, non-transferable, revocable license to use, copy, modify, and distribute the Skills (including modified versions) in connection with the Services. + +3. Restrictions + +You will not, and will not permit others to: +Use the Skills to develop, train, improve, or benchmark a model or dataset intended to compete with the Services. +Use the Skills to build or enhance a competing product or service that provides substantially similar functionality to the Services. +Remove or alter copyright, trademark, or attribution notices in the Skills. +Represent modified Skills as official or endorsed by Pigment. +Distribute the Skills (including modified versions) without retaining all applicable copyright, trademark, and attribution notices included in the original Skills. +Distribute the Skills (including modified versions) without including a copy of this License. +Distribute modified versions of the Skills without clearly indicating that changes were made and that such modifications are not endorsed by Pigment. + +4. Ownership + +The Skills are licensed, not sold. Pigment retains all rights, title, and interest in and to the Skills, including all intellectual property rights. + +5. Feedback + +If You provide suggestions, ideas, or feedback about the Skills or Services (“Feedback”), You grant Pigment a perpetual, irrevocable, worldwide, royalty-free right to use and incorporate Feedback into its products and services without any obligation. + +6. No Warranty + +THE SKILLS ARE PROVIDED “AS IS” AND “AS AVAILABLE.” PIGMENT DISCLAIMS ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING IMPLIED WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND NON-INFRINGEMENT. +You are responsible for evaluating outputs produced using the Skills, including for accuracy, safety, and compliance. + +7. Limitation of Liability + +TO THE MAXIMUM EXTENT PERMITTED BY LAW, PIGMENT WILL NOT BE LIABLE FOR ANY INDIRECT, INCIDENTAL, SPECIAL, CONSEQUENTIAL, EXEMPLARY, OR PUNITIVE DAMAGES, OR ANY LOSS OF PROFITS, REVENUE, DATA, OR GOODWILL, ARISING OUT OF OR RELATED TO THE SKILLS OR THIS LICENSE. IN ALL CASES, PIGMENT’S TOTAL LIABILITY UNDER THIS LICENSE WILL NOT EXCEED USD $100 (OR THE EQUIVALENT IN LOCAL CURRENCY). + +8. Termination + +This License is effective until terminated. It will terminate automatically when you stop using the Services or if You breach this License. Upon termination, You must stop using the Skills and delete all copies in Your possession or control, including modified versions. + +9. Compliance with Law + +You will use the Skills in compliance with applicable laws and regulations, including those related to data protection and export controls. + +10. Trademark / No Endorsement + +This License does not grant You any rights to use Pigment’s trademarks or branding. You may not imply endorsement by Pigment. + +11. Governing Law + +This License and any dispute or claim (including non-contractual disputes or claims) arising out of or in connection with it or its subject matter or formation shall be governed by and construed in accordance with the laws of England and Wales. The parties irrevocably agree that the courts of England and Wales shall have exclusive jurisdiction to settle any such dispute or claim. diff --git a/plugins/pigment/README.md b/plugins/pigment/README.md new file mode 100644 index 00000000..91494de2 --- /dev/null +++ b/plugins/pigment/README.md @@ -0,0 +1,65 @@ +# pigment + +Pigment planning and modeling guidance for Cline, with optional Pigment MCP registration. + +## What It Does + +Installs Pigment skills for analyzing workspace data, modeling applications, writing Pigment formulas, building boards and views, integrating external data, planning cycles, securing access rights, optimizing performance, and applying FP&A or workforce planning patterns. + +The plugin can also register the Pigment MCP server when the user provides their workspace MCP URL. Pigment MCP gives Cline live access to the user's Pigment workspace through Pigment's OAuth flow. Default Pigment MCP tools are read-oriented for analysis. Pigment Advanced MCP tools can create or edit modeling objects when the user enables advanced tools in Pigment. + +## Install + +```bash +cline plugin install pigment +``` + +For local development from this repository: + +```bash +cline plugin install ./plugins/pigment --cwd . +``` + +## Enable Pigment MCP + +Pigment generates a workspace-specific MCP URL under Pigment settings: + +```text +Settings > Integrations > MCP +``` + +Set that URL before installing or re-enabling the plugin: + +```bash +export CLINE_PIGMENT_MCP_URL="https://pigment.app/api/mcp/public/your-mcp-id" +cline plugin install pigment +``` + +If `CLINE_PIGMENT_MCP_URL` is not set, the plugin still installs the Pigment skills and safety rule, but it does not create a Pigment MCP settings entry. + +## Example Usage + +After installation, ask Cline: + +```text +Review this Pigment formula for correctness and performance before I apply it. +``` + +With Pigment MCP configured, ask: + +```text +List my Pigment applications, find the revenue metrics in the FP&A model, and explain which ones are enabled for AI data access. +``` + +## Requirements + +- A Pigment workspace. +- Pigment MCP enabled in `Settings > Integrations > MCP` when live workspace access is needed. +- Pigment OAuth in Cline's MCP flow after the MCP server is registered. +- Advanced MCP tools enabled in Pigment only when Cline should help create or edit modeling objects. + +## Security Notes + +Pigment data and model changes can affect financial planning workflows. The plugin adds a rule that requires explicit confirmation before Cline uses advanced MCP tools for writes, imports, access-right changes, board or view edits, scenario or snapshot changes, and deletions. + +The bundled skills are licensed by Pigment for use with Pigment services. See `LICENSE.pigment-skills`. The bundled markdown has been format-normalized for this repository's validation rules and is not represented as an official or endorsed Pigment distribution. diff --git a/plugins/pigment/index.ts b/plugins/pigment/index.ts new file mode 100644 index 00000000..b7e48b03 --- /dev/null +++ b/plugins/pigment/index.ts @@ -0,0 +1,63 @@ +import type { AgentPlugin } from "@cline/sdk" + +const PIGMENT_MCP_URL_ENV = "CLINE_PIGMENT_MCP_URL" + +type PigmentMcpUrlConfig = + | { status: "unset" } + | { status: "valid"; url: string } + | { status: "invalid"; reason: string } + +function readPigmentMcpUrl(): PigmentMcpUrlConfig { + const raw = process.env[PIGMENT_MCP_URL_ENV]?.trim() + if (!raw) { + return { status: "unset" } + } + + try { + const url = new URL(raw) + if (url.protocol !== "https:") { + return { status: "invalid", reason: "URL must use https" } + } + return { status: "valid", url: url.toString() } + } catch { + return { status: "invalid", reason: "URL could not be parsed" } + } +} + +const plugin: AgentPlugin = { + name: "pigment", + manifest: { + capabilities: ["mcp", "rules", "skills"], + }, + setup(api) { + const pigmentMcpUrl = readPigmentMcpUrl() + if (pigmentMcpUrl.status === "invalid") { + throw new Error( + `Invalid ${PIGMENT_MCP_URL_ENV}: ${pigmentMcpUrl.reason}. Set it to the https URL from Pigment Settings > Integrations > MCP, or unset it to install skills only.`, + ) + } + + if (pigmentMcpUrl.status === "valid") { + api.registerMcpServer({ + name: "pigment", + transport: { + type: "streamableHttp", + url: pigmentMcpUrl.url, + }, + }) + } + + api.registerRule({ + id: "pigment-workspace-safety", + source: "pigment", + content: [ + "When working with Pigment, use the bundled Pigment skills for formulas, modeling, views, boards, imports, planning cycles, performance, and access rights.", + "Do not invent Pigment application IDs, block IDs, metric names, dimension names, view IDs, or formula syntax. Read available workspace context through Pigment MCP tools when they are configured, or ask the user for the missing details.", + "Treat Pigment Advanced MCP tools as workspace-changing operations. Before creating or editing dimensions, metrics, formulas, calendars, imports, boards, views, access rights, scenarios, snapshots, or deleting anything, explain the intended change and get explicit user confirmation.", + "Never print, store, commit, or ask the user to paste OAuth tokens or credentials. Pigment MCP authentication should happen through Cline's MCP OAuth flow after the user provides their workspace MCP URL.", + ].join("\n"), + }) + }, +} + +export default plugin diff --git a/plugins/pigment/package.json b/plugins/pigment/package.json new file mode 100644 index 00000000..c0a0f272 --- /dev/null +++ b/plugins/pigment/package.json @@ -0,0 +1,21 @@ +{ + "name": "pigment", + "version": "0.0.0", + "private": true, + "type": "module", + "description": "Cline plugin with Pigment planning, modeling, formula, and workspace skills.", + "cline": { + "plugins": [ + { + "paths": [ + "./index.ts" + ], + "capabilities": [ + "mcp", + "rules", + "skills" + ] + } + ] + } +} diff --git a/plugins/pigment/skills/analyzing-pigment-data/SKILL.md b/plugins/pigment/skills/analyzing-pigment-data/SKILL.md new file mode 100644 index 00000000..a577a136 --- /dev/null +++ b/plugins/pigment/skills/analyzing-pigment-data/SKILL.md @@ -0,0 +1,225 @@ +--- +name: analyzing-pigment-data +description: Always use this skill when querying, exploring, or analyzing existing data in a Pigment workspace. Covers the analysis workflow, query formulation, data concepts, analysis patterns, ambiguity handling, and result interpretation. +metadata: + skill_path: /analyzing-pigment-data/SKILL.md + base_directory: /analyzing-pigment-data + includes: + - "*.md" +--- + +# How to Use This Skill + +This `SKILL.md` is self-contained. Read it fully before performing any data analysis. + +# Analyzing Pigment Data + +This skill teaches you how to answer analytical questions using data that already exists in a Pigment workspace. It covers how to discover what data is available, how to formulate effective queries, how to interpret results, and how to structure multi-step analyses. + +This skill is for read-only exploration and analysis. If the user wants to create, modify, or configure application objects (metrics, dimensions, formulas, boards), use the relevant modeling skills instead. + +--- + +## When to Use This Skill + +- Answer data questions - "What were Q4 sales by region?" +- Explore available data - "What metrics exist in this app?" +- Understand metric structure - "What dimensions does Revenue have?" +- Compare values - "Compare actual vs budget for EMEA" +- Find top contributors - "Which products drive the most revenue?" +- Analyze trends - "Show me headcount over the last 12 months" +- Cross-metric analysis - "How do Sales and Costs compare by department?" + +--- + +## Core Concepts + +Pigment organizes data in a multidimensional model: + +- Metrics - Named data blocks containing values (numbers, text, dates, booleans). A metric is the primary unit of analysis. +- Dimensions - The axes of a metric. A metric dimensioned by `Country` and `Month` stores one value per country per month. +- Items - The members of a dimension. `France`, `Germany`, `US` are items of the `Country` dimension. +- Properties - Attributes on dimension items. The `Country` dimension may have a `Region` property grouping countries into `EMEA`, `AMER`, `APAC`. Property dimensions (e.g. `Country > Region`) can be used as regular dimensions for breakdowns and filters. + +When analyzing data, you query a metric and optionally: +- Break down (pivot) by one or more dimensions to see values at a finer grain +- Filter by dimension items to narrow the scope (e.g. only `Country` = `France`, `Germany`) + +--- + +## Analysis Workflow + +Follow this sequence for every analytical question: + +### Step 1: Identify the application + +Use `get_applications` to list available applications and obtain application IDs. + +Never fabricate IDs - always retrieve them from tool responses. + +### Step 2: Discover available metrics + +Use `get_ai_metrics` to list AI-enabled metrics in the application. Only metrics with AI Search enabled can be queried via natural language. + +If a metric the user mentions is not in the list, possible reasons: +- AI data access is not enabled on that metric +- The user does not have access to it +- The metric does not exist + +In these cases, inform the user and suggest they check their Pigment workspace settings. + +### Step 3: Understand metric structure + +Use `get_metric_description` to inspect a metric before querying it. This reveals: +- Which dimensions the metric has (and therefore which breakdowns and filters are valid) +- The data type (number, text, date, boolean, dimension) +- Available scenarios (if any) +- Dimension items and properties + +Always call this before your first query on a metric. It prevents invalid queries and helps you formulate precise requests. + +### Step 4: Query the data + +Use `query_data` to retrieve data using natural language. Formulate your query by specifying: +- The metric to analyze +- Optional breakdowns (dimensions to pivot by) +- Optional filters (dimension items to include or exclude) + +### Step 5: Interpret and present results + +After receiving data: +- Highlight the key findings concisely +- Compute derived values yourself if needed (ratios, percentages, rankings) +- Suggest follow-up analyses when patterns warrant deeper investigation + +--- + +## Query Formulation Rules + +### One metric per query + +If the user asks about multiple metrics (e.g. "Compare Sales and Costs"), query each metric separately and combine the results yourself. Never request derived expressions like "Sales / Costs" in a single query. + +### No value-based filtering at query time + +The query tool cannot filter by metric values (e.g. "top 10", "greater than 1M", "largest change"). Instead: +1. Fetch the raw data with appropriate dimensional filters +2. Apply sorting, ranking, or thresholds yourself after receiving the results + +Item-based filtering is supported (e.g. filter to `Country` = `France`). + +### Manage data volume + +Queries have limits on the number of values returned. If a query exceeds the limit: +1. Narrow item filters - restrict to fewer dimension items +2. Reduce breakdowns - use fewer pivot dimensions (the tool returns aggregated values for omitted dimensions) +3. Inform the user - if scope cannot be reduced while still answering the question, explain the limitation and ask how to refine + +### Text metrics have special rules + +Text data does not support numeric aggregation: +- All base dimensions must be included as breakdowns or single-item filters +- Filters on text metrics can only use one item per dimension + +Use `get_metric_description` to check which dimensions are required. + +--- + +## Analysis Patterns + +### Top contribution analysis + +Find the main contributors to a metric value. + +Example: "Find the top 5 countries contributing to Sales" +- Query `Sales` broken down by `Country` +- Sort the results by value and take the top 5 +- Optionally drill deeper: for each top country, break down by another dimension + +### Variance analysis (compare two items) + +Compare two items of the same dimension within a metric. + +Example: "Compare Sales in FY24 vs FY25 by Product" +- Query `Sales` broken down by `Product`, filtered to `Year` = `FY24` +- Query `Sales` broken down by `Product`, filtered to `Year` = `FY25` +- Compute the difference and highlight significant variances + +Always specify: the metric, the dimension to compare on, the two items, and any breakdown dimensions. + +### Cross-metric comparison + +Compare two different metrics along shared dimensions. + +Example: "Compare Sales and Costs by Department" +- Query `Sales` broken down by `Department` +- Query `Costs` broken down by `Department` +- Present side-by-side and compute derived values (e.g. margin) + +Breakdowns must only use dimensions shared by both metrics. + +### Time series analysis + +Analyze how a metric evolves over time. + +Example: "Show Revenue trend by Month for the last year" +- Query `Revenue` broken down by `Month` (with appropriate time filters) +- Identify trends, shifts, seasonality, or outliers + +Always specify: the metric, the time dimension, and the time range. + +--- + +## Multi-Step Analysis + +Complex questions often require a discovery phase followed by deeper analysis. Break these into explicit steps. + +Example: "Analyze Sales performance - find the top regions and drill into their best products" + +1. Step 1 (discovery): Query `Sales` broken down by `Region` -> identify top 3 regions +2. Step 2 (deep dive): For each top region, query `Sales` broken down by `Product`, filtered to that region -> identify top products +3. Step 3 (synthesis): Combine findings into a coherent narrative + +Each step should be self-contained: specify the metric, breakdowns, and filters explicitly. Do not rely on implicit context from previous steps when formulating queries. + +--- + +## Handling Ambiguity + +User requests are often imprecise. Follow these rules in priority order: + +1. Never assume - if more than one valid interpretation exists, ask the user to clarify +2. Use context - infer meaning from recently mentioned metrics or dimensions +3. Use exploration tools - call `get_ai_metrics` or `get_metric_description` to identify likely matches +4. Ask rather than guess - when in doubt, request clarification. Contextualize your question so the user understands what is ambiguous. + +### Common ambiguity sources + +| Ambiguity | Example | How to resolve | +|-----------|---------|----------------| +| Metric name | "Show me revenue" (multiple revenue metrics exist) | List the candidates and ask which one | +| Time range | "Last quarter" (fiscal vs calendar? which year?) | Ask for clarification or check available time dimension items | +| Comparison target | "Compare against plan" (Budget? Forecast? Target?) | List available scenario/version items | +| Breakdown level | "By region" (geographic region? business region?) | Check available dimensions and ask if ambiguous | +| Scope | "Sales performance" (all products? all countries?) | Ask if they want the full scope or a specific subset | + +Querying tools is expensive. Do not call them until you are confident the user's intent is clear. + +--- + +## Presenting Results + +- Be concise - lead with key findings, not raw data +- Match depth to the question - simple questions get short answers; complex analyses get structured responses +- Highlight what matters - surface the most significant numbers, changes, or outliers +- Suggest next steps - when findings reveal something interesting, propose follow-up analyses +- Be transparent - explain what you queried and any limitations in the data + +--- + +## Cross-References + +- Understanding application structure: modeling-pigment-applications skill +- Building dashboards from analysis results: designing-pigment-boards skill +- Writing formulas for computed metrics: writing-pigment-formulas skill + diff --git a/plugins/pigment/skills/creating-and-editing-pigment-views/SKILL.md b/plugins/pigment/skills/creating-and-editing-pigment-views/SKILL.md new file mode 100644 index 00000000..f19817eb --- /dev/null +++ b/plugins/pigment/skills/creating-and-editing-pigment-views/SKILL.md @@ -0,0 +1,123 @@ +--- +name: creating-and-editing-pigment-views +description: Always use this skill when creating or editing Views, or needing to pick a View. +metadata: + skill_path: /creating-and-editing-pigment-views/SKILL.md + base_directory: /creating-and-editing-pigment-views + includes: + - "*.md" +--- + +# How to Use This Skill + +Progressive Disclosure Pattern: This `SKILL.md` provides an overview. Most details live in supporting files. + +This file alone is often not sufficient + +Required workflow: + +1. Read this file first - Understand available resources and when to use them +2. Identify relevant topics - Match your task to any of the supporting documents +3. Read supporting files - Use available Cline file-reading or search tools to access detailed documentation +4. Explore as needed - Use available Cline listing and search tools to discover additional resources in this directory (some might not be explicitly mentioned in this file) + +## UI and tool semantics (Views) + +These notes align the Pigment UI with view creation and edition tools + +### `values` (value fields) + +Each entry describes what appears in the cells : + +- Metrics -> one metric +- Tables -> one value field per table metric +- List -> one value per property + +Value labels in Pivot panel in the UI are different: + +- for Tables: "Metrics" +- for Lists: "Properties" + +### `metricsLocation` (Metric & Table views) + +API enum (C# `MetricsLocation`): `Columns` (1), `Rows` (2), `Pages` (3) - which axis carries the metrics in the pivot. Dimensions go under `rows`, `columns`, or `pages`. Metrics are chosen in `values`; they are not duplicated into the arrays for dimensions. Their axis is set only via `metricsLocation`. + +KPI rule: `metricsLocation` for a KPI View MUST NOT be `Rows`. KPIs have no row pivots, so Rows yields a broken layout. Use `Columns` (default) or `Pages`. + +### `pivotFieldId` + +Every pivot field placed in `pages`, `rows`, or `columns` gets a stable id (GUID) assigned by the server. `pivotFieldId` in other structures (filters, sorts, etc.) points to that pivot - read it back from the `tool:create_view` / `tool:update_view_pivots` response, do not invent it. + +### `listPropertyPath` on pivots (grouping / hierarchy) + +`ListPropertyPath` is the technical name of properties (e.g. in the UI: `Month > Year`, `Country > Region`). + +### Other + +- Do not create views on sublists. +- The widget's `display_type` (KPI / Grid / Chart) is not stored on the View; configure it on the Board widget. + +--- + +# CRITICAL RULES + +- Value and pivot ids - Assigned by the server. For a NEW pivot/value, omit `id`. To KEEP an existing one, echo back the id from a prior `tool:create_view` / `tool:update_view_*` / `tool:get_view` response. Never invent UUIDs. +- Same dimension on Pages and on Rows/Columns is SUPPORTED - When the user asks to "put X on Pages", add to Pages without removing X from Rows/Columns. Page selectors then narrow which modalities appear on the row/column axis. Do not treat this as a conflict. See [view_components.md](./view_components.md) for OK patterns vs. anti-patterns. +- "Filter" from users -> Page Selector first - Restricting to a dimension item (Year, Country, Version, ...) = `pages` + default item, not `filters[]`. View Filters only for top-N-by-metric, exclusion, or logic Page Selectors cannot express. See [view_filtering.md](./view_filtering.md). +- New View (greenfield) - Two-step flow: + 1. Call `tool:create_view`. Leave `pivotLayout` null to let the server build a sensible default layout for the underlying block. To override, send a complete `pivotLayout` with all three axes (`rows`, `columns`, `pages`) populated - each entry is a simplified pivot seed (`dimensionId` + optional `listPropertyPath`); use an empty array for an axis with no pivot. Half-specified layouts are rejected. Values and hidden-dim aggregations are created with sensible defaults; refine them in step 2 if needed. + 2. Iterate with `tool:update_view_pivots`, `tool:update_view_values` (and the other `update_view_*` tools) to refine the configuration. None of those refinements are accepted by `tool:create_view`. +- Editing an existing View - Use the field-specific variants directly on the View id: + - Values (add/remove value fields, `showValueAsConfiguration`) -> `tool:update_view_values`. Echo back existing value ids to keep them stable. + - Pivot edits (rows / columns / pages / metricsLocation) -> `tool:update_view_pivots`. + - Filters -> `tool:update_view_filters`. Sorts -> `tool:update_view_sorts`. Chart config -> `tool:update_view_chart_config`. + - Metadata, hidden-dim aggregations, template -> `tool:update_view`. + + If a Draft was auto-created, the agent should: + - wire the widget to display it via `tool:update_view_widget_overrides` so only this user sees it + - tell the user they can save the Draft in the Board UI. +- Bulk-save protocol if `tool:save_draft_views` is available - after creating or editing one or more Draft Views: + 1. List the draft view names and ask the user for explicit confirmation before saving. + 2. Once confirmed, call `tool:save_draft_views` once with all draft view IDs. + 3. Report each result: view name, resulting ID, and whether it was merged or promoted to a new view. +- When editing a View in the context of a widget on a Board, you must: + - use the draft + override workflow to allow safe, user-specific preview before committing changes that affect all users. + - read: [view_widgets.md](../designing-pigment-boards/view_widgets.md). +- Name (first signal) - "View 1" and similar are often placeholders. Prefer `create_view` with a real name and pivots aligned to this widget and other widgets on the same board unless the existing View already fits. +- Shared / external View (other users, other boards) - Prefer Draft (or a new View) before overwriting something others rely on or displayed in another board, except if asked explicitely. +- Table views - per-view metrics - When adding or removing metrics on Table block views, call `tool:update_view_values` with the full desired value list: add a value entry for new metrics, remove value entries that are not relevant. Prefer removal over hiding - hidden metrics may still compute. Keep a metric hidden (`displayed: false`) only when the view still depends on it, such as for value-field filtering, sort-by-metric-value, or as an advanced-aggregator operand (ratio, growth, etc.). Do not plan a separate step to update the Table block's metric membership first. Do not add every table metric to each view and hide the rest; configure only the metrics each view should show. + +# Definitions + +## Views + +A View is how a Block (Metric, List, Table) is shown: pivots, filters, sort, display. One Block, many Views. + +## Draft Views + +A private working copy to preview edits before they hit an existing view. On boards, use a Draft + widget overrides when changing the live View behind a widget-see [view_widgets.md](../designing-pigment-boards/view_widgets.md). Not a substitute for `create_view` when you need a new View. Save via bulk-save protocol - list the draft names, wait for user confirmation, then call `tool:save_draft_views` once with all draft view IDs + +--- + +# Reuse vs create + +`tool:get_block_views` helps spot candidates; there is no hard rule to "find similar views" before you create. Creating is normal when names are generic or pivots do not match the board. Details: [relevant_views.md](../designing-pigment-boards/relevant_views.md), [view_design_process.md](./view_design_process.md). + +# View naming + +Grid - no widget suffix. Chart - add chart type, e.g. `... - Waterfall`. KPI - suffix ` - KPI`. + +--- + +# View Design Process + +Must read: [view_design_process.md](./view_design_process.md). + +# View components, filters, sort, aggregators + +- [view_components.md](./view_components.md) +- [view_filtering.md](./view_filtering.md) +- [view_sorting.md](./view_sorting.md) +- [view_display_modes.md](./view_display_modes.md) +- [view_pivoting.md](./view_pivoting.md) +- [view_aggregators.md](./view_aggregators.md) diff --git a/plugins/pigment/skills/creating-and-editing-pigment-views/view_aggregators.md b/plugins/pigment/skills/creating-and-editing-pigment-views/view_aggregators.md new file mode 100644 index 00000000..addec26c --- /dev/null +++ b/plugins/pigment/skills/creating-and-editing-pigment-views/view_aggregators.md @@ -0,0 +1,408 @@ +# View Aggregators and Totals + +This guide explains how aggregation works in Pigment Views: where aggregation applies, what creates visible totals, and when to use simple vs advanced aggregation. + +--- + +## 1. Core mental model + +Aggregation in a View exists at two levels: + +1. Metric default aggregators + - Defined on the Metric + - One for temporal dimensions + - One for non-temporal dimensions + +2. View-level overrides + - Defined on the View + - Used either for: + - visible pivots on Rows / Columns + - hidden dimensions, including dimensions placed only on Pages + +### Critical distinction +- Visible pivot aggregation controls subtotals and grand totals on the grid. +- Hidden dimensions aggregation does not create new total cells. It only determines how visible cells are computed when some metric dimensions are not on Rows / Columns. + +--- + +## 2. Where aggregation applies + +| Dimension placement | Visible on grid? | Pivot aggregation allowed? | Uses hiddenDimensionsAggregations? | Creates visible totals? | +|---|---:|---:|---:|---:| +| Rows | Yes | Yes | No | Yes | +| Columns | Yes | Yes | No | Yes | +| Pages | No | No | Yes | No | +| Not shown | No | No | Yes | No | + +### Practical rule +- If a dimension is on Rows or Columns, use pivot aggregation when you want visible totals. +- If a dimension is only on Pages or not shown, use hiddenDimensionsAggregations. +- If the Metric defaults are already correct, prefer `Default` / no override. + +--- + +## 3. Totals + +### Subtotals +A subtotal is an aggregated value for a visible grouped level on a Row or Column pivot. + +### Grand totals +A grand total is the aggregation across all visible modalities on the relevant axis or axes. + +### Important +- Rows / Columns aggregation can create visible subtotal and grand total cells. + +--- + +## 4. Metric default aggregators + +Every Metric has two default aggregators, set together: + +- Temporal default + - Common choices: + - `Sum` for additive flows (`Revenue`) + - `Last` for snapshots (`Headcount`) + +- Non-temporal default + - Common choices: + - `Sum` for additive measures + - `Any` / `First` for reference-like values + +For rate / percentage / ratio metrics and growth / relative variance metrics, no simple Metric default produces a correct value. These must be aggregated at the View level with Advanced Aggregators (Ratio or Growth). See section 7 and section 10. + +### `Default` +At View level, `Default` means: inherit the Metric's own default aggregator. + +--- + +## 5. View-level aggregation + +### A. Visible pivot aggregation +Use for dimensions on Rows or Columns. + +This controls: +- rollup on the visible axis +- subtotals / grand totals +- aggregation per value field on that pivot + +### B. Hidden dimensions aggregation +Use for dimensions: +- only on Pages +- or not shown in the View + +This controls: +- how Pigment collapses dimensions that are not on Rows / Columns +- how visible cells are computed before visible totals are shown + +--- + +## 6. Simple aggregators: practical reference + +### Decimal / Integer +Common: +- `Sum` +- `Avg` +- `Min` +- `Max` +- `First` +- `Last` +- `FirstNonBlank` +- `LastNonBlank` +- `FirstNonZero` +- `LastNonZero` +- `Count` +- `CountAll` +- `CountUnique` +- `CountBlank` +- `Median` +- `Stdevp` +- `Stdevs` +- `Blank` +- `Default` + +Typical use: +- `Sum` for additive metrics +- `Last` for snapshots + +For rate / percentage / ratio metrics, do not use simple aggregators (`Avg`, `Sum`, ...). Use Advanced Aggregator Ratio at the View level - see section 7. + +### Boolean +Common: +- `Any` +- `All` +- `First` +- `Last` +- `FirstNonBlank` +- `LastNonBlank` +- `Count` +- `CountAll` +- `CountUnique` +- `CountBlank` +- `Blank` +- `Default` + +### Text +Common: +- `TextList` +- `First` +- `Last` +- `FirstNonBlank` +- `LastNonBlank` +- `Count` +- `CountAll` +- `CountUnique` +- `CountBlank` +- `Blank` +- `Default` + +### Date +Common: +- `Min` +- `Max` +- `First` +- `Last` +- `FirstNonBlank` +- `LastNonBlank` +- `Count` +- `CountAll` +- `CountUnique` +- `CountBlank` +- `Blank` +- `Default` + +### Dimension / Permission-like / Access-right-like values +Common: +- `First` +- `Last` +- `FirstNonBlank` +- `LastNonBlank` +- `Count` +- `CountAll` +- `CountUnique` +- `CountBlank` +- `Blank` +- `Default` + +### Rare / specialized +- `OnlyOneNotNull` +- `BitAnd` + +Use only when the business meaning is explicit. + +--- + +## 7. Recommended patterns + +### Additive metrics +Examples: +- Revenue +- Cost +- Units + +Recommended: +- Temporal: `Sum` +- Non-temporal: `Sum` + +### Snapshot metrics +Examples: +- Headcount +- Ending Inventory +- Balance Sheet values + +Recommended: +- Temporal: `Last` +- Non-temporal: often `Sum`, depending on business meaning + +### Metrics calculating a Rate / percentage / ratio +Examples: +- Conversion Rate +- Utilization % +- Margin % + +Recommended: Advanced Aggregator Ratio + +Workflow (Table views): +1. The ratio must exist as a Metric with a Pigment formula (for example `GM% = Gross Margin / Revenue`). Do not fake it by adding the same operand Metric twice as separate value fields. +2. Add the ratio Metric and both operand Metrics to the Table. +3. In the View, include three value fields: ratio Metric plus both operands. Configure Advanced Aggregator Ratio on the ratio Metric's value field; operands are the two operand Metrics' value fields. + +Do not use Advanced Aggregators on a Calculated Item value field to stand in for a cross-metric ratio-Calculated Items are for dimension-level derived rows/columns; configure aggregators on Metric value fields instead. + +Rates and growth-style percentages that roll up across pivots must use Advanced Aggregators wherever the view aggregates. + +### Metrics calculating a Growth or a Relative Variance + +Examples: +- YoY growth % between 2 metrics +- Actual vs Budget % variance + +Recommended: Advanced Aggregator Growth (A-B/B) + +### Reference / lookup values +Examples: +- Owner +- Status +- Category + +Recommended: +- `First`, `Last`, `Any`, or `Blank` + +--- + +## 8. Advanced aggregation + +Advanced aggregation performs a mathematical operation using two value fields. + +Examples: +- `Ratio` +- `Product` +- `Sum` +- `Difference` +- `AbsoluteDifference` +- `Growth` +- `AbsoluteGrowth` + +### Applicability + +| View type | Simple aggregation | Advanced aggregation | +|---|---:|---:| +| View on a Metric | Yes | No | +| View on a Table | Yes | Yes | +| View on a List | Not applicable in the same way | No | + +### Constraints +Advanced aggregation: +- is only supported on Views on Tables +- requires exactly 2 operands +- operands must be metric value fields +- operands must be numeric +- cannot directly self-reference + +### Important +Advanced aggregation is a View configuration, not a new Metric. + +Advanced Aggregators apply to Metric value fields on Table views. They are not the right lever on Calculated Item value fields when the goal is a ratio between two Metrics-define a ratio Metric with a formula, then set the Advanced Aggregator on that value field. + +Do not duplicate an operand Metric as an extra value field and slap an Advanced Aggregator on the duplicate to imitate a ratio; create the ratio Metric and use its value field as the target. + +--- + +## 9. Mapping to tool fields + +### Visible pivot aggregation +Applies to: +- Rows +- Columns + +Configured with: +- `aggregationConfigurations` on the pivot field + +Meaning: +- defines how a value field aggregates on that visible pivot +- controls visible totals + +### Hidden dimensions aggregation +Applies to: +- dimensions only on Pages +- dimensions not shown in the View + +Configured with: +- `hiddenDimensionsAggregations` + +``` +HiddenDimensionsAggregation: + valueFieldId: + temporalDimensionsAggregator: + otherDimensionsAggregator: +``` + +Meaning: +- defines aggregation across hidden temporal dimensions +- defines aggregation across hidden non-temporal dimensions +- does not create visible totals + +### Default inheritance +Configured with: +- `aggregator: Default` + +Meaning: +- use the Metric's own default behavior + +--- + +## 10. Common mistakes + +- Trying to use pivot aggregation on Pages + - Page-only dimensions use `hiddenDimensionsAggregations` + +- Expecting hidden-dimension aggregation to create subtotal rows + - It only affects visible cell calculation + +- Using `Sum` for snapshot metrics across time + - Usually `Last` is correct + +- Using `Sum` for rates / percentages metrics + - Use Advanced Aggregator (type Ratio) on all dimensions, with the same numerator and denominator as in the metric formula. + +- Using `Sum` for growth / relative variance metrics + - Use Advanced Aggregator (type Growth A-B/B) on all dimensions, with the same A and B as in the metric formula. + +- Using advanced aggregation on a Metric View + - Advanced aggregation is for Table Views only + +- Using Advanced Aggregators on Calculated Items to mimic a ratio between two Metrics + - Prefer a dedicated ratio Metric plus Advanced Aggregators on its value field. Keep Calculated Items for derived dimension rows/columns where that pattern fits. + +- Duplicating the same operand Metric as a second value field to simulate a ratio + - Does not replace a proper ratio Metric; create the ratio Metric with a Pigment formula and wire operands explicitly. + +- Overriding aggregation when Metric defaults are already correct + - Prefer `Default` / no override unless the View needs different behavior + +--- + +## 11. Quick decision tree + +### Step 0 +Does the View need a ratio, percentage, or relative variance built from two Metrics (same idea as A / B or (A - B) / B)? +- Yes -> Ensure a dedicated ratio Metric with a Pigment formula exists; add it and both operands to the Table; then configure Advanced Aggregators on the ratio value field (see section 7). Continue to Step 1 for pivot-level aggregation choices. +- No -> Continue + +### Step 1 +Is the dimension on Rows or Columns? +- Yes -> use pivot aggregation if you want visible totals +- No -> continue + +### Step 2 +Is the dimension only on Pages? +- Yes -> use hiddenDimensionsAggregations +- No -> continue + +### Step 3 +Is the dimension not shown anywhere? +- Yes -> use hiddenDimensionsAggregations + +### Step 4 +Are Metric defaults already correct? +- Yes -> keep `Default` / no override +- No -> define a View-level override + +### Step 5 +Do you need a math operation between two metrics? +- Yes -> use advanced aggregation on a Table View +- No -> use simple aggregation + +--- + +## 12. Summary + +- Metric defaults are the baseline. +- Rows / Columns use pivot aggregation and can create subtotals / grand totals. +- Pages and other hidden dimensions use hiddenDimensionsAggregations. +- Hidden dimensions aggregation does not create visible totals. +- Advanced aggregation is only for Table Views with two numeric value fields. +- Safe defaults: + - `Sum` for additive metrics + - `Last` for snapshots + - Advanced Aggregator Ratio for rates / percentages / ratios + - Advanced Aggregator Growth (A-B/B) for growth / relative variance diff --git a/plugins/pigment/skills/creating-and-editing-pigment-views/view_components.md b/plugins/pigment/skills/creating-and-editing-pigment-views/view_components.md new file mode 100644 index 00000000..08385b58 --- /dev/null +++ b/plugins/pigment/skills/creating-and-editing-pigment-views/view_components.md @@ -0,0 +1,119 @@ +# View Components + +In the UI we say Values (what appears in the cells) and Pages / Rows / Columns. + +## Step 1: Data & Layout + +### Values (cells) + +That is what shows in the cells. It depends on the Block: + +- Metric Block: The Metric itself +- Table Block: The Metrics within the Table +- List Block: The List properties + +Configuration options: + +- `displayed`: Controls whether the value is visible in the View (UI: eye icon). On Table views, prefer removing irrelevant metrics from `values` rather than hiding them - hidden metrics may still compute. Keep a metric hidden only when the view still depends on it, such as for value-field filtering, sort-by-metric-value, or as an advanced-aggregator operand (ratio, growth, etc.). +- Formatting: Can specify number format, decimal places, currency, etc. +- Order: Multiple values can be displayed in a specific sequence + +Example (Table views): For a Table containing "Revenue", "Cost", and "Profit", to show only "Revenue" and "Profit" on a given view, include only those metrics in `values`. To drop "Cost", remove its value entry; do not leave it with `displayed: false`. + +### Pivots (Rows, Columns & Pages) + +Dimensions used to organize and break down data: + +- Rows: Primary breakdown Dimension (e.g., Products, Departments, Accounts). Users read top-to-bottom for comparisons. +- Columns: Secondary breakdown, often time (e.g., Months, Quarters). Users read left-to-right for trends. +- Pages: Dimensions the user views one value at a time via a selector (e.g., Country, Year, Scenario, Version). + +CRITICAL - Same dimension on Pages and on Rows or Columns + +In Pigment, the same dimension may appear on Pages and on Rows or Columns at the same time. Example: Month on Columns and Month > Year (grouping) on Pages-year(s) chosen in the Page Selector determine which months are shown as columns. Page Selectors narrow which modalities appear on the row/column axes according to the user's selection (single- or multi-select depends on the page configuration). Do not ask the user to "resolve a conflict" when they request this; it is supported behavior, not a mistake. + +When the goal is to restrict what appears on rows/columns, prefer Pages (with Default items on the relevant page selector) rather than substituting with View Filter objects that duplicate the same narrowing-unless you truly need a [view_filtering.md](./view_filtering.md) filter type. + +When the goal is to compare a specific subset of modalities on an axis (e.g. "compare FY 24 and FY 25", "Actuals vs Budget", "Baseline vs Optimistic"), put that dimension on both Pages and Rows/Columns, with the compared modalities as multi-select Default items on the page selector. The axis lays them out side-by-side; the page selector lets the user swap the compared set without editing the View. + +Examples: OK patterns vs. anti-patterns + +1. Goal: show Country in rows and focus the view on France. + - Anti-pattern: Country only in rows, plus a View Filter in the spirit of _Keep - Country - is in - France_. + - OK pattern: Country on Pages and rows; set France as the Default item on the Country page selector. + +2. Goal: show Country in rows and only countries in region EMEA. + - Anti-pattern: Country only in rows, plus a View Filter like _Keep - Country > Region - is in - EMEA_. + - OK pattern: Country in rows and Country > Region (grouping) on Pages; set EMEA as the Default item on that page selector. + +3. Goal: show a comparison of a specific subset of modalities of dimension D side-by-side on an axis (e.g. Actuals vs Budget, Baseline vs Optimistic scenario, FY 24 vs FY 25), with some other breakdown on the other axis. + - Anti-pattern: D only on Rows or Columns, no page selector - the compared modalities are hard-coded in the View and the user cannot swap the compared set from the Board. + - OK pattern: D on the axis AND on Pages; set the compared modalities (e.g. [Actuals, Budget]) as multi-select Default items on the D page selector. + +Rows/Columns vs. Pages: use this to decide what must appear together on the grid for comparison (Rows / Columns) versus what is driven by Page selectors. That guidance does not forbid putting the same dimension on Pages and on an axis when page selections should narrow the visible rows/columns-see the CRITICAL block above. + +### Pivot Field Types + +Pivot fields can have different types (kinds) depending on their configuration: + +1. Dimension Pivot: A simple pivot on a Dimension + - Has `dimensionId` only + - Displays modalities directly from that Dimension + +2. Grouping Pivot: Groups data by following List Properties to a target Dimension + - Has both `dimensionId` AND `listPropertyPath` + - Allows hierarchical grouping (e.g., Month -> Quarter -> Year) + +3. Scenario Pivot: Special pivot for scenarios + - Has no `dimensionId` (null) + - Used for scenario selection + +4. Joined Pivot aka Mapped Dimensions. Uses a mapping Metric to join data. + - Has `dimensionId` and `mappingMetricId` + +5. Slice Pivot, aka Data Slice: Uses a slice configuration + - Has `dimensionId` and `sliceConfigurationId` + +### How List Properties Work in Pivots (Grouping) + +When you add a List Property path to a pivot field, it transforms from a simple Dimension pivot into a Grouping pivot. This allows you to aggregate data along Dimension hierarchies. + +Example Hierarchy: + +- You have a Metric defined on Dimension Month +- Quarter is a Dimension Property of List Month +- Year is a Dimension Property of List Quarter + +To group by Year: + +```json +{ + "dimensionId": "", + "listPropertyPath": ["quarter", "_year"] +} +``` + +What happens: + +1. Starts at the source Dimension (Month) +2. Follows the List Property path: Month -> Quarter -> Year +3. Groups data by the target Dimension (Year) +4. Displays aggregated values at the Year level + +Important notes: + +- ListPropertyPath contains friendly names of Dimension properties (the display names shown to users) +- Each step navigates one level in the Dimension hierarchy +- The source Dimension must exist and be valid for the View's underlying Block +- The List Property path must be valid (each Property must exist on the respective Dimensions) + +Configuration: + +```json +{ + "dimensionId": "8f301e67-dda4-4276-bc1b-4db418b8b3ff", + "listPropertyPath": ["quarter", "_year"] +} +``` + +This creates a row/column that shows data grouped by Year, even though the underlying Metric is defined on Month. diff --git a/plugins/pigment/skills/creating-and-editing-pigment-views/view_design_process.md b/plugins/pigment/skills/creating-and-editing-pigment-views/view_design_process.md new file mode 100644 index 00000000..73603a45 --- /dev/null +++ b/plugins/pigment/skills/creating-and-editing-pigment-views/view_design_process.md @@ -0,0 +1,28 @@ +# View Design Process + +## Step 1: Define your ideal View + +Before any lookup, be clear on: + +1. Data (which Metric, List, or Table?) +2. Breakdown (which Dimensions in Pages, Rows, Columns - and for Table, where metrics sit) +3. View filtering and sorting + +Use [view_components.md](./view_components.md), [view_filtering.md](./view_filtering.md), [view_sorting.md](./view_sorting.md). [view_display_modes.md](./view_display_modes.md) for `display_type` / block rules. + +## Step 2: Optional scan of existing Views + +Call `tool:get_block_views` on the block (use `display_intent` when the tool allows). Read [relevant_views.md](../designing-pigment-boards/relevant_views.md) for how to read results - this is a light pass, not a hard prerequisite. + +Names first: e.g. "View 1" -> very likely a not nicely formatted re-usable view. Unless its pivots already match this widget and is consistent with the rest of the board, prefer `tool:create_view` with a real name and coherent layout. Meaningful names (e.g. _Revenue by Region - Monthly_) are a strong reuse signal. + +## Step 3: Reuse, or create + +- Strong name + pivot fit (including board-wide pages and story) -> reuse; wire the widget to that View. +- Otherwise -> `tool:create_view`, then iterate with `tool:update_view_pivots` (placement, calculated items, custom display, styling, aggregations) and the other `update_view_*` tools (filters, sorts, chart config, ...). When `tool:create_view` is called with `pivotLayout` null, the server picks a sensible default layout; to override, send a complete `pivotLayout` with all three axes populated (empty array = no pivot on that axis). Editing the live View behind a board widget -> call the matching `update_view_*` tool directly on that View id - if a Draft was created, pair the edit with `tool:update_view_widget_overrides` so this user sees the Draft on the widget (see [view_widgets.md](../designing-pigment-boards/view_widgets.md)). + +Templates (Grid only): If `tool:get_all_view_templates` exists, after a new View in Grid mode, pick and apply a template silently when one clearly fits; else skip. + +## Step 4: Validate + +After create/update, the response should match what you sent; dropped fields may mean sanitization. Before wiring a widget, confirm `display_type` and block rules ([view_display_modes.md](./view_display_modes.md), [view_widgets.md](../designing-pigment-boards/view_widgets.md)). On errors, [view_troubleshooting.md](./view_troubleshooting.md). diff --git a/plugins/pigment/skills/creating-and-editing-pigment-views/view_display_modes.md b/plugins/pigment/skills/creating-and-editing-pigment-views/view_display_modes.md new file mode 100644 index 00000000..25af95d1 --- /dev/null +++ b/plugins/pigment/skills/creating-and-editing-pigment-views/view_display_modes.md @@ -0,0 +1,9 @@ +# Display Modes + +The widget sets `display_type` (KPI / Grid / Chart). The View may hold chart config for graph types; no chart config in the View -> typical grid behavior. + +Chart config is applied after creation via `tool:update_view_chart_config`, using pivot ids from `tool:create_view` response, `tool:get_view` response or others. + +- KPI: No row pivots. `metricsLocation` MUST NOT be `Rows` (use `Columns` or `Pages`). Not for List blocks. +- Grid (Table / List): Tabular. Only mode for List blocks. +- Chart: Chart kind lives on the View; not for List blocks. diff --git a/plugins/pigment/skills/creating-and-editing-pigment-views/view_filtering.md b/plugins/pigment/skills/creating-and-editing-pigment-views/view_filtering.md new file mode 100644 index 00000000..c64bb7c3 --- /dev/null +++ b/plugins/pigment/skills/creating-and-editing-pigment-views/view_filtering.md @@ -0,0 +1,131 @@ +# Narrowing data in a View + +## Terminology (do not confuse) + +| Pigment feature | UI label | API / tool | When to use | +| --- | --- | --- | --- | +| Page Selector | Pages | `pages[]` + default items | Default when user asks to "filter", "focus on", "for Year X", "show only France", pick Version/Scenario | +| View Filter | Filters (on row/column pivots) | `filters[]` | Exclusion, top-N by metric value, property-based rules Page Selectors cannot express | +| Board Page Selector | Board Pages | `update_board` page config | Same as Page Selector, but shared across widgets on a Board | + +## Interpreting user language + +When the user says "filter", "for 2024", "focus on EMEA", "Actuals only" - they almost always mean a Page Selector (View `pages` with a default item), not a View Filter object. + +Use View Filters only when the request is clearly value-based (e.g. "top 10 suppliers by cost") or exclusion logic Page Selectors cannot do. + +Both can apply: e.g. "top 10 suppliers for Year 2024" -> Year via Page Selector; top 10 via View Filter (ValueField). + +### Page Selectors (Pages) + +The simplest way to reduce what's shown: configure Default Items on a Page Selector to restrict which data is displayed. For example, setting Default Items to "Q1 2024" on a Quarter Page Selector means only Q1 data is displayed - no View Filter configuration needed. + +Grouping pivots can also be used in Pages. Selecting a value on a Grouping Page Selector narrows base items (e.g., Months) whose property chain matches the selected target (e.g., Year = FY 2024 -> only Months with `_year` = FY 2024). + +### View Filters (API `filters[]`) + +For more control than Page Selectors, use View Filters. There are several types. +Filters are applied after creation via `tool:update_view_filters`, using pivot ids from the `tool:create_view` response. + +CRITICAL REQUIREMENT: The `pivotFieldId` in filters MUST reference a pivot from the rows or columns arrays, NOT from the pages array. + +### PivotField Filters (By Items) + +Used to exclude Dimension items based on which modalities are present in a pivot. + +Note: To narrow to specific items (inclusion), prefer Page Selectors instead. Use PivotField Filters only when you need to exclude items or apply complex filtering logic. + +Configuration: + +```json +{ + "type": "PivotField", + "pivotFieldFilteringOption": { + "pivotFieldId": "", // MUST be from rows or columns, NOT pages! + "compareOperator": "IsIn", + "modalityIds": ["", ""], + "variableIds": [] + } +} +``` + +### ValueField Filters (By Value) + +Filter rows/columns based on Metric values (e.g., "show only products where Revenue > 1000"). + +CRITICAL REQUIREMENT: When there are pivots on the opposite axis from the filtered pivot, you MUST provide projections for each of those pivots. + +Why projections are needed: + +- When filtering rows by value, but columns exist, you need to specify WHICH column value to use for comparison +- Example: If filtering "Product" rows by "Revenue > 1000" and "Month" is in columns, you must specify which month (e.g., "January 2024") to use for the comparison +- Without valid projections, the filter will be silently removed during View creation + +Configuration: + +```json +{ + "type": "ValueField", + "valueFieldFilteringOption": { + "pivotFieldId": "", // Must be innermost pivot on its axis + "valueFieldId": "", + "compareOperator": "Gt", + "values": ["1000"], + "variableIds": [], + "projections": [ + // REQUIRED if opposite axis has pivots + { + "pivotFieldId": "", + "modalityId": "" // Which modality to use for comparison + } + ] + } +} +``` + +Example scenario: + +- Rows: Product Dimension +- Columns: Month Dimension +- Want to filter: "Show only products where Quantity Sold > 100" +- You MUST specify which month to use for comparison (e.g., the first month modality) + +### PivotListProperty Filters + +Filter based on List Property values. + +Configuration: + +```json +{ + "type": "PivotListProperty", + "pivotListPropertyFilteringOption": { + "pivotFieldId": "", // MUST be from rows or columns, NOT pages! + "listPropertyPath": ["propertyName"], + "compareOperator": "Eq", + "values": ["value"], + "variableIds": [] + } +} +``` + +Example - Filtering by Product Name: + +If you want to filter to show only "Choco Bites" product: + +1. Ensure the Product Dimension appears in rows or columns (not just pages) +2. Use the pivotFieldId from that row/column pivot (e.g., from the columns array) +3. Use the Property name (typically `"_name_XXXXXX"` where XXXXXX is a suffix) + +```json +{ + "type": "PivotListProperty", + "pivotListPropertyFilteringOption": { + "pivotFieldId": "da327057-0d83-4ef4-9ec6-ba3934f6ce5f", // From columns array + "listPropertyPath": ["_name_D81JNJ"], + "compareOperator": "Eq", + "values": ["Choco Bites"], + "variableIds": [] + } +} +``` diff --git a/plugins/pigment/skills/creating-and-editing-pigment-views/view_pivoting.md b/plugins/pigment/skills/creating-and-editing-pigment-views/view_pivoting.md new file mode 100644 index 00000000..a57754dc --- /dev/null +++ b/plugins/pigment/skills/creating-and-editing-pigment-views/view_pivoting.md @@ -0,0 +1,180 @@ +This guide defines how Dimensions ("pivots") are ordered and split between rows, columns and pages for boards. + +It covers: + +1. Ordering pivots Dimensions +2. Allocating them to Rows vs Columns + +--- + +# 1. Ordering Rules + +## Global Ordering Principles + +1. Parent Dimensions before child Dimensions +2. Dimensions Order: Time -> Business -> Metric -> Comparison or Scenario + +### Example + +Input: + +- Month > Year +- Scenario +- Segment +- Country +- Country > Region +- Month + +Reordered: + +- Month > Year +- Month +- Segment +- Country > Region +- Country +- Scenario + +## How Order Maps To Display + +Within an axis, the order of pivots determines how the data nests: + +- Rows: the first pivot is the outermost (leftmost) grouping; each subsequent pivot nests inside it, the last being the most granular. +- Columns: the first pivot is the top-most header band; each subsequent pivot nests beneath it. +- Pages: order changes the order in which the page selectors appear in the UI but has no impact on the data grouping. + +Reordering pivots on an axis changes the grouping hierarchy of the rendered data, not just their listing. + +--- + +# 2. Special Behavioral Rules + +## Filtering ("by metric value") + +Filtering overrides all display rules: + +- They must be placed last (most granular position) +- If multiple filtering pivots exist: + - Only the first is guaranteed to work + - Others may lose filters (known limitation) + +## Grouping Dimensions + +Related dimensions (parent-child or same hierarchy) must always be allocated together. They cannot be split between rows and columns. + +### Building a hierarchy in Rows + +To expose a multi-level grouping on the same entity, add each level as its own pivot in Rows, ordered from the shallowest path to the deepest (parent chain before children). Do not skip intermediate levels if you want the full drill-down in the grid. + +Example on an `Entity` list: `Entity > Grouping L1`, then `Entity > Grouping L2`, then `Entity > Grouping L3`, and so on - one pivot per level, all in Rows, respecting the global ordering (parents before children on that chain). + +### Tree layout vs tabular layout (Grid) + +For a Grid widget, the product can render the same row pivots either as tabular row headers (one column per pivot level) or as a treeview (single hierarchy column with indentation / expand-collapse). If `create_view` does not accept this display mode, recommend to the user to do it manually in the UI. + +--- + +# 3. Display-Type Driven Allocation + +Pivot allocation depends primarily on the display type. + +--- + +## 3.1 KPI + +- All pivot Dimensions -> columns +- `metricsLocation` MUST be `Columns` (or `Pages`) - never `Rows`. KPI views have no row pivots, so Rows produces a broken layout. Default to `Columns`. + +--- + +## 3.2 Pie Chart + +- Rows define slices (series) +- Dimensions in columns are aggregated + +### Rules + +- All pivots Dimensions -> rows + +--- + +## 3.3 Line Chart & Bar Chart & Combined Chart + +- Columns: horizontal axis +- Rows: series +- If you need to create a comparison, Dimension should be placed in Rows + +### With time dimension + +- Time dimensions -> columns +- All others -> rows + +### Without time dimension + +- First non-comparison Dimension -> columns +- Others -> rows + +--- + +## 3.4 Grid + +If you need to create a comparison, Dimension should be placed in Columns + +### With calendar dimension + +- Calendar Dimensions -> columns +- Others -> rows + +### Without calendar dimension + +- First pivot Dimension (with its parent Dimension) or Comparison Dimension -> columns +- All others -> rows +- Keep related Dimensions together + +### Example 1 + +revenue by segment, country, region + +Ordered: + +- Segment +- Country > Region +- Country + +Allocation: + +- Columns: Segment +- Rows: Country > Region, Country + +### Example 2 + +Ordered: + +- Country > Region +- Country +- Segment + +Allocation: + +- Columns: Country > Region, Country +- Rows: Segment + +## 3.5 Waterfall Variation + +- Similar to grid behavior + +## 3.6 Waterfall Contribution + +- All pivot Dimensions -> rows +- Dimensions in columns are aggregated + +--- + +# 4. Summary Heuristics + +1. Always group pivots first +2. Order: Time -> Business -> Comparison +3. Apply display-type rules +4. Handle filters last +5. Ensure groups Dimensions remain together (in Rows or in Columns) + +--- diff --git a/plugins/pigment/skills/creating-and-editing-pigment-views/view_sorting.md b/plugins/pigment/skills/creating-and-editing-pigment-views/view_sorting.md new file mode 100644 index 00000000..11d18b8a --- /dev/null +++ b/plugins/pigment/skills/creating-and-editing-pigment-views/view_sorting.md @@ -0,0 +1,133 @@ +# Sorting + +UI: order of rows/columns (and chart categories). Tool payloads use `pivotFieldId` and `sorts` (same pivot list as filters). + +Sorts are applied after creation via `tool:update_view_sorts`, using pivot ids from the `tool:create_view` response. + +Sorting controls how data is ordered in the View. It applies to both Grid and Chart display (e.g., bar order in a bar chart). Multiple sorting options can be applied, with the first option having the highest priority. + +### Types of Sorting + +There are two types of sorting options: + +### 1. Sort by Property (ByProperty) + +Sorts data based on a Property value of a Dimension (e.g., sort by Name, Code, or any other Dimension Property). + +When to use: + +- Sorting List items alphabetically by name +- Sorting by a Dimension Property like "Code" or "Category" +- Ordering data by manual Dimension order (when `property_friendly_name` is null) + +Configuration: + +```python +PydanticSortingOption( + order=Order.Asc, # or Order.Desc + type=SortingOptionType.ByProperty, + by_property_sorting_option=PydanticByPropertySortingOption( + pivot_field_id="", # The pivot field to sort + property_friendly_name="Name", # Friendly name of property to sort by (or None for manual order) + ), +) +``` + +Example use cases: + +- Sort products alphabetically by product name +- Sort employees by employee code +- Sort departments by a custom "Priority" Property + +### 2. Sort by Metric Value (ByMetricValue) + +Sorts data based on Metric values (e.g., sort products by their revenue). + +When to use: + +- Sorting by calculated values (revenue, cost, profit, etc.) +- Ranking items by performance Metrics +- Ordering data by aggregated values + +Configuration: + +```python +PydanticSortingOption( + order=Order.Desc, # or Order.Asc + type=SortingOptionType.ByMetricValue, + by_metric_value_sorting_option=PydanticByMetricValueSortingOption( + pivot_field_id="", # The pivot whose modalities are sorted + value_field_id="", # The metric to sort by + projections=[ # Define which modality to use for sorting + PydanticSingleModalityProjection( + pivot_field_id="", + modality_id="", # Can be None for null modality + ) + ], + subtotal_pivot_field_ids=None, # Optional: for sorting on subtotals + ), +) +``` + +Important notes about projections: + +- Projections specify which specific modality to use when sorting by Metric values +- All pivot fields on the opposite axis must be either projected or included in subtotals +- For example, if sorting rows by a Metric, all column pivot fields must be projected +- Each projection selects a specific modality (or null modality) for a pivot field + +Example use cases: + +- Sort products by total revenue (descending) +- Sort regions by sales performance +- Rank employees by their productivity Metrics + +### Multiple Sorting Options + +You can apply multiple sorting options to a View. They are applied in order, with the first option having the highest priority. + +Example: + +```python +sorts=[ + # Primary sort: by category (ascending) + PydanticSortingOption( + order=Order.Asc, + type=SortingOptionType.ByProperty, + by_property_sorting_option=PydanticByPropertySortingOption( + pivot_field_id=category_pivot_id, + property_friendly_name="Name", # Friendly name + ), + ), + # Secondary sort: by revenue (descending) + PydanticSortingOption( + order=Order.Desc, + type=SortingOptionType.ByMetricValue, + by_metric_value_sorting_option=PydanticByMetricValueSortingOption( + pivot_field_id=product_pivot_id, + value_field_id=revenue_value_field_id, + projections=[...], + ), + ), +] +``` + +This would first sort by category name (A-Z), then within each category, sort products by revenue (highest to lowest). + +### Common Sorting Patterns + +1. Alphabetical sorting of List items: + - Use `ByProperty` with `property_friendly_name="Name"` (friendly name) + - Order: `Asc` for A-Z, `Desc` for Z-A + +2. Top N analysis (e.g., top 10 products by revenue): + - Use `ByMetricValue` with `order=Order.Desc` + - Combine with filters to limit to top N items + +3. Time-based sorting: + - Use `ByProperty` with the time Dimension's natural order + - Set `property_friendly_name=None` to use manual/natural order + +4. Multi-level sorting: + - Apply multiple sorting options in priority order + - First sort establishes primary grouping, subsequent sorts refine within groups diff --git a/plugins/pigment/skills/creating-and-editing-pigment-views/view_troubleshooting.md b/plugins/pigment/skills/creating-and-editing-pigment-views/view_troubleshooting.md new file mode 100644 index 00000000..a3182356 --- /dev/null +++ b/plugins/pigment/skills/creating-and-editing-pigment-views/view_troubleshooting.md @@ -0,0 +1,95 @@ +# Troubleshooting + +## View Creation Fails (Pivot Fields) + +Symptoms: + +- Error: "Dimension ID not found" +- Error: "Invalid pivot field configuration" +- Pivot field is silently removed during View creation + +Solutions: + +1. Verify Dimension IDs exist in the application +2. Check that Dimension is actually a Dimension of the underlying Metric/List +3. Ensure pivot field kind matches (Dimension vs Scenario) +4. For Grouping pivots with `listPropertyPath`: + - Verify the source Dimension exists in the List cache + - Check that each Property in the path exists on the respective Dimensions + - Ensure the List Property path uses friendly names (the display names shown to users) + - Verify the Dimension is part of the allowed Dimensions (determined by Metrics in valueFields) + +## Filters Are Being Removed (Silently Sanitized) + +Symptoms: + +- Filter was sent to `tool:update_view_filters` but is missing from the updated View +- Filter appears to be ignored +- No error message, but the filter just doesn't appear + +Root Cause: +The filter failed validation during view sanitization and was removed. This happens when: + +1. Wrong pivotFieldId source (MOST COMMON): The `pivotFieldId` in the filter references a pivot from the pages array instead of from rows/columns. + - Why it fails: Filters are only validated against `DimensionalPivotFields`, which only includes pivots from rows and columns + - How to fix: Always use a pivotFieldId from the rows or columns arrays, never from pages + - Example: If filtering on Product Dimension, ensure Product is in rows or columns, then use that pivot's ID + +2. Missing projections for ValueField filters: If filtering on a pivot in one axis (e.g., rows) and there are pivots on the opposite axis (e.g., columns), you MUST provide projections for each opposite-axis pivot. + +3. Invalid pivot reference: The `pivotFieldId` doesn't reference the innermost pivot on its axis. + +4. Invalid value field reference: The `valueFieldId` doesn't exist in the View's value fields. + +5. Invalid comparison operator: The operator doesn't match the Metric type (e.g., using "Contains" on a numeric Metric). + +6. Invalid List Property path: For PivotListProperty filters, the Property path doesn't exist on the Dimension. + +Solutions: + +1. CRITICAL: Always use pivotFieldId from rows or columns, NEVER from pages: + - Check that the Dimension you want to filter on appears in rows or columns + - If it only appears in pages, you need to add it to rows or columns first + - Use the pivotFieldId from that row/column entry + +2. Always provide projections for ValueField filters when the opposite axis has pivots: + + ```json + "projections": [ + { + "pivotFieldId": "", + "modalityId": "" + } + ] + ``` + +3. Use the GetBlockViews tool to inspect an existing similar view to see how filters are structured + +4. Verify all IDs refer to an existing pivot and the pivots exist in rows/columns (not pages!) + +## View Shows No Data + +Symptoms: + +- View created but displays empty + +Solutions: + +1. Check filters - may be filtering out all data +2. Verify underlying Metric/List has data +3. Check `show_empty_rows` and `show_empty_columns` settings +4. Ensure value fields are set to `displayed: true` + +## Performance Issues + +Symptoms: + +- View takes a long time to load +- Browser becomes unresponsive + +Solutions: + +1. Add more aggressive filters to reduce data volume +2. Reduce number of breakdowns +3. Set `show_empty_rows: false` and `show_empty_columns: false` +4. Use pages with `single_modality: true` to limit data diff --git a/plugins/pigment/skills/designing-pigment-boards/SKILL.md b/plugins/pigment/skills/designing-pigment-boards/SKILL.md new file mode 100644 index 00000000..b34cd99c --- /dev/null +++ b/plugins/pigment/skills/designing-pigment-boards/SKILL.md @@ -0,0 +1,238 @@ +--- +name: designing-pigment-boards +description: Always use when creating or editing a Board. This skill includes supporting files in this directory - explore as needed. +metadata: + skill_path: /designing-pigment-boards/SKILL.md + base_directory: /designing-pigment-boards + includes: + - "*.md" +--- + +# How to Use This Skill + +Progressive Disclosure Pattern: This `SKILL.md` provides an overview. Most details live in supporting files. + +This file alone is often not sufficient + +Required workflow: + +1. Read this file first - Understand available resources and when to use them +2. Identify relevant topics - Match your task to any of the supporting documents +3. Read supporting files - Use available Cline file-reading or search tools to access detailed documentation +4. Explore as needed - Use available Cline listing and search tools to discover additional resources in this directory (some might not be explicitly mentioned in this file) + +# Pigment Board Design Knowledge Base + +This skill provides comprehensive guidance for designing Boards in Pigment. It covers board structure, layout rules, widget sizing standards, content conventions, and proven board design patterns for common business use cases. + +Board design in Pigment is the practice of translating business questions into clear, structured, and visually coherent dashboards. A well-designed Board improves readability, adoption, decision-making speed, and executive trust. + +This skill focuses on what a Board should contain and how it should be laid out, not on modeling or formula logic. + +--- + +## When to Use This Skill + +Use this skill whenever you need to: + +- Design a new Board from scratch +- Structure a Board logically with sections and hierarchy +- Apply consistent layout rules using Pigment's 12-column grid +- Choose appropriate widget sizes for KPIs, charts, and grids +- Ensure governance and consistency across Boards +- Translate business needs into dashboard structure +- Follow best practices for executive-ready dashboards + +This skill should be used after modeling is done and before or during Board creation. + +--- + +## Supporting documents + +When doing the following tasks, you MUST read these documents: + +- When creating or editing a Board: + - Must read to the end: [board_design_rules.md](./board_design_rules.md) + - Must read: [board_pages.md](./board_pages.md) + - Apply the inline widget sizing rules in the Widget Sizing section below. + +- When you need a View: + - Read [relevant_views.md](./relevant_views.md) and [view_widgets.md](./view_widgets.md) (CRITICAL for widgets). + - Use `tool:get_block_views` to see what exists. Reusing is optional: generic names (e.g. _View 1_) often mean you should create with `tool:create_view`. Do not block on a long "search for similar" pass. + +--- + +## Core Principles of Board Design in Pigment + +- Boards tell a story, not just display data +- Structure and hierarchy matter more than visual density +- Consistency across Boards improves usability and trust +- Boards describe _what should be displayed_, not _how it is calculated_ +- Simplicity beats completeness for executive and operational dashboards +- Do not answer with too much detail to avoid overloading the user's chat. +- If the user asks to avoid new modeling blocks (e.g. no new Metrics, Lists, or structural changes), creating a Table block can still be appropriate: a Table bundles existing Metrics for visualization and/or input on a Board. It's a layout container, not a new structural object dimension. + +--- + +### Design Principles + +[board_design_rules.md](./board_design_rules.md) - Board design principles +Covers: + +- Global Board Structure +- Widget Sizes Guidelines +- Column Layout Strategy + +--- + +## Content Guidelines + +### Supported Widget Types + +- GOOD: Text widgets - Titles, descriptions, explanatory content +- GOOD: View widgets - Data visualizations (Grids, Charts, KPIs) +- GOOD: Spacer widgets - Visual separation between sections +- BAD: Do not use other widget types (ActionButton, Image) unless explicitly asked + +### Text Widget Usage + +Use text widgets for: + +- Section titles and subtitles +- Explanatory text and commentary + +Do NOT use text widgets for describing intended data visualizations. Use actual View widgets instead. + +### View Widget Usage + +Use View widgets for data visualizations (Grids, Charts, KPIs). + +WARNING: CRITICAL: Every View widget requires a View ID. There is NO such thing as: + +- BAD: "View ID: Not applicable" +- BAD: "Using the Metric directly for KPI/Chart display" +- BAD: Referencing only a Block ID without a View + +Even for a simple KPI showing a single Metric value, you must: + +1. Create or find a View that references the Metric +2. Configure the View appropriately +3. Reference that View ID in the View widget + +### KPI Widget Usage + +- No row pivots +- `metricsLocation` MUST NOT be `Rows` - use `Columns` (default) or `Pages`. KPIs have no row pivots, so Rows produces a broken layout. +- The KPI widget displays as many columns as the underlying View. +- Do not put a Dimension in columns unless it has very few items. +- To display several Metrics side-by-side, prefer a single KPI Widget on a View with multiple Metrics in `values` (and `metricsLocation: Columns`) over multiple KPI Widgets - even if individual single-Metric Views already exist. +- Not available for List blocks (KPI requires a Metric or Table block). + +### Spacer Widget Usage + +Use spacer widgets for: + +- Visual separation between sections +- Standard size: `width=12`, `height=1` + +### Prioritization Rules + +- Prioritize Blocks that directly support the Board's purpose +- Avoid unnecessary information +- Aim for clarity, hierarchy, and narrative flow +- Design Boards that are: visually clean, easy to scan, logically ordered + +--- + +## Board Creation Workflow + +Follow this 4-step workflow when creating a Board: + +### Step 1: Define Board Purpose and Plan Board Pages + +1. Define board purpose (1-2 sentences) + - Example: "Track Q1 2024 actual performance against budget" + +2. Use Search tool to check what Dimensions your Metrics have + +3. Plan Board Page Selectors (not View Filters - defaults applied in Step 4): + - Time Page Selector (Month, Quarter, Year) - only if Metrics have time Dimensions + - Version Page Selector (Actuals, Budget, Forecast, or combinations) - only if Metrics have Version + - Scenario Page Selector (Default or multiple scenarios) - only if Metrics have Scenario + - Other dimensional Page Selectors as needed + - See [board_pages.md](./board_pages.md) for detailed guidance + +4. Before treating Board Page Selectors as shared context for every widget, verify that each View you intend to place on the Board includes a compatible page for every dimension you will set at board level (e.g. if the board should narrow by Year, each View must have Year in Pages-or a grouping page that resolves to Year-see board_pages.md). If a View is missing that dimension in Pages, edit or create the View first; the board cannot force a dimension onto a View that does not expose it in Pages. + +### Step 2: Create Board Structure + +1. Create a board with (in board settings): + - Board name and description + - Icon and color + +2. Add sections and widgets: + - Section titles and subtitles (text widgets) + - View widgets for data visualizations + - Spacer widgets between sections + +### Step 3: Find or create Views, then add widgets + +1. Identify Blocks for the story. + +2. For each Block, `tool:get_block_views` - pick a reusable View only if name + pivots fit this board; otherwise `tool:create_view` (see [relevant_views.md](./relevant_views.md)). Ensure Pages align with [board_pages.md](./board_pages.md). + +3. Add View widgets that reference those View IDs. + +### Step 4: Update Board Pages + +1. Use `tool:update_board` to set Board Page Selectors +2. Apply the Time, Version, Scenario, and other Page Selector defaults you defined in Step 1 +3. Set default selected items for each Board Page at board level (e.g. default Year to FY25 / 2025 when that is the intended analytical context) +4. Confirm each View widget is linked to the Board Pages you care about (widgets do not "inherit" a dimension the View never had in Pages-see board_pages.md, Board-to-Widget Page Compatibility Rule) + +Key Points: + +- Plan Board Pages in Step 1, but apply defaults and selections in Step 4 (after Views are added) +- Only define Board Page Selectors for dimensions that at least one View exposes; for every widget that should follow a board-level Page Selector, that widget's View must include a compatible page on that dimension +- View widgets link to Board Page Selectors when their View has a matching page; they do not automatically narrow for dimensions absent from the View's Pages + +--- + +## Learning Path: Read in This Order + +### 1. START HERE: Board Structure + +Based on the list of relevant Blocks to display, focus on: + +- Board structure (title, description, sections hierarchy) +- For each section, selecting appropriate widgets (View, Text, Spacer, etc.) to display data and provide context + +--- + +### 2. THEN: Content & Widgets + +Focus on: + +- Use View widgets to display data from Metrics, Lists, or Tables +- Use Text widgets for titles, descriptions, and context +- Use Spacer widgets for visual separation + +--- + +### 3. FINALLY: Widget Sizing + +You MUST follow these height guidelines. When a data widget has a title, add 1 to the minimum height. + +| Widget type | Height | +|---|---| +| Text (title only) | 2 | +| Text (title + subtitle) | 3 | +| Spacer | 1 | +| KPI without title | 4-6 | +| KPI with title | 5-7 | +| Chart without title | 11-18 | +| Chart with title | 12-18 | +| Grid without title | 11-24 | +| Grid with title | 12-24 | + +Chart/Grid height depends on data complexity (rows, columns, legends, axis labels). diff --git a/plugins/pigment/skills/designing-pigment-boards/board_design_rules.md b/plugins/pigment/skills/designing-pigment-boards/board_design_rules.md new file mode 100644 index 00000000..66ccfe3a --- /dev/null +++ b/plugins/pigment/skills/designing-pigment-boards/board_design_rules.md @@ -0,0 +1,228 @@ +# Board Design Principles + +## Basic Board Setup + +- Never include a title or description in the Board as a Text widget, use the properties of the Board instead +- Define an icon that matches the Board's intent +- Set the full-width property of the Board to true (recommended default) +- Never use H1, bold, italic, or underline for section titles + +--- + +## Column Layout Strategy + +### Understanding the 12-Column Grid + +- Board content spans across 12 columns +- Each widget width can be defined from 1 to 12 columns +- Choose between wide (12 columns), medium (10 columns), or narrow (8 columns) layouts, by leaving 1 or 2 columns of margin on each side +- Balance cognitive bandwidth vs information density + +### When to Use Wide 12-Column Layout + +Use full 12 columns when: + +- Horizontal comparison is the primary task +- Designing monitoring dashboards +- Displaying 3+ numeric KPIs in a row +- Showing large numeric grids +- Presenting time series charts that benefit from horizontal resolution +- Users need to scan the screen left-to-right continuously + +Typical use cases: + +- Executive KPI dashboards +- Budget vs forecast comparisons +- Monthly trend charts with dense data + +### When to Use Narrow 10 or 8-Column Layout + +Use 10 or 8 columns (centered with margins) when: + +- The Board reads top-to-bottom and resembles a document more than a dashboard +- Displaying 1-2 charts per section +- Content is narrative in nature +- Widgets contain text explanations +- The goal is insight, not monitoring + +Typical use cases: + +- Department deep-dives +- Review screens +- Approval flows +- Scenario explanations +- Manager-focused operational Boards + +### Layout Consistency Rule + +Critical: Never alternate between wide and narrow layouts within a single Board. Choose one layout strategy early and maintain it throughout the entire Board design. + +--- + +## Board Content Organization + +### Section-Based Structure + +Always structure Boards using Sections to optimize information architecture and content hierarchy. + +Section characteristics: + +- Sections combine multiple related widgets +- Each section stacks vertically below the previous one +- Sections typically span the full width of the Board (12, 10, or 8 columns depending on your layout choice) + +Side-by-side sections (use sparingly): + +- Only use when necessary: Maximum 2 sections side-by-side, no more +- Each section spans half the Board width +- Both sections must have the same height +- Each side-by-side section typically contains only a single widget due to limited width +- This is a rare layout pattern - use only when absolutely needed + +Section separation: + +- Always use a spacer widget between sections +- This creates clear visual separation and structure +- See Widget Height Guidelines below for spacer sizing + +### How to Build a Section + +Step 1: Section Header (Text Widget) + +- Create a text widget spanning the full width of the section +- Include a short, descriptive title explaining what the section contains +- Use H2 text style for the title +- Optional: Add a subtitle for additional context (use paragraph text style in the same widget) +- See Widget Height Guidelines below for section header sizing + +Step 2: Section Content (Data Widgets) + +- Display widgets below the section title +- Use data widgets: Grids, Lists, Charts, KPIs - all related to the section topic +- Content can span multiple rows +- Limit to maximum 3 widgets per row +- When using multiple widgets per row: Make them symmetrical with matching height and width + +Widget Titles (Optional) + +When to omit widget titles: + +- When omitting a title, keep the text content, but set the widget boolean property `show_title` to false +- Widget content is self-explanatory from its data +- Section title and context make widget purpose clear +- Most of the time, widget titles are not needed - avoid redundant visual noise + +When to use widget titles: + +- Content spans multiple rows and structure is complex +- Widget titles add necessary hierarchy + +Widget title consistency rule: + +- If using widget titles on one widget in a section, use them on all widgets in that section (per-section basis) +- Exception: KPI widgets directly below the section title don't need titles - they feel integrated with the section header + +### When to Skip Section Titles + +Default: Always use section titles - they're essential for organizing and structuring Board content. + +Exceptions (use rarely): + +1. Very simple Boards: + - Board has only a couple of widgets + - Thinking in terms of sections is still good practice, but titles may be unnecessary + - Widget titles alone may be sufficient for structure + +2. Opening KPI section: + - The very first section contains only a row of KPIs + - KPIs introduce the Board context on their own + - An additional section title may be redundant + +--- + +## Global Board Structure + +### Standard Board Pattern + +For reporting, monitoring, and comparison Boards, use this classic structure: + +1. KPI Overview Section (may not need a title) + - Display high-level KPIs + - Provide general context and overview of main indicators + +2. Visual Analysis Section + - Show charts for visual data representation + - Enable trend and pattern recognition + +3. Detailed Data Section + - Present grids with complete data + - Allow deep-dive analysis + +### Adapt to User Needs + +Important: No one-size-fits-all approach exists. + +- Base Board structure on user needs and intent +- Match the design to user expectations +- Many layouts are valid based on content and Board purpose +- Maintain structure, organization, and readability above all + +--- + +## Core Design Principles + +### Principle 1: Optimize Reading Flow + +Prioritize readability above all: + +- Structure Boards as simply as a document +- Consider how users will read and scan the Board +- Organize content left-to-right, top-to-bottom +- Stack sections sequentially + +Avoid common mistakes: + +- Don't leave massive white gaps +- Don't misalign widgets +- Don't create unnecessary visual noise + +Apply design fundamentals: + +- Keep everything symmetrical +- Make content instantly parseable +- Leverage graphic design best practices +- Follow worldwide UX/UI design principles + +### Principle 2: Create Clear Information Hierarchy + +Structure the Board effectively: + +- Create strong information hierarchy through sections +- Use section organization to achieve great layout +- Prevent user confusion through clear grouping + +Ensure clarity: + +- Users should easily identify which content relates to which group +- Make title-to-section relationships obvious +- Group related information together visually + +### Principle 3: Use Familiar Patterns + +Optimize for functional design and usability, not creativity. + +User expectations: + +- Users need to identify data pieces as fast and easily as possible +- Users don't want to re-learn visual systems for every Board +- Users value speed and efficiency above everything +- Users shouldn't get confused, misread data, or waste time due to unconventional design + +Design approach: + +- Board design is not a creative exercise +- Focus on efficient, well-structured layouts +- Follow standard interface guidelines users already know +- Don't re-invent existing design patterns + +Outcome: Boards should follow the simple structure of an easy-to-read dashboard or document - no more, no less. diff --git a/plugins/pigment/skills/designing-pigment-boards/board_pages.md b/plugins/pigment/skills/designing-pigment-boards/board_pages.md new file mode 100644 index 00000000..669e4efd --- /dev/null +++ b/plugins/pigment/skills/designing-pigment-boards/board_pages.md @@ -0,0 +1,176 @@ +# Board Page Selectors + +Board Pages in the UI are Board Page Selectors - default selections applied at the Board level (not View Filters). + +## What They Are + +Board Page Selectors define the analytical context users expect when opening the Board. Which widgets actually follow a given Board Page Selector depends on each widget's View, not on the Board alone. + +### Board-to-Widget Page Compatibility Rule (critical) + +A board-level page selector on dimension D affects only widgets whose underlying View has a compatible page on D: + +- A simple Page on D (`dimensionId` = D's ID), or +- A Grouping Page whose `listPropertyPath` resolves to D (see Board Pages and Grouping Pivots in a View's Pages below). + +To make one board-level selector (e.g. Year) drive multiple widgets together, every target widget's View must include that same page (or a compatible grouping page on D). If a View has no Year in Pages (e.g. only Version, or no time page at all), the board's Year selector does not filter that widget's data. + +Workflow implication: Before relying on Board Pages, verify or edit each widget View so Pages align with the dimensions you want at board level. Creating the Board first and only then discovering mismatched Views is a common source of "the board filter does nothing." + +Key rule (configuration): Board Page selectors come from the union of Pages on the Views you add to the Board. You cannot invent a new Page selector on the Board that no View has. In case of comparison, you must configure default selected items for each Board Page at board level (default modalities = compared items). When the user names specific modalities in the prompt - either a single value ("for FY 26") or a subset to compare ("FY 26 and FY 27", "Actuals vs Budget") - those exact modalities are the defaults, single- or multi-select accordingly. + +After defaults are set, users can change Board Page selections; widgets whose Views are linked and compatible will update accordingly. + +### Unlinking a Page at the Widget Level + +Widgets whose Views do have a compatible page can still opt out of the Board Page by Unlinking that page. For example, if the Board has a Board Page for `Year`, a widget can unlink its `Year` Page so it is no longer driven by the board-level Year selector; that page then behaves as a normal view-level page. + +Note: "No effect from the board Year selector" is different from unlinking: if the View never had a compatible Year page, the board Year filter never applied to that widget-unlinking is not required to explain that behavior. + +### Page Selector Visibility + +Page selectors can be shown, minimized, or hidden. Never hide a page selector unless the user explicitly asks for it. + +--- + +## Page Selector strategy by Board purpose + +| Board Purpose | Time Page Selector | Version Page Selector | Scenario Page Selector | +| ------------------------- | ------------------ | ----------------------- | ---------------------------------------- | +| Monthly Review | Month=Current | Version=Actuals | Scenario=Default | +| Quarterly Business Review | Quarter=Current | Version=Actuals,Budget | Scenario=Default | +| Annual Planning | Year=Next | Version=Budget,Forecast | Scenario=Default | +| Variance Analysis | Month=Current | Version=Actuals,Budget | Scenario=Default | +| Scenario Planning | Year=Current | Version=Forecast | Scenario=Baseline,Optimistic,Pessimistic | +| Executive Overview | Quarter=Current | Version=Actuals | Scenario=Default | +| YTD Performance | Year=Current | Version=Actuals,Budget | Scenario=Default | +| Year-over-Year / Period Comparison | Year=FY N-1, FY N (multi) | Version=Actuals | Scenario=Default | + +Adapt based on which Dimensions your Views actually have. Skip any column that doesn't apply. Values like "Current" and "Next" are conceptual - resolve them to actual modality IDs from the Dimension's items. + +--- + +## Common Filter Dimensions + +### 1. Time-Related Dimensions + +Choose based on Board purpose. Only if Views have time Dimensions: + +- Single period: `Month=Jan 24`, `Quarter=Q1 24`, `Year=FY 24` +- Comparison (when the user asks to compare specific periods, e.g. "FY 24 vs FY 25", "YoY", "this year vs last year"): multi-select default with exactly those periods, e.g. `Year=FY 24, FY 25`. Use this whenever the same dimension is on a chart axis to span those modalities - the Board Page lets the user change the compared set without editing the View. + +### 2. Version Dimension + +Only if Views have a Version Dimension: + +- Single version: `Version=Actuals` or `Version=Budget` or `Version=Forecast` +- Comparison: `Version=Actuals,Budget` (for variance analysis) +- Multi-version: `Version=Actuals,Budget,Forecast` + +### 3. Scenario Dimension + +Only if Views have a Scenario Dimension: + +- Single scenario: `Scenario=Default` +- Comparison: `Scenario=Baseline,Optimistic` (for scenario planning) + +### 4. Other Dimensions + +Any business Dimensions your Views have in Pages: Region, Department, Product Line, etc. + +--- + +## Examples + +### Standard: Quarterly Variance Analysis + +``` +Board Purpose: Compare actual vs budget performance for Q1 24 + +Board Pages: +- Time: Quarter=Q1 24 +- Version: Actuals,Budget +- Scenario: Default +``` + +### Edge Case: Views With Only Time Dimensions + +``` +Board Purpose: Track monthly sales trends + +Views have: Month Dimension +Views don't have: Version, Scenario + +Board Pages: +- Time: Month=Jan 24 to Dec 24 +(No Version or Scenario filters needed) +``` + +### Edge Case: Views With No Standard Dimensions + +``` +Board Purpose: Product catalog dashboard + +Views have: Product, Category, Region +Views don't have: Time, Version, Scenario + +Board Pages: +- Region=All +- Category=All +(Only filter the Dimensions that actually exist) +``` + +### Edge Case: Mixed Dimension Availability + +``` +Board Purpose: Mixed KPI dashboard + +Some Views have: Year, Version, Scenario +Other Views have: Only Region, Product +Some Views have: No Dimensions at all + +Board Pages: +- Year=FY 24 +- Version=Actuals +- Scenario=Default + +Note: A Board Page only affects widgets whose Views have a compatible +page on that dimension. Widgets whose Views lack Year / Version / Scenario +in Pages are not filtered by those board-level selectors-align Views first +if you need every widget to follow the same board context. +``` + +--- + +### Board Pages and Grouping Pivots in a View's Pages + +A Board Page on Dimension D can drive a View Page if that Page is: + +- A simple Page on D (`dimensionId` = D's ID), or +- A Grouping Page whose `listPropertyPath` resolves to D (e.g., base Dimension Month with property path `["_year"]` leading to Year). + +When a Board Page selects a value on the target Dimension (e.g., Year = FY 2024), the Grouping Page filters its base Dimension items (e.g., Months) to only those whose property chain matches the selected value (Months whose `_year` = FY 2024). + +Example: Board Page on Year driving a Month-based Grouping Page + +``` +Metric base Dimension: Month +Month has a Dimension Property _year -> Year + +View Page: + dimensionId = + listPropertyPath = ["_year"] (Grouping Page targeting Year) + +Board Page: + pageIdentifier: { pageIdentifierType: "Dimension", dimensionId: } + defaultModalityReferences: [{ type: "Fixed", fixedValue: }] + +Result: Only Months whose _year = FY 2024 are included. +The user sees a Year filter, even though the View is modeled at Month level. +``` + +When to use Grouping Pages: + +Use when Views are modeled at a fine grain (Month, Store, Employee) but you want Board-level filters on higher-level Dimensions (Year, Region, Division). This requires clean Dimension Properties defining the hierarchy. + +Important: The `listPropertyPath` in the Grouping Page must correctly resolve to the same target Dimension as the Board Page's `dimensionId`. Use technical property names, not display names. diff --git a/plugins/pigment/skills/designing-pigment-boards/relevant_views.md b/plugins/pigment/skills/designing-pigment-boards/relevant_views.md new file mode 100644 index 00000000..eb47d9fc --- /dev/null +++ b/plugins/pigment/skills/designing-pigment-boards/relevant_views.md @@ -0,0 +1,206 @@ +# Finding Relevant Views + +Balance reuse and creation. Reusing a well-named, well-fitted View preserves formatting work. Creating with `create_view` is normal and often the preferred path when modeling new app features. + +Name before score: A title like "View 1" (or other generic default) is a weak reuse candidate: it is often the product's placeholder. Prefer a new View with a clear name and pivots that fit this widget and other widgets on the same board, unless the listed View already matches. + +Key concept: Display mode (KPI, Grid, Chart) is set on the Widget, not in the View. Judge pivot fit for your `display_intent`, not past widget usage. + +--- + +## Step 1: Define Your Intent + +Before searching, clarify what you need: + +``` +Block: [Metric/List/Table name] +Display Mode: [KPI/Grid/Chart type] +Breakdowns: [Dimensions for rows/columns] +Pages: [Required item filters] +Purpose: [What question does this answer?] +``` + +Example: + +``` +Block: Revenue Metric +Display Mode: Grid (Table) +Breakdowns: Rows=Product Line, Columns=Month +Pages: Year=2024, Version=Actuals,Budget +Purpose: Compare actual vs budget revenue by product line over time +``` + +## Step 2: Retrieve and Evaluate Existing Views + +Retrieve all Views for the Block using `tool:get_block_views`. Pass your display intent using the `display_intent` parameter: +- `Kpi` - for single-value KPI display (Metric and Table blocks only) +- `Grid` - for table/grid display with rows and columns +- `Chart` - for chart/data visualization (optionally specify `chart_type`: Bar, Line, Pie, Waterfall, Org, Geo, Combined) (Metric and Table blocks only) + +List blocks only support Grid display. When calling `tool:get_block_views` for a List, use `display_intent: Grid` or omit it. Do not pass `Kpi`, `Chart`, or any `chart_type` - the API will return an error. + +When a `display_intent` is provided, results are sorted by compatibility and limited to the top relevant Views. Focus your evaluation on the top results. + +Refer to the tool description for the full list of returned fields. + +### Evaluation criteria (in priority order) + +1. Pivot Configuration Match (most important) + +Check if rows, columns, and pages align with your intent: + +- Perfect - all Dimensions align exactly. Select and use as-is. +- Close - most Dimensions align, minor edits needed (swap rows/columns, remove or add 1 pivot, change Quarter to Month). Select and edit. +- Partial - some Dimensions align but significant differences. Consider if editing effort is worth it. +- Poor - completely different Dimensions. Skip. + +Extra pivots are free to ignore (except for KPIs): for Grid and Chart intents, if a View has all the Dimensions you need _plus_ extra ones, treat it as a Perfect match - you simply remove the extra pivots with minimal effort. However, for KPI intent, any extra row pivot is a problem since KPIs must have no row pivots. Extra column pivots are acceptable (they produce multiple KPI cards). + +Pivot matching examples: + +``` +Intent: Revenue KPI (no breakdowns) + +View A: Revenue (no pivots) -> Perfect - use as-is +View B: Revenue by Region (Region in Rows) -> Close - remove Region pivot +View C: Revenue by Product and Month (Product in Rows, Month in Cols) -> Partial - remove both pivots +``` + +``` +Intent: Revenue by Product (rows) and Month (columns) + +View A: Product (rows), Month (columns) -> Perfect - use as-is +View B: Month (rows), Product (columns) -> Close - swap rows/columns +View C: Product (rows), Quarter (columns) -> Close - change Quarter to Month +View D: Region (rows), Month (columns) -> Partial - change Region to Product +View E: Product (rows), no columns -> Close - add Month to columns +``` + +2. Display Mode Compatibility (inferred from configuration) + +- KPI intent: View must have no row pivots. Views with 1-2 Column pivots can work well (they produce multiple KPI cards). Multiple Page pivots are fine - pages act as filters and don't affect the KPI layout. +- Grid intent: View with Row/Column pivots. Page pivots are fine (they act as filters). Note: Grid <-> Spreadsheet conversion is NOT supported - these are fundamentally different display modes. +- Chart intent: View with `chartTypes`, or 1-2 Dimensions suitable for visualization. Views with >2 pivots in Rows or Columns need trimming for readability. Views with a time Dimension (Month, Quarter) are good candidates for trend charts. + +Hints in View data: + +- `chartTypes` non-empty -> likely Chart +- Several `rows`/`columns` populated -> likely Grid +- No Row Pivots and no `chartTypes` -> likely KPI +- View name containing "Chart", "Trend", "Line", "Bar" suggests chart usage. + +3. Board Usage + +- 5+ Boards: proven, well-maintained - strong signal +- 2-4 Boards: validated by multiple users +- 1 Board: may be specialized +- 0 Boards: possibly outdated or abandoned + +4. Recency + +- Last 30 days: actively maintained +- Last 3 months: likely still relevant +- Over 1 year: check carefully, may be outdated + +5. Formatting & Customization + +Views with high `formatOverridesCount` or `conditionalFormattingCount` are worth reusing to preserve that effort. + +6. Name & Description + +- Descriptive names (e.g. *Revenue by Product - Monthly*) support reuse. +- Description text (when present): check it matches your intent (target board, slice, question). A mismatch lowers reuse even with similar pivots. +- Generic names (*View 1*, *Test*) -> usually create a new View unless pivots are already a Perfect match; do not over-optimize for placeholder names. + +## Step 3: Select the Best View + +``` +Does pivot configuration match your intent? +-- Perfect or Close match -> Candidate +-- Partial match -> Continue evaluation, weigh editing effort +-- Poor match -> Skip + +Among candidates, pick the most valuable one: +1. Prefer richer Views: a View with formatting, conditional formatting, or more + customization is a better starting point than a bare-bones exact match. + Removing an extra pivot is cheap; recreating formatting from scratch is not. +2. Board usage (prefer higher - signals quality and maintenance) +3. Recency (prefer more recent) + +Note: the selected View's Dimensions must be a SUPERSET of the intended +Dimensions. A View missing a Dimension you need is harder to fix than a View +with extra Dimensions you can remove. +``` + +If evaluating the match or picking the best View gets too hard, refer to [scoring_relevant_views.md](./scoring_relevant_views.md) for a detailed scoring system. + +## Step 4: Reuse or create + +- Good match (from Step 3) -> use it. +- Close match, worth editing -> Draft from that View, or new View via `create_view` if edits would mangle a shared/poorly named View - use [How to assess the cost of edits](#how-to-assess-the-cost-of-edits). +- No / weak match (including generic default names) -> `create_view`; you can still mirror a Block's better conventions. +- Default: prefer creating over long user prompts when the choice is unclear. + +Convention (new views): time Dimensions often in Columns; entity Dimensions in Rows. For tables, metrics in Rows. Name and description should reflect the board story. + +Naming Drafts (when you edit a live View in preview): clear name + description so the user can review the Draft. + +--- + +## How to assess the cost of edits + +These edits modify the View's pivot configuration to support your intended display mode. + +| Pattern | Edit | Difficulty | +| --------------------------- | --------------------------------------------------------------- | -------------------------------------- | +| Grid -> KPI | Remove all Row pivots (Column pivots may stay) | Easy | +| KPI -> Grid | Add Dimensions to Rows and/or Columns | Easy | +| Swap layout | Move Row dims to Columns and vice versa | Easy | +| Change chart type | Bar -> Line, Line -> Area, etc. | Easy | +| Change granularity | Replace one Dimension with another (Quarter -> Month) | Moderate | +| Adjust filters | Add/remove Dimensions from Pages | Moderate | +| Reorder multiple Dimensions | Rearrange several dims across Rows/Columns/Pages | Moderate | +| Grid <-> Chart conversion | Restructure pivots for a different display paradigm | Hard | +| Add complex calculations | Add YoY%, variance, or custom formulas | Hard | +| Complete restructuring | All Dimensions different from intent | Hard - find another View or create new | +| Grid <-> Spreadsheet | Not supported - these are fundamentally different display modes | Impossible | + +--- + +## Examples + +### Example 1: Revenue KPI + +Intent: Revenue Metric, KPI display, no breakdowns, Pages: Year=2024 Version=Actuals + +| View Name | Pivot Config | Boards | Last Updated | +| ----------------- | ------------ | ------ | ------------ | +| Revenue KPI | None | 8 | 15 days ago | +| Revenue by Region | Rows=Region | 6 | 10 days ago | +| Total Revenue | None | 0 | 6 months ago | + +Decision: Select "Revenue KPI" - perfect pivot config, high usage, recently updated. + +If it didn't exist: Select "Revenue by Region" over "Total Revenue". Despite needing a pivot removal (Region from Rows), it has high Board usage (6 vs 0) and is more recent. A View named "by Region" works fine as a KPI once you remove the pivot and set KPI display mode at widget level. + +### Example 2: Sales Grid + +Intent: Sales Metric, Grid display, Rows=Product Line, Columns=Quarter + +| View Name | Pivot Config | Boards | Last Updated | +| ---------------------------- | -------------------------- | ------ | ------------ | +| Sales by Product - Quarterly | Rows=Product, Cols=Quarter | 2 | 60 days ago | +| Sales by Product - Monthly | Rows=Product, Cols=Month | 5 | 20 days ago | +| Product Performance | Rows=Product, Cols=Region | 1 | 90 days ago | + +Decision: Select "Sales by Product - Quarterly" - exact pivot match outweighs the higher usage of the Monthly variant. Alternative: "Sales by Product - Monthly" if flexible on Month vs Quarter. + +--- + +## Special Cases + +No good match - default to creating with `create_view`; only loop in the user for a product-style choice if both paths are high-effort and ambiguous. + +Multiple equally good matches - break ties: pivot closeness, board usage, recency, then name clarity (favor informative names over *View 1*). + +Outdated View (>1 year): If the Block is still active and the View is still on Boards, it's probably fine. If it's on 0 Boards, create a new view. diff --git a/plugins/pigment/skills/designing-pigment-boards/scoring_relevant_views.md b/plugins/pigment/skills/designing-pigment-boards/scoring_relevant_views.md new file mode 100644 index 00000000..7d1eec3e --- /dev/null +++ b/plugins/pigment/skills/designing-pigment-boards/scoring_relevant_views.md @@ -0,0 +1,42 @@ +# Scoring System for View Selection + +Use this when evaluating multiple Views and the best choice isn't obvious from the criteria in [relevant_views.md](./relevant_views.md). + +## Scoring Table + +| Criterion | Weight | Score Range | How to score | +| --------------------------- | ------- | ----------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Pivot Configuration | HIGHEST | 0-10 | 10=Perfect match, 8-9=Close (swap dims, remove 1 pivot), 6-7=Moderate edits (add/remove multiple pivots), 4-5=Partial (change Dimensions), 0-3=Poor (completely different) | +| Configuration Compatibility | HIGH | 0-5 | 5=Perfect for intended display mode, 3-4=Compatible with minor edits, 1-2=Moderate restructuring, 0=Incompatible | +| Board Usage | MEDIUM | 0-5 | 5=5+ Boards, 3=2-4 Boards, 1=1 Board, 0=no Boards | +| Recent Updates | MEDIUM | 0-5 | 5=last 30 days, 3=last 3 months, 1=last 6 months, 0=over 1 year | +| Formatting & Customization | MEDIUM | 0-3 | 3=Heavy formatting, 2=Some, 1=Minimal, 0=None | +| Name/Description | LOW | 0-2 | 2=Clear and matches intent, 1=Somewhat clear, 0=Unclear or misleading | + +## Configuration Compatibility Details + +- KPI intent + no pivots -> 5 points +- KPI intent + 1 pivot to remove -> 3-4 points +- Grid intent + good row/column setup -> 5 points +- Grid intent + KPI View (need to add pivots) -> 3-4 points +- Chart intent + chartConfig + 1-2 Dimensions -> 5 points +- Chart intent + >2 pivots (need trimming) -> 3 points + +## How to Use + +1. Score each candidate View across all criteria +2. A total score of 20+ indicates a strong candidate +3. Pivot Configuration dominates - a View scoring 8+ on pivots with low scores elsewhere is usually better than a View scoring 5 on pivots with high scores elsewhere +4. Select the highest-scoring View whose editing effort is acceptable + +## Worked Example + +Intent: Revenue KPI, no breakdowns + +| View | Pivot (0-10) | Compat (0-5) | Usage (0-5) | Recency (0-5) | Formatting (0-3) | Name (0-2) | Total | +| -------------------------------------------------- | ------------ | ------------ | ----------- | ------------- | ---------------- | ---------- | ----- | +| Revenue KPI (no pivots, 8 boards, 15d ago) | 10 | 5 | 5 | 5 | 1 | 2 | 28 | +| Revenue by Region (Rows=Region, 6 boards, 10d ago) | 8 | 4 | 5 | 5 | 1 | 1 | 24 | +| Total Revenue (no pivots, 0 boards, 6mo ago) | 10 | 5 | 0 | 1 | 0 | 1 | 17 | + +"Revenue KPI" wins clearly. "Revenue by Region" is a solid fallback despite needing a pivot edit - its high usage and recency compensate. "Total Revenue" scores poorly on usage and recency, suggesting it may be abandoned. diff --git a/plugins/pigment/skills/designing-pigment-boards/view_widgets.md b/plugins/pigment/skills/designing-pigment-boards/view_widgets.md new file mode 100644 index 00000000..2143ced5 --- /dev/null +++ b/plugins/pigment/skills/designing-pigment-boards/view_widgets.md @@ -0,0 +1,36 @@ +# View Widgets + +When creating or editing View Widgets on a Board, each widget must point to a View. How you configure the widget depends on whether the View needed modifications. + +## Display Modes + +See [view_display_modes.md](../creating-and-editing-pigment-views/view_display_modes.md) for display mode definitions. + +Board-level page filters: If the Board should drive several widgets with the same page selector (e.g. Year = FY25), each underlying View must include a compatible page on that dimension. Otherwise the board selector will not filter that widget. See [board_pages.md](./board_pages.md) (Board-to-Widget Page Compatibility Rule). + +WARNING: CRITICAL Display Type / Block Type Rules: + +The widget `display_type` MUST match the underlying block type: + +- Widgets on views on List blocks -> MUST use the List display type +- Widgets on views on Metric blocks -> MUST NOT use the List display type (use Table, Chart, Kpi, or Spreadsheet instead) +- Widgets on views on Table blocks -> MUST NOT use the List display type (use Table, Chart, Kpi, or Spreadsheet instead) + +## Creating a View Widget + +WARNING: CRITICAL (order of operations - every View widget): Do not create or point a View widget, or call `tool:update_view_widget_overrides`, until the underlying View (or Draft) is valid for the `display_type` you set on the widget and for the Block (the Display Type / Block Type Rules above, plus [view_display_modes.md](../creating-and-editing-pigment-views/view_display_modes.md)). + +Read the View (or Draft) and confirm block <-> `display_type` and view_display_modes rules (e.g. Kpi -> no row pivots and `metricsLocation` MUST NOT be `Rows` (use `Columns` or `Pages`); List blocks only List display). If invalid, fix or create a suitable View (`create_view` or Draft) before the widget. Do not trust `get_block_views` candidates without this check. + +## Changing a View that is already on this Board + +When the user asks to modify the View currently shown on a View widget (same board context): + +1. If the ask is a new visualization on the block (not changing this widget's current View), use `create_view` instead. +2. Otherwise call `tool:update_view` / `tool:update_view_chart_config` / `tool:update_view_filters` / `tool:update_view_sorts` directly on the View id. +3. If a Draft was auto-created , leave the widget bound to the original View id and call `tool:update_view_widget_overrides` so the widget displays the Draft for the current user until they save or discard in the Pigment Board UI. +4. If the edit applied directly (no Draft fork - e.g. you're editing a View you just created), nothing else to do; the widget already shows the latest content. + +The agent does not replace the organization-wide widget target or save Drafts on the user's behalf unless the product explicitly allows it - user validation happens in the UI. + +See also `skill:creating-and-editing-pigment-views` (CRITICAL RULES). diff --git a/plugins/pigment/skills/integrating-external-data/SKILL.md b/plugins/pigment/skills/integrating-external-data/SKILL.md new file mode 100644 index 00000000..e44275de --- /dev/null +++ b/plugins/pigment/skills/integrating-external-data/SKILL.md @@ -0,0 +1,196 @@ +--- +name: integrating-external-data +description: Always use this skill when creating new lists to import CSV data into, importing CSV files into Pigment, mapping CSV columns to properties, deciding whether to import into dimensions vs transaction lists, configuring cross-application imports, troubleshooting data import issues, or importing excel files. This skill includes supporting files in this directory - explore as needed. +metadata: + skill_path: /integrating-external-data/SKILL.md + base_directory: /integrating-external-data + includes: + - "*.md" +--- + + +# Integrating External Data + +This skill provides guidance for importing external data into Pigment applications efficiently. + +## When to Use This Skill + +There are three broad use-cases. + +1. Direct CSV Import Workflow + +- Create lists for CSV import - Creating new dimensions or transaction lists that will receive CSV data +- Import CSV files - Loading data from CSV into dimensions or transaction lists +- Map CSV columns - Matching columns to properties using semantic matching +- Decide import targets - Choosing between dimensions and transaction lists + +2. Direct Excel import Workflow + +- Excel import - Importing an excel spreadsheet + +3. Generic information on how to create import connector in Pigment + +- Configure P2P imports - Moving data between Pigment applications +- Optimize import performance - Scoping and filtering strategies +- Troubleshoot imports - Resolving connector issues and data quality problems + +# Direct CSV Import Workflow + +## Step 1: Identify Data Type + +- [ ] Determine if master data (entities like customers, products) or transactional data (events like orders, sales) +- [ ] Search this SKILL.md for relevant section +- [ ] Read documentation files listed + +## Step 2: Decide Import Target + +Use Decision Framework: + +| Data Characteristic | Import To | Reason | +| --------------------------------------------- | -------------------- | --------------------------------------- | +| Master data (customers, products, employees) | Dimension | Relatively static, used as dimension | +| Transactional data (orders, sales, movements) | Transaction List | High volume, time-stamped events | +| Static entities with properties | Dimension | Need to maintain properties/hierarchies | +| Granular event-based data | Transaction List | Aggregate to metrics using formulas | + +## Step 3: Map Columns & Import + +- [ ] Map CSV columns to properties (semantic matching handles translations/abbreviations/synonyms) +- [ ] Create missing properties if needed +- [ ] Configure and execute import +- [ ] Validate results + +## Prerequisites + +From modeling-pigment-applications skill: + +- Core Pigment concepts (dimensions, metrics, transaction lists, sparsity) +- When to use dimensions vs transaction lists + +If unfamiliar -> Use modeling-pigment-applications skill first + +Read: [./data_import_csv.md](./data_import_csv.md) + +Quick Decision: + +- Master data (customers, products, employees) -> Dimension +- Transactional data (orders, sales, movements) -> Transaction List + +# Direct Excel Import Workflow + +Excel imports are to be done according to the instructions in [./excel_import.md](./excel_import.md) + +# Generic information on how to create import connector in Pigment + +Questions: + +- "What integration types are available?" +- "When should I use CSV vs API vs connectors?" +- "What are P2P imports?" + +Read: [./integration_overview.md](./integration_overview.md) + +### Semantic Column Matching + +Scenario: CSV columns don't exactly match property names + +Examples: + +- Translations: "Pays" -> "Country" +- Abbreviations: "Emp" -> "Employee" +- Synonyms: "Client" -> "Customer" + +Read: [./data_import_csv.md](./data_import_csv.md) - "Semantic Column Matching" section + +Key principle: Match by meaning, not exact name + +--- + +## Common Import Patterns + +### Pattern 1: Master Data Import + +Scenario: Importing customer, product, or employee master data + +Steps: + +- [ ] Import into Dimension (not transaction list) +- [ ] Map CSV columns to dimension properties +- [ ] Use semantic matching for column names +- [ ] Create properties if they don't exist + +### Pattern 2: Transactional Data Import + +Scenario: Importing orders, sales, or inventory movements + +Steps: + +- [ ] Import into Transaction List (not dimension) +- [ ] Each CSV row = one transaction +- [ ] Aggregate transaction list to metrics using formulas (BY modifier) + +### Pattern 3: Semantic Column Matching + +Scenario: CSV columns don't exactly match property names + +Approach: + +- Use semantic matching (meaning, not exact name) +- System handles translations, abbreviations, synonyms automatically +- Case-insensitive matching ("email" -> "Email") + +--- + +## Quick Reference Tables + +### Decision Framework: Dimension vs Transaction List + +| Use Dimension When | Use Transaction List When | +| -------------------------------------------- | --------------------------------------------- | +| Master data (customers, products, employees) | Transactional data (orders, sales, movements) | +| Relatively static data | High volume, time-stamped events | +| Used as dimension in metrics | Need to aggregate to metrics | +| Need to maintain properties/hierarchies | Granular event-based data | + +### Semantic Matching Examples + +| CSV Column | Matches Property | Type | +| ------------ | ---------------- | -------------------- | +| "Pays" | "Country" | Translation (French) | +| "Emp" | "Employee" | Abbreviation | +| "Client" | "Customer" | Synonym | +| "email" | "Email" | Case variation | +| "PRODUCT_ID" | "Product ID" | Case + spacing | + +--- + +## Documentation Files + +- [./integration_overview.md](./integration_overview.md) - Integration patterns and best practices +- [./data_import_csv.md](./data_import_csv.md) - CSV import to dimensions and decision framework +- [./excel_import.md](./excel_import.md) - Excel import guidelines + +--- + +## Cross-References + +Before Integration: + +- modeling-pigment-applications - Dimensions, metrics, transaction lists + +After Integration: + +- writing-pigment-formulas - Aggregating transaction lists (BY modifier) +- optimizing-pigment-performance - Import performance optimization + +--- + +## Critical Notes + +- Always determine data type first - Master vs transactional drives all decisions +- Use semantic matching - Column names don't need to match exactly +- Import to dimensions for master data - Customers, products, employees +- Import to transaction lists for events - Orders, sales, movements +- Validate after import - Check data quality and completeness +- Performance matters - Large transaction lists need aggregation formulas +- Document your decision - Explain dimension vs transaction list choice diff --git a/plugins/pigment/skills/integrating-external-data/data_import_csv.md b/plugins/pigment/skills/integrating-external-data/data_import_csv.md new file mode 100644 index 00000000..2f507be8 --- /dev/null +++ b/plugins/pigment/skills/integrating-external-data/data_import_csv.md @@ -0,0 +1,151 @@ +# CSV Data Import + +## Overview + +CSV import populates dimensions or transaction lists with external data. Each CSV row represents an item, each CSV column maps to a property. + +--- +## Working with File Attachments + +The user may attach files (CSVs, PDFs) to the conversation. All attached files remain available throughout the conversation. + +### File Identification +Each file has: +- File Name: The user-visible name (e.g., "sales_data.csv") +- Metadata: Size, row count, columns, preview data + +### Handling Multiple Files with the Same Name + +When multiple files share the same name: + +1. Examine the metadata, order and preview to differentiate files +2. If genuinely ambiguous, use the `ask_user` tool to clarify and give it the name, metadata, preview data and order + +## 1. CSV Analysis + +Understanding the CSV structure before import. + +### Column Filtering + +Exclude from import: +- Empty columns (no header or no data) +- Columns with technical/generated names: `_1`, `_2`, `_internal_id`, `sys_key`, `row_id`, names starting with `_` +- System timestamps: `created_at`, `updated_at`, `last_modified`, `updated_timestamp` +- Free-text comments (unless explicitly requested) + +CRITICAL: Do not create properties for excluded columns. + +### Column Classification + +| Column Type | Characteristics | Examples | +|-------------|-----------------|----------| +| Categorical (low cardinality) | Values repeat across rows | Country: "France", "Germany", "France" | +| Unique (high cardinality) | Distinct value per row | Order ID, Description, Amount | +| Numeric | Numbers, quantities, amounts | Price, Quantity, Total | +| Temporal | Dates, timestamps | Order Date, Created At | + +### Number/Date Format Detection + +After filtering the relevant columns, examine the preview rows to detect: + +- Date columns: If any, determine the appropriate date format before calling the MCP tool +- Number columns: If any, determine the appropriate number format before calling the MCP tool + +If dates are ambiguous (e.g., '01/02/2023' could be Jan 2nd or Feb 1st) or if the preview doesn't contain enough examples to confidently determine the format, ask the user for confirmation. + +--- + +## 2. Dimension vs Transaction List + +### Recognition Patterns + +| CSV Pattern | Likely Type | Rationale | +|-------------|-------------|-----------| +| Mostly categorical columns, no unique ID per row | Dimension | Master data (products, customers, regions) | +| Has a unique ID column + categorical columns | Transaction List | Transactional data (orders, sales, movements) | + +--- + +## 3. Categorical Columns and Referenced Dimensions + +Categorical columns (low cardinality, repeated values) can be handled in two ways: + +| Approach | When to Use | Example | +|----------|-------------|---------| +| Create separate dimensions | For reusability across the application | Create dimension "Country", then TL "Sales" with property "Country" referencing it | +| Simple Text properties | For quick import without reusability needs | Create TL "Sales" with property "Country" as Text | + +Creating separate dimensions enables: +- Reuse across multiple lists +- Hierarchies (Country -> Region) +- Additional properties on the dimension itself (Country.Population, Country.Currency) + +CRITICAL: When categorical columns are detected, ALWAYS present both options to the user. Do not assume which approach to use. Choosing Text properties when dimensions are needed later requires recreating the list. + +### After the user chooses: + +If user chooses "Create separate dimensions": +- First, create one dimension per categorical column (e.g., dimension "Country", dimension "Product", dimension "Customer") +- Then, create the target list with properties of type Dimension referencing them +- Finally, import the CSV + +If user chooses "Text properties": +- Create the target list with Text properties +- Import the CSV directly + +--- + +## 4. Mapping Rules + +### 4.1 New Dimension or Transaction List + +When creating a new target for import: + +Properties match CSV columns: Create one property per CSV column (after filtering). The mapping is 1:1 - each CSV column maps to the property with the same name. + +No uniqueness constraints for transaction lists: Do not create properties with uniqueness constraints. The preview only shows a sample; duplicates may exist in the full data and would cause import failure. + +Example: +``` +CSV columns (filtered): Date, Customer, Product, Amount +-> Create TL "Sales" with properties: Date, Customer, Product, Amount +-> Mapping: {"Date": "Date", "Customer": "Customer", "Product": "Product", "Amount": "Amount"} +``` + +### 4.2 Existing Dimension or Transaction List + +When importing into an existing target: + +Semantic matching: Match CSV columns to existing properties using: + +| Match Type | CSV Column | Property | +|------------|------------|----------| +| Exact | "Country" | "Country" | +| Translation | "Pays", "Produit" | "Country", "Product" | +| Abbreviation | "Dept", "Qty", "Amt" | "Department", "Quantity", "Amount" | +| Synonym | "Client", "Item", "Location" | "Customer", "Product", "Region" | + +Missing properties: If a CSV column has no matching property, propose adding new properties to the target (one per relevant column). + +Dimension-type properties: If a property references another dimension (e.g., property "Country" references dimension "Country"), map the CSV column directly to that property. The backend resolves references automatically. + +--- + +## 5. Import Scope + +Rule: Import only into the target requested by the user. + +When importing into a dimension or transaction list that has properties referencing other dimensions: + +| Action | Correct? | +|--------|----------| +| Import into TL "Sales" with mapping `{"Country": "Country", "Product": "Product", ...}` | GOOD: Yes | +| Import separately into dimension "Country", then dimension "Product", then TL "Sales" | BAD: No | + +The backend handles reference resolution internally. Do not perform multiple imports on referenced dimensions. + +--- + +## 6. Post-Import Verification + +CRITICAL: After every import, fetch the target dimension or transaction list and verify that the number of items is greater than 0. If the count is 0, the import has failed. Inform the user and suggest using the Pigment UI to perform the import manually. \ No newline at end of file diff --git a/plugins/pigment/skills/integrating-external-data/excel_import.md b/plugins/pigment/skills/integrating-external-data/excel_import.md new file mode 100644 index 00000000..444c8edc --- /dev/null +++ b/plugins/pigment/skills/integrating-external-data/excel_import.md @@ -0,0 +1,414 @@ +# Excel -> Pigment Modeling Specification + +## Purpose + +Take one or more Excel files (of any complexity) and produce a comprehensive, implementation-ready +modeling specification for rebuilding the entire workbook as a Pigment application. The spec must +be detailed enough that a Pigment modeler can build the app without referring back to the Excel. + +## Why This Skill Exists + +Excel models used in enterprise planning are often messy, undocumented, and fragile. They contain +implicit dimensional structures, hidden cross-references, hardcoded assumptions, and accumulated +technical debt. Converting them to Pigment requires careful reverse-engineering - not just listing +the sheets, but understanding the *intent* behind the structure and mapping it to Pigment's +multidimensional paradigm. This skill codifies that reverse-engineering process. + +--- + +## Workflow Overview + +The workflow has 4 phases. Complete them in order. Each phase produces artifacts that feed the next. + +``` +Phase 1: DISCOVERY -> Inventory the Excel structure +Phase 2: ANALYSIS -> Identify dimensions, data and calculation flows, errors +Phase 3: MAPPING -> Map Excel concepts and data to Pigment +Phase 4: SPECIFICATION -> Write the detailed modeling spec +``` + +--- + +## Phase 1: DISCOVERY - Inventory the Excel Structure + +Goal: Build a complete picture of what's in the file before making any modeling decisions. + +### 1.1 Read the Excel File(s) + +For each sheet, including hidden, collect: + +- Sheet metadata: name, dimensions (used range), row/column count +- Sheet intent: brief description of the sheet role +- Merged cells: count and locations (merged cells are a strong signal of reporting/dashboard sheets) +- Headers: identify header rows (may not be row 1 - look for the first row where most cells are non-empty strings) +- Data types per column: sample 20+ rows to determine types (numeric, text, date, boolean, dimension) +- Formulas: count, complexity, cross-sheet references +- Named ranges: list all, flag any that resolve to #REF! +- Data validation / dropdowns: list ranges with validation rules +- Independent grids on this sheet: note every separate contiguous data or report block (different headers, spacing, or unrelated topics); assign a region id for the spec (section 1.2) +- Charts: count embedded chart objects; note sheet vs. regions (section 1.2). For each, capture section 1.4: intention, format, source data range. + +### 1.2 Classify Each Sheet + +Assign each sheet to one of these categories: + +| Category | Description | Pigment mapping | +|---|---|---| +| DATA_SOURCE | Flat tabular data with headers and rows (imports from ERP, HRIS, etc.) | Transaction List or Dimension import | +| REFERENTIAL | Lookup/mapping table (accounts, org hierarchy, FTE types) | Dimension List with properties | +| CALCULATION | Sheet dominated by formulas computing intermediate or final results | Metrics with formulas | +| REPORTING | Dashboard/summary with merged cells, charts, formatting | Board / Table / View | +| INPUT | Cells meant for user entry (highlighted, validated, sparsely filled) | Input Metrics on Boards | +| PARAMETERS | Configuration sheet (reference year, dropdown lists, settings) | Settings folder, Dimension items, or Input metrics | +| NAVIGATION | Menu/separator sheet | For navigation boards | +| MACRO_SUPPORT | Objective, Role, VBA timing, logging, or macro control sheet | Modeled only if core calculation/validation process | + +Heuristics for classification: +- If >80% of non-empty cells are formulas -> CALCULATION +- If merged cells > 10 and sheet has "Reporting", "Overview", "Summary", "Dashboard" -> REPORTING +- If sheet has flat headers in row 1 and uniform data below -> DATA_SOURCE or REFERENTIAL +- If sheet name starts with "Source_" or "Data" -> DATA_SOURCE +- If sheet name contains "Master Data", "Ref_", "List", "Mapping" -> REFERENTIAL +- If sheet has data validation or highlighted input areas -> INPUT +- If sheet contains navigation buttons or links -> NAVIGATION +- If sheet has embedded charts (see section 1.4) -> strong signal for REPORTING (or mixed regions) + +Multiple grids on one tab. A single sheet may contain several independent data or report areas (e.g. two unrelated tables one under the other, a small parameters block next to a ledger, one pivoted block and one flat table on the same tab). Do not assume *one sheet -> one* Pigment Transaction List, metric, or Table. Split the tab into logical regions (give each a stable id in the spec, e.g. `SheetName + cell range` or `SheetName::Region_A`), classify each region separately (it may be DATA_SOURCE, pivoted facts, REPORTING, etc.), and plan separate TLs, metrics, Tables, charts, and/or Boards as needed. Data source imports and build order in the written spec (Phase 4 - Excel migration annex and Implementation) apply per region, not only per sheet name. + +### 1.3 Map Cross-Sheet Dependencies + +Build a dependency graph: +- For each formula with "!", record source_sheet -> target_sheet +- For named ranges, record which sheets reference which +- Identify circular dependencies (rare but critical) +- Identify the "flow direction": data sources -> calculations -> reporting + +Output: a dependency summary showing which sheets feed which. + +### 1.4 Chart inventory (for REPORTING / dashboards) + +When a sheet contains embedded charts, record each chart in a compact list. section 3.6 and spec section 8 carry Pigment design (pivot, metrics, board placement). + +For each chart: + +- Intention: what it is meant to convey (e.g. "revenue by month", "mix by product", Actual vs Budget by cost center). +- Format: Excel chart type (line, column, bar, stacked column, pie, doughnut, area, scatter, combo, waterfall, ...). +- Source data range: workbook reference for the chart's data - cell range(s), pivot chart source, or named range(s) Excel binds to the series and categories. + +--- + +## Phase 2: ANALYSIS - Identify Dimensions, Data Flows, Errors + +### 2.1 Extract Candidate Dimensions + +Scan DATA_SOURCE and REFERENTIAL sheets for columns that are categorical (repeating values), per logical region when a tab holds multiple grids (section 1.2). +Apply the cardinality analysis from Pigment fundamentals: + +``` +For each column: + cardinality_ratio = unique_values / total_rows + + If ratio < 0.8 and values repeat -> candidate Dimension + If ratio approx 1.0 and text -> candidate Text property + If numeric -> candidate Number property / Metric value + If date -> candidate Date property + If boolean or yes/no -> candidate Boolean property +``` + +Cross-validate dimensions across sheets. If "Cost Center" appears as a column in 5 different source sheets, +it's almost certainly a dimension. If "Cost Center" also appears in a REFERENTIAL sheet with properties like +"Cost Center name", "Entity", "Division" -> that's the dimension definition. + +Canonical names: master data wins. When different Excel labels refer to the same entity (abbreviation vs full name, "Region A" in a DATA_SOURCE vs "Region A - North" on a REFERENTIAL row, typos, or legacy synonyms), do not treat each spelling as a different Pigment item. Pick the name (or code) defined on the master / REFERENTIAL sheet as the single canonical value for specs, imports, `tool:add_list_items` rows, and `tool:set_metric_input` coordinates. List every alias discovered in the workbook in the spec (Excel raw value -> canonical master name) so execution never forks duplicate members. + +### 2.2 Identify Hierarchies + +Look for patterns that indicate parent-child hierarchies: +- REFERENTIAL sheets with columns like "L4 -> L3 -> L2 -> L1 -> Account" (multi-level account hierarchy) +- Organization tables with "Cost Center -> Entity -> Division" +- Named ranges with "List_xxx" that enumerate dimension items +- Columns whose values are a subset of another column's values + +### 2.3 Identify the Time Dimension + +Look for: +- Column headers that are years (2025, 2026, ...) or months (Jan, Feb, Jul-25, ...) +- Columns named "Year", "Month", "Quarter", "Period" +- Named ranges referencing years +- Fiscal year patterns (FY26, Q1/26, H1) + +Determine the time granularity needed: annual, quarterly, monthly, or mixed. +Note any fiscal year offsets (e.g., FY starting in July). + +### 2.4 Identify Metrics vs Properties + +Long-format / "database" columns (each numeric column is one measure or each row adds one fact with dimension values in dedicated columns): + +- If the column represents a *measure* that varies by the dimensional combinations (Amount, FTE count, Cost) and each row is a grain you will keep in Pigment -> it is usually modeled as a Property (Dimension or Transaction) and aggregated into metrics with formulas +- If the column represents a *fixed attribute* of a dimension item (Account code, Sort order) -> it's a Property on a Dimension or on list items. + +Pivoted / "matrix" layouts (two categorical axes: one dimension along rows, another along columns - e.g. products down, months across - with numeric values only in the interior cells). A sheet may contain more than one such block; treat each as its own mapping target (section 1.2). + +- This shape matches a Metric or Table view in Pigment: row header dimension x column header dimension (often including time in the column headers). +- The practical load path for that grid is usually `set_metric_input`: each non-empty cell becomes a coordinate (row item, column item) plus a value. See section 3.8 for volume: very large grids may require multiple `set_metric_input` calls (chunking) or a redesign (unpivot -> Transaction List + formulas). + +Table-shaped DATA_SOURCE (header row + many data rows; every column is a field, no measure "spread" across a second dimension in the header row); scope to one logical region if the sheet has several (section 1.2): + +- Model as Transaction List (or Dimension items + properties when it is master data). +- To import rows: use `add_list_items` only - successive tool calls with at most 100 rows per `rows` payload until all Excel rows are loaded (large sheets require many batches). + +Calculated properties on dimension or transaction lists (formula columns on REFERENTIAL or table-shaped sheets, not only CALCULATION sheets): + +- Inspect each column: if every populated cell uses the same formula (same structure; only row-relative references differ), create the property, exclude that column from import, and set a list property formula once input properties on that list exist. +- If formulas vary materially across rows (different functions, layouts, or row-specific logic), do not force one property formula - import the column values with `add_list_items` as an input property instead. +- In the spec, label each list property input or calculated (with the Pigment formula when calculated). + +### 2.5 Detect Errors and Red Flags + +This is critical for building user trust. Actively look for: + +| Error type | How to detect | Severity | +|---|---|---| +| #REF! in named ranges | Named range attr_text contains "#REF!" | HIGH - broken references, likely from deleted sheets | +| #REF! in formulas | Cell values containing "#REF!" | HIGH - broken calculations | +| #DIV/0!, #VALUE!, #NAME? | Cell values or formula errors | MEDIUM - formula bugs | +| "Check to zero" cells != 0 | Cells labeled "check to zero" or "control" with non-zero values | HIGH - model doesn't balance | +| Missing dimension values | Null/empty cells in categorical columns | MEDIUM - incomplete data | +| Inconsistent naming | Same entity spelled differently across sheets (e.g., "Nearshoring PL" vs "Nearshoring Poland") | MEDIUM - must normalize to the master / REFERENTIAL name (or code) before import; see section 2.1 | +| Orphan sheets | Sheets not referenced by any formula and not referencing others | LOW - possibly obsolete | +| Excessive merged cells | >50 merged cell ranges in a sheet | LOW - complicates parsing | +| 100% empty Projet column | A dimension column that is empty in all rows | HIGH - missing mandatory dimension | +| Broken named ranges ratio | >30% of named ranges are #REF! | HIGH - significant technical debt | +| Hardcoded values in formulas | Formulas containing literal numbers instead of cell references | LOW - reduces flexibility | +| Circular references | Formulas that reference themselves (directly or indirectly) | HIGH - may produce incorrect results | + +Report errors prominently at the start of the spec. This builds confidence. + +### 2.6 Detect Calculation Logic Patterns + +Scan CALCULATION sheets for common patterns: +- Aggregation: Use views to naturally aggregate across dimension properties, or BY mapping in formula +- VLOOKUP/INDEX-MATCH/XLOOKUP: Dimension properties in formulas or ITEM() +- IF/IFS/SWITCH: IF/SWITCH formulas +- OFFSET/INDIRECT: Combination of properties and BY mappings +- SUMPRODUCT/SUMIFS: BY modifier +- Cumulative (running totals): -> CUMULATE function +- Year-over-year/shifts: SELECT modifier (e.g. `Revenue[SELECT: Month - 12]`) +- Sensitivity toggles: Cells that select an item in a list -> Input metric of type Dimension + +--- + +## Phase 3: MAPPING - Map Excel Concepts to Pigment Concepts + +### 3.1 Dimension List Design + +For each identified dimension, define: +- Name: follow Pigment naming conventions +- Source: which Excel sheet/column provides the items +- Properties: columns from the REFERENTIAL sheet that become properties +- Property types: apply cardinality analysis to determine Dimension-type vs Text/Number +- Unique property: which property identifies items for imports (usually Code or Name) +- Aliases: any non-master labels seen in DATA_SOURCE or CALCULATION sheets -> map each to the canonical name/code from this REFERENTIAL (section 2.1) + +### 3.2 Transaction List Design + +For each distinct table-shaped DATA_SOURCE region on a tab (see section 1.2 and section 2.4 - not pivoted matrices; those belong under Metric Design): + +- Region / Source: which Excel sheet and range (or named area) defines this single grid +- Name: follow Pigment naming conventions (one Transaction List per region, not necessarily per sheet) +- Dimension-type properties: which columns link to which Dimension Lists (values normalized to canonical master names per section 2.1) +- Numeric properties: which columns hold measure values +- Expected row count: approximate volume for performance planning (informs the load path in section 3.8) + +### 3.3 Metric Design + +For each metric (calculated or input), including each pivoted fact block when a tab holds several grids (section 1.2) - each block may be its own metric or feed a distinct structure: + +- Name follow Pigment naming conventions +- Type: Number, Date, Text, Boolean, Dimension +- Structure (dimensions): which Dimension Lists define the metric structure (max 5 recommended) +- Formula logic: describe in natural language + pseudo-Pigment syntax +- Aggregator: Sum, Avg, Advanced Aggregator, etc. +- Data source: if populated from a Transaction List, specify the BY aggregation +- Input vs Calculated: is this user-entered or formula-driven? +- Planned write path: for user-entered (input) metrics, state whether values should be loaded via `tool:set_metric_input` (moderate cell counts) or derived from lists/metrics instead (see section 3.8). Pivoted Excel grids (section 2.4) are the primary case for `tool:set_metric_input`. +- Excel region (when the tab has multiple blocks): sheet + range for this metric's source grid (section 1.2) + +### 3.4 Table Design + +For reporting views that group metrics (P&L, Summary). One Excel tab may contain several unrelated report layouts (side-by-side blocks, stacked summaries): model one Pigment Table (and associated views) per distinct reporting region (section 1.2), not one Table per sheet by default. + +For each table: +- Excel region (optional but required when the sheet has multiple grids): sheet name + range identifying this report block (section 1.2) +- Name: follow Pigment naming conventions +- Metrics included: list the metrics +- Calculated items: any derived rows (e.g., "Net Revenue = Gross Sales - Trade Investment") +- View pivot (target layout): for the table's primary View, specify which dimensions belong on rows, columns, and pages (page = selectors / filters that scope the whole view without adding a row or column axis). Align with how the Excel report is read left-to-right and top-to-bottom. +- Sorting: default sort on row and/or column axes where it matters (e.g. hierarchy order, chronological time, custom account sequence); note any Excel-implied order to preserve. +- Filtering: view- or axis-level filters to mirror the workbook (slices, excluded members, "actuals only", version, etc.) and whether null/zero rows or columns should be hidden for parity with Excel. + +### 3.5 Board Design (high-level) + +For each REPORTING sheet, suggest: +- Board name: mapped from the Excel sheet name +- Widgets: metrics, tables, charts (each specified in section 3.6), KPI cards, text +- Page Selector: dimensions in page selectors at board level +- Layout: ordered widget list + +### 3.6 Chart design (Board-ready visualizations) + +Expand section 1.4 (intention, format, source data range) into a full Pigment plan for each chart (and any chart implied by a REPORTING region without a formal chart object but built from a pivot chart range): + +- Analysis - what is shown: clear description for implementers (trend, composition, variance, ranking, correlation, etc.) and the business question the chart answers. +- Excel chart format -> Pigment chart type: map Excel line / column / bar / stacked / pie / doughnut / area / scatter / combo (etc.) to the closest Pigment visualization; call out secondary axis, stacked vs clustered, and 100% layouts where relevant. +- Underlying data: which Pigment metrics, Table, or View must power the chart (names / ids); ensure those blocks exist before the chart is wired. +- Chart pivot (mirror View configuration): specify dimensions on rows, columns, and pages (selectors); which field drives series / color / legend; category axis ordering (time order, hierarchy, custom); filters (versions, scenarios, top members) so the dataset matches the Excel slice. +- Formatting parity (when important): notable colors, data labels, axis titles - only what affects readability or sign-off. +- Board placement: target Board (section 3.5), slot in the widget order, and spatial relationship to neighboring tables (e.g. "chart above Table X as in Excel"). + +### 3.7 Calendar Configuration + +Based on the time analysis: +- Granularity: monthly, quarterly, annual +- Range: start year to end year +- Fiscal year: does it align with calendar year? +- Properties: Quarter, Half-Year, Year as properties of Month + +### 3.8 Data load mapping + +Goal: After the conceptual mapping (lists, metrics, calendars), the implementation plan must name how data lands in Pigment when an agent executes the migration. Pick tools by volume and shape (grid inputs vs tabular rows). + +| Need | MCP tool | When to use | Limits / notes | +|------|----------|-------------|----------------| +| Metric input values (user-entered cells on a defined metric structure) | `tool:set_metric_input` | Pivoted Excel grids (section 2.4) and other low-to-moderate coordinate sets: each call carries rows of dimension values + cell values. | Not for table-shaped fact sheets (use lists). Very large pivots: chunk multiple calls or unpivot -> TL + formulas (section 2.4). | +| Append rows to a Dimension or Transaction List | `tool:add_list_items` | Table-shaped list imports: data is supplied as headers + row arrays in the tool payload. | At most 100 rows per tool call (maximum length enforced on the `rows` argument). Larger sheets: only repeated calls with <=100 rows each until the dataset is exhausted. | + +Guidelines + +- Pivoted sheets (section 2.4) -> `tool:set_metric_input` on the target input metric; table-shaped sources -> `tool:add_list_items` in batches of <=100 rows per call. +- For every table-shaped DATA_SOURCE region (section 1.2), the spec must spell out `tool:add_list_items` batching (row order, headers, and how many calls for the expected row count). +- `tool:add_list_items` and `tool:set_metric_input` payloads must use canonical dimension item names/codes from master data (section 2.1) +- Formula-driven metrics are not "written" row-by-row with these tools; they are defined with formulas (other MCP tools). Reserve `tool:set_metric_input` for input metrics. +- Calculated list properties (section 2.4): omit from `add_list_items`; set the property formula after the list and its input properties exist. + +--- + +## Phase 4: SPECIFICATION - Write the Detailed Modeling Spec + +### 4.1 Spec document structure (aligned with `tool:rewrite_specifications`) + +The written spec MUST follow this outline: ## Summary, ## Architecture, then one `## Application: ...` block per Pigment application, each with Blocks and Folders -> Views -> Boards + +Cross-read [modeling_architecture_design.md] (Pillars 3-4: data cycle, governance, Hub vs spokes) and [modeling_principles.md] (Hub pattern, Library folder) before fixing Architecture and deciding which blocks live in Hub vs spoke apps. + +```markdown +# Pigment Modeling Specification +## Source: [Excel filename(s)] +## Generated: [date] + +## Summary +[Narrative goal of the Excel migration.] Key outcomes, scope, and risks (data quality, complexity, deadlines). + +## Architecture +Applications & Hub decision: +- Single app vs multi-app: State whether the workspace uses one application or Hub + one or more spokes (use cases, ownership, sensitivity per architecture skill). +- Hub / master data:: a Hub holds shared dimensions (referential / master lists, Version if used, organizational & chart-of-accounts dimensions reused across apps, Calendar-related structural choices when centralized). Spoke apps hold use-case-specific metrics, transaction lists loaded for that process, tables, and reporting views/boards, unless transactional data must be shared across apps (then Hub placement may apply - state why). +- Excel mapping: Which REFERENTIAL / DATA_SOURCE regions (section 1.2) land in Hub dimensions vs spoke TLs; call out canonical names (section 2.1). +- Data flow: Short diagram in words: Excel -> imports (section Data source imports) -> lists/metrics -> tables -> views/charts -> boards; mention Library / `PUSH_` / `PULL_` if multi-app sharing applies. + +## Excel migration annex (traceability - optional subsection before Application blocks) +Use this block so Excel specifics are not lost; keep it after Architecture and before the first `## Application`. + +### Error & quality report +[Brief reference to Phase 2 findings - or "clean".] + +### Multi-grid & chart index +- Regions per sheet (section 1.2); charts (section 1.4 recap: intention, format, source range). + +### Data source imports (execution) +- Per Excel region (section 1.2): pivoted -> `tool:set_metric_input`; table-shaped -> `tool:add_list_items` in batches of <=100 rows; order after target blocks exist. +- Do not merge unrelated regions on one tab into a single list/metric. + +--- + +## Application: `Hub` - *omit this heading when everything fits a single non-Hub app; use `Hub` or `Master data` display name when a separate Hub is required* + +### Blocks and Folders + +| Action | Object | Type | Role | Reason | +|--------|--------|------|------|--------| +| create/reuse/edit/delete | `name` (mention if known) | dimension, transaction list, metric, table, block folder, board folder | input/processing/output / shared master | e.g. shared referential from Excel REFERENTIAL sheets; Version dimension in Hub per scenarios skill | + +*Do not list view or board rows here - use Views and Boards below.* + +### Views (when this app has reporting widgets) + +| # | Action | View | Block | Rows | Columns | Pages | Chart type | +|---|--------|------|-------|------|---------|-------|------------| +| 1 | create/reuse/... | `View name` | `Metric or table` (metric/table) | ... | ... | ... | Grouped Bar chart, Stacked Bar chart, Line chart, Combined chart, Pie Chart, Waterfall chart, or `-` | + +*For charts, align View name with Boards widget plan; chart type column matches Pigment chart types.* + +### Boards (when relevant) +Repeat per board with a `### Board: ...` heading. Widget plan: one row per widget, screen order (left -> right, top -> bottom). + +| # | Action | Widget | View | Widget type | Width x Height | +|---|--------|--------|------|-------------|----------------| +| 1 | create/edit | Label | same `View` as in Views or `-` | Grid, Chart, List, KPI, spacer, action, image, single text, dimensioned text | e.g. 12x3 | + +--- + +## Application: `[Primary planning / use-case app]` - *repeat the three subsections for each spoke* + +### Blocks and Folders +[Same table pattern - TLs and metrics consumed by this use case, folders per modeling_principles.] + +### Views +[Same Views table - include chart types on views that feed Chart widgets.] + +### Boards +[Same Boards widget plan - reference section 3.6 / Excel chart intentions.] + +--- + +## Application: `[Second spoke if any]` +*Same three subsections - omit when not applicable.* + +--- + +## Implementation +### Phase 1 (and further phases as needed) +- Build order: Hub shared dimensions (and shared blocks) before spoke metrics that reference them; imports after target lists/metrics exist. +- Call out Excel region -> Pigment object for major loads. +- Reporting: underlying metrics/tables/views before Board chart/table widgets. + +## Notes (optional) +Open questions, parity exceptions. + +## Appendix +- Sheet classification & multi-grid tab index (section 1.2) +- Named ranges inventory (errors #REF!, etc.) +- Cross-sheet dependency summary +- Full error list +``` + +### 4.2 Guidelines for a Successful Plan + +- Be specific. Don't write "create a metric for costs" - write `CALC_Total_Cost` dimensioned by `CostCenter x AccountREP x Month x Version` with formula `LOAD_Budget_Charges.Total_Cout_KEUR[BY SUM: ...]`. +- Include the Excel-to-Pigment mapping for every element. The reader should be able to trace any Pigment object back to the exact Excel sheet/cell/column it came from. +- Flag decisions. When the mapping is ambiguous (e.g., a column could be a dimension or a property), explain both options and recommend one with reasoning. +- Use Pigment naming conventions from the modeling-pigment-applications skill. +- Challenge structures with >5 dimensions. If a metric would need 6+ dimensions, suggest alternatives (properties, mapped dimensions). +- Always propose to model "the Pigment way". Importing an Excel file does not mean useing the literal idiosyncracies of Excel, and you should always plan to use idiomatic Pigment patterns instead of a direct translation. + +--- + +## Critical Reminders + +- Written spec shape: Phase 4 output must match `rewrite_specifications` (Summary, Architecture with Hub vs spoke, then per-app Blocks and Folders -> Views -> Boards). Shared master / referential dimensions from Excel usually belong in a Hub when multiple apps or reuse are anticipated - see modeling_architecture_design.md Pillar 4. +- Always read the writing-pigment-formulas skill before writing formula pseudo-code. Pigment has its own formula language - never use Excel syntax in the spec. +- Never assume a clean Excel. Headers may not be in row 1. Data may start at row 7. Columns may be hidden. Sheets may be named misleadingly. +- The error report is not optional. Even if the file is clean, state that explicitly. If there are errors, report them prominently - this builds trust. +- Cross-validate dimensions across all sheets. A dimension that appears in 1 source sheet is suspect; one that appears in 5 is reliable. +- One canonical name per entity. When labels differ, use the master / REFERENTIAL name (or code) everywhere (section 2.1). +- Multiple grids on one tab (section 1.2): plan separate Pigment objects and import sequences per region - don't assume one sheet -> one TL or one Table. diff --git a/plugins/pigment/skills/integrating-external-data/integration_overview.md b/plugins/pigment/skills/integrating-external-data/integration_overview.md new file mode 100644 index 00000000..72047290 --- /dev/null +++ b/plugins/pigment/skills/integrating-external-data/integration_overview.md @@ -0,0 +1,44 @@ +# Integration Overview + +## Introduction + +Data integration in Pigment involves importing data from external sources, moving data between applications, and managing data flows. This guide provides an overview of integration patterns and best practices. + +## Integration Types + +### CSV Imports + +- Manual file uploads +- Scheduled imports +- One-time data loads + +### API Integrations + +- Real-time data sync +- Scheduled pulls +- Webhook triggers + +### Native Connectors + +- Pre-built integrations +- Automated data sync +- Configuration-based + +### P2P Imports + +- Between Pigment applications +- Within same workspace +- Async data movement + +## Best Practices + +1. Validate data before import +2. Use transaction lists for high-volume data +3. Scope imports for performance +4. Monitor import success rates +5. Handle errors gracefully +6. Document data mappings + +## See Also + +- [CSV Data Import](./data_import_csv.md) - Detailed guide for importing CSV data diff --git a/plugins/pigment/skills/modeling-pigment-applications/SKILL.md b/plugins/pigment/skills/modeling-pigment-applications/SKILL.md new file mode 100644 index 00000000..fc442c58 --- /dev/null +++ b/plugins/pigment/skills/modeling-pigment-applications/SKILL.md @@ -0,0 +1,179 @@ +--- +name: modeling-pigment-applications +description: Always use this skill when designing or modifying Pigment applications. Provides the mental model of a Pigment app (Application, Dimensions, Calendars, Metrics, Transaction Lists, Tables), the core concepts (dimension list vs property vs transaction list, metric vs table, sparsity, scope), the canonical order of architecture decisions, a minimal viable application pattern and pointers to deeper-dive docs (architecture, naming, hierarchies, calendars, principles, folders, subsets, performance). +metadata: + skill_path: /modeling-pigment-applications/SKILL.md + base_directory: /modeling-pigment-applications + includes: + - "*.md" +--- + +# Modeling Pigment Applications + +Core concepts and architecture for designing Pigment applications. Read first for any modeling task. Jump to the linked deep dives only when the task requires the detail. + +## When to Use This Skill + +Read this skill before: + +- Designing or restructuring an Application (dimensions, metrics, tables, transaction lists, folders) +- Choosing dimension vs property, metric vs transaction list, table vs standalone metric +- Reviewing whether an existing structure is sound (sparsity, scope, T&D safety) +- Onboarding to an unfamiliar workspace before changing anything + +--- + +## Mental Model + +Pigment is an in-memory, sparse, multidimensional engine. An Application is a graph of typed blocks plus orthogonal cycle layers. Block types: + +- Folders -- organizational only, never logic, optional +- Dimensions (includes the Version dimension) -- analysis axes, items + properties +- Calendar dimensions -- Month / Quarter / Year, plus Date +- Metrics -- multidim grids, typed Number / Date / Text / Dim / Bool +- Transaction Lists (TL) -- high-volume facts, NOT structural +- Tables -- group metrics sharing dimensions +- Boards / Views -- UX layer over metrics and tables + +Plus two orthogonal layers that apply across blocks: + +- Native Scenarios -- ad-hoc what-if, app-level feature +- Snapshots -- point-in-time copy of the app + +Invariants the agent must respect: + +- A metric cell is identified by one item per structural dimension. Empty cells are not stored (sparsity). +- Only dimension lists can sit in a metric structure. Transaction lists never can. Aggregate them inside formulas with `BY`. +- The Version Dimension is the planning-cycle dimension (Budget, Actual, Forecast). Whenever you design or modify a planning metric or anything time-bound, also consult `skill:planning-cycles-pigment-applications`. +- Folders are inert. Placement affects discovery and governance, not calculation. + +--- + +## Core Concepts + +### Block types + +| Concept | What it is | When to pick it | +|---|---|---| +| Dimension list | Analysis axis with unique items and properties. Usable as a structural axis. | Country, Product, Employee, Account, anything you will slice metrics by | +| Property | Column on a dimension. Type Number / Date / Text / Boolean / Dimension. | Static attribute of an item | +| Metric | Multidimensional, sparse, typed grid. Sourced by input, import, or formula. | Anything you compute, plan, or report | +| Transaction List | High-volume row store (orders, journal entries). Items not unique. Not structural. | Granular facts to aggregate into metrics | +| Table | Group of metrics sharing dimensions, with calculated rows or columns. | P&L, Balance Sheet, Cash Flow, multi-metric reporting | +| Calendar | App-level time dimensions (Month, Quarter, Year) plus Date type. | Always reuse. Never re-create time dimensions | +| Version Dimension | Custom Dimension holding Budget / Actual / Forecast plus switchover and gating Boolean metrics. | Any planning cycle, Actual vs Plan layering, cross-version variance | +| List Subset | Constrained view of a parent list. Power tool with irreversible data loss on membership change. | Prefer filters or a separate list unless the subset use case is clear | + +### Data visualization types + +| Concept | What it is | When to pick it | +|---|---|---| +| View | Configured visual of a Metric or Table (filters, sort, breakdown, display mode). Reusable across Boards. | Whenever you want to show data the same way in multiple places | +| View display mode | How a View renders its data: `Grid` (pivot table), `Chart` (bar / line / pie / etc.), `KPI` (single big number). | Grid for tabular breakdown, Chart for trend or comparison, KPI for a headline metric | +| Board | Container page that lays out one or more Widgets. The unit of user-facing reporting, dashboards, and input screens. | Any user-facing dashboard, report, or input form | +| Widget | An element placed on a Board. Most commonly renders a View; also supports text, image, button, separator. | Anything embedded on a Board | + +### Engine vocabulary + +- Block: any first-class object in an app (Dimension, Metric, Transaction List, Table, Calendar, Board, Folder). +- Item: a row of a list (Dimension or Transaction List). +- Structural dimension: a Dimension used to define a metric's grid. +- Cell: one value at one combination of structural items. +- Sparsity: only non-blank cells are stored and processed. +- Scope: the dimensional context in which a formula evaluates. + +### Sibling decisions the agent gets wrong most often + +- Dimension vs Property. Values repeat across rows AND you need to slice metrics by them, use a Dimension. Free text, measure, or boolean, use a Property. +- Dimension vs Transaction List. Need to slice metrics by it with unique items, use a Dimension. Granular events, high volume, not structural, use a Transaction List. +- Metric vs Transaction List as source of truth. Aggregated planning numbers belong in a Metric. Atomic events from ERP or CRM belong in a Transaction List, then aggregate with `BY`. +- Table vs standalone Metric. Multiple metrics share dimensions and you want calculated items or a single view, use a Table. One isolated KPI or intermediate calc stays standalone. + +--- + +## Architecture: Decisions in Order + +Decide in this order. Reversing causes rework. + +1. Application boundary. One app vs a Hub-and-domain-apps topology. The Hub holds shared Dimensions (Entity, GL, FX, Version, Time) and shared reference/actuals data. Domain apps reference Hub content via shared Blocks (Library), typically metrics with a Push/Pull naming convention. +2. ``` +2. Dimensional structure. List the slicing axes. Target 5 or fewer structural dimensions per metric. Challenge anything above. +3. Calendar. Pick fiscal year, granularity (Month or Quarter), and date range. Use the existing app calendar. Never roll your own time list. +4. Version Dimension. If the app holds any planning cycle (Budget, Forecast, Actual), build a Version Dimension and define switchover and gating Booleans now, not later. See `skill:planning-cycles-pigment-applications`. +5. Data sourcing per metric. Input vs Import vs Formula. Data from ERP or CRM lands in a Transaction List, then aggregates into a metric with `BY`. +6. Metric layering. `INPUT_` then `CALC_` then `OUTPUT_` or `RES_`. Reporting metrics stay thin. No cross-cutting calc inside them. +7. Tables for reporting. Group metrics sharing dimensions. One centralized reporting metric per financial view (P&L, Balance Sheet). +8. Folders. Map blocks to numeric-prefixed folders (Dimensions, Library, Data Loads, Assumptions, Reporting). Never place blocks at the root level. +9. Governance. Naming, T&D safety (no direct item refs in formulas), Access Rights, audit. + +--- + +## Minimal Viable Application + +Concrete shape of a well-formed micro-app. Generalize from this pattern. + +``` +Folders: + 01. Dimensions + 02. Library + 03. Data Loads + 04. Reporting + +Dimensions: + Country items: FR, US, UK property: Region [Dim] + Product items: P1, P2 property: Category [Dim] + Calendar Month, Quarter, Year (existing) + Version items: Budget FY26, Reforecast Q2 FY26, ... (Actual is optional) + properties: Start_Month, End_Month, Switchover_Month, + Active_Version [Bool], Lock_Version [Bool], + Version_Type [Dim] (MP02-safe Actual ref) + Boolean metrics: Is_Version, Is_Actual, Is_Plan (layer Actual vs Plan) + +Transaction List: + LOAD_Sales props: Country [Dim], Product [Dim], Date, Amount [Number] + +Metrics: + DATA_Sales_Amount [Country x Product x Month x Version] + LOAD_Sales.Amount [BY SUM: LOAD_Sales.Country, LOAD_Sales.Product, + TIMEDIM(LOAD_Sales.Date, Month)] + INPUT_Budget [Country x Product x Month x Version] user input, scoped via Version_Type = "Budget" (MP02-safe) + CALC_Variance [Country x Product x Month x Version] DATA_Sales_Amount - INPUT_Budget + OUTPUT_Net_Revenue [Country x Product x Month x Version] thin reporting metric + +Table: + P&L (rows = OUTPUT_ metrics, dims = Country x Month x Version) +``` + +--- + +## Critical Rules + +- Architecture before blocks. Dimensional structure is the single most expensive decision. Design before building. +- Only dimension lists can be structural. Transaction lists never. Aggregate with `BY`. +- Never use `.` or `:` in names. They break formula references. +- Never place a block at the root level. +- Never reference an item directly in a formula when T&D is in use. Use an input metric of type Dimension. +- List Subsets are not a default. Membership change deletes data irreversibly. Prefer filters. +- Reuse the app calendar. Never create parallel time dimensions. +- Always model planning cycles with a Version Dimension. Consult `skill:planning-cycles-pigment-applications` whenever a planning cycle is in scope. + +--- + +## Deeper Dives + +Open only when the task requires the detail. + +| Need | Doc | +|---|---| +| Engine, sparsity, dimension list vs transaction list, BY pattern, IFDEFINED vs ISBLANK | [./modeling_fundamentals.md](./modeling_fundamentals.md) | +| Hierarchies, ragged hierarchies, mapped dimensions, time-dependent hierarchy, dimension explosion | [./modeling_dimensions_and_hierarchies.md](./modeling_dimensions_and_hierarchies.md) | +| Calendars, fiscal year, date range, time dimension mechanics | [./modeling_time_and_calendars.md](./modeling_time_and_calendars.md) | +| End-to-end architecture design (5 pillars, Hub pattern, UX, data flow, governance) | [./modeling_architecture_design.md](./modeling_architecture_design.md) | +| Naming conventions (prefixes, casing, character rules) | [./modeling_naming_conventions.md](./modeling_naming_conventions.md) | +| Modeling principles, T&D safety, data loading strategy | [./modeling_principles.md](./modeling_principles.md) | +| Folder placement decisions | [./modeling_working_with_folders.md](./modeling_working_with_folders.md) | +| List Subsets: safe patterns and data-loss risks | [./modeling_subsets.md](./modeling_subsets.md) | +| Design-time performance (1G cells, masks, table consolidation) | [./modeling_performance_considerations.md](./modeling_performance_considerations.md) | +| Centralized Reporting Metric (P&L, Balance Sheet aggregation) | [`../solving-specific-use-cases/finance_nexus_financial_statements.md`](../solving-specific-use-cases/finance_nexus_financial_statements.md) | +| FX / currency conversion (Hub pattern, AVG vs END, entity mapping) | [`../solving-specific-use-cases/fx_currency_conversion.md`](../solving-specific-use-cases/fx_currency_conversion.md) | +| Modifier syntax (BY, ADD, REMOVE, SELECT, FILTER) | [`../writing-pigment-formulas/formula_modifiers.md`](../writing-pigment-formulas/formula_modifiers.md) | diff --git a/plugins/pigment/skills/modeling-pigment-applications/modeling_architecture_design.md b/plugins/pigment/skills/modeling-pigment-applications/modeling_architecture_design.md new file mode 100644 index 00000000..cab9d676 --- /dev/null +++ b/plugins/pigment/skills/modeling-pigment-applications/modeling_architecture_design.md @@ -0,0 +1,307 @@ +# Pigment Architecture Design + +Purpose: Design a complete Pigment architecture by systematically gathering requirements and making architectural decisions across five pillars. Architecture serves as a high-level blueprint for builders and developers: it defines what needs to be built without prescribing every detail. + +Core principle: Architecture precedes building. Dimensional structure is the foundation that defines UX, performance, calculations, and data handling. Getting it wrong requires rebuilding everything. + +--- + +## What an Architecture Defines + +An architecture in Pigment is a high-level blueprint that follows Pigment best practices and aligns with how the Pigment engine works. It must define: + +| Component | Description | +| ------------------------- | ---------------------------------------------------------------------------------------------- | +| Dimensional structure | Which dimensions are in metric structures; granularity; use of properties vs mapped dimensions | +| Data flows | How data moves through the system | +| Applications | Which applications to create and how they relate (hub vs spokes) | +| Functionalities | Use cases and how data flows within each | +| End user activity | Which functionality end users interact with, especially input data | + +The dimensional structure of the matrix is the most critical architectural decision. It must be determined before starting an application because it defines UX, performance, calculations, and data handling. + +--- + +## Architectural Process + +Work through each pillar systematically: gather requirements and make decisions before proceeding to the next. Do not skip pillars. + +--- + +## Pillar 1: Dimensional Structure + +Objective: Define which dimensions belong in metric structures before building begins. + +For detailed guidance on dimensions, properties, and hierarchies, see [modeling_dimensions_and_hierarchies.md](./modeling_dimensions_and_hierarchies.md) and [modeling_fundamentals.md](./modeling_fundamentals.md). + +### Key Decisions + +- Which dimensions are required in the structure? +- What is the granularity of each dimension (month vs quarter, country vs region, etc.)? +- Can any dimensions be replaced with properties or mapped dimensions? + +### Standard Dimensions to Consider + +| Dimension | Typical use | +| --------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Version | Present on almost every metric (Actuals, Budget, Forecast, etc.) | +| Time | Managed by Pigment Calendar native feature. Month is most common; week/day in some cases; quarter/year are usually properties of month, aggregated in views | +| Organization | Cost center, department, or team level; in finance, often a mix of cost center and department | +| Chart of accounts | For finance use cases (cost types) | +| Other | Country, product, employee, or use-case-specific dimensions | + +### Design Rules + +- Minimize dimensions - Rarely more than 5 per metric. If more than 5, challenge whether dimensions can be inferred from properties or use mapped-dimension features. +- Use properties for aggregations - What you report on is not necessarily what goes in the structure. Example: plan at month level -> only month in structure; quarter and year are properties of month, aggregated in views. Example: country in structure, region as property of country. +- Use mapped dimensions when mappings change over time - e.g. employee moving cost centers: create a metric by Employee x Month x Version with type Dimension (Cost Center); use this in views to display data in the correct cost center over time. +- Access rights drive structure - If access is driven at country level, country must be in the structure. +- Challenge any structure with more than 5 dimensions. + +### Questions to Ask When Information Is Missing + +- At what level of granularity will users input data for each business dimension? +- At what level of granularity will calculations occur? +- Do dimension item mappings change over time (e.g. employee moving cost centers)? +- What level drives user access rights? +- Are there organizational hierarchies with varying granularity needs (some departments drive access at lower levels)? + +--- + +## Pillar 2: End User UX + +Objective: Let desired user experience drive architectural decisions. Pigment is highly flexible; architecture choices must be guided by the desired end user experience. + +### Input and Planning UX + +- On which dimensions do users need to input? +- How should dimensions be organized in the view (columns vs rows)? +- Do users need to see parent dimensions while inputting? +- Do users need simultaneous widgets with aligned dimensions? +- Does data need pre-population? + +These requirements directly influence which dimensions go in the metric structure and how tables are organized. + +### Reporting and Display + +- What are the key final reports (e.g. P&L)? +- What variations are shown (budget vs forecast vs actuals)? +- What KPIs and calculations are required? +- Can calculations use calculated items and "show value as" features? + +Best practice: Do not create aggregated metrics when not necessary. +Avoid creating aggregated metrics when view capabilities can achieve the same result. Views offer powerful aggregation, grouping, and calculation features (grouping by properties, Show Value As, Calculated Items) that eliminate the need for many "helper" or "reporting" metrics. +Why this matters: + +- Reduces model complexity - fewer metrics = easier maintenance +- Improves performance - fewer calculations to compute and store +- Better separation of concerns - model focuses on core logic; views handle presentation and ad-hoc analysis +- More flexible - users can pivot/group/calculate in views without rebuilding the model + +### When to Use Views Instead of Creating Metrics + +Anti-pattern (unnecessary metric): + +``` +Metric: Revenue (by Country) +Metric: Revenue by Region <-- UNNECESSARY +``` + +Best practice (use view grouping): + +- Keep only `Revenue (by Country)` +- In the view, group by `Country.Region` property (either in Rows, Columns or Pages) +- The view will automatically aggregate Revenue to the Region level + +Rule: If a dimension has a parent hierarchy or property (e.g., Country -> Region, Employee -> Department), do not create separate metrics for parent levels. Use view grouping or pivoting. + +Best practice: Maximize use of calculated items and show value as: + +- Use calculated items for calculations between items or modalities of a dimension (e.g., FY 2027 vs FY 2026, Actual vs Forecast) in report views. +- Use show value as for display transformations: temporal comparisons, cumulative series, offsets, differences from another item or metric, and derived presentations of a single metric when the view supports it (e.g., YTD). Avoid extra metrics when the display is sufficient and not needed downstream. + +### Workflow and Access Rights + +- If access rights need to change between workflow steps, separate logic and input into different metrics. + +### Questions to Ask When Information Is Missing + +- Can you provide wireframes or mockups of the desired input screens? +- Can you share examples of current reports before Pigment? +- What variations need to appear as columns in reports? +- What KPIs or ratios are required, and how are they calculated? +- Do certain calculations require separate metrics vs calculated items? +- What workflow steps exist, and do access rights need to change between steps? + +--- + +## Pillar 3: Data and Its Cycle + +Objective: Understand data sources, granularity, and transformation requirements. + +### Data Sources and Where They Load + +| Data type | Typical load target | Examples | +| -------------------------------- | ---------------------------------------------- | ---------------------------------------------------- | +| Transactional data | Transaction lists | GL from ERPs, HR roster, CRM (e.g. Salesforce) | +| Metadata | Dimensions (loaded separately; requires codes) | Products, cost centers, chart of accounts | +| Historical outputs / budgets | Direct to metrics | When detail level doesn't justify a transaction list | + +Exception: Transactional data can be loaded into a dimension list when that list is useful in the structure; this is rare. + +### Data Granularity + +Data granularity is critical. When available data granularity does not match planning/input granularity, the architecture must accommodate through aggregation, allocation, or programmatic generation. If data cannot support requirements, challenge those requirements. + +### Design Rules + +- Minimum transformation in Pigment - Pigment is not an ETL tool; cleaning and transformation should be done outside Pigment before loading. +- Load only what is required - Do not add properties or blocks "just in case." Unused data creates bloated, confusing applications. +- Separate metadata from transactional loading - Metadata requires codes and verification; it typically comes from a different source than transactional data. Loading metadata from transactional data often yields only names without robust structure. + +### Questions to Ask When Information Is Missing + +- What are all data sources (ERPs, HR, CRM, etc.)? +- At what granularity is data available for each dimension? +- At what granularity do users need to plan? +- What is the gap between available data granularity and planning requirements? +- Where will data transformation occur (before or within Pigment)? +- What metadata is available for each dimension? +- What historical data needs to be loaded, and is it transactional or already aggregated? + +--- + +## Pillar 4: Governance and Security + +Objective: Determine application structure based on ownership, access, and sensitivity. + +For detailed guidance on application structure, hub pattern, and folder organization, see [modeling_principles.md](./modeling_principles.md). For Access Rights design, see `skill:securing-pigment-applications` ([securing_access_rights.md](../securing-pigment-applications/securing_access_rights.md)). + +### Key Decisions + +- How many applications are needed? +- What belongs in the hub vs specific applications? +- Where do shared dimensions reside? + +### Application Separation Criteria + +| Criterion | Action | +| ---------------------------------------------- | ----------------------------------------------- | +| Different teams managing different use cases | Separate applications | +| Sensitive data (e.g. individual employee data) | Independent application (strict access control) | +| Different planning cycles | Potentially separate applications | + +### Hub and Dimension Placement + +- Every workspace needs a hub application, even when starting with a single use case. The hub contains dimensions shared across applications and simplifies adding new connected applications. +- Dimensions used by multiple applications -> place in hub, with one exception: when dimension items are created automatically or manually by end users or business logic within a specific application, that dimension should reside in that application and be shared outward. +- Transactional data typically loads into the application where it is used; if it must be accessed by users across multiple applications, it can be placed in the hub. + +### User Experience + +- Users cannot natively navigate between applications; manual links on boards are required. Architect each application to provide a complete journey for a specific user type, minimizing cross-application navigation. + +### Common FP&A Pattern + +| Application | Role | +| ---------------------- | -------------------------------------------------------------------------------------------------------------------------------- | +| Hub | Shared dimensions and resources | +| Workforce planning | Independent (sensitive, individual-level data) | +| Revenue planning | Separate (different team/cycle) | +| FP&A application | Actuals, P&L, plan vs actuals, light planning (e.g. OPEX); pulls planning data from workforce and optionally other planning apps | + +### Questions to Ask When Information Is Missing + +- Which teams will manage which use cases? +- What data is sensitive and requires restricted access? +- Do any use cases involve individual-level data? +- Which dimensions are shared across multiple use cases? +- Are any dimensions created automatically by business logic within a specific use case? +- Which users need access to which data? +- Do users need to navigate between applications? + +--- + +## Pillar 5: Planning Cycle + +Objective: Design version management and historical plan protection. The planning cycle (version/scenario management) is why customers adopt EPM tools: to plan budgets, reforecasts, and compare against actuals. + +For detailed guidance on scenarios vs version dimensions, see `skill:planning-cycles-pigment-applications`. + +### Version Dimension + +- Create a version dimension (e.g. Actuals, Budget, Forecast). It should exist in the structure of almost every metric in the application. Critical for FP&A; less so for supply chain or SPM. + +### Dual Challenge: Completed Versions + +When a planning version is complete: + +1. Numbers must be fixed as an unchangeable reference. +2. Numbers must remain available for reuse in new planning cycles. + +### Snapshots vs Live Versions + +| Approach | Use | +| ----------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Snapshots | Fixed copies; numbers never change. Accessed via data slices. Reports must use data slices to leverage snapshots. Snapshots cannot serve as the basis for new planning runs. | +| Live versions | Maintain a live version of historical plans (alongside snapshots) to enable replanning from historical data. | + +Constraint: Avoid too many live versions; max ~10 recommended (actual limit depends on volume and other factors). Pigment is a live calculation engine. + +### Protecting Historical Plans + +Build logic so that changes to numbers, formulas, or data do not retroactively impact historical plans: + +- Dated loads - When loading employee rosters or similar data, date the loads so switchover logic prevents new loads from impacting historical budgets. +- Historical data reloads - If reloading past-period data that was used to generate historical plans, implement logic to isolate these changes (e.g. effective dating of transactions, copies/imports from transaction lists or metrics). + +### Version Initialization + +Determine how new versions will be initialized: + +- Import from another version +- Bulk copy from an existing version +- Clone all data (Clone data 2) +- Re-import from a scenario +- Start from scratch + +### Report-Driven Design + +- Examine the customer's final reports, especially variations/versions displayed as columns. +- Determine whether the standard combination of data slices + calculated items with versions and snapshots suffices, or if a custom solution is required. + +### Questions to Ask When Information Is Missing + +- What versions/scenarios are needed? +- How many live versions will exist concurrently (max ~10 recommended)? +- How should completed plans be protected from changes? +- Do users need to replan based on historical budgets? +- When loading new data, should it impact historical plans? +- How will new planning cycles be initialized? +- Do transaction lists need effective dating? + +--- + +## Validation Before Finalizing + +Before locking the architecture, validate: + +- [ ] Can the dimensional structure support all required inputs and calculations? +- [ ] Does the structure enable the desired UX? +- [ ] Does data granularity match structural requirements? +- [ ] Are governance and security requirements met? +- [ ] Is version management adequate for the planning cycle? +- [ ] Are there fewer than ~10 live versions? +- [ ] Have all dimensions been challenged for necessity? + +--- + +## See Also + +- [./modeling_fundamentals.md](./modeling_fundamentals.md) - Core concepts (dimensions, properties, metrics vs transaction lists) +- [./modeling_dimensions_and_hierarchies.md](./modeling_dimensions_and_hierarchies.md) - Hierarchies, dimension vs property +- [./modeling_principles.md](./modeling_principles.md) - Folder structure, hub, Library +- [./modeling_naming_conventions.md](./modeling_naming_conventions.md) - Naming conventions +- `skill:planning-cycles-pigment-applications` - Scenarios vs version dimension, planning cycles +- [./modeling_time_and_calendars.md](./modeling_time_and_calendars.md) - Time dimensions and calendars +- `skill:securing-pigment-applications` - Access rights design and governance diff --git a/plugins/pigment/skills/modeling-pigment-applications/modeling_dimensions_and_hierarchies.md b/plugins/pigment/skills/modeling-pigment-applications/modeling_dimensions_and_hierarchies.md new file mode 100644 index 00000000..6ae0dc58 --- /dev/null +++ b/plugins/pigment/skills/modeling-pigment-applications/modeling_dimensions_and_hierarchies.md @@ -0,0 +1,266 @@ +# Dimensions and Hierarchies + +Complete guide for dimensional modeling, hierarchy implementation, and strategic decision-making. + +## When to Read This + +- Designing metric dimensional structures +- Creating hierarchies with dimension-type properties +- Deciding whether to add dimensions to structure or use properties +- Working with multi-level or ragged hierarchies +- Handling dynamic relationships (mapped dimensions, time-dependent hierarchy) + +## Prerequisites + +Understand from [modeling_fundamentals.md](./modeling_fundamentals.md): dimensions, properties, dimension-type properties, hierarchy terminology. + +--- + +## Part 1: Dimensional Concepts + +### Core Concept: Source vs Target Dimensions + +Compare source data dimensions to desired target: + +- Same Dimensions: No action required +- More/Different Dimensions: Map or allocate data +- Fewer Dimensions: Sum or remove dimensions + +### Dimensional Transformation Cases + +Case 0: No Modification + +- Source and target have identical dimensions +- Example: `Unit Price x Quantity` (both `Product x Warehouse`) + +Case 1: Aggregation + +- Use `BY` modifier with operators (SUM, AVERAGE, MIN, MAX, COUNT) +- Example: `'Order List'.Amount [BY SUM: 'Order List'.Product, 'Order List'.Month]` +- Aggregates from `Order x Product x Customer x Month` to `Product x Month` + +Case 2: Aggregation via Properties + +- Aggregate to parent: `'Inventory Level' [BY SUM: Warehouse.Region, Product, Month]` +- Allocate to children: `'Shipping Rate by Region' [BY: Warehouse.Region] x 'Order Quantity'` +- Note: `BY CONSTANT` distributes single value to multiple items + +Case 3: Add/Remove Dimensions + +- Add: `'Actual Sales' [ADD CONSTANT: Version]` - Copies value across new dimension +- Remove: `'Sales' [REMOVE SUM: Product]` - Aggregates and drops dimension + +Same logical dimension twice (mirror dimension): When one list must appear twice in a structure (e.g. Months as Time and as Cohort month, Company in rows and columns), use List Subsets so a single source list is reused. Subsets behave as separate dimensions and require explicit mapping; see [List Subsets](./modeling_subsets.md) for when to use them and for data-loss and remapping rules. + +Modifier syntax reference: [formula_modifiers.md](../writing-pigment-formulas/formula_modifiers.md) + +--- + +## Part 2: Building Hierarchies + +### Core Concept + +Dimension-type properties create hierarchies without adding dimensions to metric structure. Creates parent-child relationships enabling aggregation at any level without changing metric dimensions. + +WARNING: Important: Properties are NOT structural elements of metrics. Only dimension lists (regular dimensions) can be used in a metric's structure. Properties are attributes of dimensions used for grouping, aggregation, and navigation-but they don't define the dimensional grid where data is stored. + +### Hierarchy Examples + +Product: Product -> Category -> Line + +- Product has "Category" property (dimension-type) +- Category has "Line" property (dimension-type) +- Metric: `Product x Month` (not `Product x Category x Line x Month`) + +Organizational: Employee -> Department -> Division +Geographic: Store -> Region -> Country + +### Property Chaining + +Syntax: `Dimension.Property.Property.Property...` + +Examples: + +- `Product.Category.Line` - 2 levels up +- `Employee.Department.Division` - 2 levels up + +Formula aggregation: + +```pigment +'Product Revenue' [BY SUM: Product.Category, Month] +'Product Revenue' [BY SUM: Product.Category.Line, Month] +'Sales' [BY SUM: Product.Category, Store.Region, Month] +``` + +### Benefits + +- Report at any level without restructuring +- 5-level hierarchy = 0 added dimensions +- Change hierarchies by updating properties, not formulas + +### Mapped Dimensions (Dynamic Relationships) + +Also called time-dependent hierarchy, slowly changing hierarchy, or slowly moving dimensions. Parent-child relationships can change across periods; past periods keep the old parent, future ones the new. The mapping metric usually varies by time, sometimes by Version. + +When to Use: + +Dimension-type properties for static relationships: + +- Product -> Category (rarely changes) +- Store -> Region (fixed) + +Mapped Dimensions for dynamic relationships (most often varying by Month; sometimes by Version): + +- Cost Center -> Department (re-org: historical actuals stay under the old department) +- Employee -> Team (changes monthly) +- Product -> Promotion (varies by period) +- Customer -> Segment (behavior-based) + +Implementation (modeler workflow): + +1. `tool:create_metric` - mapping metric, Dimension-typed, structured on the source dimensions only (e.g. `Cost Center x Month` -> Department). +2. Populate values with a formula on the mapping metric, or manually via `tool:set_metric_input`. +3. Views - add a Mapped Dimension (Joined Pivot) on the core metric or Table; do not add the parent dimension to every underlying metric. + +Example: Salary (`Employee x Month`) mapped to Team via mapping metric. Employees automatically aggregate to correct Team per month. + +Comparison: + +- Property: Static - updating the parent moves all history with the child; fine when the relationship never changes +- Mapped Dimension: Dynamic - views only, varies by period or version; core metrics stay lean + +Reference: [Mapped Dimensions docs](https://kb.pigment.com/docs/mapped-dimensions) + +### Ragged/Unbalanced Hierarchies + +Types: + +1. Variable Depth: Some paths shorter than others (2 vs 5 levels) +2. Multiple Parents: Item belongs to multiple categories (avoid double-counting) +3. Optional Levels: Some items have blank intermediate properties + +Recommended Approach: Accept Blanks + +Create full hierarchy, accept blank intermediate levels. + +Example: + +``` +Employee: John -> Manager: Jane -> Director: Bob -> VP: Sarah +Employee: Alice -> Manager: [BLANK] -> Director: Bob -> VP: Sarah +Employee: Charlie -> Manager: [BLANK] -> Director: [BLANK] -> VP: Sarah +``` + +Formula: `'Salary' [BY SUM: Employee.Manager.Director.VP, Month]` +Result: All roll up correctly despite blanks + +Multiple Parents: + +- Approach 1 (Recommended): One primary parent via property, track others separately +- Approach 2: Transaction list for true multi-parent scenarios + +### Performance + +- 2-3 levels: Excellent +- 4-5 levels: Very good +- 6+ levels: Test with real data + +Workflow: + +1. Identify hierarchy structure +2. Create dimensions top-to-bottom +3. Add properties bottom-to-top +4. Test property chains +5. Create metrics with base dimensions only + +--- + +## Part 3: Decision Framework - Property vs Dimension in Structure + +### The Core Question + +Add dimension to metric structure OR reference through property? + +Remember: Only dimension lists can be added to metric structure. Properties and transaction lists cannot be structural elements. + +### Decision Tree + +``` +Is this dimension needed for... + +-- Direct user input at this level? -> YES = Add to structure +-- Calculations at this specific level? -> YES = Add to structure +-- Filtering that reduces data volume? -> YES = Consider structure +-- Only reporting/grouping? -> YES = Use property +-- Only drill-down/navigation? -> YES = Use property +``` + +### Use Dimension-Type Property When + +- Hierarchy/Grouping Only: No direct input, only for aggregation +- Reporting Flexibility: View data at multiple levels +- Parent-Child Relationship: Clear hierarchy, input at child level +- Many-to-One: Multiple children to one parent +- Dimension Explosion Risk: Already 6+ dimensions +- Frequently Changing: Relationships change regularly + +### Add to Metric Structure When + +- Direct Input Required: Users input at this level +- Calculation Logic: Formulas need values at this level +- Significant Filtering: Dimension scopes most calculations +- Independent Dimension: No parent-child relationship (Time, Scenario) +- Dense Data: Most combinations have values +- Allocation Required: Top-down planning +- Cross-Dimensional Calculations: Reference specific items in formulas + +### Examples + +Sales with Product/Category: + +- Input at Product level +- Category only for grouping +- Decision: Property (`Product x Store x Month`, Product.Category property) + +Headcount Planning: + +- If input/calculation at Department level -> Add to structure +- If only employee input -> Use property + +### Trade-offs + +| Aspect | In Structure | Property | +| --------------- | ------------ | ---------- | +| Performance | Slower | Faster | +| Flexibility | Fixed | Dynamic | +| Input | Direct | Child only | +| Maintenance | Harder | Easier | +| Sparsity | Risk | Better | + +### Best Practices + +1. Start minimal: fewest dimensions, properties for hierarchies +2. Test performance with real data +3. Document decisions +4. Review as model evolves + +### Common Mistakes + +- Over-Dimensionalizing: Adding all hierarchy levels (10+ dimensions) +- Under-Using Properties: Missing flexible reporting opportunities +- Inconsistent Patterns: Properties for some hierarchies, dimensions for similar ones +- Ignoring Input Patterns: Wrong granularity level + +### When to Reconsider + +- Complex formulas working around properties -> Add to structure +- Many metrics at different levels -> Use properties +- Performance slow with many dimensions -> Move to properties + +--- + +## Cross-References + +- Modifier syntax: [formula_modifiers.md](../writing-pigment-formulas/formula_modifiers.md) +- Performance: [modeling_performance_considerations.md](./modeling_performance_considerations.md) +- Standards: [modeling_principles.md](./modeling_principles.md) diff --git a/plugins/pigment/skills/modeling-pigment-applications/modeling_fundamentals.md b/plugins/pigment/skills/modeling-pigment-applications/modeling_fundamentals.md new file mode 100644 index 00000000..e0256b44 --- /dev/null +++ b/plugins/pigment/skills/modeling-pigment-applications/modeling_fundamentals.md @@ -0,0 +1,248 @@ +# Pigment Core Concepts + +## 1. General Vision: Pigment as a Multidimensional Engine + +Pigment is an in-memory multidimensional engine designed for planning, simulation, and real-time analysis. It is built on four core principles: + +- Everything is multidimensional: Metrics exist within a dimensional space, where a cell represents a combination of elements from a set of dimensions. + +- Real-time calculations: Changes to inputs, parameters, or imports trigger instant recalculations of dependent formulas. + +- Granular models + scenarios: Supports detailed models (transactions, employees) alongside native version/scenario support. + +- Collaborative interface: Data is exposed through boards containing multi-metric tables, charts, and workflows. + +--- + +## 2. Pigment Modeling Building Blocks + +Pigment applications are built from four core components: + +- Dimensions: Analysis axes containing items (Product, Employee, Month) with properties +- Metrics: Multidimensional grids for calculations, inputs, and reporting +- Transaction Lists: High-volume granular data storage (orders, invoices, movements) +- Tables: Containers organizing multiple metrics with shared dimensions for analysis + +### 2.1 Dimensions and Properties + +Dimensions are analysis axes (Month, Account, Product, Employee) containing items (rows) and Properties (columns). + +#### Property Data Types + +Properties can be Number, Boolean, Text, Date, or Dimension type (referencing another dimension). + +Property Type Selection: + +- Number: Monetary values, quantities, measurements, ratios +- Date: Temporal fields, timestamps +- Text: Free-form descriptive content +- Dimension: Categorical, hierarchical, reference fields that group/classify items +- Boolean: True/false flags + +Selection Rule: + +Use Dimension type when property values reference or categorize items (Department, Region, Category, fields ending in Id/Code). Use Text/Number/Date/Boolean for measures, attributes, and free-form content (Discount %, Lead time, descriptions). + +#### Cardinality-Based Property Type Selection + +Use data-driven cardinality analysis to determine property types: + +Step 1: Calculate Cardinality Ratio + +``` +For each property: + cardinality_ratio = (number of unique values) / (total number of rows) +``` + +Step 2: Apply Classification Rules + +| Cardinality Pattern | Data Characteristics | Property Type | Action Required | +| --------------------------------------------- | ------------------------------------------------------- | ------------------ | ---------------------------------------------------------------------- | +| Categorical (< 0.8 with repeating values) | Values repeat across rows, indicating categories/groups | Dimension | Create parent dimension with unique values as items, then reference it | +| Numeric | Numbers: integers, decimals, percentages, quantities | Number/Integer | Use directly | +| Temporal | Dates, timestamps, time values | Date | Use directly | +| Binary | True/false, yes/no, 0/1 | Boolean | Use directly | +| High-cardinality (> 0.8, mostly unique) | Unique per row, free-form text, descriptions | Text | Use directly | + +Key Rule: If ANY value repeats across rows (ratio < 1.0), check if it's categorical. Even at 0.5-0.8 cardinality, repeating values indicate categories -> use Dimension type. + +Step 3: Detect Hierarchical Patterns + +Indicators that suggest Dimension type: + +- Dot notation in property names (e.g., `Parent.Child`, `Group.Subgroup`) +- Property names ending in common grouping terms +- Values that represent categories, groups, or classifications + +Step 4: Workflow for Dimension-Type Properties + +When cardinality analysis indicates Dimension type: + +1. Extract unique values from the property +2. Create parent dimension using those unique values as items +3. Reference parent dimension ID when creating child dimension property +4. Populate child dimension data + +Example Cardinality Analysis: + +| Property | Unique Values | Total Rows | Ratio | Pattern | Type | +| ----------- | ------------- | ---------- | ----- | -------------------- | ------------- | +| Name | 10 | 10 | 1.0 | Unique identifiers | Text | +| GroupA | 4 | 10 | 0.4 | Repeating categories | Dimension | +| GroupB | 7 | 10 | 0.7 | Repeating categories | Dimension | +| Rate | 8 | 10 | 0.8 | Numeric measures | Number | +| Description | 10 | 10 | 1.0 | Free-form text | Text | + +#### Other Property Considerations + +Unique Properties: Dimensions can have properties marked as "unique" (e.g., Code, Name) to identify items for imports and integrations. + +Calendar: Each application includes a calendar made up of time dimensions and properties. Always reuse these existing calendar dimensions instead of creating new ones to ensure consistency and optimal performance. + +### 2.2 Metrics + +Metrics are the central blocks for modeling, data input, calculation, and reporting. Like properties, a metric is defined by a data type (Number, Boolean, Text, Date, Dimension). + +Metric Type Selection Guide: + +- Number: Financial values, KPIs, quantities, forecasts, calculations +- Date: Milestones, deadlines, temporal tracking fields +- Text: Status indicators, categorical outputs, labels +- Dimension: Reference fields linking to dimension items +- Boolean: Flags, toggles, binary states + +Anatomy of a Metric: + +- Structure: The structure is defined by dimension lists creating a multidimensional grid (e.g., Country x Product x Month). Remember: transaction lists CANNOT be used as structural dimensions-only regular dimensions (dimension lists) are allowed. + +- Data Input: Can be manual (overrides computed values), imported, or computed via formula. + +- Scenarios: Tracks multiple versions of the same metric. + +- Sparsity: Only relevant combinations store values; blanks are not stored. + +Default Aggregators: + +Metrics define how values aggregate across dimensions (Sum, Avg, Min, Max, Count, First/Last Non Blank). + +Common Use Cases: + +- Input & Calculation: Capturing assumptions and performing intermediate logic (e.g., seasonality adjustments, allocation rules). + +- Reporting: KPIs, dashboards, and analytical views across multiple dimensions. + +- Planning: Financial planning (budget, cash flow), workforce planning (headcount, capacity), supply chain planning (inventory, orders), sales planning (forecasts, quotas), and operational planning (utilization, resources). + +### 2.3 Transaction Lists + +Transaction Lists store granular, atomic records (e.g., invoices, orders, HR events, inventory movements) and act as the "fact tables" of the system. + +Key Characteristics: + +- Non-Structural: They cannot be used as structural dimensions in a Metric. WARNING: Critical: A metric must NEVER be dimensioned over a Transaction List. Transaction Lists are fundamentally different from Dimensions (dimension lists). They are data containers, not structural axes. + +- High Volume: Designed to hold millions of rows, unlike standard Dimensions. + +- No Unique ID: Unlike Dimensions, they do not require a unique identifier per item. + +#### Dimension list vs Transaction list: impact on metrics + +Common points: Both are types of Lists in Pigment. Both contain Items and can have Properties. Both organize and structure data in the model. + +Differences: + +- Dimension list: Defines the structure of your data (analysis axes: Country, Product, Month, etc.). Each Item must be unique. Can be used as dimensions in Metrics and Tables. Reusable across the Application. +- Transaction list: Stores events or transactional data (orders, HR records, accounting entries). Items do not need to be unique. Cannot be used as dimensions in Metrics-only referenced in formulas. Designed for high-volume, transactional data. + +A Transaction List (TL) is not a dimension. It therefore cannot be a structural dimension of a metric. Only dimension lists can appear in a metric's structure. Transaction lists are referenced inside metric formulas to aggregate their properties toward dimensions (via the `BY` modifier). + +Correct pattern: With a Transaction List `Sales` that has properties `Country` and `Month` (both of type Dimension) and a numeric property `Value`, the correct syntax for a metric that aggregates by Country and Month is: + +```pigment +Sales.Value[BY SUM: Sales.Country, Sales.Month] +``` + +The metric is dimensioned by Country x Month (or by the dimensions referenced by those properties), and the formula references the TL and aggregates with BY. + +Anti-pattern: Using the list itself as a dimension in BY: + +```pigment +Sales.Value[BY SUM: Sales] // BAD: Error: Sales is not a dimension +``` + +This causes a syntax error or invalid behavior: `Sales` is not a dimension, so you cannot aggregate "by Sales". You must list the TL's properties that are of type Dimension (or expressions such as `TIMEDIM(...)` for dates) in the BY clause. + +Primary Usage: + +- Data Imports: The main destination for granular ERP/CRM/data warehouse data. + +- Aggregation: Data is rolled up into Metrics using formulas (e.g., `Orders.Amount [BY SUM: Orders.Product, Orders.Month]`). + +- Drill-Down: Allows users to view underlying details contributing to aggregated metrics. + +Best Practices: + +- Never dimension a metric over a Transaction List. Unlike regular Dimensions (dimension lists), Transaction Lists cannot be used in a metric's dimensional structure. Only dimension lists can be structural elements of metrics. +- Use transaction lists for high-volume transactional data instead of creating massive dimensions +- Always aggregate transaction list data to metrics for analysis and reporting +- Consider versioning strategies when transaction data needs historical tracking + +### 2.4 Tables + +Tables are containers that group multiple metrics together, enabling unified analysis and consistent dimensional structure. + +Key Characteristics: + +- Shared Dimensions: All metrics in a table share at least a common dimension +- Calculated Items: Create derived rows/columns using formulas across table metrics +- Single View: Present related metrics together (Revenue, Cost, Margin in one table) + +Primary Usage: + +- Financial Statements: P&L, Balance Sheet, Cash Flow with multiple line items +- Planning Models: Combine inputs, calculations, and outputs in one structure +- Reporting: Multi-metric dashboards with consistent dimensional breakdown + +Table vs Standalone Metrics: + +| Aspect | Table | Standalone Metrics | +| -------------------- | ----------------------------------------- | -------------------------------------- | +| Structure | Shared dimensions across all metrics | Each metric has independent dimensions | +| Calculated Items | Formula-based rows/columns across metrics | Not available | +| Presentation | Unified view of related metrics | Individual metric views | +| Use Case | Financial models, complex reports | Simple KPIs, intermediate calculations | + +Best Practices: + +- Use tables for metrics that belong together logically (P&L Metrics) +- Leverage calculated items for derived metrics (Revenue - Cost = Margin) +- Keep intermediate calculations as standalone metrics, not in reporting tables + +--- + +## 3. Sparsity: Core Engine Principle + +Pigment is a sparse engine: it only stores and processes cells with actual values. Empty cells remain blank (not zeros or FALSE). + +Core principle: Only relevant dimensional combinations carry values. + +Why it matters: + +- Not every product sells in every country in every period +- Not every employee works in every department +- Sparse storage enables billions of potential cells with only millions actually stored + +Advantages over dense engines: + +- Faster calculations (only populated cells processed) +- Better scalability (less storage) +- No dimension limits (competitors often cap at 10-12 dimensions) + +Working with sparsity: + +- Use `IFDEFINED` - Checks values exist without densifying +- Avoid `ISBLANK` - Returns TRUE/FALSE for ALL cells, creates density +- Leave ELSE blank - `IF(condition, value)` without ELSE preserves sparsity +- Avoid `IF(..., 0)` patterns - Fills sparse cells unnecessarily + +For detailed performance optimization: See [modeling_performance_considerations.md](./modeling_performance_considerations.md) diff --git a/plugins/pigment/skills/modeling-pigment-applications/modeling_naming_conventions.md b/plugins/pigment/skills/modeling-pigment-applications/modeling_naming_conventions.md new file mode 100644 index 00000000..8032c511 --- /dev/null +++ b/plugins/pigment/skills/modeling-pigment-applications/modeling_naming_conventions.md @@ -0,0 +1,430 @@ +# Pigment Naming Conventions + +Consistent naming improves auditability, simplifies formulas, and signals functionality. + +## Principles + +1. Consistent - Same convention throughout the model +2. Descriptive - Names explain the element's purpose +3. No abbreviations - Use full words (`Headcount` not `HC`) +4. Documented - Create an Application Guide Board explaining your convention + +--- + +## Three-Tier Naming Framework: Technical Name, Friendly Name, and Display Name + +Pigment uses three distinct naming layers for blocks (metrics, dimensions, tables, transaction lists). Understanding each layer helps organize models for both system integration and user experience. + +#### Quick Visual Overview + +``` +Block: Revenue Metric + +Technical Name -> XDTA_REV_INPUT_Annual_Sales <- Used in formulas only (XDTA_ is an example of a prefix for technical names that will be generated by server upon creation of the block) + down +Friendly Name -> REV_INPUT_Annual_Sales <- Agent refers to this when display name not available + down +Display Name -> Annual Sales Revenue (US$) <- What end users see in Views/Boards and agents +(en-US) +Display Name -> Ventes Annuelles (US$) <- What French users see +(fr-FR) +``` + +Each layer serves a specific purpose: +- Technical: System plumbing +- Friendly: Internal communication (model navigation) +- Display: User experience (Views, Boards, dashboards visible to business users) + +### 1. Technical Name (Not Shown to Users) + +Purpose: Internal system reference, used in formulas and API calls. + +Characteristics: +- Unique identifier for the block within the application +- Should never be displayed to end users +- Used in formula references: `Revenue_Total`, `CALC_Net_Margin`, `INPUT_Budget_Amount` +- System-of-record for data integrity + + +### 2. Friendly Name (Primary UI Name) + +Purpose: Default human-readable name shown throughout the Pigment UI. + +Characteristics: +- Still descriptive and follows naming conventions (e.g., Snake_Case for metrics) +- Unique within the application +- Visible in block lists, explorers, and UI selectors +- Appears in Views and Boards by default +- Fallback in the agent when display name is not available + +### 3. Display Name (Localizable, User-Friendly) + +Purpose: Localized, business-friendly name optimized for end-user consumption in Views and Boards. + +Characteristics: +- Localizable - Can have different names per language/locale (today only `en-US` or `fr-FR`) +- More natural language than the technical friendly name +- More descriptive, user-focused phrasing +- Optional - if not set, falls back to friendly name +- Shown in Views, Boards, and widgets to business users +- Can include more spaces, capital letters, and marketing-friendly phrasing +- What the agent typically uses to refer to blocks in conversation + +### Decision Matrix: When to Use Each Name + +| Writing a formula => technical name +| Agent discussing block => Display Name if available, otherwise friendly name +| View/Board header for users => Display Name if available, otherwise friendly name +| API/integration call => after tool / api documentation +| Multi-language support => display name + +### Practical Examples + +Example 1: Headcount Metric + +``` +Technical Name: EE_CALC_Total_Headcount +Friendly Name: EE_CALC_Total_Headcount (appears in block lists) +Display Name: + en-US: Total Headcount (FTE) + fr-FR: Total d'Effectifs (ETP) + +Agent behavior: +- References block as: "Total Headcount (FTE)" in English and "Total d'Effectifs (ETP)" in French +- User sees in Board: "Total Headcount (FTE)" (English) or "Total d'Effectifs (ETP)" (French) +``` + +### Best Practices + +1. Always set friendly names following your naming convention (e.g., two-prefix system for metrics) +2. Set a display name on every block by default, in both `en-US` and `fr-FR` +3. Skip the display name only when the block will never be surfaced to any end user (purely internal/technical block, that will never appear in any View, Board, Table) and if no simple, clear display name exists without being confusing or overly complicated, skip it entirely - do not force a display name for compliance. +4. Use clear, non-technical language in display names - "Forecast Revenue" instead of "CALC_REV_Forecast" + +### Agent Guidance + +When the modeler agent discusses blocks: +- Refer to blocks by display name in conversation: "I created the metric Forecast Revenue" +- Never use technical names in user-facing explanations unless explaining formula integration +- When creating or updating display names, ensure they are descriptive and localizable where applicable + +--- + +## Character Rules + +| Character | Allowed | Usage | +| -------------------- | ------- | --------------------------- | +| Letters, Numbers | Yes | Standard naming | +| Underscores `_` | Yes | Preferred separator | +| Spaces | Yes | Use sparingly | +| Parentheses `()` | Yes | Units: `($)`, `(#)`, `(%)` | +| Square brackets `[]` | Yes | Sort control: appears first | +| Hyphens `-` | Yes | Status: `-TEST`, `-OLD` | +| Periods `.` | No | Breaks formula referencing | +| Colons `:` | No | Breaks formula referencing | + +## Sort Order + +Pigment sorts: special chars -> numbers -> letters (A-Z). + +- `[Name]` or `01_Name` -> force to top +- `ZZ_Name` -> push to bottom (archived) + +## By Element Type + +### Applications + +Use a numeric prefix (single digit with space) as an optional example for ordering. Not mandatory. + +Example: + +``` +0. Hub # Shared resources +1. Core Reporting # Use-case specific +2. Workforce Planning +ZZ_ POC Revenue Planning # Archived +``` + +### Folders + +Numeric prefix (single digit e.g. 0., 1., 2. for top level; two digits e.g. 10., 11., 20. for data/themed). Numbering restarts at each folder level. + +Core Principles: + +- Numbering always restarts at each folder level (subfolders never inherit parent numbering) +- Folder order follows the logical order of operations of the use case +- `0.` folder is optional and reserved for technical, transversal, or summary content +- Never use `0.` for business logic +- Maximum recommended depth: 5 levels +- Structure should be clear, simple to navigate, and follow model structure +- Rework folder structure throughout project to ensure model clarity and ease of navigation +- Do not create a folder named "Security" (with or without a prefix). A Security folder already exists by default in every application. + +Generic Pattern (Application Root - Level 1): + +``` +0. Administration # Optional: governance, conventions, documentation +1. Structural Definitions # Dimensions, hierarchies, master data +2. Data Ingestion # Imports, staging, connections +3. Processing/Calculation # Business logic, transformations +4. Outputs/Consumption # Reports, dashboards, exports +``` + +Generic Pattern (Within Any Folder - Level 2+): + +``` +0. Summary or Technical # Optional: overview, config, utilities +1. First Logical Step +2. Second Logical Step +3. Third Logical Step +``` + +Example: Hub Administration Structure + +``` +0. Administration + 0. Maintenance # Logs, patches, system tasks + 1. Convention # Color codes, naming rules + 2. Automations # Scripts, job definitions + 3. Licences # Keys, subscriptions + 4. Documentation # Design docs, architecture + +1. Dimensions + 1. Chart of Accounts # CoA structure, mappings + 2. Organisation # Business units, hierarchies + 3. Currency & FX # Rates, conversion logic + 4. Version # Scenarios: Actual, Budget, Forecast + +2. Data Integration + 0. Configuration # Source definitions, mappings + 1. Import # Raw data landing (ERP, HR, CRM) + 2. Transformation # Cleansing, harmonization + 3. Quality & Controls # Reconciliation, validation +``` + +Example: Deeper Nesting + +``` +2. Data Integration + 0. Configuration + 0. Summary + 1. Source Systems + 2. Mapping Rules + 1. Import + 1. ERP + 2. HRIS + 2. Transformation + 1. Cleansing + 2. Enrichment + 3. Quality & Controls + 1. Reconciliation + 2. Exceptions +``` + +### Dimension Lists + +PascalCase, descriptive, no prefix. If the application already uses another +style (e.g. Snake_Case) for dimensions, keep that style for consistency. + +| Good | Avoid | +| -------------- | ------------ | +| `Department` | `DIM_Dept` | +| `CashCategory` | `Cats` | +| `GLAccount` | `Accts.List` | + +### Transaction Lists + +Snake_Case with purpose-based prefix `LOAD_`, describing the data source or business process: + +| Good | Avoid | +| -------------------------- | ------------------------------------ | +| `LOAD_Sales_Orders` | `SalesOrders` (no prefix) | +| `LOAD_Employee_Events` | `EE_Events` (unclear abbreviation) | +| `LOAD_Inventory_Movements` | `Inv.Movements` (period breaks refs) | + +### Properties + +Snake_Case, clear names: + +| Good | Avoid | +| -------------- | ----------- | +| `Sort_Order` | `SO` | +| `Account_Type` | `Acct.Type` | +| `Is_Active` | `Active?` | + +### Metrics + +Snake_Case with type prefix: + +#### Two-Prefix System + +For complex models, combine Prefix A (business process) + Prefix B (utilization type): + +- Prefix A: Business process context (`REV_`, `EE_`, `TBH_`, `SC_`) +- Prefix B: How the block is used (`INPUT_`, `CALC_`, `OUTPUT_`) +- Combined: `REV_INPUT_Growth_Rate` or `EE_CALC_Total_Salary` + +Common Prefix A options (business process): + +- `REV_` - Revenue-related +- `EE_` - Existing Employees +- `TBH_` - To Be Hired +- `SC_` - Sales Capacity +- Or use folder acronyms/numbers + +#### Prefix B: Utilization Type (Complete List) + +| Prefix | Purpose | Example | +| --------- | ------------------------------------------ | ---------------------- | +| `INPUT_` | User-entered data | `INPUT_Budget_Amount` | +| `CALC_` | Intermediate calculations | `CALC_Gross_Margin` | +| `OUTPUT_` | Reporting metrics | `OUTPUT_Net_Revenue` | +| `RES_` | Final results (for display) | `RES_Total_Headcount` | +| `MAP_` | Lookups/mappings | `MAP_Account_Sign` | +| `ASM_` | Assumptions | `ASM_Growth_Rate` | +| `DATA_` | Aggregated data with simple transformation | `DATA_Employee_Count` | +| `PUSH_` | Shared to other apps | `PUSH_Revenue_Total` | +| `PULL_` | Pull data from other apps | `PULL_WF_Headcount` | +| `ADM_` | Admin/config | `ADM_Switchover_Date` | +| `SET_` | General settings | `SET_Default_Rate` | +| `FIL_` | Filter metrics for tables | `FIL_Active_Only` | +| `VAR_` | Variable management | `VAR_Scenario_Switch` | +| `ARM_` | Access rights management | `ARM_Read_Cost_Center` | +| `BPM_` | Board permissions | `BPM_Budget_Access` | +| `LOCK_` | Lock/restriction controls | `LOCK_Prior_Year` | +| `AUT_` | Automations | `AUT_Monthly_Refresh` | +| `REP_` | Reporting-specific | `REP_Executive_KPI` | +| `EXP_` | Export metrics | `EXP_Billing_Data` | +| `CTRL_` | Data quality controls | `CTRL_Balance_Check` | +| `M2M_` | Metric-to-metric operations | `M2M_Revenue_Mapping` | +| `WIP_` | Work in progress (not implemented) | `WIP_New_Logic` | +| `KPI_` | KPI display metrics | `KPI_Monthly_Growth` | + +Suffixes for units: + +- `($)` - Currency +- `(#)` - Count +- `(%)` - Percentage + +Suffixes for status: + +- `-TEST` - Temporary +- `-OLD` - Deprecated + +Calculation chains: + +Add sequence numbers inside the name of the block: + +``` +CALC_01_Base_Revenue +CALC_02_Growth_Adjustment +CALC_03_Final_Revenue +``` + +Combining prefixes: + +You can combine Prefix A + Prefix B for clarity: + +``` +REV_PLAN_INPUT_Growth_Rate # Revenue planning input +REV_ACT_DATA_Historical # Revenue actuals data +EE_ASM_INPUT_Merit_Tenure # Existing employees assumption +``` + +Formula referencing notes: + +- Use single quotes `'` when starting a metric name with a number: `'01_Revenue` +- Typing a space in formulas indicates two separate objects +- Never use periods `.` or colons `:` in metric names - they break referencing + +### Tables + +Use the `[TBL] ` prefix (square brackets + space) to sort tables to the top of folders. The brackets ensure correct sort order in Pigment. + +``` +[TBL] Cash_Flow_Summary +[TBL] Variance_Analysis +[TBL] PnL_by_Department +[TBL] EXP_Cost_Center_Billing +``` + +Without the bracket prefix, use descriptive Snake_Case names: + +``` +Cash_Flow_Summary +Variance_Analysis +PnL_by_Department +``` + +### Boards + +Use numeric numbering with logical grouping (single digit for top level, two digits for sub-groups): + +``` +0. Admin + 0. Application overview + 1. Parameters + 2. Maintenance + +10. Data + 11. Current Transactions + 12. Most Updated Employee Detail + +20. Assumptions + 21. Merit Increase + 22. Taxes + +30. Planning + 31. Department Planning + 32. Existing Employee Management + 33. TBH Management + +40. Results + 41. Actual Headcount & FTE + 42. Workforce Reporting +``` + +Board folders: Boards are not stored in the Blocks area. They have their own folder structure (under the application's Boards section). Do not create a block folder named "Board" or "Boards" to hold boards-boards live in their own hierarchy, separate from Blocks. + +Alternative: Purpose prefix + function (no numbering): + +``` +CF_Input # Cash flow input +CF_Dashboard # Cash flow dashboard +Budget_Entry # Budget data entry +Executive_Summary # Executive reporting +``` + +### Views + +Include target user and purpose. Use prefixes for clarity: + +``` +KPI_Executive_Summary +Graph_Monthly_Trend +Chart_Department_Variance +Department_Head_Requests_To_Validate +``` + +## Anti-Patterns + +| Bad | Problem | Fix | +| ----------------- | ------------------ | ------------------- | +| `Rev.Total` | Period breaks refs | `Revenue_Total` | +| `Dept:Sales` | Colon breaks refs | `Dept_Sales` | +| `HC` | Unclear | `Headcount` | +| `Copy of Revenue` | Default name | Delete or rename | +| Inconsistent folder order | Hard to navigate | Use numeric prefixes (0., 1., 2. or 10., 11., 20.) consistently | + +## Quick Reference + +| Element | Style | Prefix | Example | +| ---------------- | ---------- | --------- | -------------------------- | +| Application | Title Case | Optional `#.` | `1. Workforce Planning` | +| Folder | Title Case | `#.` or `##.` | `0. Settings`, `10. Data Loads` | +| Dimension | PascalCase | None | `CostCenter` | +| Transaction List | Snake_Case | `LOAD_` | `LOAD_Sales_Orders` | +| Property | Snake_Case | None | `Account_Type` | +| Metric | Snake_Case | See [Metrics](#metrics) | `CALC_Net_Revenue`, `PUSH_Revenue_Total` | +| Table | Snake_Case | Optional `[TBL] ` | `[TBL] Variance_Analysis` | +| Board | Snake_Case | Abbrev or numbering | `CF_Dashboard`, `0. Admin` | diff --git a/plugins/pigment/skills/modeling-pigment-applications/modeling_performance_considerations.md b/plugins/pigment/skills/modeling-pigment-applications/modeling_performance_considerations.md new file mode 100644 index 00000000..bec7846b --- /dev/null +++ b/plugins/pigment/skills/modeling-pigment-applications/modeling_performance_considerations.md @@ -0,0 +1,52 @@ +# Modeling Performance Considerations + +## Introduction + +Performance considerations should be part of modeling decisions from the start. This guide covers dimensional design, table management, and architectural patterns that affect performance. + +For comprehensive performance optimization guidance including formula optimization, sparsity management, and troubleshooting slow calculations, see the `skill:optimizing-pigment-performance`. + +## Dimension Growth Impact + +### The 1G Cells Problem + +Having a billion cells in a metric, even if most are blank, can cause increased processing time, memory usage, and slower recalculation or rendering in both formulas and views. + +When dimensions grow, total possible cells grow exponentially: + +- 1,000 products x 1,000 customers x 1,000 months = 1 billion cells + +### Mitigation Strategies + +1. Use properties instead of dimensions where possible +2. Use transaction lists for high-cardinality data +3. Apply access rights to limit scope +4. Use formula-driven subsets to limit active data only when appropriate; see [List Subsets](./modeling_subsets.md) for when to use subsets vs filters and for data-loss risks and safe patterns. + +## Combination Masks / Boolean scoping + +Limit which dimension combinations are valid to reduce cell count. + +A combination mask is a technique to ensure calculations only occur for valid combinations of dimension items, improving performance by reducing unnecessary data. +Example with a Multidimensional Metric: Suppose you have a Boolean metric called `IsValidCombination` with multiple dimensions (for example, Product and Region). You can use it in your formula like this: + +`IF(IsValidCombination, SalesAmount, BLANK)` + +Here, `IsValidCombination` is TRUE only for valid product-region pairs. The formula calculates `SalesAmount` only where the combination is valid, leaving other cells blank. This approach helps keep your model efficient. + +## Table Consolidation Strategies + +- Split large tables into focused smaller tables +- Use hidden metrics for intermediate calculations +- Consolidate related metrics into tables + +## View Management + +- Limit dimensions in views (3-5 max) +- Use filters to reduce data volume +- Consider view truncation limits + +## See Also + +- `skill:optimizing-pigment-performance` +- [Modeling Principles](./modeling_principles.md) diff --git a/plugins/pigment/skills/modeling-pigment-applications/modeling_principles.md b/plugins/pigment/skills/modeling-pigment-applications/modeling_principles.md new file mode 100644 index 00000000..75b9089f --- /dev/null +++ b/plugins/pigment/skills/modeling-pigment-applications/modeling_principles.md @@ -0,0 +1,463 @@ +# Become a Pigment Pro: Modeling Principles & Palette + + +## Proper operation ordering +Modeling involves CRUD operations performed in the correct order (solve sub-components, then create target with sub-components assigned). +When constructing a target block -> check if the target sub-components exist -> if they do not exist, create them -> finally create the target with the sub-components assigned. + +This avoids creating blocks and then modifying them. + +For example: +- When a new dimension/list is needed with properties that clearly represent entities (e.g. Product, Customer, Country, Region, SKU, Store, Employee), you must: +First, search for any existing dimensions that match or are closely related. +If none exist, create new dimensions for these entities. +Then make the properties Dimension-typed referencing those new dimensions. + +- When creating a metric. Check if its assigned dimensions exist. Create those which do not exist. Then, create the target metric with dimensions assigned. + +Resolving objects in the live application: Reading skills and using `grep` on documentation is not enough to know which metrics, lists, and properties exist in the user's workspace or to obtain their IDs. Use `tool:search` (Application Expert)-the intended, fast path to retrieve workspace inventory and disambiguate blocks; combine with `kind` / `regexp` filters when several names are similar. + +## 1. Folder Structure + +Never create blocks in "No Folder". Every new block (metric, dimension list, transaction list, table) must be created in an explicit folder. "No Folder" is a system default placeholder, not a valid target; blocks left there are hard to find and clutter the application. Before creating any block, determine the target folder and create (or assign) the block there. For how to choose the right folder and where to place each block type, see [Working with Folders](./modeling_working_with_folders.md). + +For comprehensive guidance on folder structure and organization, see [Working with Folders](./modeling_working_with_folders.md). + +--- + +## 2. The Library Folder: Sharing Metrics + +The Library folder helps track data flow and manages security between applications. + +- Push Metrics: Sanitized output metrics shared _to_ other applications. Name with `PUSH_` prefix (see [Naming Conventions - Metrics](./modeling_naming_conventions.md#metrics)). + +- Pull Metrics: Data received _from_ other applications. Name with `PULL_` prefix (see [Naming Conventions - Metrics](./modeling_naming_conventions.md#metrics)). + +Process for Sharing: + +1. Duplicate the metric to be shared and rename with `PUSH_` prefix. + +2. Move to the Library folder and replace the formula with a reference to the original metric. + +3. Sanitize: Use `REMOVE` modifiers to strip dimensions not needed outside the app. + +4. Share: Toggle "Share this Block" in settings. + +5. Pulling: In the destination app, create a `PULL_` metric referencing the shared metric and use `RESETACCESSRIGHTS()` to ensure users can see the data. + +--- + +## 3. Naming Conventions + +Consistent naming aids navigation and formula writing. When you create blocks, use the current application's naming conventions in priority. Boards (Pigment dashboards) have their own folders; do not create a block folder for Boards. + +For Applications, Folders, Blocks (metrics and lists), Tables, and Boards naming-including numeric prefixes, metric prefixes (e.g. PUSH_, PULL_, CALC_, INPUT_), and the [TBL] table prefix-see [Naming Conventions](./modeling_naming_conventions.md). + +--- + +## 4. Formula Best Practices + +- Formatting: Use line breaks, tabs, and comments (`//`) to make formulas readable. + +- Structure: Indent sub-calculations and end parentheses at the start of a new line. + +Deployment-safe and maintainable formulas + +MP02 (hard constraint): Do not hard-code values or dimension items in formulas - especially on Time and Version. Do not use `DATE(...)` for planning period bounds or embed fixed periods in metric names. Never write `Dimension."Item"` in formulas; use a `VAR_` input metric of type Dimension or Date, or a structural boolean (e.g. IsActual). + +When a formula depends on a specific member: (1) Create a `VAR_` input metric of type Dimension. (2) Set its default to the item - the only place the literal may appear. (3) Reference the metric in formulas. (4) Expose it on a Board. For Version semantics (actual vs plan), prefer IsActual / `'Version Type'` - see `skill:planning-cycles-pigment-applications`. + +Metric names: Use relative temporal labels (`'Next Period Forecast'`, `'Budget Current Year'`) - not absolute years or months (`Forecast 2026`). User mentions of a period in the request are context; do not copy them into formulas or names. + +Exceptions: User may explicitly accept a one-off hard-code after you propose the compliant alternative. Stable type/class/category dimensions may stay hard-coded in formulas (e.g. `'FX Rate Types'."AVG"`) - not Time, Version, countries, or planning periods. + +See MP02 - No Hard-Coding (section 8). Formula patterns: `skill:writing-pigment-formulas` ([formula_modifiers](../writing-pigment-formulas/formula_modifiers.md), [formula_writing_workflow](../writing-pigment-formulas/formula_writing_workflow.md)). + +--- + +## 5. Data Loading: Where does it go? + +- Transaction Lists: For transactional data (multiple fields, frequent reloads, generated IDs). Examples: Order transactions, inventory movements, customer interactions, employee events. + +- Dimension Lists: For Meta Data (Products, Customers, Suppliers, Warehouses, Employees, Accounts). + +- Metrics: For analytical data already structured by dimensions (e.g., Sales by Product x Region x Month, Inventory levels by Warehouse x Product x Week). A metric must never be dimensioned over a Transaction List-only dimension lists define metric structure; see [modeling_fundamentals](./modeling_fundamentals.md) for details. + +--- + +## 6. Multi-Application Architecture + +Using multiple applications (Distributed Planning) allows for segregation of duty, cleaner models, and different planning cadences. + +The Hub: +A mandatory application containing shared Dimensions (Time, Country, etc.) and central settings (FX rates). It acts as the single source of truth. + +For comprehensive guidance on multi-application architecture and the Hub pattern, see [Modeling Architecture Design](./modeling_architecture_design.md). + +--- + +## 7. Multi-Dimensional Modeling + +Pigment uses modifiers to handle dimension mismatches between source and target metrics. + +- Aggregation: Use `[BY SUM: Dimension]` to aggregate data (e.g., Employee -> Country). + +- Allocation: Use `[BY CONSTANT: Dimension]` or `[ADD CONSTANT: Dimension]` to apply a value across a new dimension. + +- Mapping: Use `[BY: Mapping_Metric]` when transforming dimensions based on a relationship (e.g., Shipping Rate by Region applied to Warehouse, or Discount Rate by Customer Segment applied to Order). + +- Remove: Use `[REMOVE SUM: Dimension]` to strip a dimension from the structure. + +--- + +## 8. Pigment Modeling Best Practices Rules + +The Pigment Modeling Best Practices consists of 28 rules organized into three categories: MG (Modeling in General), MS (Modeling for Speed), and MP (Modeling for Posterity). + +### MG - Modeling in General + +#### MG01 - Explicit Dimensions: Always Define Dimension Alignment + +Always use explicit modifiers when the grain or dimensions differ between source and target. In BY, specify only the dimensions you are transforming (aggregating or allocating); do not re-list dimensions that are already on the metric and unchanged (avoid over-explicit BY). + +For comprehensive guidance on dimension alignment, modifiers, and formula writing, see [Writing Pigment Formulas - Formula Modifiers](../writing-pigment-formulas/formula_modifiers.md). + +#### MG02 - Structure: Keep Dimensions Minimal + +The dimensional structure of every Metric is of capital importance. Give it a lot of thought. It needs to have the right Dimensions, no more. If a Metric has more than 8-10 Dimensions, check if you really need all those Dimensions. + +Questions to ask: +- Are they needed for reporting purposes? +- Are they independent Dimensions, or can we leverage Dimension Properties instead? +- Is the user experience still good with so many selectors? +- Are the calculations performant enough with so many Dimensions? + +Dimension Properties as Dimensions: + +When creating a dimension with properties, never default all properties to Text. Prefer creating reusable property dimensions over Text properties for categorical or enumerable fields. Then assign those property dimensions to your target dimensions. + +Workflow: +- Create the dimension-properties first (or reuse existing ones) +- Then create your target dimensions and assign them their properties + +Use Dimension Properties instead: + +Using Properties of existing Dimensions as Dimensions in the Pivot or Page selector is unlimited, and this is the preferred option whenever possible. + +Examples: +- Metrics by Employee: Contract Type isn't required in the structure if it's a Property of the Employee Dimension +- Metrics by Product: Category isn't required in the structure if Category is a Property of the Product Dimension +- Metrics by Warehouse: Region isn't required in the structure if Region is a Property of the Warehouse Dimension +- Metric by Month: Year Dimension isn't required in its structure as it's a Property of Month Dimension + +#### MG04 - Justify Metrics: Think Twice Before Creating + +Leverage existing Metrics: + +Before creating a new Metric, check if existing Metrics, Views, or Properties are available to meet your needs. This avoids having unnecessary Metrics that consume storage and resources, and require maintenance and documentation. + +Justify Metric creation: + +You should only create a Metric when it's essential: +- For importing or inputting data +- To break down complex calculations for clarity or performance +- For specific Boards or reporting needs +- To share and use in other Applications + +Unnecessary Metrics creation leads to heavy, hard-to-audit models and hidden maintenance costs. + +When Views Can Replace Metrics: Avoid "Display-Only" Metrics + +Modeling and Views are different teams, but in practice modeling and reporting are tightly linked and shouldn't be designed in isolation. Many requirements that seem to require new Metrics can actually be handled by Views, which are more flexible, performant, and easier to maintain. + +Critical Principle: Only create Metrics when the dimensionality or calculation is needed for downstream calculations. If the requirement is purely for display or visualization, use Views instead. + +Common Anti-Patterns: Creating Metrics That Views Can Handle + +1. Aggregated Metrics "For Display Only" + +Anti-Pattern: +- Revenue metric is structured by `Country x Month` +- Country dimension has a `Region` property (dimension-type) +- Modeler creates a new metric: `Revenue by Region` with formula `Revenue [BY: Country.Region]` +- This metric is only used for visualization, not in any downstream calculations + +Correct Approach: +- Keep Revenue metric structure as `Country x Month` +- In Views, add `Country.Region` to pages, and to rows or columns +- Views automatically aggregate Revenue by Region using the metric's default aggregator +- No additional metric needed + +Why This Matters: +- Views can pivot by dimension-type properties without changing metric structure +- Multiple aggregation levels can be shown in different Views of the same metric +- Metric structure remains minimal and performant +- See `skill:creating-and-editing-pigment-views` for details + +2. Percentages, Variations, and Cumulates "For Display Only" + +Anti-Pattern: +- Creating metrics like `EBIT%`, `EBIT Growth YoY`, or `Cumulated Revenue` when these are only needed for display +- These metrics are not referenced in any downstream calculations + +Correct Approach: +- For percentage-like metrics built from two base metrics (a ratio A / B, or relative variance / growth such as (A - B) / B): create or reuse a dedicated Metric with a Pigment formula (for example GM% = Gross Margin / Revenue), add it to the Table together with both operand Metrics, then use Views on Tables with Advanced Aggregators on that ratio Metric everywhere it must roll up correctly. That order (Metric on the Table first, then View configuration) is the default for cross-metric ratios and relative variances. +- Show value as has many modes. Only % of ... metric and % growth from ... metric overlap the rollup behavior that Advanced Aggregators (Ratio / Growth) address-use Advanced Aggregators on Table views for those cases instead of those two SVA modes. All other Show value as options (cumulative, % of grand total, % of parent, and the rest) stay on Show value as. +- Calculated Items suit derived rows or columns on a dimension (for example a total row). They are not a substitute for a dedicated ratio Metric between two Metrics: do not replace the ratio Metric with a Calculated Item on the value axis, duplicate an operand Metric as an extra value field, or try to "fix" rollups by putting Advanced Aggregators on that Calculated Item. +- See [MG09 - Ratios and percentage-like metrics: create the Metric, then Advanced Aggregators](#mg09---ratios-and-percentage-like-metrics-create-the-metric-then-advanced-aggregators) and [View design process](../creating-and-editing-pigment-views/view_design_process.md) for the workflow. + +Examples of View-Based Calculations: +- Two-metric ratio, rate, percentage, growth, or variance on a Table: dedicated ratio Metric with formula on the Table, then View on a Table -> Advanced Aggregators (Ratio, Growth, or Absolute growth) on that ratio Metric with operands on the two base Metrics (see MG09). Prefer this over Show value as -> % of ... metric or % growth from ... metric for those rollups. +- % of grand total, % of parent, and other share-of-axis modes: Show value as (not Advanced Aggregators) +- Cumulative and other SVA modes: Show value as as appropriate +- Other derived dimension logic: Calculated Items where appropriate + +3. Mapped Dimensions for Reporting + +Anti-Pattern: +- Creating aggregated metrics when the requirement is to report across different dimensional structures +- Example: Creating `Revenue by Region` metric when Revenue is by `Country x Month` and Country has Region property + +Correct Approach: +- When the parent-child relationship varies by period, create a mapping metric and use Mapped Dimensions in Views (Joined Pivot) +- When the relationship is static, use dimension-type properties in Views for grouping without changing metric structure +- Multiple Views can show the same metric at different aggregation levels + +4. Filtering and Sorting Requirements + +Anti-Pattern: +- Creating separate metrics for filtered or sorted views +- Example: Creating `Top 10 Products Revenue` metric + +Correct Approach: +- Use View filters (by items or by value) to restrict visible data +- Use View sorting to order data by metric value or properties +- Use "top N" or "bottom N" filters for ranking analysis +- See `skill:creating-and-editing-pigment-views` + +5. Variance Analysis Considerations + +When designing for variance analysis (Actuals vs. Plan, Plan vs. Budget, etc.), consider: +- Are Actuals and Plan data in the same metrics or separate? +- What type of variance analysis is typically done? +- Can Views handle the comparison using Show Value As or Calculated Items? + +Decision Framework: Metric vs. View + +Create a Metric when: +- GOOD: Data needs to be imported or input +- GOOD: Calculation is needed for downstream formulas +- GOOD: Metric needs to be shared across Applications +- GOOD: Calculation logic is complex and benefits from being in the model +- GOOD: Dimensionality is required for calculation accuracy + +Use a View when: +- GOOD: Requirement is purely for display/visualization +- GOOD: Aggregation can be done via dimension-type properties +- GOOD: Calculation is a percentage, ratio, growth, or variance +- GOOD: Filtering or sorting is the main requirement +- GOOD: Multiple aggregation levels are needed from the same base metric + +Key View Capabilities to Leverage + +Before creating a metric, consider if Views can handle the requirement using: +- Dimension-type properties for hierarchical reporting (see `skill:creating-and-editing-pigment-views`) +- Advanced Aggregators on Views on Tables for two-metric ratios, percentages, and growth or relative variance-after the ratio Metric exists on the Table (see [MG09 - Ratios and percentage-like metrics: create the Metric, then Advanced Aggregators](#mg09---ratios-and-percentage-like-metrics-create-the-metric-then-advanced-aggregators) and `skill:creating-and-editing-pigment-views`) +- Calculated Items for derived dimension logic where they fit +- Filters for data restriction (by items, by value, top/bottom N) (see `skill:creating-and-editing-pigment-views`) +- Sorting for data ordering (by metric value, by property) (see `skill:creating-and-editing-pigment-views`) +- Page selectors for user-controlled filtering (see `skill:creating-and-editing-pigment-views`) + +Reference Documentation + +For comprehensive guidance on View capabilities, see: +- `skill:creating-and-editing-pigment-views` - Definitions, draft workflow, and where to read next +- `skill:creating-and-editing-pigment-views` - Step-by-step configuration (reuse, draft, validate) +- `skill:creating-and-editing-pigment-views` - Pivots, filters, and sorting; [Pivoting rules](../creating-and-editing-pigment-views/view_pivoting.md) and [Display modes](../creating-and-editing-pigment-views/view_display_modes.md) for layout and widget constraints + +#### MG05 - Simple Flows: One-Way Data Flow + +Ensure one-way data flow: + +Design data to flow towards a central consolidation point that combines Actuals and Planning data for reporting on Boards. Reference the correct Block directly and use the dependency diagram to maintain clarity and simplicity. + +For comprehensive guidance on financial statement modeling and data flow patterns, see [Core P&L reporting (Nexus pattern)](../solving-specific-use-cases/finance_nexus_financial_statements.md). + +#### MG06 - Block Usage: Proper Role Assignment + +Understanding Block roles is key in deciding which type of Block to use and when to use it. For definitions and characteristics of each block, see [modeling_fundamentals section 2 - Building Blocks](./modeling_fundamentals.md#2-pigment-modeling-building-blocks). + +- Dimension: Represents business structure +- Transaction List: Handles transactional data loads +- Metric: Manages end-user inputs and calculation logic +- Table: Serves as the user interface for inputs and reporting, and is placed on Boards + +You should manage end-user planning inputs and logic primarily within Metrics and Tables. Metrics offer the right Dimensions, scenario Applications, full auditability, and accurate execution. Reporting is based on Metrics. End-user inputs should only be used in Lists if there is a valid exceptional reason. + +#### MG07 - Imports: Lists First, Metrics Second + +Importing data into Lists offers greater flexibility. Lists allow you to create additional Properties for data cleaning and transformation. Importing data into a Metric and changing its structure can result in data loss. This is avoided when you import data into Lists. + +Benefits of List imports: +- Importing into Lists as the TEXT data type allows you to extract codes from a chain of characters using functions like LEFT(), RIGHT(), CONTAIN(), and MID() +- You can then identify Dimension Items afterward using the ITEM() function +- Lists support scoped imports, enabling selective updating and cleaning of Dimension intersections + +When Metric imports are useful: +- If you need to leverage the Clone data to feature +- To load historical planning data during implementation +- To import large volumes of non-transactional data with consistent structure that doesn't require Item creation or drill-down features (can significantly improve calculation speed) + +Apart from these exceptions, always use Lists for data loading. This approach is safer, more powerful, and enhances model clarity for future audits. + +#### MG08 - Iterative Functions: Use Only for Calculations + +To support iterative calculations, Pigment provides two functions: + +- PREVIOUS(): Creates an iterative calculation while referencing the same Metric +- PREVIOUSOF(): Creates an iterative calculation while referencing another Metric + +Note: `PREVIOUSBASE()` is deprecated. Use `PREVIOUSOF()` instead. + +These functions perform sequential calculations. For example, if PREVIOUS() is used on a Calendar Dimension, the calculation will first complete January before calculating February. + +Critical rule: These functions are designed to be used for the purpose of performing iterative calculation only, and not to facilitate user inputs. Using PREVIOUS() and PREVIOUSOF() to project user input assumptions onto other Items prevents you from deleting initial input Items, and as a result complicates model maintenance. + +Example: Setting assumptions for FY23 and calculating subsequent years prevents the deletion of FY23 from the Calendar Dimension. This contradicts the goal of keeping your Calendar as small as possible. + +For comprehensive guidance on iterative calculations (PREVIOUS vs PREVIOUSOF, circular dependencies, configuration, when to use), see [Iterative Calculation (PREVIOUS & PREVIOUSOF)](../writing-pigment-formulas/functions_iterative_calculation.md). For performance optimization (subsetting, FILLFORWARD, CUMULATE), see [Performance - Iterative Calculations](../optimizing-pigment-performance/performance_iterative_calculations.md). For when and how to use List Subsets (including data-loss risks and safe patterns), see [List Subsets](./modeling_subsets.md). + +#### MG09 - Ratios and percentage-like metrics: create the Metric, then Advanced Aggregators + +The problem: A ratio Metric (A / B) or relative variance ((A - B) / B) evaluates correctly at the finest grain shown in the view. If the Table view rolls it up like an ordinary additive measure, aggregated totals are wrong (for example sum of ratios instead of ratio of sums). + +Step 1 - Create or locate the ratio Metric. The ratio must be a real Pigment Metric with a formula (for example GM% = Gross Margin / Revenue). Add it to the Table with both operand Metrics. Avoid: duplicating an operand Metric as a second value field to simulate the ratio; using a Calculated Item on the value axis instead of that Metric for a cross-metric ratio; relying on Show value as -> % of ... metric or % growth from ... metric as the default substitute for Advanced Aggregators on Table views (those two SVA modes overlap the rollup shape but Advanced Aggregators are the explicit fix). + +Step 2 - Configure Advanced Aggregators in each Table view where the ratio must roll up. Add value fields for the ratio Metric and both operand Metrics. Set Ratio, Growth, or Absolute growth on the ratio Metric's value field with the two operand value fields as operands (same A and B as in the formula). Apply on Rows, Columns, and Hidden dimensions aggregation as needed. Details: [View aggregators](../creating-and-editing-pigment-views/view_aggregators.md). + +Show value as remains appropriate for many single-metric or axis-relative displays (cumulative, % of grand total, % of parent, YoY-style references where that mode fits, and other SVA options not listed here). + +Agent rule (Tables only): When a View on a Table shows such a ratio or relative-variance Metric, you must verify the dedicated Metric exists (create it if not), ensure it and both operands are on the Table, then configure Advanced Aggregators as above. This does not apply to view types without Advanced Aggregators (such as Views on Metrics); use a Table view when correct rollups are required. + +#### MG10 - Security: Start Restrictive + +Pigment is most likely to contain sensitive information. Always start with the most restrictive security settings. Grant authorizations only when necessary and continuously evaluate the need for each permission. Regularly review and challenge the necessity of granted permissions to maintain a high level of security and minimize risks. + +Key Principles: +- Use the Restrict domain feature +- Use the Group feature to give Members access to only the required Applications +- Give access to only relevant Boards per Role. Board permissions should be set to None for all non-Admin Roles +- Share the minimum number of Blocks in the Library. Regularly check the Library to ensure there aren't any unnecessary shared Blocks +- Setup centralized, clear, and robust access rights rules that can be easily audited + +For comprehensive guidance on access rights and security, see: +- `skill:securing-pigment-applications` ([securing_access_rights.md](../securing-pigment-applications/securing_access_rights.md)) +- [Performance - Access Rights](../optimizing-pigment-performance/performance_access_rights.md) + +#### MG11 - Sharing: Only What's Necessary + +Create targeted Metrics for sharing: + +When sharing Application outputs, create dedicated Metrics to be shared and reused in different Applications. This establishes a single source of truth and reduces the risk of errors. Use clear and explicit naming for shared Metrics to avoid duplication and confusion. + +Avoid unnecessary Dimensions: + +Exclude unnecessary Dimensions from these shared Metrics to enhance security and simplify the Workspace. Regularly review and update shared Blocks in the Workspace Library. This practice maintains optimal organization, ensures data hygiene, reduces maintenance costs, and minimizes the risk of errors. + +#### MG12 - Planning Cycle: Use Version Dimension + +The Scenario feature that Pigment offers natively, as opposed to a classical Dimension that would be called Scenario or Version, is a robust tool designed to facilitate "What if?" analyses. It allows you to model each planning cycle exercise (Actual, Budget, Forecast) as separate Scenarios. It doesn't support cross-Scenario calculations or data referencing through formulas, but it allows for powerful comparisons between Scenario Snapshots and live data. + +Recommendation: For more flexibility and to fully accommodate your planning needs, using a normal Dimension to model your planning cycle is recommended. Begin by creating a Version Dimension in your central hub Application. Incorporate this Dimension into your Metrics structure, either for input data (usually in Tables) or for building your calculation logic. Maintain a live version of data that is regularly updated, and couple it with the Clone data to functionality, which replicates inputs across various planning phases. Finally, configure read and write access rights to effectively manage visibility and editing permissions for the different planning stages. + +ALWAYS read `skill:planning-cycles-pigment-applications` before structuring any planning metric. The Version Dimension is foundational: switchover semantics, the IsActual / IsPlan / IsVersion Boolean metrics, and Actual/Plan layering must be wired correctly upfront. Treat that skill as a required companion to this one, not an optional reference. + +### MS - Modeling for Speed + +Performance optimization rules. For comprehensive guidance, see: +- [Performance - Sparsity Deep Dive](../optimizing-pigment-performance/performance_sparsity_deep_dive.md) - MS01, MS02 +- `skill:writing-pigment-formulas` - MS03, MS05, MS06, MS10 +- [Iterative Calculation (PREVIOUS & PREVIOUSOF)](../writing-pigment-formulas/functions_iterative_calculation.md) - When and how to use PREVIOUS/PREVIOUSOF (MS10) +- [Performance - Iterative Calculations](../optimizing-pigment-performance/performance_iterative_calculations.md) - Optimizing iterative calculations (MS10) + +Quick Reference: +- MS01 - Sparse Engine: Avoid `IF(..., 0)` or `ISBLANK` (which returns False) as they fill sparse cells with data. Use `ISDEFINED` or leave `ELSE` blank. For the underlying concept, see [Sparsity in modeling_fundamentals](./modeling_fundamentals.md#3-sparsity-core-engine-principle). +- MS02 - Cardinality: Minimize the number of items in dimensions used in metrics. High cardinality slows performance. +- MS03 - Calculate Once: Calculate a value in one metric and reference it elsewhere. Don't repeat formulas. +- MS04 - Aggregate Loads: Aggregate Transaction List data into a single `DATA_` staging metric, then reference that metric. +- MS05 - Scope: Filter data _early_ in the formula (using `FILTER` or `SELECT`) before aggregating. +- MS06 - Split Metrics: If a formula is too complex to read in one go, split it into multiple metrics to avoid timeouts. +- MS07 - Monitor Time: Formulas should take seconds. If >15 seconds, investigate. +- MS08 - Small Dimensions: Keep Calendars and Versions lean. Archive historical data to static snapshots. +- MS09 - Dependency: Understand that independent metrics calculate in parallel. Break live calculations (e.g., disable Auto Save) if necessary. +- MS10 - Heavy Functions: Avoid heavy functions like `CUMULATE`, `MOVINGSUM`, or text manipulation (`FIND`, `SUBSTITUTE`) on large lists. +- MS11 - Engine vs. View: Prefer Calculated Items and Show value as where they replace redundant metrics-but for two-metric ratio or relative-variance rollups on Tables, create the ratio Metric on the Table first and use Advanced Aggregators (not SVA's % of ... metric or % growth from ... metric for that job, and not duplicate value fields or Calculated Items on the value axis as a substitute). Keep all other SVA modes on Show value as where they apply. +- MS12 - Split Access Rights: Split security rules into smaller, dimension-specific metrics rather than one complex rule. + +### MP - Modeling for Posterity + +- MP01 - Readability: Indent formulas and use comments (`//` or `/* */`). + +- MP02 - No Hard-Coding: Hard constraint - see section 4. + +- MP03 - Naming: Adhere strictly to the naming convention. + +- MP04 - Limit Views: Keep public views under 10 per block. + +- MP05 - Admin Boards: Create documentation boards for maintenance tasks (e.g., "Start new planning cycle"). + +- MP06 - Hygiene: Regularly delete unused apps, boards, snapshots, and inactive members. + +- MP07 - Dynamic Variables: Use `VAR_` metrics for workspace-wide variables (e.g., Current Month). + +- MP08 - Production Changes: Use "Test & Deploy" or duplicate metrics to test new formulas before replacing the old ones. + +- MP09 - Next Modeler: Build simply and document via text widgets on boards so others can understand the model. + +- MP10 - Direct Security: Apply Access Rights directly to blocks/lists rather than relying on inheritance, which is harder to audit. + +- MP11 - Import Best Practices: Load unique IDs, remove zeros, map dates twice (as date and time dimension), and scope imports. + +--- + +## 9. When Test & Deploy is used + +When Test & Deploy is enabled (deployment across environments, e.g. Dev -> Prod), MP02 (section 4) is always enforced - hard-coded dimension items break deployment across environments. Test & Deploy adds one additional hard constraint (Rule 2 below). Determine Test & Deploy context before applying Rule 2. + +### MP02 - always enforced + +MP02 applies whether or not Test & Deploy is active. See section 4. + +Disallowed patterns (never emit in formulas unless the user explicitly overrides after you propose a `VAR_` alternative): + +```pigment +Country."France" +IF(Country = Country."France", 1, 0) +'Sales'[FILTER: Country = Country."France"] +IF(Country = Country."France", 'Revenue', 0) +'Revenue'[SELECT SUM: Country = Country."France"] +IF(Version = Version."Budget", 'Actual_Revenue', 'Plan_Revenue') +'Revenue'[FILTER: Country = Country."France" OR Country = Country."UK"] +``` + +Agent behavior: Refuse to produce or keep such formulas. Propose compliant alternatives per section 4 (`VAR_` metrics, property-based filters, IsActual). + +### Determining Test & Deploy context + +- Test & Deploy status: Consider Test & Deploy active when the user or application context indicates use of Test & Deploy, or when the user mentions deploying from Dev/Staging to Production or working across multiple environments. If unclear, assume T&D is active when the user refers to multiple environments or deployment. +- Dimension connectivity (for Rule 2): Use information from the user or application context to know whether each dimension is connected (synchronized across environments) or disconnected (items may differ between environments). If connectivity is unknown and Rule 2 might apply, ask the user before creating or modifying a property. + +### Rule enforcement + +| Test & Deploy status | Enforcement | +| -------------------- | ----------- | +| Active | MP02 (section 4) is a hard constraint. Rule 2 is a hard constraint. | +| Not active | MP02 (section 4) is still a hard constraint. Rule 2 does not apply. | + +### Rule 2 - No disconnected dimension as property type on a connected dimension (T&D only) + +Terminology: Connected and disconnected describe whether items in the dimension are synchronized across environments (connected) or may differ between them (disconnected). + +When Test & Deploy is active, a disconnected dimension must not be used as the type of a property on a connected dimension. Using a disconnected dimension as the property type on a connected one can cause deployment failures or inconsistent structure across environments. + +Agent behavior: When T&D is active and you create or modify dimension properties, ensure the property type is not a disconnected dimension when the host dimension is connected. If whether items are connected or disconnected is unknown, ask the user before proceeding. diff --git a/plugins/pigment/skills/modeling-pigment-applications/modeling_subsets.md b/plugins/pigment/skills/modeling-pigment-applications/modeling_subsets.md new file mode 100644 index 00000000..be8f1507 --- /dev/null +++ b/plugins/pigment/skills/modeling-pigment-applications/modeling_subsets.md @@ -0,0 +1,364 @@ +# List Subsets + +Guide for using List Subsets only when they create clear modeling value, and avoiding pitfalls-especially irreversible data loss, hidden complexity, and unnecessary operational overhead. + +## When to Read This + +- User asks to create or use a subset (sublist), or to "limit a list" / "use only some items" in structures +- Designing cohort models, intercompany matrices, or "same dimension twice" (e.g. Time + Cohort month) +- Considering subsets for performance (iterative calculations) or for dropdown UX +- User mentions security/filtering goals that might be confused with subsets + +Tool note: Subset creation is available via agent tools; update filter and delete are not yet exposed. This document prepares the skill for full tool coverage and guides design regardless of how the subset is created (UI or future tools). + +--- + +## Purpose + +Help modelers (and the agent) use List Subsets only when they create clear modeling value, while avoiding common pitfalls-especially irreversible data loss, hidden complexity, and unnecessary operational overhead. + +--- + +## Core mental model + +How the agent must think about subsets: + +- A Subset is effectively a separate Dimension in structures and formulas. + - Metrics dimensioned by the subset and metrics dimensioned by the parent are different shapes. + - You cannot freely substitute one for the other without explicit mapping. +- A Subset: + - Inherits items/properties/order/sharing behavior from its Parent list. + - Does NOT automatically "map back" to the parent dimension in calculations. +- Data loss behavior: + - If an item is deselected from a subset, Pigment permanently deletes associated datapoints in metrics that use that subset as a dimension. + - This deletion: + - Affects only subset-dimensioned metrics (not metrics on the parent list). + - Typically applies across all scenarios where the subset-dimensioned metric stores data. + - Is not reversed if the item is later re-added to the subset; deleted cells do not come back. +- Therefore: Subsets are a power tool, not a default pattern. + - Use them when they deliver clear modeling or performance benefits, with a plan to manage the risks. + +--- + +## Agent Decision Checklist (before proposing a Subset) + +Before recommending a subset, the agent must go through this checklist: + +1. Is the user's goal actually security or filtering? + - If the goal is: + - "who can see/edit what?" -> use Access Rights / Permissions, not subsets. + - "only show active items / specific subset in a view" -> use filters or Boolean/list properties. + - -> In these cases, do NOT propose subsets. +2. Is the user's goal just "a smaller list to input data on"? + - If yes, and: + - there will be manual inputs, and + - membership is likely to change, and + - there is no strong iterative/performance need, + - -> Prefer a separate regular list. +3. Is there an iterative / recursive / cycle-heavy calculation where only some members matter? + - If yes (e.g. `PREVIOUS`, `PREVIOUSOF`, complex iterative steps), and a subset significantly reduces members: + - -> Subsets can be considered, with explicit mapping back to parent where needed. +4. Is the same logical dimension needed twice in one metric (mirror dimension)? + - E.g. Months as both Time and Cohort, or Company in rows and columns. + - -> Subset can be a good fit (see "Strongly recommend" cases). +5. Will any metrics on the subset contain manual inputs, and can the client manage safe-update workflows? + - If manual inputs on subset + changing membership + weak governance: + - -> Either use Pattern A (STORE/CALC) or avoid subsets and propose a separate list. + +Only if at least one valid use case (below) is clearly met and the risks are manageable should the agent recommend subsets. + +--- + +## Decision policy: When to recommend Subsets + +### GOOD: Strongly recommend (high-value, low-regret) + +1. Mirror dimensions (same logical list used twice in a structure) + +Typical examples: + +- Cohort modeling: Months as "Time" + Months as "Cohort month". +- Intercompany elimination / consolidation: Company in rows + Company in columns. +- Other "matrix" / relationship models (e.g. Origin vs Destination). + +Why subsets are appropriate: + +- Keep a single "source of truth" list while enabling the same logical dimension twice. +- Avoids maintaining two independent lists that must be kept in sync. +- Still requires awareness that the "second copy" is a separate dimension shape. + +2. Restricted dropdown lists for UX/control (when the gap is huge) +- Example: selecting a Supplier from a curated subset instead of thousands of suppliers. + +Agent guidance: + +- Use the Subset as the selection UI dimension only. +- Store resulting data on the Parent dimension, using mapping (see Pattern B). +- This preserves UX while keeping most data attached to the more stable parent list. + +--- + +### GOOD: Recommend with caveats (only if it materially improves performance) + +1. Performance optimization for iterative/recursive calculations + +Especially for: + +- Time-based iterations: `PREVIOUS()`, `PREVIOUSOF()`. +- Recursive or cycle-heavy patterns where computation cost approx #cycles x #members. +- Cases where many members are structurally present but logically unused for the iterative logic. + +Agent guidance: + +- Subsets do not reduce storage size for metrics on the parent list. +- They reduce compute only where the subset itself is used as a dimension. +- Use a subset to: + - Restrict the number of members along which the iterative calc is performed. + - Then remap results downstream to the parent dimension (Pattern C) if needed. + +Only propose this if: + +- The parent list is large *and* +- There's a clearly identified recursive/iterative metric where limiting members yields meaningful performance gain. + +--- + +## Decision policy: When NOT to recommend Subsets + +### WARNING: Usually avoid (prefer filtering, properties, or another list) + +1. "Focused analysis subsets" (Ex: active entities only, Subset of countries only, etc.) + +If the goal is just to analyze a smaller portion of a dimension or to simplify reporting: + +- Prefer: + - View filters (on Time, Status, Region, etc.). + - Boolean/list properties + filters. + - Security filters if access restriction is needed. + +Reason: + +- The data loss and governance overhead of subsets are rarely worth it for pure analysis. +- Filters are reversible and safer. + +2. Time subsets (e.g. only forecast months, only open periods) +- Tempting use case: subset of Time for "open" or "forecasted" periods. +- However: + - Same data-loss behavior applies to any metrics on the Time subset. + - Filtering or helper metrics (e.g. "Is Forecast Month") are often safer. + +Agent rule: + +- For Time, default to filters or helper metrics, and use subsets only in specific performance/mirror-dimension scenarios with explicit acknowledgement of risk. + +--- + +### GOOD: Prefer creating another regular list when... + +Use a regular list instead of a subset when: + +- You need a stable "modeling list" where items should not disappear regularly. +- Membership logic is expected to change often (new criteria, business rules, etc.). +- You plan to store manual inputs at that level and cannot accept deletion risk. +- The "subset" would effectively become a quasi-dimension with its own lifecycle. + +Agent heuristic: + +If all of the following are true: + +- Users will input data on the target structure, and +- Membership will be edited / changed over time, and +- There is no strong iterative or mirror-dimension need, + +-> Recommend a new regular list, not a subset. + +--- + +## Hard warnings the agent must always surface + +When the user is planning to create or heavily use a subset, the agent must highlight these risks explicitly. + +### Warning 1 - Data loss is real, cross-scenario, and not auto-reversible + +- When an item is deselected from a Subset, Pigment permanently deletes associated datapoints in metrics that use that Subset as a dimension. +- This deletion: + - Typically affects all scenarios of those subset-dimensioned metrics. + - Is not reversed if the item is later re-selected in the subset. +- This is especially dangerous when: + - Subset membership is formula-driven. + - Users are entering manual inputs in subset-dimensioned metrics. + +Agent must: + +- Warn clearly that this behavior is structural and irreversible (without backup). +- Suggest a mitigation such as Pattern A (STORE/CALC) or storing on the parent list instead. + +--- + +### Warning 2 - Subset and parent are different dimensions (remap explicitly) + +- Parent list and subset are distinct dimensions in structures and formulas. A metric dimensioned by the subset is not interchangeable with one on the parent until you remap dimensions. +- For the common case - same list, natural 1:1 between subset item and parent item - use the native modifiers (no aggregator): + - TOPARENTLIST - expression on the subset -> same values on the parent (parent items not in the subset are blank). + - TOSUBSET - expression on the parent -> same values on the subset (parent rows outside the subset are dropped). +- When you need a custom mapping, several subsets feeding one block, or the compiler rejects the subset modifiers for your structure, use explicit mapping properties and `[BY: ...]` (see Patterns B and C below). +- The agent must not assume a subset-dimension metric can be used where a parent-dimension metric is expected without `TOPARENTLIST`, `TOSUBSET`, or an equivalent `BY` mapping. + +--- + +### Warning 3 - Operational overhead and governance + +Safe subset usage often requires: + +- Storage / helper metrics (e.g. STORE/CALC pattern). +- Manual or scheduled imports / actions. +- Board workflows (buttons, clear documentation). +- Owners who understand what happens when membership changes. + +Agent guidance: + +- Avoid proposing subsets if the client clearly lacks the appetite or governance to maintain: + - storage mechanics, + - periodic syncs, + - and clear change-control around membership. + +--- + +## Safe implementation patterns + +### Pattern A - Safe "formula-driven subset" without losing manual inputs + +Use when: + +- Subset membership logic may change over time, and +- There are (or will be) manual inputs in metrics dimensioned by the subset, and +- You need to avoid wiping historical inputs when membership changes. + +Goal: Decouple "membership logic" from the actual "subset membership storage" so: + +- Additions can be automated, +- Removals are deliberate and controlled. + +Steps (on the Parent list) + +1. Create two metrics on the Parent list: + - `CALC_Subset` (Boolean, formula) - determines if item *should* be in subset now. + - `STORE_Subset` (Boolean, manual/storage) - the metric actually linked to subset membership. +2. In `CALC_Subset`, implement logic and exclude stored members: + - Example structure: + + ``` + IF( + 'Parent'.'Some Property' = ..., + TRUE, + FALSE + ) + [EXCLUDE: 'STORE_Subset'] + ``` + + - Purpose: + - `CALC_Subset` flags new candidates based on logic. + - Existing stored members are preserved and not overwritten. +3. Create a Metric-to-Metric import: + - Source: `CALC_Subset` + - Target: `STORE_Subset` + - Critical: "Clear values before import" = OFF + - Expose this as: + - A button on a board, and/or + - A scheduled action. +4. Link the Subset membership to `STORE_Subset`: + - The subset uses `STORE_Subset` as its condition (e.g. formula referencing that metric). + +Behavior + +- New members: + - When logic says TRUE and `STORE_Subset` was FALSE, import sets them to TRUE -> added to subset. +- Existing members: + - Stay in the subset unless explicitly turned off in `STORE_Subset`. +- Removals: + - Are deliberate actions (manual untick or specific process) with visible consequences. + +When this pattern is not needed + +- If all metrics using the subset are fully formula-driven (no manual inputs and acceptable to recompute from scratch), you can: + - Drive membership directly from a formula metric, and + - Accept that membership changes may wipe data (which will be recomputed). + +--- + +### Pattern B - Restricted dropdown UX while storing on the parent dimension + +Use when: + +- You want a curated subset for user selections, but +- You want to store and maintain data on the parent dimension, not on the subset. + +Approach + +- Use subset as the selection dimension (e.g. dropdown in a table or form). +- Use a mapping to re-align chosen subset item back to the parent dimension where the data is stored. + +Implementation (high-level) + +1. Have a Parent list (e.g. `Supplier`) and a Subset (e.g. `Supplier_Subset`). + +2. When a user picks a value using the subset (e.g. `Selected Supplier (Subset)` metric dimensioned by `Supplier_Subset`), realign it back to the parent using the native remap TOPARENTLIST modifier: + + ``` + 'Selected Supplier on Parent' = + 'Selected Supplier (Subset)'[TOPARENTLIST: 'Supplier_Subset'] + ``` + + See [TOPARENTLIST and TOSUBSET](../writing-pigment-formulas/formula_modifiers.md#toparentlist-and-tosubset-list-subsets). + +Why this is safer + +- The UX benefit comes from the curated subset dropdown. +- Most persistent data lives on the parent list, which: + - Is less likely to have items removed, + - Avoids heavy dependence on subset membership for stored values. + +--- + +### Pattern C - Remap subset items back to parent list (general recipe) + +Use when: + +- You compute something at subset level but need it at parent level, or +- You combine several subsets into a parent-level report. + +Steps + +1. Use TOPARENTLIST when a single metric on the subset must appear on the parent with natural 1:1 item identity: + + ``` + 'Metric on Parent' = + 'Metric on Subset'[TOPARENTLIST: 'Subset'] + ``` + + See [TOPARENTLIST and TOSUBSET](../writing-pigment-formulas/formula_modifiers.md#toparentlist-and-tosubset-list-subsets). + +2. If different subsets map into the same parent, centralize mappings and avoid duplicating this structure in many places. + +Agent guidance + +- Prefer TOPARENTLIST / TOSUBSET for straight subset <-> parent remaps when the compiler allows them (see [formula_modifiers.md](../writing-pigment-formulas/formula_modifiers.md#toparentlist-and-tosubset-list-subsets)). +- For custom or multi-subset shapes, aim for reusable mapping metrics/properties rather than many one-off mapping formulas, and use `[BY: ...]` consistently. + +--- + +## Summary for the agent + +- Treat every subset as its own dimension shape with: + - Separate data storage, + - Non-reversible deletion behavior on membership changes, + - No implicit interchange with the parent dimension - use TOPARENTLIST, TOSUBSET, or mapping + BY when you need both shapes in formulas. +- Default patterns: + - For security or filtering -> use security/filters, not subsets. + - For small input lists with changing membership -> prefer a regular list. + - For mirror dimensions or targeted iterative performance -> subsets are often appropriate; remap subset <-> parent with TOPARENTLIST / TOSUBSET or explicit BY mapping as needed. + - For dropdown UX -> use subsets only as the selection dimension, store data on the parent. +- When proposing subsets, always: + - Surface the data loss warning, + - Explain the need for an explicit dimensional remap (TOPARENTLIST / TOSUBSET or mapping + BY), + - And, if manual inputs are involved, recommend Pattern A or an alternative design. diff --git a/plugins/pigment/skills/modeling-pigment-applications/modeling_time_and_calendars.md b/plugins/pigment/skills/modeling-pigment-applications/modeling_time_and_calendars.md new file mode 100644 index 00000000..e514fead --- /dev/null +++ b/plugins/pigment/skills/modeling-pigment-applications/modeling_time_and_calendars.md @@ -0,0 +1,465 @@ +# Time and Calendar Modeling + +Complete guide for calendar configuration and time dimension usage in Pigment. + +## When to Read This + +Read when: + +- Setting up application calendars +- Configuring fiscal years or time ranges +- Understanding time dimension hierarchy +- Working with time dimensions in metrics and formulas +- Using time-based functions (CUMULATE, LAG, YEARTODATE) + +--- + +## Part 1: Calendar Configuration + +### Overview + +Every Pigment application has a Calendar that defines the time structure for all time-based operations. The calendar determines how periods are organized, how fiscal years are structured, and which time dimensions are available for use in metrics and formulas. Use `tool:calendar_get` to read the current calendar configuration and `tool:calendar_create` to set up a new one. + +### Calendar Types + +Pigment supports two calendar types: + +Gregorian Calendar: + +- Standard calendar type used by most applications +- Based on Gregorian calendar system with configurable fiscal year settings +- Standard month-based periods (January through December) +- Configurable fiscal year starting month +- Supports all time dimensions: Day, Week, Month, Quarter, Half, Year +- Most common choice for financial planning and reporting + +Weekly Calendar: + +- Calendar type organized around weeks rather than months +- Week-based periods +- Configurable first day of week (Sunday, Monday, etc.) +- Supports Week and Day time dimensions +- Less common, used for specific operational planning + +When to use: + +- Gregorian: Standard financial planning, month/quarter/year reporting, most business scenarios +- Weekly: Operations planning requiring week-level granularity, retail/operations with weekly cycles + +### Fiscal Year Configuration + +For Gregorian calendars, you can configure when the fiscal year starts. This is critical for financial planning applications. + +Common Fiscal Year Starts: + +- January (Calendar Year): Fiscal year aligns with calendar year +- April: Common in many countries (e.g., UK, Japan) +- July: Common in some industries +- October: US Federal fiscal year + +Impact: + +- Affects how `TIMEDIM(..., 'Year')` returns fiscal year items +- Affects `YEARTODATE` calculations (resets at fiscal year start) +- Affects quarter boundaries (Q1 starts at fiscal year start month) + +Example (Fiscal year starting in April): + +- FY 2026 = April 2025 to March 2026 +- Q1 FY 2026 = April, May, June 2025 +- Q2 FY 2026 = July, August, September 2025 + +Setting Fiscal Year Start: +Configure the fiscal year starting month when creating or editing the calendar. Use `tool:calendar_create` to set this when creating a new calendar. + +Best Practice: Set the fiscal year start early in application setup. Changing it later can affect existing formulas and data. + +### Calendar Date Ranges + +Every calendar has a start date and end date that define the range of periods available. + +Start Date: + +- Earliest period available in the calendar +- Typically set to cover historical data needs +- Common: 2-3 years before current year + +End Date: + +- Latest period available in the calendar +- Typically set to cover planning horizon +- Common: 3-5 years after current year + +Example: + +- Start Date: January 1, 2020 +- End Date: December 31, 2027 +- Covers: 8 years of periods + +Extending Calendars: +You can extend a calendar's date range without affecting existing data. + +When to extend: + +- Planning horizon needs to go further into the future +- Historical analysis requires earlier periods +- Application scope expands + +How to extend: + +- Use `tool:calendar_expand` to update Start Date or End Date +- Existing data remains unchanged + +Important: Extending forward is safe. Extending backward may create new periods, but existing data is preserved. You cannot shorten the calendar date range (remove periods) if data exists in those periods. + +### Distinguishing Actuals from Plan periods + +To distinguish historical Actuals from future Plan periods in a planning model, use the Version Dimension Switchover pattern documented in [`../planning-cycles-pigment-applications/SKILL.md`](../planning-cycles-pigment-applications/SKILL.md) (skill: `planning-cycles-pigment-applications`). The pattern uses a Switchover Month (or Year) Property on the Version Dimension plus the IsVersion / IsActual / IsPlan Boolean Metrics to layer Actuals and plan data per Version. Calendars do not handle this; do not use Calendar tools for versioning. + +### Time Dimensions Selection + +When setting up a calendar, you select which time dimensions to include: + +Standard Time Dimensions: + +- Year: Annual periods +- Half: Semi-annual periods (H1, H2) +- Quarter: Quarterly periods (Q1, Q2, Q3, Q4) +- Month: Monthly periods (required for Gregorian calendars) +- Week: Weekly periods (optional) +- Day: Daily periods (optional) + +Best Practice: Include only the time dimensions you need. More dimensions mean more complexity and potential performance impact. Use `tool:calendar_add_time_dimension` to add a dimension and `tool:calendar_remove_time_dimension` to remove one. + +Extra Time Dimensions: + +In addition to standard time dimensions, calendars can include extra time dimensions: + +- DayOfWeek: Monday, Tuesday, etc. +- MonthOfYear: January, February, etc. (regardless of fiscal year) +- QuarterOfYear: Q1, Q2, Q3, Q4 (regardless of fiscal year) +- WeekOfYear: Week 1-52/53 +- HalfOfYear: H1, H2 (regardless of fiscal year) + +Use Cases: + +- Seasonal analysis (MonthOfYear for seasonality) +- Day-of-week patterns (DayOfWeek for operations) +- Week-based reporting (WeekOfYear) + +### Calendar Setup Best Practices + +1. Set Fiscal Year Early: +Configure the fiscal year starting month during initial application setup. Changing it later can require formula updates. + +2. Plan Date Range Carefully: +Set start and end dates to cover historical data needs and planning horizon with buffer for future extensions. Common Pattern: 3 years historical + 5 years forward = 8-year range. + +3. Include Only Needed Dimensions: +Select only the time dimensions required for your use case: + +- Financial planning: Year, Quarter, Month (most common) +- Operations planning: May need Week or Day +- Strategic planning: Year, Half, Quarter + +4. Reuse Calendar Dimensions: +Always use the built-in calendar dimensions provided by your application's calendar. Do not create custom time dimensions. Calendar dimensions are optimized and consistent across the application. + +5. Actuals vs Plan separation: +To distinguish historical Actuals from Plan periods, use the Version Dimension Switchover pattern (see `skill:planning-cycles-pigment-applications`). Do not rely on the calendar's Actual vs Forecast toggle for this purpose. + +6. Document Calendar Settings: +Document your calendar configuration: Fiscal year start month, date range, selected time dimensions, Actual vs Forecast settings. + +### Common Setup Patterns + +Standard Financial Planning: + +- Type: Gregorian +- Fiscal Year: January (calendar year) or April/July/October +- Dimensions: Year, Quarter, Month +- Date Range: 3 years historical + 5 years forward + +Operations Planning: + +- Type: Gregorian or Weekly +- Dimensions: Month, Week (or Week, Day for weekly) +- Date Range: 1 year historical + 2 years forward +- Extra Dimensions: DayOfWeek (if needed) + +Strategic Planning: + +- Type: Gregorian +- Dimensions: Year, Half, Quarter +- Date Range: 5 years historical + 10 years forward + +--- + +## Part 2: Time Dimensions and Hierarchy + +### Overview + +Time dimensions are the building blocks of temporal analysis in Pigment. They represent different levels of time granularity (Year, Quarter, Month, Week, Day) and are organized in a hierarchical structure. + +### Time Dimension Hierarchy + +Time dimensions form a hierarchical structure where each level contains the levels below it: + +``` +Year + --- Half (optional) + --- Quarter + --- Month + --- Week (optional, may overlap months) + --- Day +``` + +Hierarchy Levels: + +Year: + +- Top level of the hierarchy +- Contains all periods in a fiscal or calendar year +- Used for annual reporting and strategic planning + +Half: + +- Semi-annual periods (H1, H2) +- Contains Quarters +- Used for mid-year reporting + +Quarter: + +- Quarterly periods (Q1, Q2, Q3, Q4) +- Contains Months +- Most common for financial reporting + +Month: + +- Monthly periods (January, February, etc.) +- Contains Weeks (in Gregorian calendars) +- Standard granularity for most planning + +Week: + +- Weekly periods +- Contains Days +- Used for operations planning + +Day: + +- Daily periods +- Lowest level of granularity +- Used for detailed operational analysis + +### Parent-Child Relationships + +Each time dimension has a parent-child relationship: + +- Parent: Contains the child dimension (e.g., Year contains Quarters) +- Child: Belongs to the parent dimension (e.g., Quarter belongs to a Year) + +Example: + +- Q1 2026 is a child of Year 2026 +- Q1 2026 contains Months: Jan 2026, Feb 2026, Mar 2026 +- Jan 2026 contains Weeks (if Week dimension is enabled) + +### Standard Time Dimensions + +Year Dimension: + +- Purpose: Annual periods for strategic planning and annual reporting +- Top level of hierarchy +- Defined by fiscal year start (if fiscal year differs from calendar year) +- Contains Half, Quarter, Month, Week, Day dimensions +- Example Items: FY 2026, Calendar Year 2026 +- Use Cases: Annual budgets/forecasts, year-over-year comparisons, strategic planning + +Half Dimension: + +- Purpose: Semi-annual periods for mid-year reporting +- Contains two halves per year (H1, H2) +- Optional dimension (not required) +- Example Items: H1 FY 2026, H2 FY 2026 +- Use Cases: Mid-year reviews, semi-annual reporting, strategic planning + +Quarter Dimension: + +- Purpose: Quarterly periods for financial reporting and planning +- Contains four quarters per year (Q1, Q2, Q3, Q4) +- Most common time dimension for financial planning +- Example Items: Q1 FY 2026, Q2 FY 2026 +- Use Cases: Quarterly financial reporting, quarterly planning cycles, QoQ analysis + +Month Dimension: + +- Purpose: Monthly periods for detailed planning and reporting +- Contains 12 months per year +- Required dimension for Gregorian calendars +- Example Items: Jan 2026, Feb 2026, Mar 2026 +- Use Cases: Monthly planning/budgeting, monthly reporting, month-over-month analysis + +Week Dimension: + +- Purpose: Weekly periods for operations planning +- Contains approximately 52-53 weeks per year +- Optional dimension +- More common in operations planning than financial planning +- Example Items: Week 1 2026, Week 2 2026 +- Use Cases: Operations planning, retail planning, weekly capacity planning + +Day Dimension: + +- Purpose: Daily periods for detailed operational analysis +- Contains 365-366 days per year +- Lowest level of granularity, optional dimension +- Example Items: 2026-01-01, 2026-01-02 +- Use Cases: Daily operations tracking, detailed transaction analysis, day-level reporting + +### Extra Time Dimensions + +Extra time dimensions provide additional ways to analyze time patterns without being part of the main hierarchy. + +DayOfWeek: + +- Not hierarchical, repeats across all weeks +- Items: Monday, Tuesday, Wednesday, Thursday, Friday, Saturday, Sunday +- Use Cases: Identifying weekday vs weekend patterns, operations planning, retail analysis + +MonthOfYear: + +- Not hierarchical, repeats across all years +- Items: January, February, March, etc. +- Use Cases: Seasonal pattern analysis, month-over-month comparisons across years + +QuarterOfYear: + +- Not hierarchical, repeats across all years +- Items: Q1, Q2, Q3, Q4 +- Use Cases: Quarterly pattern analysis, quarter-over-quarter comparisons across years + +WeekOfYear: + +- Not hierarchical, repeats across all years +- Items: Week 1, Week 2, ..., Week 52, Week 53 +- Use Cases: Week-based pattern analysis, week-over-week comparisons across years, operations planning + +HalfOfYear: + +- Not hierarchical, repeats across all years +- Items: H1, H2 +- Use Cases: Semi-annual pattern analysis, half-over-half comparisons across years + +### Time Dimension Relationships + +Hierarchical Relationships: +Year -> Half -> Quarter -> Month -> Week -> Day + +Example: +Year 2026 contains: + +- H1 2026 and H2 2026 +- Q1 2026, Q2 2026, Q3 2026, Q4 2026 +- Jan 2026, Feb 2026, ..., Dec 2026 +- Week 1 2026, Week 2 2026, ..., Week 52 2026 +- 2026-01-01, 2026-01-02, ..., 2026-12-31 + +Aggregation Relationships: + +- Aggregating up: Summing child periods into parent periods + - Example: Summing months to get quarterly totals + - Example: Summing quarters to get annual totals + +- Aggregating down: Allocating parent periods to child periods + - Example: Distributing annual budget across months + - Example: Allocating quarterly targets to monthly targets + +Dimension Alignment: + +When using time dimensions in metrics: + +- Granularity: Choose the most granular time dimension needed +- Aggregation: Use aggregation functions to roll up to higher levels +- Allocation: Use allocation functions to distribute to lower levels + +Example: Metric with `Month` dimension can be aggregated to `Quarter` or `Year`. Metric with `Quarter` dimension can be allocated to `Month`. + +### Using Time Dimensions in Metrics + +Choosing Time Granularity: + +Select the most granular time dimension needed for your use case: + +- Strategic Planning: Year or Quarter +- Financial Planning: Month or Quarter +- Operations Planning: Week or Day + +Best Practice: Start with the granularity you need. You can always aggregate up, but you cannot create detail that doesn't exist. + +Time Dimension in Metric Structure: + +Time dimensions are used like any other dimension in metric structures: + +Example Structures: + +- `Product x Month` - Monthly product sales +- `Product x Quarter` - Quarterly product sales +- `Product x Year` - Annual product sales +- `Product x Month x Region` - Monthly product sales by region + +Aggregation Across Time: + +Use aggregation functions to roll up time periods: + +- SUM: Sum values across time periods +- AVERAGE: Average values across time periods +- MAX/MIN: Find maximum/minimum across time periods + +Example: Sum monthly sales to get quarterly totals, average monthly values to get quarterly averages. + +### Best Practices + +1. Use Calendar Dimensions: +Always use the built-in calendar dimensions from your application's calendar. Do not create custom time dimensions. Calendar dimensions are optimized, consistent, and work seamlessly with time functions. + +2. Choose Appropriate Granularity: +Select the most granular time dimension needed. Too granular creates unnecessary complexity and performance impact. Too coarse means missing detail needed for analysis. + +3. Understand Hierarchy: +Understand how time dimensions relate hierarchically to support aggregation, allocation, time functions, and dimensional alignment. + +4. Consider Extra Dimensions: +Use extra time dimensions (DayOfWeek, MonthOfYear) for pattern analysis: Seasonal patterns, day-of-week patterns, week-based patterns. + +5. Document Time Structure: +Document which time dimensions are used in your application to help team members understand the time structure and support formula writing. + +### Troubleshooting + +Calendar Not Available: + +- Check Application Settings -> Calendar +- Ensure calendar is configured +- Verify time dimensions are selected + +Fiscal Year Not Working as Expected: + +- Verify fiscal year starting month is set correctly +- Check that formulas use calendar dimensions (not custom dimensions) +- Ensure TIMEDIM uses the correct time dimension + +Date Range Too Short: + +- Extend calendar date range in Settings -> Calendar +- Update Start Date or End Date as needed +- Existing data is preserved when extending + +--- + +## See Also + +- [modeling_fundamentals.md](./modeling_fundamentals.md) - Core modeling concepts +- [modeling_principles.md](./modeling_principles.md) - Best practices and standards +- [functions_time_and_date.md](../writing-pigment-formulas/functions_time_and_date.md) - Time-based functions +- [functions_lookup.md](../writing-pigment-formulas/functions_lookup.md) - TIMEDIM function diff --git a/plugins/pigment/skills/modeling-pigment-applications/modeling_working_with_folders.md b/plugins/pigment/skills/modeling-pigment-applications/modeling_working_with_folders.md new file mode 100644 index 00000000..87344794 --- /dev/null +++ b/plugins/pigment/skills/modeling-pigment-applications/modeling_working_with_folders.md @@ -0,0 +1,137 @@ +# Working with Folders + +When building Applications, there are certain folders and structures we use in almost every use case Application. Typical exceptions are a data hub and specific Admin applications. + +This folder structure allows us to order the Blocks intentionally, separate out the Blocks that serve important functions, and maintain flexibility to add new Blocks as the Application scope and complexity evolves. + +Folder naming conventions (numeric prefixes, order): see [Naming Conventions - Folders](./modeling_naming_conventions.md#folders). + +### System folders in every application + +Every Pigment application has a few system folders that always exist: + +- No folder - A default placeholder for blocks that are not assigned to any folder. Do not create new blocks in "No folder"; always assign them to an explicit folder (see [Placing new blocks](#placing-new-blocks)). Blocks left there are hard to find and clutter the app. +- Security - Exists by default in every application. It cannot be renamed. You can add blocks (e.g. AR metrics, mapping metrics) and create subfolders inside it. Do not create a separate folder named "Security" in the Blocks tree-use this one. +- Calendar - Created automatically by the application when you create a Calendar. It contains the time dimension blocks (e.g. Month, Quarter, Year) defined by that calendar. This folder can be renamed (e.g. add a numeric prefix like `0. Calendar` to order it at the top). + +Example folder structure: + +``` +- Boards + - 0 + - All Boards +- Blocks + - All Blocks + - > 0. Settings + - > 1. Dimensions + - > 2. Library + - > 10. Data Loads + - > 11. Data Checks + - > 20. [Themed] Data + - > 21. [Themed] Assumptions + - > 22. [Themed] Forecast/Results +``` + +Boards vs Blocks: Boards have their own folder structure under the application's Boards section. They are not placed in the Blocks area. Do not create a block folder named "Board" or "Boards"-boards are organized in their own hierarchy (e.g. `0. Admin`, `10. Data`, `20. Planning`). Use the naming conventions for board folders (see [Naming Conventions - Boards](./modeling_naming_conventions.md#boards)). + +Flexibility in folder structure + +Do not treat the structure above as rigid. Existing applications may use different patterns-for example a functional structure (e.g. `2.1 Process step 1`, `2.2 Process step 2`) or theme-based numbering. When working in an app, observe the existing folder names and hierarchy and align with them. If the app has no clear structure, propose one following the patterns below; otherwise place new blocks in the folder that best matches their purpose. + +## The "0X" Folders (Settings & Setup) + +These appear at the top of the application and contain frequently referenced blocks. + +- 0. Settings: Contains all blocks used for settings in the application-any blocks created to configure/tailor the application or for technical purposes. Examples include mapping metrics, variable metrics (e.g., "Load Month"), and configuration blocks. + +- 1. Dimensions: Contains all dimensions that are specific to this application. Having this dimension folder appear at the top makes it convenient to quickly find all of your application-specific dimensions as you're modeling. + +- 2. Library: Used to visualize the inflows (Pull) and outflows (Push) metrics between your application and others in the workspace. It usually contains both Push and Pull metrics: + - Push metrics: Sanitized versions of end result metrics that need to be shared with other applications + - Pull metrics: Receiving blocks from other applications' Push metrics + - Name them with prefixes as per [Naming Conventions - Metrics](./modeling_naming_conventions.md#metrics) (e.g. `PUSH_`, `PULL_`). + +- 3. Assumptions: Dedicated folder for model assumptions (e.g., conversion rates, growth percentages, allocation factors). If a model requires more than a few assumptions, it can be clearer to keep them with the "themed" folders (see below), but if you have fewer assumptions, an assumptions-specific folder helps keep them accessible. + +## Data Folders (1X Range) + +In the 1X range, create folders for data loads and checks to help consolidate and validate the data being used in your application. + +- 10. Data Loads: All transaction lists containing data coming from external sources, as well as the metrics used to aggregate data from those transaction lists (staging metrics). + +- 11. Data Checks: Metrics used to check the quality of the data, fill missing mappings, run number checks, and display validation results on boards. + +## Themed Folders (2X Range and Beyond) + +In the 2X range and beyond, create themed folders based on the calculation step or business process of the model. + +Example patterns: + +Pattern 1: Data -> Assumptions -> Results +- 20. [Theme] Data: Metrics aggregating relevant data from transaction lists +- 21. [Theme] Assumptions: Metrics used to input or calculate assumptions +- 22. [Theme] Forecast/Results: Forecast or calculation result metrics + +Pattern 2: Load -> Data -> Assumptions -> Results +- 20. [Theme] Load: Transaction lists for this theme +- 21. [Theme] Data: Aggregated data metrics +- 22. [Theme] Assumptions: Assumption metrics +- 23. [Theme] Forecast/Results: Result metrics + +Pattern 3: Combined Load & Data +- 30. [Theme] Load & Data: Transaction lists and aggregated metrics together +- 31. [Theme] Assumptions: Assumption metrics +- 32. [Theme] Forecast/Results: Result metrics + +Example applications: + +| Supply Chain Planning | Sales Operations | Product Management | +| :------------------- | :--------------- | :----------------- | +| > 0. Settings | > 0. Settings | > 0. Settings | +| > 1. Dimensions | > 1. Dimensions | > 1. Dimensions | +| > 2. Library | > 2. Library | > 2. Library | +| > 10. Data Loads | > 10. Data Loads | > 10. Data Loads | +| > 11. Data Checks | > 11. Data Checks| > 11. Data Checks | +| > 20. Inventory Data | > 20. Pipeline Data | > 20. Feature Data | +| > 21. Inventory Assumptions | > 21. Pipeline Assumptions | > 21. Feature Assumptions | +| > 22. Inventory Forecast | > 22. Pipeline Forecast | > 22. Feature Adoption | + +By structuring folders in this way, all frequently referenced blocks are easily visible at the top (the 0X folders), and the rest of the blocks are organized thematically in an easy-to-navigate way. + +## Subfolders + +Creating subfolders is useful when a folder would otherwise contain too many blocks or when you need to segment blocks by sub-purpose (e.g. by process step, by data source, or by report type). Use a numeric prefix with a dot between the main and secondary level: `main.secondary` (e.g. `2.1 Process step 1`, `2.2 Process step 2`, `10.1 ERP Loads`, `10.2 CRM Loads`). Numbering restarts at each level (see [Naming Conventions - Folders](./modeling_naming_conventions.md#folders)). Subfolders keep the block list scannable and make it easier to find the right place for new blocks. + +## Reorganizing folders and blocks + +You can rename folders and move blocks between folders or subfolders to improve organization. If you see folders that are poorly named, inconsistent, or overloaded, you can propose: + +- Adding subfolders - Split a crowded folder into subfolders (e.g. `10. Data Loads` -> `10.1 ERP`, `10.2 CRM`) and move blocks into the appropriate subfolder. +- Renaming folders - Suggest clearer or more consistent names (e.g. align with naming conventions, fix typos, or match the app's theme). +- Moving blocks - Move blocks (metrics, lists, tables) into the folder or subfolder that best matches their type and purpose (use the [Placing new blocks](#placing-new-blocks) mapping). Moving a block does not change its formula or content; it only changes where it appears in the folder tree. + +When proposing reorganization, respect the application's existing pattern where one exists; otherwise suggest a structure that follows the conventions (numeric prefixes, logical grouping) and offer to rename or move as needed (the user or the UI can perform renames and moves). + +## Placing new blocks + +Rule: Never create a block in "No Folder". Always create (or move) every new block into an explicit folder. + +Before creating any block: + +1. List the existing Blocks folders (and subfolders) in the application. +2. Choose the folder that best matches the block's type and purpose, using the mapping below. If the application uses a different structure (e.g. functional steps), pick the folder that corresponds to the block's role in that structure. +3. If no folder fits (e.g. a new theme), create a new folder following the naming conventions and the app's existing pattern, then create the block inside it. + +Block type and purpose -> suggested folder: + +| Block type / purpose | Suggested folder (adapt to existing app structure) | +| -------------------- | -------------------------------------------------- | +| Dimension (list) | Folder that groups dimensions (often named like "1. Dimensions" or similar). | +| Transaction list | Data loads folder (e.g. "10. Data Loads") or themed "Load" folder for the relevant process. | +| Metric - settings, mappings, config (SET_, MAP_, VAR_, ADM_) | Settings-type folder (e.g. "0. Settings"). | +| Metric - shared out (PUSH_) | Library folder (e.g. "2. Library"). | +| Metric - pulled from other app (PULL_) | Library folder (e.g. "2. Library"). | +| Metric - staging from transaction lists (DATA_) | Data loads folder or themed "Data" folder. | +| Metric - assumptions (ASM_) | Dedicated assumptions folder (e.g. "3. Assumptions") or themed "Assumptions" folder. | +| Metric - inputs, calcs, outputs (INPUT_, CALC_, OUTPUT_, RES_, etc.) | Themed folder that matches the process (e.g. "21. [Theme] Assumptions", "22. [Theme] Forecast/Results"). If no theme applies, use the folder that is closest in purpose (Data, Assumptions, or Results). | +| Table | Themed folder that matches the table's use (input, reporting, process step), or a dedicated tables folder if the app has one. | diff --git a/plugins/pigment/skills/optimizing-pigment-performance/SKILL.md b/plugins/pigment/skills/optimizing-pigment-performance/SKILL.md new file mode 100644 index 00000000..829c5db3 --- /dev/null +++ b/plugins/pigment/skills/optimizing-pigment-performance/SKILL.md @@ -0,0 +1,144 @@ +--- +name: optimizing-pigment-performance +description: Always use this skill when troubleshooting slow calculations or timeouts, analyzing profiler output to identify bottlenecks, understanding scope propagation, managing sparsity, optimizing formula performance, improving iterative calculations, optimizing access rights performance, conducting systematic performance audits, auditing a Pigment application (modeling, formula hygiene, folders, boards, governance), cleaning unused dimensions, metrics, tables, properties, or boards, identifying dead or stale boards, or removing unused metrics. Modeler-agent skill for Performance Insights tools (performance_profile_change, get_top_blocks_by_performance), then classify and fix. Provides the optimization loop, audit vs cleaning modes, and routing to deep dives. Always profile before formula changes; never optimize from assumptions. +metadata: + skill_path: /optimizing-pigment-performance/SKILL.md + base_directory: /optimizing-pigment-performance + includes: + - "*.md" +--- + +# Optimizing Pigment Performance + +Profiler-driven performance optimization, sparsity-aware patterns, and application audit/cleaning for the modeler agent. Read this overview first, then the matching deep dive. + +## When to Use This Skill + +- Slow calculations, timeouts, or profiler analysis +- Scope loss, densification, iterative horizons, AR overhead, calendar iteration +- Systematic performance audit on an application +- App audit (modeling, formulas, folders, boards, governance) with severity-tagged findings +- Cleaning unused dimensions, metrics, tables, properties, or boards (deletion only) +- Identifying dead or stale boards, removing unused metrics +- Board loads slowly but profiler shows fast metric compute (rendering vs compute) +- Planning cycle grew many scenarios/versions and inputs feel much slower + +--- + +## Performance Mental Model + +Every optimization follows the same loop. Skipping the profile step is the most common failure mode. + +1. Profile (mandatory). Call `tool:get_top_blocks_by_performance` and/or `tool:performance_profile_change` (see below). No optimization without tool output. +2. Classify the bottleneck. Scope loss, sparsity densification, iterative horizon, AR overhead, calendar iteration, formula shape, or board-render overhead (widget count, heavy views). +3. Apply the right pattern. Scope-first, `BY` over `ADD`, `ISDEFINED` over `ISBLANK`, etc. +4. Re-profile, compare, document the gain. + +### Core Principles + +1. Scope first. Start formulas with scoping clauses (`FILTER`, `EXCLUDE`, `IFDEFINED`). +2. Preserve sparsity. Use `ISDEFINED` instead of `ISBLANK`. Use `BLANK` instead of `0` or `FALSE`. +3. Filter early, defer aggregations. Apply `FILTER`/`EXCLUDE` before computation; push `REMOVE` to the end of the chain. +4. Profile systematically. Measure before and after every change. +5. Understand scope propagation. Know when scope is lost (`REMOVE`, `CUMULATE`, AR). + +### Performance profiling tools (modeler agent) + +Requires Performance Insights (`use_performance_tools`). Profiling tools and output parsing: [./performance_profiling.md](./performance_profiling.md). + +| Tool | When to use | Input highlights | +|---|---|---| +| `tool:performance_profile_change` | A specific slow user action is known; you have (or can get) its `change_id` from audit trail | `change_id` (UUID). Returns execution chain with duration, scope, dependencies. | +| `tool:get_top_blocks_by_performance` | Exploratory app-wide triage: which blocks cost the most over a period | `scenario_id`, `criteria` (`ExecutionCount`, `ExecutionTimeAvgMs`, `ExecutionTimeSumMs`, `CombinedCardinality`), `range_start`, `range_end`, `top_n`. Returns ranked blocks with cardinality and job stats. | + +Workflow: Start with `get_top_blocks_by_performance` when the hotspot block is unknown. Use `performance_profile_change` after reproducing a slow change to analyze scope propagation and the execution chain for that change. + +--- + +## Audit vs Cleaning (Application Hygiene) + +Two modes; never mix in the same pass. + +| Mode | Purpose | Output | Never | +|---|---|---|---| +| Audit | Diagnostic, non-destructive | Findings with HIGH / MEDIUM / LOW + proposed fixes | Deletes, renames, refactors | +| Cleaning | Deletion only | Removes unused objects in strict order | Formula edits, renames, folder moves | + +### Severity (canonical) + +| Severity | Meaning | +|---|---| +| HIGH | Breaks critical rules, blocks T&D, or data-loss risk | +| MEDIUM | Performance, maintainability, or governance harm | +| LOW | Cosmetic or minor hygiene | + +### Deletion Order (canonical) + +Cleaning uses two independent axes executed in this order: + +1. DEAD boards first. Classify boards (ACTIVE / STALE / DEAD) from usage analytics. Delete DEAD boards (tag, notify, contestation window, delete). STALE boards are not deleted automatically but signal future cleanup. +2. Recompute structural usage after board deletion. +3. Structural objects in order: Dimensions, Metrics, Tables, Properties. Hide, observe, delete. Recompute usage after each pass; iterate until no new candidates. + +System truth (settings, dependency graph, usage analytics) defines "unused", not agent judgment. Always validate deletions with the user before irreversible removal. + +### Scenario Cardinality + +Version and scenario proliferation multiplies work per input (roughly linear in active scenario count, often felt as much worse when scope and AR compound). When an application grows from 3 to 12 scenarios, expect at least ~4x more computation and plan for higher perceived slowdown. Audit scenario cardinality as a structural performance factor; recommend subsetting inactive scenarios or archiving historical versions. See also `skill:modeling-pigment-applications` for version/scenario architecture. + +| Need | Doc | +|---|---| +| Full app audit (modeling, UX, governance, cleanup candidates) | [./performance_auditing_application.md](./performance_auditing_application.md) | +| Deletion workflow, unused definitions, board usage rules | [./performance_cleaning_application.md](./performance_cleaning_application.md) | + +--- + +## Bottleneck Routing + +| Signal | Read | +|---|---| +| Profiling (tools, parse output, report) | [./performance_profiling.md](./performance_profiling.md) | +| Rank app hotspots over a time window | `tool:get_top_blocks_by_performance` (see tools table above) | +| Scope loss after `REMOVE`, `CUMULATE`, AR | [./performance_scoping_patterns.md](./performance_scoping_patterns.md) | +| Unexpected metric size, `ISBLANK` / `ISNOTBLANK` | [./performance_sparsity_deep_dive.md](./performance_sparsity_deep_dive.md) | +| Formula shape (`IF` vs `FILTER`, `BY` vs `ADD`) | [./performance_formula_optimization.md](./performance_formula_optimization.md) | +| `PREVIOUS`, `PREVIOUSOF`, `CUMULATE` horizons, calendar iteration | [./performance_iterative_calculations.md](./performance_iterative_calculations.md) | +| AR-heavy formulas, `ISDEFINED(User)` wrapper | [./performance_access_rights.md](./performance_access_rights.md) | +| Board slow to load, profiler shows fast compute | [./performance_troubleshooting_workflow.md](./performance_troubleshooting_workflow.md) (board-render fork); `skill:designing-pigment-boards` | +| Many scenarios/versions, inputs much slower | [./performance_scoping_patterns.md](./performance_scoping_patterns.md) (scenario cardinality); `skill:modeling-pigment-applications` | +| Where to start a systematic audit | [./performance_troubleshooting_workflow.md](./performance_troubleshooting_workflow.md) | + +--- + +## Quick Reference: Common Anti-Patterns + +For detailed patterns with examples, see [./performance_formula_optimization.md](./performance_formula_optimization.md) and [./performance_sparsity_deep_dive.md](./performance_sparsity_deep_dive.md). + +| Anti-Pattern | Fix | +|---|---| +| `ISBLANK` / `ISNOTBLANK` for sparsity gates | `ISDEFINED` / `IFDEFINED` / `IFBLANK` / `EXCLUDE`, or `BY` on dimension-typed metric | +| No scoping at formula start | Add `FILTER` or `EXCLUDE` first | +| Early `REMOVE` in chain | Defer `REMOVE` to end; use `BY` with mappings | +| Long dense horizons in `PREVIOUS` | Subset time dimension | +| AR formula without `ISDEFINED(User)` guard | Wrap in `IFDEFINED(User, ...)` | +| Fast profiler, slow board load | Check widget count, view filters, displayed volume; see troubleshooting workflow | +| Many active scenarios/versions | Subset inactive scenarios; archive historical versions | + +--- + +## Cross-References + +- modeling-pigment-applications: dimensional design, principles, folders, version/scenario architecture +- writing-pigment-formulas: syntax, modifiers, functions +- securing-pigment-applications: AR patterns and AR metric construction +- designing-pigment-boards: board design (audit section) + +--- + +## Critical Rules + +- Always profile with performance tools first before formula changes (or ask user for `change_id` if tools disabled). +- Audit is diagnostic; cleaning is deletion only. +- DEAD boards first, then structural objects in order (Dimensions, Metrics, Tables, Properties). Recompute usage between passes. +- Validate cleanup with the user before irreversible deletions. +- ISDEFINED over ISBLANK; scope early; document profiler findings. diff --git a/plugins/pigment/skills/optimizing-pigment-performance/performance_access_rights.md b/plugins/pigment/skills/optimizing-pigment-performance/performance_access_rights.md new file mode 100644 index 00000000..24ef9267 --- /dev/null +++ b/plugins/pigment/skills/optimizing-pigment-performance/performance_access_rights.md @@ -0,0 +1,115 @@ +# Performance Access Rights + +How access rights affect computation performance and the `ISDEFINED(User)` wrapper pattern. For AR metric construction, rule design, and governance patterns, see `skill:securing-pigment-applications`. + +--- + +## How AR Affects Performance + +Basic flow: User requests data, system retrieves metric data, applies AR rules (most costly step), filters cells, returns visible data. + +High impact scenarios: + +- Many metrics with AR, each requiring AR evaluation +- Complex multi-dimensional AR rules +- Large user base (AR computed for many users) +- Dimension-heavy AR on high-cardinality dimensions + +Low impact scenarios: few metrics with AR, simple user-only rules, small user base, static AR. + +--- + +## The ISDEFINED(User) Pattern + +### The Problem + +Without `ISDEFINED(User)`, AR is computed for all users in the workspace, including users without application access. + +```pigment +// Anti-pattern: computes AR for all workspace users +'Revenue with AR' = 'Revenue'[AR: 'Revenue AR Rules'] +``` + +If the workspace has 500 users but only 50 have app access, 90% of computation is wasted. + +### The Solution + +```pigment +'Revenue with AR' = + IFDEFINED(User, + 'Revenue'[AR: 'Revenue AR Rules'] + ) +``` + +`IFDEFINED(User, ...)` checks if the current user has application access. AR computation only runs for relevant users. + +Always use this pattern when a metric has AR applied and the application has a subset of total workspace users. + +--- + +## AR Optimization Patterns + +### Scope Before AR + +Filter and aggregate before applying AR to reduce the dataset size for AR computation: + +```pigment +// Anti-pattern: AR on all data, then filter +'Result' = 'Revenue'[AR: 'AR Rules'][FILTER: 'Product'.'Active' = TRUE] + +// Optimized: filter first, then AR +'Result' = 'Revenue'[FILTER: 'Product'.'Active' = TRUE][AR: 'AR Rules'] +``` + +### Apply AR Once + +Do not apply AR at every step of the computation chain. Apply it once at the end: + +```pigment +// Anti-pattern: AR at every step +'Step 1' = 'A'[AR: 'Rules'] +'Step 2' = 'Step 1' + 'B'[AR: 'Rules'] + +// Optimized: AR once at end +'Step 1' = 'A' +'Step 2' = 'Step 1' + 'B' +'Final' = 'Step 2'[AR: 'Rules'] +``` + +### Apply AR Consistently (Not Selectively) + +Applying AR selectively to "save performance" does not significantly improve performance. The expensive operation is the AR join, which happens regardless of how many metrics have AR. Applying AR to all metrics on a dimension enables row-level filtering, which is faster than cell-level filtering. + +### Aggregate Before AR + +```pigment +// Anti-pattern: AR at transaction level, then aggregate +'Customer Total' = 'Transaction Amount'[AR: 'Transaction AR'][BY: 'Transaction'.'Customer'] + +// Optimized: aggregate first, then AR (if AR rules are at Customer level) +'Customer Total' = 'Transaction Amount'[BY: 'Transaction'.'Customer'][AR: 'Customer AR'] +``` + +--- + +## AR and Consolidation + +Access rights are applied after consolidation, not before. Consolidation performance is not affected by AR. + +--- + +## Best Practices Summary + +1. Always wrap AR in `IFDEFINED(User, ...)` to skip computation for users without app access. +2. Apply AR consistently across all metrics on a dimension (enables row-level filtering). +3. Apply AR once at the end of the computation chain, not at every step. +4. Scope before AR -- filter and aggregate before applying AR. +5. Keep AR rules simple -- prefer user-only rules when possible. For AR rule design, see `skill:securing-pigment-applications`. + +--- + +## See Also + +- [Performance Formula Optimization](./performance_formula_optimization.md) - General formula optimization including AR +- [Performance Sparsity Deep Dive](./performance_sparsity_deep_dive.md) - ISDEFINED patterns +- `skill:securing-pigment-applications` - AR rule design, governance, and construction patterns diff --git a/plugins/pigment/skills/optimizing-pigment-performance/performance_auditing_application.md b/plugins/pigment/skills/optimizing-pigment-performance/performance_auditing_application.md new file mode 100644 index 00000000..3526e1d0 --- /dev/null +++ b/plugins/pigment/skills/optimizing-pigment-performance/performance_auditing_application.md @@ -0,0 +1,218 @@ +# Audit a Pigment App (Modeling, UX & Governance) + +Purpose: Run a structured audit of a Pigment app to find issues in modeling, formulas, metric hygiene, folder structure, boards, governance, and cleanup-and to propose actionable improvements with severity. Use other skills for deep-dive checks (formulas, performance, boards); this doc defines scope, heuristics, output format, and delegation. + +Scope: Models, metrics, folders, boards, governance artifacts. Optimize for future maintainers: explicit structure, treat hardcodes/temporary metrics/oversized boards as technical debt. Assume the app will scale in users, time horizon, and scenario complexity. + +--- + +## 1. Output Expected from the Audit + +Produce: + +1. Structured audit report grouped by: Modeling | Performance | UX | Governance | Cleanup +2. Actionable recommendations per issue: what to change, why it matters, how to fix it +3. Severity - use HIGH / MEDIUM / LOW as defined in [SKILL.md](./SKILL.md#severity-canonical) +4. Delegated skill references when a finding comes from or should be deepened in another skill (e.g. Formula Optimization / Performance skill, Board design skill, Application Cleaning skill for deletion workflow) + +--- + +## 2. Modeling & Formula Audit + +### 2.1 Formula Quality & Anti-Patterns + +Use: `skill:writing-pigment-formulas` and `skill:optimizing-pigment-performance` for detailed formula and performance checks. + +Identify: + +- Repeated logic that should be centralized (helper metrics, "compute once, reference many") +- Overly complex formulas (deep nesting, long IF chains) +- Metrics doing multiple jobs (split into focused metrics) +- Excessive use of volatile or heavy functions (e.g. PREVIOUS, dynamic scenario logic) where simpler patterns exist + +Flag: + +- Poor formula layout (hard to read, no structure, no comments) +- Missing logical decomposition into helper metrics + +Recommend: Centralize repeated logic; introduce helper/shared metrics; improve readability and structure. Reference formula workflow and performance patterns when proposing changes. + +### 2.2 Hardcoding & Time & Date Risks + +Use: [`../modeling-pigment-applications/modeling_principles.md`](../modeling-pigment-applications/modeling_principles.md) (sections 4 and 9 for deployment-safe formulas and Test & Deploy when used); [`../modeling-pigment-applications/modeling_time_and_calendars.md`](../modeling-pigment-applications/modeling_time_and_calendars.md) and `skill:writing-pigment-formulas` for time/date functions. + +Detect: + +- Hardcoded numbers, dates, scenario names, version names in formulas +- Inline time logic instead of Time & Date functions (e.g. fixed dates instead of STARTOFMONTH, TIMEDIM) +- Business rules embedded directly in formulas instead of assumptions or parameters + +Recommend: Use assumption metrics, parameters, or centralized time/date logic; avoid literals that differ across environments or break when data changes. + +--- + +## 3. Metric Hygiene & Cleanup + +For strict deletion workflow and machine-readable definition of "unused", use [performance_cleaning_application.md](./performance_cleaning_application.md). The audit surfaces candidates and recommendations; the application cleaning doc defines order of deletion, observation period, and board usage rules. + +### 3.1 Unused Metrics (Audit View) + +Identify metrics that: + +- Are not referenced anywhere (no other metric, table, or board references them) - aligns with application cleaning's "unused" for metrics when downstream + UI exposure = 0 +- Look like legacy remnants of past refactors + +Recommend: Flag as deletion candidates. For actual deletion, follow [performance_cleaning_application.md](./performance_cleaning_application.md): DEAD boards first, then structural objects (dimensions -> metrics -> tables -> properties) with recompute between passes. Validate with user before deletion when uncertain. + +### 3.2 Temporary / Copy Metrics + +Search for metrics: + +- Names starting with ZZ, TMP, COPY +- Names containing: remove, TBD, test + +Assess: Still needed or should be deleted/renamed? Flag as technical debt if left in place without a clear purpose. Deletion, when decided, follows the application cleaning workflow (no renaming in Phase 1 cleaning). + +### 3.3 Shared Metrics Validation + +Check shared metrics: + +- Are they actually reused by other applications or blocks? +- Or shared "just in case" with no consumers? + +Recommend: Un-share unused shared metrics; consolidate duplicates where appropriate. See Library folder usage in [modeling_principles.md](../modeling-pigment-applications/modeling_principles.md). + +--- + +## 4. Folder & Structural Audit + +### 4.1 Metric Folder Structure + +Use: [modeling_principles.md](../modeling-pigment-applications/modeling_principles.md) (OX folders, themed folders). + +Identify: + +- Overloaded folders (too many blocks, mixed purposes) +- Inconsistent layering (e.g. inputs mixed with outputs, core calcs mixed with technical helpers) + +Enforce logical layers: + +- Inputs +- Assumptions +- Core calculations +- Outputs +- Technical / Helper (and consider hiding non-business blocks from end users) + +Note: Folder organization is Phase 2 / optional in [performance_cleaning_application.md](./performance_cleaning_application.md); audit can recommend it as hygiene, but it is not part of deletion-only cleaning. + +### 4.2 "No Folder" Cleanup + +Prevention: When creating blocks, always assign a folder. Never create in "No Folder". See [Working with Folders](../modeling-pigment-applications/modeling_working_with_folders.md) - Placing new blocks. + +Identify: Blocks (metrics, lists) not assigned to any folder. + +Action: Classify and either move to the appropriate folder or flag for deletion/archival. Deletion, when applied, follows the application cleaning doc (order, observation). + +--- + +## 5. Board & UX Audit + +Use: `skill:designing-pigment-boards` for layout, naming, and patterns. + +### 5.1 Board Folder Structure + +Check: Logical grouping of boards by business purpose (e.g. Input / Review / Analysis / Admin). Flag flat or deeply nested structures that hurt findability. + +Recommend: Clear separation by purpose; consistent naming (e.g. prefixes like IN-, REV-, ADM- if adopted). See [modeling_naming_conventions.md](../modeling-pigment-applications/modeling_naming_conventions.md). + +### 5.2 Board Naming Conventions + +Identify: Ambiguous or inconsistent board names. + +Recommend: Business-oriented naming; align with [modeling_naming_conventions.md](../modeling-pigment-applications/modeling_naming_conventions.md) and board skills. + +### 5.3 Board Size & Performance + +Flag boards with: More than ~20 widgets (heuristic; actual threshold depends on widget complexity, data volume, and view types). + +Risks: Slower load times, harder maintenance, poor UX. + +Recommend: Split into focused boards; use navigation boards where appropriate. + +### 5.4 Unused Boards (Audit vs Cleaning) + +Audit: Flag boards that appear unused (e.g. no recent use, or only admin viewers). For deletion and classification (ACTIVE / STALE / DEAD), use [performance_cleaning_application.md](./performance_cleaning_application.md): definition is usage-based (view_count, unique_non_admin_viewers, time window); DEAD boards follow tag -> notify -> contestation -> delete. Audit does not replace the application cleaning workflow. + +--- + +## 6. Governance & Access Rights + +Use: `skill:securing-pigment-applications` and [modeling_principles.md](../modeling-pigment-applications/modeling_principles.md) (MS12, MP10, security). + +Identify: + +- Helper or technical metrics visible to business users (should be hidden or access-restricted) +- Missing access restrictions on sensitive metrics or lists + +Recommend: Role-based access cleanup; hide or restrict non-business metrics; ensure AR rules are applied where data is sensitive (no "security by Board only"). + +--- + +## 7. App Cleanup & Maintenance (Validate with User) + +For strict deletion workflow and definitions, see [performance_cleaning_application.md](./performance_cleaning_application.md). Below is the audit-oriented view. + +### 7.1 Restored Blocks + +Pigment's Restore block feature restores previously deleted blocks by creating a copy in a system-created folder (typically named "Restored blocks" or similar). This folder can accumulate blocks over time and is easy to overlook. + +Identify: The system folder containing restored blocks, and the blocks inside it. + +Action: Propose cleaning this folder if the user confirms they no longer need the restored blocks: either move blocks that are still needed to an appropriate folder (see [Working with Folders](../modeling-pigment-applications/modeling_working_with_folders.md) - Placing new blocks), or delete unused ones. Validate with the user before removing any restored blocks. If deletion is confirmed, align with the application cleaning doc (order, observation, logging). + +### 7.2 Snapshot Cleanup + +Identify: Old or unused snapshots. + +Explain: Impact on space and performance (see MP06 in [modeling_principles.md](../modeling-pigment-applications/modeling_principles.md)). + +Action: Validate with the user before deletion. Snapshot cleanup is not part of the structural/board cleaning order in the application cleaning doc but is part of general hygiene. + +--- + +## 8. Expert Audit Heuristics + +- Optimize for future maintainers: Prefer explicit structure over convenience. +- Technical debt: Treat hardcodes, temporary metrics (ZZ/TMP/COPY/test/TBD), and oversized boards as debt; call them out and prioritize by severity. +- Scale assumption: Assume the app will grow in users, time horizon, and scenario complexity; flag design choices that will not scale. +- Delegation: When a finding is about formula quality or performance, point to the relevant skill. When it is about deletion of unused objects, point to [performance_cleaning_application.md](./performance_cleaning_application.md) for workflow and definitions. + +--- + +## 9. Severity Classification + +Canonical severity definitions are in [SKILL.md](./SKILL.md#severity-canonical). Use HIGH / MEDIUM / LOW as defined there. + +--- + +## 10. Delegated Skill References + +When findings come from or require deeper use of another skill, state it explicitly in the report: + +- Formula quality / optimization: `skill:writing-pigment-formulas`, `skill:optimizing-pigment-performance` (including [performance_troubleshooting_workflow.md](./performance_troubleshooting_workflow.md)) +- Time & dates / T&D risks: [modeling_principles.md](../modeling-pigment-applications/modeling_principles.md) (sections 4 & 9 for T&D), [modeling_time_and_calendars.md](../modeling-pigment-applications/modeling_time_and_calendars.md), `skill:writing-pigment-formulas` (time/date functions) +- Folder structure / governance: [modeling_principles.md](../modeling-pigment-applications/modeling_principles.md). Naming: [modeling_naming_conventions.md](../modeling-pigment-applications/modeling_naming_conventions.md) +- Boards / UX: `skill:designing-pigment-boards` (or advanced boards skill) +- Access rights: `skill:securing-pigment-applications` ([securing_access_rights.md](../securing-pigment-applications/securing_access_rights.md)) +- Application cleaning (deletion workflow): [performance_cleaning_application.md](./performance_cleaning_application.md) - for definition of "unused", order of deletion, observation period, board usage-based cleaning + +--- + +## See Also + +- [modeling_principles.md](../modeling-pigment-applications/modeling_principles.md) - Folder structure, MP06 hygiene +- [modeling_naming_conventions.md](../modeling-pigment-applications/modeling_naming_conventions.md) - Naming conventions (including Applications ZZ\_) +- [performance_cleaning_application.md](./performance_cleaning_application.md) - Deletion-only application cleaning, unused definitions, mandatory order, boards by usage +- [performance_troubleshooting_workflow.md](./performance_troubleshooting_workflow.md) - Performance audit methodology +- `skill:writing-pigment-formulas` - Formula workflow and quality +- `skill:designing-pigment-boards` - Board structure and naming diff --git a/plugins/pigment/skills/optimizing-pigment-performance/performance_calendar_considerations.md b/plugins/pigment/skills/optimizing-pigment-performance/performance_calendar_considerations.md new file mode 100644 index 00000000..74759ddd --- /dev/null +++ b/plugins/pigment/skills/optimizing-pigment-performance/performance_calendar_considerations.md @@ -0,0 +1,50 @@ +# Calendar Performance Considerations + +## Introduction + +Time-based calculations can significantly impact performance, especially over long horizons with fine granularity. This guide covers performance patterns for time-dimensioned calculations. + +## Time Horizon Impact + +### Granularity Tradeoffs + +- Monthly: 12-60 periods (fast) +- Weekly: 52-260 periods (moderate) +- Daily: 365-1,825 periods (slow) + +### Subsetting Time Dimensions + +Limit iterative calculations to relevant periods: + +- Current fiscal year only +- Rolling 90 days +- Last 12 months + +## Iterative Calculations Over Long Horizons + +### Problem + +PREVIOUS, PREVIOUSOF, and CUMULATE over many periods: + +- 5 years daily = 1,825 sequential calculations +- Performance degrades significantly + +### Solutions + +1. Subset to recent periods +2. Use monthly instead of daily +3. Pre-compute starting points +4. Use FILLFORWARD when possible + +## Daily vs Monthly Granularity + +Consider if daily granularity is truly needed: + +- Planning: Usually monthly sufficient +- Actuals: May need daily +- Forecasting: Weekly or monthly often adequate + +## See Also + +- [performance_iterative_calculations.md](./performance_iterative_calculations.md) - Iterative calculation optimization +- [modeling_time_and_calendars.md](../modeling-pigment-applications/modeling_time_and_calendars.md) - Calendar configuration and time dimension structure diff --git a/plugins/pigment/skills/optimizing-pigment-performance/performance_cleaning_application.md b/plugins/pigment/skills/optimizing-pigment-performance/performance_cleaning_application.md new file mode 100644 index 00000000..57930789 --- /dev/null +++ b/plugins/pigment/skills/optimizing-pigment-performance/performance_cleaning_application.md @@ -0,0 +1,189 @@ +# Pigment Application Cleaning -- Agent Rules and Workflows + +Purpose: Define a strict, deletion-focused approach to cleaning a Pigment application for execution by an agent. Cleaning = deletion only; "unused" is determined from Pigment system truth (settings, dependency graphs, usage analytics), not human judgment. Formula refactoring, optimization, renaming, and folder organization are out of scope for this cleaning phase. + +--- + +## Core Principles + +- Cleaning = Deletion. Only actions that remove objects from the application belong to the core cleaning phase. Renaming, folders, naming alignment are optional hygiene (Phase 2), not covered here. +- System truth over human opinion. "Unused" is determined exclusively from Pigment settings, dependency graphs, and usage analytics. No interviews, no inferred intent. +- No formula changes. Formulas are optimization; explicitly excluded from cleaning. + +--- + +## Cleaning Phases (Scope of This Doc) + +| Phase | Scope | Mandatory | +| ----------- | ----------------------------------- | ----------------------- | +| Phase 1 | Deletion of unused objects | Yes -- this document | +| Phase 2 | Renaming, folders, naming alignment | Optional (not cleaning) | + +This document covers Phase 1 only. + +--- + +## Canonical Deletion Order + +Cleaning proceeds in two axes, in this sequence: + +1. DEAD boards first -- classify and delete usage-based DEAD boards (see Board Cleaning below). +2. Recompute structural usage -- board deletions may unlock structural objects that are now unused. +3. Structural objects in order: Dimensions, Metrics, Tables (Transaction Lists), Properties. For each type: identify unused, hide, observe, delete. +4. Recompute and iterate -- deletion of one layer may unlock deletion in the next. Repeat until no new candidates appear. + +Changing this order increases the risk of false positives (deleting an object whose consumer has not yet been removed). + +--- + +## Phase 1 -- Board Cleaning (Usage-Based, First) + +Boards are evaluated by usage analytics only, not by structural references. Board cleaning runs first because boards are frequently the primary trigger that enables deeper structural cleaning. + +### Source of Truth for Boards + +- `last_viewed_at` +- `view_count` (rolling window) +- `unique_viewers` +- Viewer role (admin vs business user) + +A board viewed only by admins is not considered used for cleaning. + +### Board Classification + +| Category | Criteria | +| ---------- | ------------------------------------------------------------------ | +| ACTIVE | Viewed by >= 1 business user in the time window | +| STALE | Viewed only by admins/builders in the time window (not auto-deleted, but a strong signal for future cleanup) | +| DEAD | `view_count = 0` OR `unique_non_admin_viewers = 0` in the time window | + +Seasonality or annual-only usage must be explicitly whitelisted (clarify with solution architect). + +### Board Deletion Workflow (DEAD only) + +1. Auto-tag `TO_BE_DELETED` +2. Notify owner (if any) +3. Contestation window (duration agreed with solution architect) +4. Delete + +STALE boards are not automatically deleted. They are flagged for review and often allow downstream structural cleaning once confirmed removable. + +--- + +## Phase 1 -- Structural Cleaning (Settings-Based) + +### Source of Truth + +For all structural objects use Pigment Settings only: + +- Dependency graphs +- Explicit references +- UI exposure metadata + +No interviews, no inferred intent. + +### Object Processing Order (Mandatory) + +1. Dimensions +2. Metrics +3. Tables (Tables / Transaction Lists) +4. Properties (list fields) + +--- + +## Definition of "Unused" (Machine-Readable) + +### Dimensions + +A dimension is unused only if all are true: + +- Not referenced by any table +- Not referenced by any metric +- Not referenced by any property +- Not used in any board or page (axis, filter, segmentation) + +| Usage count | Action | +| ----------- | ---------------------- | +| 0 | Candidate for deletion | +| > 0 | Do not touch | + +There is no "rarely used" concept. + +--- + +### Metrics + +A metric is unused if: + +- Not referenced by any other metric +- Not displayed in any board or page +- Not used in any configured export + +Note: Being inside a table is not considered usage for this definition. + +| Downstream references | UI exposure | Action | +| --------------------- | ----------- | ---------------------- | +| 0 | 0 | Candidate for deletion | +| >= 1 | any | Keep | + +No merging, renaming, or formula edits in this phase. + +--- + +### Tables (Transaction Lists / Tables) + +A table is unused if: + +- No downstream consumers (metrics, boards, other tables) +- No active input updates (for input tables) +- No connected exports or integrations + +Special case: Tables used only as historical or staging layers are candidates only if no boards consume them. + +--- + +### Properties (List Fields) + +A property is unused if: + +- Not referenced in any formula +- Not used as filter or display field +- Not visible in the UI + +Optional strong signal: never populated with data. + +Process: Hide -> Observe -> Delete. No type changes, no renaming in this phase. + +--- + +## Deletion Workflow (All Structural Objects) + +1. Identify unused object via settings (dependency + exposure). +2. Disable / hide object. +3. Observation period (e.g. minimum one business cycle; duration agreed with solution architect, e.g. 30 / 60 / 90 days). +4. Ask the user for explicit confirmation before final deletion after observation. +5. Log deletion event (who / what / when / why). + +Silence during the observation window is a cleanup signal, not approval to delete. For contested or seasonal use, clarify with solution architect: observation window, whitelisting of seasonal/annual boards, minimum deletion batch size per run. Before irreversible deletion, get explicit user confirmation. + +--- + +## Agent Execution Logic (Conceptual) + +When executing a cleaning task: + +1. Start with boards -- classify all boards (ACTIVE / STALE / DEAD) using usage analytics. +2. Delete DEAD boards -- follow board deletion workflow. +3. Recompute structural usage -- after board deletion, re-evaluate all structural objects. +4. Process structural objects in order -- Dimensions, Metrics, Tables, Properties. +5. For each object type -- identify unused, hide, observe, delete. +6. Log all actions -- maintain an audit trail. +7. Iterate -- deletion of one layer may unlock deletion in the next. + +--- + +## See Also + +- [performance_auditing_application.md](./performance_auditing_application.md) - Full app audit (surfaces candidates; this doc defines deletion workflow) +- [modeling_principles.md](../modeling-pigment-applications/modeling_principles.md) - Folder structure, MP06 hygiene +- [modeling_naming_conventions.md](../modeling-pigment-applications/modeling_naming_conventions.md) - Naming conventions (including Applications ZZ\_) diff --git a/plugins/pigment/skills/optimizing-pigment-performance/performance_formula_optimization.md b/plugins/pigment/skills/optimizing-pigment-performance/performance_formula_optimization.md new file mode 100644 index 00000000..1b78548f --- /dev/null +++ b/plugins/pigment/skills/optimizing-pigment-performance/performance_formula_optimization.md @@ -0,0 +1,584 @@ +# Performance Formula Optimization + +## Introduction + +Formula optimization is about writing formulas that produce the correct result while minimizing computation time and resource usage. Small changes in formula structure can lead to dramatic performance improvements. + +This guide covers the core principles of formula optimization: scope-first, filter-early with deferred aggregations, execution order, and common anti-patterns to avoid. + +## Core Optimization Principles + +### Principle 1: Scope First + +Definition: Start formulas with scoping clauses to limit which cells are computed. + +Why it matters: Computing 1% of cells is 100x faster than computing all cells. + +Implementation: Use FILTER, EXCLUDE, or IFDEFINED at the beginning of formulas. + +### Principle 2: Filter Early, Defer Aggregations + +Definition: Apply filtering (`FILTER`, `EXCLUDE`, `SELECT`) as early as possible to shrink the dataset. Defer scope-losing aggregations (`REMOVE`) to the end of the chain to preserve scope. + +Why it matters: Filtering reduces cells without losing scope (faster at every subsequent step). `REMOVE` loses scope, so deferring it keeps downstream metrics fast. + +Implementation: Apply `FILTER`/`EXCLUDE`/`SELECT` before complex calculations. Push `REMOVE` to the end. `BY` with property mappings can replace `REMOVE + ADD` without scope loss. + +### Principle 3: Understand Execution Order + +Definition: Pigment executes formulas sequentially, left to right, modifier by modifier. + +Why it matters: The order of operations affects how much data is computed. + +Implementation: Structure formulas to minimize intermediate computation. + +## IF vs FILTER: Understanding Execution Order + +### The Problem with Sequential Execution + +Anti-pattern: + +```pigment +10[ADD: Month][FILTER: Month > VAR_Reference_Month] +``` + +Execution sequence: + +1. `10[ADD: Month]` -> Creates value 10 for every possible Month +2. `[FILTER: Month > VAR_Reference_Month]` -> Removes values where condition is false + +Problem: Computation happens for all months, even those that will be filtered out. + +Performance: If there are 24 months, computes 24 values but only keeps 12. + +### The Solution: Use IF for Conditional Creation + +Optimized pattern: + +```pigment +// VAR_Reference_Month: input metric, type Dimension +IF(Month > VAR_Reference_Month, 10) +``` + +Execution sequence: + +1. Evaluates condition `Month > VAR_Reference_Month` for each month +2. Creates value 10 only where condition is TRUE + +Performance: Computes only 12 values directly. + +Improvement: 2x faster, and scales with the selectivity of the condition. + +### When to Use IF vs FILTER + +Use IF when: + +- Creating values conditionally +- The condition is simple +- You want to avoid computing unnecessary cells + +```pigment +// Good: Only compute where needed +IF('Product'.'Active' = TRUE, 'Revenue' * 'Growth Rate') +``` + +Use FILTER when: + +- Filtering existing metric values +- You need to preserve dimensionality +- The source metric is already computed + +```pigment +// Good: Filter already-computed values +'Revenue'[FILTER: 'Product'.'Active' = TRUE] +``` + +### IFDEFINED for Boolean Conditions + +When working with boolean metrics that hold only TRUE/BLANK values (not FALSE), use IFDEFINED: + +Pattern: + +```pigment +// Boolean metric: 'Is Active' (TRUE or BLANK, never FALSE) +IFDEFINED('Is Active', 'Revenue' * 'Growth Rate') +``` + +Why better than IF: + +- Cleaner syntax +- Explicitly handles sparse boolean metrics +- Same performance as IF(ISDEFINED()) + +## Scope-First Patterns + +### Pattern 1: Filter Before Computing + +Anti-pattern: + +```pigment +// Compute first, filter later +('Revenue' * 'Growth Rate' + 'Fixed Costs')[FILTER: 'Product'.'Active' = TRUE] +``` + +Execution: + +1. Multiply Revenue x Growth Rate for all products +2. Add Fixed Costs for all products +3. Filter to active products only + +Optimized pattern: + +```pigment +// Filter first, then compute +'Revenue'[FILTER: 'Product'.'Active' = TRUE] * 'Growth Rate' + 'Fixed Costs' +``` + +Execution: + +1. Filter Revenue to active products +2. Multiply only active products x Growth Rate +3. Add Fixed Costs only for active products + +Improvement: If 20% of products are active, 5x faster. + +### Pattern 2: ISDEFINED for Sparse Metrics + +Anti-pattern: + +```pigment +// Compute for all cells +'Revenue' * 'Adjustment Factor' + 'Base Amount' +``` + +Problem: If Revenue is sparse (10% of cells), computes 90% unnecessary cells. + +Optimized pattern: + +```pigment +// Compute only where Revenue exists +IFDEFINED('Revenue', + 'Revenue' * 'Adjustment Factor' + 'Base Amount' +) +``` + +Improvement: 10x faster for 10% sparse metrics. + +### Pattern 3: Chain IFDEFINED for Multiple Metrics + +Anti-pattern: + +```pigment +// Computes even where one metric is blank +'Revenue' / 'Cost' +``` + +Problem: Computes for all cells where either metric exists, may create errors or blanks. + +Optimized pattern: + +```pigment +// Compute only at intersection +IFDEFINED('Revenue', + IFDEFINED('Cost', + 'Revenue' / 'Cost' + ) +) +``` + +Improvement: Computes only where both metrics exist. + +### Pattern 4: EXCLUDE Early + +Anti-pattern: + +```pigment +// Complex calculation, then exclude +('Revenue' * 'Growth' + 'Costs')[EXCLUDE: 'Account'.'Type' = "Test"] +``` + +Optimized pattern: + +```pigment +// Exclude first, then calculate +'Revenue'[EXCLUDE: 'Account'.'Type' = "Test"] * 'Growth' + 'Costs' +``` + +Improvement: Excludes test accounts before any computation. + +## Filter-Early Patterns + +### Pattern 1: BY Before Complex Calculations (When Equivalent) + +When downstream granularity is lower than the source, aggregate with `BY` before computing (only when mathematically equivalent): + +```pigment +// Anti-pattern: complex calculation at transaction level, then aggregate +('Transaction Amount' * 'Exchange Rate' + 'Fee')[BY: 'Transaction'.'Customer'] + +// Optimized: aggregate first, then calculate +'Transaction Amount'[BY: 'Transaction'.'Customer'] * 'Exchange Rate' + 'Fee' +``` + +Note: Only works if Exchange Rate and Fee are at Customer level, not Transaction level. + +Improvement: If 1000 transactions per customer, 1000x less computation. + +### Pattern 2: Filter Before Aggregation + +Anti-pattern: + +```pigment +// Aggregate all, then filter +'Transaction Amount'[BY: 'Transaction'.'Customer'][FILTER: 'Customer'.'Region' = "EMEA"] +``` + +Optimized pattern: + +```pigment +// Filter first, then aggregate +'Transaction Amount'[FILTER: 'Transaction'.'Customer'.'Region' = "EMEA"][BY: 'Transaction'.'Customer'] +``` + +Improvement: Aggregates only EMEA transactions, not all transactions. + +### Pattern 3: Use SELECT for Single-Item Filtering + +Anti-pattern: + +```pigment +// Filter then aggregate +'Revenue'[FILTER: Month = VAR_Reference_Month][REMOVE: Month] +``` + +Optimized pattern: + +```pigment +// VAR_Reference_Month: input metric, type Dimension +'Revenue'[SELECT: Month = VAR_Reference_Month] +``` + +Improvement: More efficient, cleaner syntax. + +## Common Formula Anti-Patterns + +### Anti-Pattern 1: Unnecessary Intermediate Calculations + +Anti-pattern: + +```pigment +// Metric 1 +'Step 1' = 'Revenue' * 'Rate' + +// Metric 2 +'Step 2' = 'Step 1' + 'Fixed' + +// Metric 3 +'Step 3' = 'Step 2'[FILTER: 'Product'.'Active' = TRUE] +``` + +Problem: Computes Step 1 and Step 2 for all products, then filters. + +Optimized pattern: + +```pigment +// Single metric with early filtering +'Result' = ('Revenue'[FILTER: 'Product'.'Active' = TRUE] * 'Rate') + 'Fixed' +``` + +Or if intermediate metrics are needed: + +```pigment +// Metric 1 with early filtering +'Step 1' = 'Revenue'[FILTER: 'Product'.'Active' = TRUE] * 'Rate' + +// Metric 2 +'Step 2' = 'Step 1' + 'Fixed' +``` + +### Anti-Pattern 2: Repeated Aggregations + +Anti-pattern: + +```pigment +// Multiple metrics each aggregating the same data +'Total Revenue' = 'Revenue'[REMOVE: Product] +'Total Cost' = 'Cost'[REMOVE: Product] +'Total Profit' = 'Total Revenue' - 'Total Cost' +``` + +Problem: Each REMOVE operation loses scope and recomputes. + +Optimized pattern: + +```pigment +// Calculate profit first, then aggregate once +'Profit' = 'Revenue' - 'Cost' +'Total Profit' = 'Profit'[REMOVE: Product] +``` + +Improvement: One aggregation instead of two. + +### Anti-Pattern 3: Dense Boolean Metrics in Conditions + +Anti-pattern: + +```pigment +// Dense boolean metric +'Has Revenue' = ISNOTBLANK('Revenue') + +// Used in condition +IF('Has Revenue', 'Revenue' * 1.1, 0) +``` + +Problem: 'Has Revenue' is dense (TRUE/FALSE everywhere), making the IF compute all cells. + +Optimized pattern: + +```pigment +// Use ISDEFINED directly +IFDEFINED('Revenue', 'Revenue' * 1.1) +``` + +Improvement: Sparse computation, no dense intermediate metric. + +### Anti-Pattern 4: Unnecessary REMOVE + +Anti-pattern: + +```pigment +// Remove dimension that's not needed downstream +'Aggregated' = 'Transaction Amount'[REMOVE: Transaction ID] +'Result' = 'Aggregated' * 'Rate' +``` + +Question: Does 'Result' or any downstream metric need Transaction ID? + +If no: + +```pigment +// Keep the dimension +'Aggregated' = 'Transaction Amount' +'Result' = 'Aggregated' * 'Rate' +// Remove only at the very end if needed for reporting +``` + +Improvement: Preserves scope through the chain. + +### Anti-Pattern 5: Complex Nested IF Statements + +Anti-pattern: + +```pigment +IF( + 'Condition 1', + IF( + 'Condition 2', + IF( + 'Condition 3', + 'Value A', + 'Value B' + ), + 'Value C' + ), + 'Value D' +) +``` + +Problem: Hard to read, hard to maintain, potentially inefficient. + +Optimized pattern: + +```pigment +// Use separate metrics for clarity +'Meets All Conditions' = 'Condition 1' AND 'Condition 2' AND 'Condition 3' +'Result' = IF('Meets All Conditions', 'Value A', 'Default Value') +``` + +Or use FILTER: + +```pigment +'Value A'[FILTER: 'Condition 1' AND 'Condition 2' AND 'Condition 3'] +``` + +Exception - when nested IF is preferable: If multiple branches (4+) would each use the same FILTER/EXCLUDE conditions with only a varying expression, a nested IF that factors the common logic can be faster than IFBLANK with repeated modifiers. Benchmarks show ~40% improvement in such cases. See [formula_conditionals_style.md](../writing-pigment-formulas/formula_conditionals_style.md) section 2.7. + +## Execution Order Optimization + +### Understanding Left-to-Right Execution + +Pigment executes formulas from left to right, applying each modifier sequentially. + +Example: + +```pigment +'Revenue'[ADD: Version][FILTER: Version = "Budget"][REMOVE: Product] +``` + +Execution sequence: + +1. Start with 'Revenue' (Product x Month) +2. `[ADD: Version]` -> Expand to Product x Month x Version +3. `[FILTER: Version = "Budget"]` -> Keep only Budget version +4. `[REMOVE: Product]` -> Aggregate to Month x Version + +Optimization opportunity: The FILTER could come before ADD to avoid expanding to all versions. + +Optimized: + +```pigment +'Revenue'[FILTER: 'Product'.'Default Version' = "Budget"][REMOVE: Product] +``` + +### Order Matters for Performance + +Anti-pattern: + +```pigment +'Metric'[ADD: Dimension A][ADD: Dimension B][FILTER: Condition] +``` + +Execution: + +1. Expand to all combinations of Dimension A +2. Expand to all combinations of Dimension B +3. Filter (but expansion already happened) + +Optimized pattern: + +```pigment +'Metric'[FILTER: Condition][ADD: Dimension A][ADD: Dimension B] +``` + +Execution: + +1. Filter first (smaller dataset) +2. Expand filtered data to Dimension A +3. Expand to Dimension B + +Improvement: Smaller expansions = faster computation. + +## Real-World Optimization Example + +### Scenario: Revenue Forecasting Model + +Original formula: + +```pigment +'Forecast' = + ('Historical Revenue'[ADD: Scenario] * 'Growth Rate' + 'New Products')[FILTER: 'Product'.'Active' = TRUE] +``` + +Problems: + +1. ADD creates values for all scenarios before filtering +2. Computation happens for inactive products +3. New Products added to all products before filtering + +Optimized formula: + +```pigment +'Forecast' = + 'Historical Revenue'[FILTER: 'Product'.'Active' = TRUE] * 'Growth Rate' + + 'New Products'[FILTER: 'Product'.'Active' = TRUE] +``` + +Improvements: + +1. Filter to active products first +2. No unnecessary ADD (Scenario comes from source metrics) +3. Both metrics filtered before computation + +Performance gain: 5x faster for 20% active products. + +### Scenario: Multi-Currency Consolidation + +Original formula: + +```pigment +'Consolidated Revenue' = + ('Revenue' * 'Exchange Rate')[REMOVE: Currency][REMOVE: Subsidiary] +``` + +Problems: + +1. Two REMOVE operations lose scope twice +2. Intermediate expansion before aggregation + +Optimized formula: + +```pigment +'Consolidated Revenue' = + ('Revenue' * 'Exchange Rate')[REMOVE: Currency, Subsidiary] +``` + +Improvements: + +1. Single REMOVE operation +2. Scope lost only once + +Performance gain: 2x faster. + +## Formula Optimization Checklist + +Before finalizing a formula, check: + +- [ ] Are scoping clauses (FILTER, EXCLUDE, ISDEFINED) at the beginning? +- [ ] Are aggregations (REMOVE, BY) deferred to the end? +- [ ] Is IF used instead of ADD + FILTER for conditional values? +- [ ] Are sparse metrics scoped with IFDEFINED? +- [ ] Are unnecessary intermediate calculations eliminated? +- [ ] Is the execution order optimized (filter before expand)? +- [ ] Are dense boolean metrics avoided in conditions? +- [ ] Is ISDEFINED used instead of ISBLANK? +- [ ] Are multiple conditions combined efficiently? +- [ ] Is the formula readable and maintainable? + +## Measuring Optimization Impact + +### Before Optimization + +1. Profile the formula: Note computation time +2. Check scope: Re-run `tool:performance_profile_change`; read effective/output scope per execution +3. Identify bottleneck: Which part is slow? + +### After Optimization + +1. Profile again: Compare computation time +2. Verify scope: Did scope improve? +3. Check correctness: Does it produce the same result? + +### Expected Improvements + +Scope optimization: + +- 0/3 -> 2/3 scope: 10-100x faster + +Sparsity optimization: + +- Dense -> Sparse: 10-50x faster + +Execution order optimization: + +- Better order: 2-10x faster + +Combined optimizations: + +- All techniques: 100-1000x faster possible + +## Best Practices Summary + +1. Scope first: Start with FILTER, EXCLUDE, or IFDEFINED +2. Filter early: Apply FILTER/EXCLUDE/SELECT before complex calculations to shrink the dataset +3. Defer aggregations: Push REMOVE to the end of the chain to preserve scope +4. Use IF for conditional creation: Don't create then filter +5. Preserve sparsity: Use ISDEFINED, not ISBLANK +6. Optimize execution order: Filter before expand +7. Profile regularly: Measure before and after +8. Keep formulas readable: Don't sacrifice clarity for micro-optimizations + +## See Also + +- [Performance Scoping Patterns](./performance_scoping_patterns.md) - Deep dive into scope preservation +- [Performance Sparsity Deep Dive](./performance_sparsity_deep_dive.md) - Sparsity management techniques +- [Performance Profiling](./performance_profiling.md) - profiling tools and output parsing diff --git a/plugins/pigment/skills/optimizing-pigment-performance/performance_iterative_calculations.md b/plugins/pigment/skills/optimizing-pigment-performance/performance_iterative_calculations.md new file mode 100644 index 00000000..fa19df73 --- /dev/null +++ b/plugins/pigment/skills/optimizing-pigment-performance/performance_iterative_calculations.md @@ -0,0 +1,477 @@ +# Performance Iterative Calculations + +## Introduction + +Iterative calculations-where each period depends on the previous period-are common in financial planning but can create significant performance challenges. Functions like PREVIOUS, PREVIOUSOF, CUMULATE, and FILLFORWARD require sequential computation that can become slow over long time horizons. + +This guide covers optimization strategies for iterative calculations, subsetting techniques, and when to use alternative approaches. For the full technical spec (circular dependencies, PREVIOUS vs PREVIOUSOF, configuration, syntax, debugging), see [Iterative Calculation (PREVIOUS & PREVIOUSOF)](../writing-pigment-formulas/functions_iterative_calculation.md). + +WARNING: IMPORTANT - PREVIOUSOF Prerequisite: +PREVIOUSOF can only be used on metrics that have iterative calculation enabled in the Pigment application settings. This configuration cannot be done via AI tools - the user must set it up in the Pigment UI. Before writing any formula with PREVIOUSOF, confirm with the user that iterative calculation is configured on the target metric. If not, instruct them to enable it first. + +Correct PREVIOUSOF pattern for period-end balances: + +Split beginning and ending metrics across the iteration cycle: + +```pigment +// Metric - 'Beginning Balance' +PREVIOUSOF('Ending Balance') + +// Metric - 'Ending Balance' +'Beginning Balance' + 'Inflow' - 'Outflow' +``` + +## Understanding Iterative Calculation Performance + +### Why Iterative Calculations Are Slow + +Sequential dependency: To compute period N, you must first compute periods 1 through N-1. + +Example (same beginning/ending pattern as above): + +Computation sequence: + +- Month 1: Beginning_1 = seed -> Ending_1 = Beginning_1 + Inflow_1 - Outflow_1 +- Month 2: Beginning_2 = Ending_1 -> Ending_2 = Beginning_2 + Inflow_2 - Outflow_2 +- Month 3: Beginning_3 = Ending_2 -> Ending_3 = Beginning_3 + Inflow_3 - Outflow_3 + +Each additional period adds one more sequential step. + +Performance impact: Cannot parallelize, must compute sequentially. + +### Scope Loss in Iterative Calculations + +Iterative calculations lose scope on the iterating dimension: + +```pigment +// Metric - 'YTD Revenue' +YEARTODATE('Monthly Revenue') +``` + +Profiler result: Scope lost on Month dimension. + +Why: If Month 3 changes, Months 3-12 must be recomputed (sequential dependency). + +### Performance Factors + +Time horizon: Longer horizons = more sequential steps + +- 12 months: Fast +- 36 months: Moderate +- 60 months (5 years): Slow +- 1,825 days (5 years daily): Very slow + +Granularity: Finer granularity = more steps + +- Monthly: 12 steps per year +- Weekly: 52 steps per year +- Daily: 365 steps per year + +Dimensions: More dimensions = more cells to iterate + +- 1 dimension (Month): Fast +- 2 dimensions (Month x Product): Moderate +- 3 dimensions (Month x Product x Region): Slow + +Data density: Dense data = more cells to compute + +- Sparse (10% cells): Fast +- Dense (90% cells): Slow + +## Optimization Strategy 1: Subset Time Dimensions + +### The Problem: Long Time Horizons + +Scenario: 5 years of daily data for inventory tracking. + +Anti-pattern: + +```pigment +// Metric - 'Beginning Daily Inventory' +PREVIOUSOF('Ending Daily Inventory') + +// Metric - 'Ending Daily Inventory' +'Beginning Daily Inventory' + 'Purchases' - 'Sales' +``` + +Performance: 1,825 sequential days x all products x all warehouses = Very slow. + +### The Solution: Subset to Relevant Periods + +Pattern: Use dimension subsets to limit the iteration window. + +Implementation: + +```pigment +// Create a subset for recent periods only +// Subset: 'Recent Days' = last 90 days + +// Metric - 'Beginning Daily Inventory' +PREVIOUSOF('Ending Daily Inventory'[FILTER: Day IN 'Recent Days']) + +// Metric - 'Ending Daily Inventory' +'Beginning Daily Inventory' + 'Purchases' - 'Sales' +``` + +Performance: 90 days instead of 1,825 days = 20x faster. + +### When to Use Subsets + +Use subsets when: + +- Historical data doesn't change +- Only recent periods need iterative calculation +- Users only interact with recent periods + +Design and risks: List Subsets have irreversible data-loss behavior when membership changes and require explicit mapping to the parent dimension. For when to recommend subsets vs filters or another list, data-loss warnings, and safe patterns (STORE/CALC, remap to parent), see [List Subsets (modeling)](../modeling-pigment-applications/modeling_subsets.md). + +Example use cases: + +- Rolling 90-day inventory +- Current fiscal year YTD +- Last 12 months cumulative +- Current quarter sequential calculations + +### Creating Effective Subsets + +Time-based subset: + +```pigment +// Property on Month dimension - 'Is Current Year' (Boolean) +Month.Year = TIMEDIM('Today', Year) + +// Metric - 'YTD Revenue' +YEARTODATE('Monthly Revenue') +``` + +Dynamic subset: + +```pigment +// Days Since Today on the Day dimension +DAYS('Today', Day.'Start Date') +``` + +## Optimization Strategy 2: Use FILLFORWARD Instead of PREVIOUS + +### When FILLFORWARD is Better + +FILLFORWARD: Non-iterative blank filling (more efficient). + +PREVIOUS: Iterative calculation (less efficient). + +Use FILLFORWARD when: + +- You only need to fill blanks with the last known value +- No calculation is needed at each step +- The logic is simple forward propagation + +### Example: Status Propagation + +Anti-pattern -- iterative carry-forward where no per-period calculation is needed: + +```pigment +// Metric - 'Current Status' (iterative enabled) +// PREVIOUSOF takes only a metric reference, not a literal. +IFBLANK('Status Input', PREVIOUSOF('Current Status')) +``` + +Problem: Iterative, computes every period even if no change. Simple carry-forward does not need iteration. + +Optimized using FILLFORWARD: + +```pigment +// Metric - 'Current Status' +FILLFORWARD('Status Input', Month) +``` + +Improvement: Non-iterative, much faster. + +### Example: Employee Assignment + +Anti-pattern -- same carry-forward pattern: + +```pigment +// Metric - 'Current Department' (iterative enabled) +IFBLANK('Department Change', PREVIOUSOF('Current Department')) +``` + +Problem: Iterative, computes every period even if no change. + +Optimized using FILLFORWARD: + +```pigment +// Metric - 'Current Department' +FILLFORWARD('Department Change', Month) +``` + +Improvement: Non-iterative, much faster. + +When PREVIOUS is required: + +- Calculation at each step (e.g., balance + inflow - outflow) +- Conditional logic at each period +- Transformations that depend on previous value + +## Optimization Strategy 3: Reduce Dimensionality + +### The Problem: High-Dimensional Iterative Calculations + +Anti-pattern: + +```pigment +// 3 dimensions: Month x Product x Region +// Metric - 'Cumulative Sales' +CUMULATE('Monthly Sales', Month) +``` + +Performance: Iterates for every Product x Region combination. + +If: 1,000 products x 50 regions = 50,000 iteration chains. + +### The Solution: Aggregate Before Iterating + +Pattern: Reduce dimensions before iterative calculation. + +Implementation: + +```pigment +// Aggregate to fewer dimensions +// Metric - 'Total Monthly Sales' +'Monthly Sales'[REMOVE: Product, Region] + +// Iterate on smaller dataset +// Metric - 'Cumulative Total Sales' +CUMULATE('Total Monthly Sales', Month) +``` + +Performance: 1 iteration chain instead of 50,000 = 50,000x faster. + +Trade-off: Less granular (total only, not by Product x Region). + +### When This Works + +Use when: + +- The cumulative total is what matters +- Product/Region-level cumulative not needed +- Reporting is at aggregate level + +Don't use when: + +- Need cumulative by Product x Region +- Granular analysis required +- Allocation would be complex + +## Optimization Strategy 4: Alternative Calculation Methods + +### Pattern 1: Pre-Compute Starting Points + +Anti-pattern: Roll forward in a single ending-balance metric from the beginning of time. + +```pigment +// Metric - 'Ending Balance' +PREVIOUSOF('Ending Balance') + 'Change' +``` + +Problem: If data goes back 10 years, iterates from year 1. + +Optimized: Use a known starting point. + +```pigment +// Metric - 'Beginning Balance' +IF(Month = 'First Month of Window', 'Imported Starting Balance', PREVIOUSOF('Ending Balance')) + +// Metric - 'Ending Balance' +'Beginning Balance' + 'Change' +``` + +Improvement: Iterate only from the window start, not all history. + +## Optimization Strategy 5: Granularity Trade-offs + +### Consider Monthly Instead of Daily + +Scenario: Cash flow forecasting with daily granularity. + +Question: Is daily granularity necessary? + +Optimized: Use monthly granularity if acceptable. + +```pigment +// Metric - 'Beginning Cash Balance' +PREVIOUSOF('Ending Cash Balance') + +// Metric - 'Ending Cash Balance' +'Beginning Cash Balance' + 'Monthly Inflows' - 'Monthly Outflows' +``` + +Improvement: ~30x fewer iterations when moving from daily to monthly (e.g. 1,825 days -> 60 months). + +## Common Iterative Calculation Patterns + +### Pattern 1: Inventory Balance + +```pigment +// Metric - 'Beginning Inventory' +PREVIOUSOF('Ending Inventory') + +// Metric - 'Ending Inventory' +'Beginning Inventory' + 'Purchases' - 'Sales' +``` + +Optimization: + +- Subset to relevant periods +- Use monthly instead of daily if possible +- Reduce product dimensionality where appropriate + +### Pattern 2: Cash Flow + +```pigment +// Metric - 'Beginning Cash Balance' +PREVIOUSOF('Ending Cash Balance') + +// Metric - 'Ending Cash Balance' +'Beginning Cash Balance' + 'Inflows' - 'Outflows' +``` + +Optimization: + +- Pre-compute starting balance for current year +- Use monthly granularity +- Consider separate metrics for different cash accounts + +### Pattern 3: Employee Headcount + +```pigment +// Metric - 'Beginning Headcount' +PREVIOUSOF('Ending Headcount') + +// Metric - 'Ending Headcount' +'Beginning Headcount' + 'Hires' - 'Departures' +``` + +Optimization: + +- Use FILLFORWARD for static assignments +- Aggregate by department before iterating +- Subset to current fiscal year + +### Pattern 4: Loan Balance + +```pigment +// Metric - 'Beginning Loan Balance' +PREVIOUSOF('Ending Loan Balance') + +// Metric - 'Ending Loan Balance' +'Beginning Loan Balance' - 'Payment' +``` + +Optimization: + +- Calculate only for active loans +- Use monthly payments instead of daily +- Pre-compute for historical periods + +## Performance Monitoring + +### Signs of Iterative Calculation Issues + +1. Timeouts: Calculation doesn't complete +2. Long computation times: >10 seconds for simple inputs +3. Profiler shows scope loss: On time dimension +4. User complaints: Slow response when updating values + +### Measuring Impact + +Before optimization: + +- Note `Duration` from `tool:performance_profile_change` +- Count number of iteration steps (time periods) +- Check dimensionality (how many chains) + +After optimization: + +- Compare computation time +- Verify correctness +- Check effective scope in profile output (avoid `no scope, full computation` on time dimension) + +Expected improvements: + +- Subsetting: 5-20x faster +- FILLFORWARD vs PREVIOUS: 10-50x faster +- Reduced dimensionality: 10-1000x faster +- Granularity change: 10-30x faster + +## Best Practices Summary + +1. Subset time dimensions: Limit iteration window to relevant periods +2. Use FILLFORWARD when possible: Non-iterative is faster +3. Reduce dimensionality: Aggregate before iterating +4. Pre-compute starting points: Don't iterate from the beginning of time +5. Consider granularity trade-offs: Monthly vs daily vs weekly +6. Use CUMULATE for simple totals: Optimized for summation +7. Split beginning/ending metrics: `Beginning = PREVIOUSOF(Ending)`, then compute `Ending` from flows +8. Profile regularly: Measure impact of optimizations +9. Accept trade-offs: Sometimes granularity or detail must be sacrificed + +## When Iterative Calculations Are Unavoidable + +Some calculations require iteration: + +Cash flow with complex logic: + +```pigment +// Metric - 'Beginning Cash Balance' +PREVIOUSOF('Ending Cash Balance') + +// Metric - 'Ending Cash Balance' +'Beginning Cash Balance' + + IF('Beginning Cash Balance' < 'Minimum', 'Credit Line Draw', 0) + + 'Inflows' - 'Outflows' +``` + +Inventory with reorder logic: + +```pigment +// Metric - 'Beginning Inventory' +PREVIOUSOF('Ending Inventory') + +// Metric - 'Ending Inventory' +'Beginning Inventory' + + IF('Beginning Inventory' < 'Reorder Point', 'Order Quantity', 0) + + 'Receipts' - 'Sales' +``` + +In these cases: + +- Optimize what you can (subsetting, dimensionality) +- Accept the performance cost +- Consider if the complexity is truly necessary + +## Calendar and Granularity Considerations + +Time horizon and granularity are the primary cost drivers for iterative calculations: + +| Granularity | Periods/year | 5-year horizon | +|---|---|---| +| Monthly | 12 | 60 (fast) | +| Weekly | 52 | 260 (moderate) | +| Daily | 365 | 1,825 (slow) | + +Key decisions: + +- Subset iterative calculations to relevant periods (current fiscal year, rolling 90 days, last 12 months). +- Consider whether daily granularity is truly needed; planning typically works at monthly, actuals may need daily, forecasting is often weekly or monthly. +- Pre-compute starting points for the iteration window so the engine does not roll forward from the beginning of time. + +For calendar configuration and time dimension structure, see [modeling_time_and_calendars.md](../modeling-pigment-applications/modeling_time_and_calendars.md). + +--- + +## See Also + +- [Iterative Calculation (PREVIOUS & PREVIOUSOF)](../writing-pigment-formulas/functions_iterative_calculation.md) - Full spec: circular dependencies, configuration, syntax, debugging +- [Performance Scoping Patterns](./performance_scoping_patterns.md) - Understanding scope loss in iterations +- [Performance Formula Optimization](./performance_formula_optimization.md) - General formula optimization +- [Time and Date Functions](../writing-pigment-formulas/functions_time_and_date.md) - FILLFORWARD, SELECT vs PREVIOUS/PREVIOUSOF diff --git a/plugins/pigment/skills/optimizing-pigment-performance/performance_profiling.md b/plugins/pigment/skills/optimizing-pigment-performance/performance_profiling.md new file mode 100644 index 00000000..8823a120 --- /dev/null +++ b/plugins/pigment/skills/optimizing-pigment-performance/performance_profiling.md @@ -0,0 +1,114 @@ +# Performance Profiling (Modeler Agent) + +Measure compute performance with profiling tools, parse their output, and report findings to the user. For scope mechanics and formula fixes, see [./performance_scoping_patterns.md](./performance_scoping_patterns.md). For the full performance loop, see [./performance_troubleshooting_workflow.md](./performance_troubleshooting_workflow.md). + +## Prerequisites + +| Tool | Use when | +|---|---| +| `tool:get_top_blocks_by_performance` | Hotspot block unknown; rank blocks app-wide over a time window | +| `tool:performance_profile_change` | Slow action reproduced; you have `change_id` from audit trail | + +## Workflow + +1. Triage (optional). `tool:get_top_blocks_by_performance` with `scenario_id`, `range_start`, `range_end`, `top_n`, and `criteria`: `ExecutionTimeSumMs` (total cost), `ExecutionTimeAvgMs` (typical cost), `ExecutionCount` (churn), `CombinedCardinality` (data volume, no execution history needed). +2. Reproduce the slow input or formula change. +3. Profile. `tool:performance_profile_change` with `change_id` (UUID). +4. Analyze using sections below; map `Blocks:` to formulas. +5. Fix one change at a time. New `change_id` -> re-profile -> compare `Duration` and scope. + +Board-render fork: Low total execution time but slow board load -> `skill:designing-pigment-boards`, not formula work. + +--- + +## Top blocks output + +``` +Top N blocks ranked by performance: +- {block_id} ({block_type}) - cardinality={n}, executions={count}, avg_ms={avg}, sum_ms={sum} +``` + +Missing `job_profile` -> widen the time window or switch `criteria`. Then profile a change on the suspect block. + +--- + +## Change profile output + +``` +Change profiled successfully. N execution(s) found. + +Executions: +1. {job_type} + - Id: {uuid} + - Blocks: Metric(`uuid`), ... + - Dimensions: uuid1, uuid2, ... + - Ready at: Xms, Executed at: Yms, Duration: Zms + - Effective scope: {text} + - Output scope: {text} + - Depends on: uuid, ... <- optional +``` + +| Field | Meaning | Tell the user | +|---|---|---| +| Ready at | Wait for dependencies | Ready at | +| Executed at | Ready + queue contention | Executed at | +| Duration | Compute time | Duration | +| Effective scope | Scope used | Effective scope | +| Output scope | Scope passed downstream | Output scope | +| Depends on | Upstream execution IDs | Dependencies | + +Block labels: `Metric(...)`, `List(...)`, `Table(...)`, `Cycle(...)`, `Block(app:...)`. + +Scope text: + +| Text | Meaning | +|---|---| +| `no change` | No cells written | +| `no scope, full computation` | Full recompute (X = 0) | +| `dim:uuid (N modalities), ...` | Scoped to N modalities per dimension | + +### X/Y notation + +- Y = count on `Dimensions:` line +- X = count of `dim:` entries in `Effective scope:` +- Target X = Y on hot paths + +| Effective | Output | Interpretation | +|---|---|---| +| `dim:...` | Same | Scope preserved | +| `dim:...` | More dims | Scope introduced downstream | +| any | `no change` | Ran, no output (still check Duration) | +| `no scope, full computation` | - | Scope-loss origin candidate | + +First `no scope, full computation` -> inspect that block's formula (`REMOVE`, `CUMULATE`, `PREVIOUS`, `RANK`). + +### Time and dependencies + +- Sort by Duration; flag > 1000 ms or dominant wall-time share. +- Contention = Executed at - Ready at (large vs Duration -> workload/queueing, not formula). +- Wall time approx max(Executed at + Duration) across executions. +- Match `Depends on` UUIDs to upstream `Id:` lines; ancestors appear earlier in the list. + +### Patterns + +| Pattern | Signature | Action | +|---|---|---| +| Cascading scope loss | Scoped runs, then first `no scope, full computation`, rest full/no change | Defer `REMOVE`/aggregations; see scoping patterns doc | +| No change, high Duration | `Output scope: no change` and Duration > 500 ms | Add earlier `FILTER`/`EXCLUDE` | +| High contention | Executed at >> Ready at on many rows | Broad scope or too many parallel branches | + +--- + +## Report to the user + +Include: execution count, approximate wall time, permission filter note, chain with natural block names, X/Y per step, slowest step, scope-loss origin, one recommendation. + +Vocabulary: Say ready at, executed at, duration, scope, dependency. Do not say execution_id, time_schedule_ms, effective_scope, clauses. + +--- + +## See Also + +- [./performance_scoping_patterns.md](./performance_scoping_patterns.md) +- [./performance_formula_optimization.md](./performance_formula_optimization.md) +- [./performance_troubleshooting_workflow.md](./performance_troubleshooting_workflow.md) diff --git a/plugins/pigment/skills/optimizing-pigment-performance/performance_scoping_patterns.md b/plugins/pigment/skills/optimizing-pigment-performance/performance_scoping_patterns.md new file mode 100644 index 00000000..3e22301e --- /dev/null +++ b/plugins/pigment/skills/optimizing-pigment-performance/performance_scoping_patterns.md @@ -0,0 +1,956 @@ +# Performance Scoping Patterns + +## Introduction + +Scoping is one of the most important performance optimization concepts in Pigment. Understanding how scope propagates, when it's lost, and how to preserve it can dramatically improve calculation performance. + +This guide provides deep technical insight into scope propagation mechanics, early scoping strategies, and scope preservation techniques. + +## Core Scoping Concepts + +### What is Scoping? + +Scoping determines which cells of a metric need to be recomputed when a change occurs. Instead of recalculating every cell in a metric, Pigment computes only the cells affected by the change. + +Example: + +- Metric: `Revenue` with dimensions `Product` (100 items) x `Month` (12 items) = 1,200 cells +- Change: Update revenue for Product "Widget A" in "January" +- Without scoping: Recompute all 1,200 cells +- With scoping: Recompute only 1 cell +- Performance gain: 1,200x faster + +### Scope Propagation + +When a metric with scope is referenced by another metric, the scope propagates downstream if no transformations break it. + +Example chain: + +``` +Input: Product "A", Month "Jan" -> scope 2/2 +down +Metric 1: 'Revenue' -> scope 2/2 (preserved) +down +Metric 2: 'Revenue' * 1.1 -> scope 2/2 (preserved) +down +Metric 3: 'Revenue' + 'Fixed Cost' -> scope 2/2 (preserved) +``` + +All metrics maintain scope, so only one cell is computed in each metric. + +## When Scope is Preserved + +### Simple Arithmetic Operations + +Scope is preserved through basic arithmetic: + +```pigment +// All preserve scope +'Metric' * 2 +'Metric' + 'Other Metric' +'Metric' - 'Baseline' +'Metric' / 'Denominator' +``` + +Condition: Both metrics must have compatible dimensions. + +### Filtering Operations + +Filtering preserves scope on the filtered dimension: + +```pigment +'Revenue'[FILTER: 'Product'.'Category' = "Electronics"] +``` + +Result: If input has scope on Product and Month, output maintains that scope. + +### Conditional Logic (with caution) + +IF statements can preserve scope if both branches maintain it: + +```pigment +IF('Condition', 'Metric A', 'Metric B') +``` + +Scope preserved if: + +- Both 'Metric A' and 'Metric B' have the same scope +- The condition doesn't require full computation + +## When Scope is Lost + +### REMOVE Modifier + +The most common cause of scope loss: + +```pigment +'Revenue'[REMOVE: Product] +``` + +Why scope is lost: To aggregate across all products, Pigment must compute all product cells, even if only one product changed. + +Scope result: `0/X` on the REMOVE dimension. + +### CUMULATE and Time Functions + +Sequential calculations lose scope on the cumulating dimension: + +```pigment +CUMULATE('Monthly Revenue', Month) +``` + +Why scope is lost: To compute Month 6, Pigment needs Months 1-5. If Month 3 changes, Months 3-12 must be recomputed. + +Same behavior: + +- `PREVIOUS`, `PREVIOUSOF` +- `YEARTODATE`, `QUARTERTODATE`, `MONTHTODATE` +- `FILLFORWARD` + +### SHIFT Operations + +Shifting on a dimension loses scope: + +```pigment +'Revenue'[SELECT: Month-1] +``` + +Why scope is lost: The output cells don't align with input cells (Month 2 output depends on Month 1 input). + +### ADD Modifier (Partial Scope Loss) + +Adding a dimension creates partial scope: + +```pigment +'Revenue'[ADD: Version] +``` + +Result: Scope `2/3` if original was `2/2`. + +Why: The original dimensions maintain scope, but the new dimension has no scope (must compute across all versions). + +## Scope Preservation Strategies + +### Strategy 1: Start Formulas with Scoping Clauses + +Principle: Filter or exclude data at the very beginning of your formula to establish scope early. + +Anti-pattern: + +```pigment +// Computation happens first, then filtering +('Revenue' * 'Growth Rate' + 'Fixed Costs')[FILTER: 'Product'.'Active' = TRUE] +``` + +Optimized pattern: + +```pigment +// Filter first, then compute +'Revenue'[FILTER: 'Product'.'Active' = TRUE] * 'Growth Rate' + 'Fixed Costs' +``` + +Why better: The filter establishes scope early, so all subsequent operations work on a smaller dataset. + +### Strategy 2: Use ISDEFINED for Early Scoping + +Pattern: Use ISDEFINED at the start of formulas to scope to defined cells only. + +Example: + +```pigment +// Without early scoping +'Revenue' * 'Adjustment Factor' + +// With early scoping +IFDEFINED('Revenue', 'Revenue' * 'Adjustment Factor') +``` + +Benefit: If 'Revenue' is sparse (only 10% of cells have values), computation is limited to those 10%. + +### Strategy 3: Defer Aggregations to the End + +Principle: Keep scope as long as possible by deferring REMOVE operations. + +Anti-pattern: + +```pigment +// Early aggregation +Step 1: 'Revenue'[REMOVE: Product] // Scope lost here +Step 2: 'Step 1' * 'Growth' // No scope +Step 3: 'Step 2' + 'Costs' // No scope +``` + +Optimized pattern: + +```pigment +// Late aggregation +Step 1: 'Revenue' * 'Growth' // Scope preserved +Step 2: 'Step 1' + 'Costs' // Scope preserved +Step 3: 'Step 2'[REMOVE: Product] // Scope lost only at end +``` + +Impact: Steps 1-2 are much faster with scope preserved. + +### Strategy 4: Use Mappings Instead of REMOVE + ADD + +Anti-pattern: + +```pigment +// Remove then add back different dimension +'Employee Salary'[REMOVE: Employee][ADD: Department] +``` + +Optimized pattern: + +```pigment +// Use mapping property +'Employee Salary'[BY: 'Employee'.'Department'] +``` + +Why better: + +- BY with mapping can preserve scope better +- Cleaner formula +- More efficient computation + +Condition: Requires a dimension-typed property mapping Employee to Department. + +### Strategy 5: Avoid Unnecessary REMOVE + +Anti-pattern: + +```pigment +// Removing dimension that could be kept +'Transaction Amount'[REMOVE: Transaction ID] +``` + +Question to ask: Do downstream metrics need the Transaction ID dimension? + +If yes: Keep it and use BY or SELECT downstream instead of REMOVE. + +If no: REMOVE is appropriate. + +## Advanced Scoping Patterns + +### Pattern 1: Conditional Scoping + +Use ISDEFINED to scope based on data presence: + +```pigment +// Only compute where both metrics have data +IFDEFINED('Metric A', + IFDEFINED('Metric B', + 'Metric A' / 'Metric B' + ) +) +``` + +Benefit: If both metrics are sparse, computation is limited to their intersection. + +### Pattern 2: Dimension-Specific Scoping + +Scope on specific dimensions while allowing others to compute fully: + +```pigment +// Scope on Product but compute all Months +'Revenue'[FILTER: 'Product'.'Active' = TRUE] +``` + +Result: Scope maintained on Product, all Months computed for active products. + +### Pattern 3: Early EXCLUDE for Performance + +Use EXCLUDE to remove unwanted data before computation: + +```pigment +// Exclude test data early +'Revenue'[EXCLUDE: 'Account'.'Type' = "Test"] * 'Growth Rate' +``` + +Benefit: Test accounts are excluded before the multiplication, reducing computation. + +### Pattern 4: Scoped Aggregation + +When aggregation is necessary, scope it to relevant subsets: + +```pigment +// Instead of aggregating everything +'Revenue'[REMOVE: Product] + +// Aggregate only relevant subset +'Revenue'[FILTER: 'Product'.'Category' = "Electronics"][REMOVE: Product] +``` + +Benefit: Smaller aggregation scope = faster computation. + +## Scope and Modifiers + +### BY Modifier Scope Behavior + +Aggregation (N->1): + +```pigment +'Transaction Amount'[BY: 'Transaction'.'Customer'] +``` + +Scope: Lost on Transaction dimension, preserved on others. + +Allocation (1->N): + +```pigment +'Budget'[BY: 'Department'.'Employee'] +``` + +Scope: Partially lost (must compute across all target items). + +### SELECT Modifier Scope Behavior + +Filtering: + +```pigment +'Revenue'[SELECT: Product = "Widget A"] +``` + +Scope: Preserved if input has scope on Product. + +Time Offset: + +```pigment +'Revenue'[SELECT: Month-1] +``` + +Scope: Lost on Month dimension (offset breaks alignment). + +### FILTER vs SELECT for Scoping + +FILTER: Preserves dimensionality and scope + +```pigment +'Revenue'[FILTER: 'Product'.'Active' = TRUE] +``` + +Result: Same dimensions, scope preserved. + +SELECT: Removes dimension, may lose scope + +```pigment +'Revenue'[SELECT: Product = "Widget A"] +``` + +Result: Product dimension removed, scope lost on Product. + +When to use each: + +- Use FILTER when you want to keep the dimension and preserve scope +- Use SELECT when you want to remove the dimension (scope loss acceptable) + +## Scope Loss: When It's Unavoidable + +### Scenario 1: Percentage of Total + +Requirement: Calculate each product's share of total revenue. + +Formula: + +```pigment +'Revenue' / 'Revenue'[REMOVE: Product] +``` + +Why unavoidable: The denominator (total) depends on all products. If one product changes, all shares change. + +Scope result: `0/X` - full recomputation required. + +Mitigation: Place this metric at the end of the computation chain. + +### Scenario 2: Ranking + +Requirement: Rank products by revenue. + +Formula: + +```pigment +RANK('Revenue'[REMOVE: Month]) +``` + +Why unavoidable: Ranking requires knowing all values to determine positions. + +Scope result: `0/X` - full recomputation required. + +Mitigation: Use only for final reporting metrics, not intermediate calculations. + +### Scenario 3: Year-to-Date Calculations + +Requirement: Calculate YTD revenue. + +Formula: + +```pigment +YEARTODATE('Monthly Revenue') +``` + +Why unavoidable: YTD for December depends on all previous months. + +Scope result: Partial scope loss on Month dimension. + +Mitigation: + +- Use subsets to limit the time range +- Consider if period-to-date is truly needed or if monthly is sufficient + +### Scenario 4: Complex Conditional Aggregations + +Requirement: Sum revenue only for products meeting dynamic criteria. + +Formula: + +```pigment +'Revenue'[FILTER: 'Revenue' > AVGOF('Revenue')][REMOVE: Product] +``` + +Why unavoidable: The average depends on all products, and the filter depends on the average. + +Scope result: `0/X` - full recomputation required. + +Mitigation: Cache the average in a separate metric if it doesn't change often. + +## Measuring Scope Impact + +### Using the Profiler + +1. Before optimization: Note effective scope text from `tool:performance_profile_change` +2. After optimization: Compare scope values +3. Look for: Increased X in X/Y notation + +Example: + +- Before: `0/3` -> After: `2/3` = Scope improved on 2 dimensions + +### Computation Time Correlation + +General rule: Better scope = faster computation. + +Expected improvements: + +- `0/3` -> `1/3`: 10-100x faster (depending on dimension size) +- `0/3` -> `2/3`: 100-1000x faster +- `0/3` -> `3/3`: 1000-10000x faster + +Note: Actual improvement depends on dimension cardinality and data sparsity. + +## Real-World Scoping Example + +### Scenario: Sales Planning Application + +Original computation chain: + +``` +1. 'Sales Input' - scope 3/3 (Product, Region, Month) +2. 'Sales with Growth' = 'Sales Input' * 'Growth Rate' - scope 3/3 +3. 'Total Sales' = 'Sales with Growth'[REMOVE: Product, Region] - scope 0/3 +4. 'Sales Share' = 'Sales with Growth' / 'Total Sales' - scope 0/3 +5. 'Allocated Costs' = 'Total Costs' * 'Sales Share' - scope 0/3 +6. 'Profit' = 'Sales with Growth' - 'Allocated Costs' - scope 0/3 +``` + +Problem: Scope lost at step 3, all subsequent steps have no scope. + +Optimized chain: + +``` +1. 'Sales Input' - scope 3/3 +2. 'Sales with Growth' = 'Sales Input' * 'Growth Rate' - scope 3/3 +3. 'Direct Costs' = 'Sales with Growth' * 'Cost Rate' - scope 3/3 +4. 'Profit' = 'Sales with Growth' - 'Direct Costs' - scope 3/3 + +// Reporting metrics only (scope loss acceptable) +5. 'Total Sales' = 'Sales with Growth'[REMOVE: Product, Region] - scope 0/3 +6. 'Sales Share' = 'Sales with Growth' / 'Total Sales' - scope 0/3 +``` + +Result: + +- Steps 1-4 maintain full scope +- Only reporting metrics (5-6) lose scope +- User inputs are 50x faster +- Reporting views still calculate correctly but aren't in the hot path + +## Best Practices Summary + +1. Scope early: Start formulas with FILTER, EXCLUDE, or ISDEFINED +2. Scope often: Add scoping clauses throughout the formula, not just at the start +3. Defer aggregations: Push REMOVE and scope-losing operations to the end +4. Use mappings: Prefer BY with mappings over REMOVE + ADD +5. Profile after changes: Re-run `tool:performance_profile_change` and compare scope text +6. Accept unavoidable loss: Some calculations require full recomputation +7. Isolate scope loss: Keep scope-losing metrics separate from the main computation chain + +## Hierarchy-Specific Performance Patterns + +### Overview + +Hierarchies implemented through dimension-type properties have specific performance characteristics. Understanding these patterns helps optimize calculations involving parent-child relationships, multi-level aggregations, and cross-hierarchy analysis. + +### Performance Characteristics of Property-Based Hierarchies + +Key Insight: Property-based hierarchies (using dimension-type properties) are generally faster than dimension-based hierarchies (adding dimensions to metric structure). + +Why Property-Based Hierarchies Perform Better: + +1. Fewer Dimensions in Structure: + - Metric: `Product x Month` (2 dimensions) + - vs. `Product x Category x Line x Month` (4 dimensions) + - Fewer dimensions = smaller cardinality = faster calculations + +2. Sparse Engine Optimization: + - Only base-level combinations need values + - Parent levels computed on-demand through aggregation + - No storage overhead for parent-level combinations + +3. Scope Preservation: + - Changes at base level maintain scope + - Aggregation to parent levels can leverage scope + - More efficient than multi-dimensional recalculation + +Performance Comparison: + +| Approach | Metric Structure | Cardinality | Calculation Speed | Storage | +| --------------- | ----------------------------------- | ------------------------- | ----------------- | ------- | +| Property-based | `Product x Month` | 1,000 x 12 = 12K | Fast | Low | +| Dimension-based | `Product x Category x Line x Month` | 1,000 x 50 x 10 x 12 = 6M | Slow | High | + +### Pattern 1: Efficient Hierarchy Aggregation + +Optimal Pattern: Use BY with property references for aggregation. + +```pigment +// Efficient: Direct property-based aggregation +'SKU Revenue' [BY SUM: SKU.Product, Month] + +// Efficient: Multi-level aggregation +'SKU Revenue' [BY SUM: SKU.Product.Category, Month] +``` + +Why Efficient: + +- Pigment optimizes property-based BY operations +- Scope can be preserved on non-aggregated dimensions +- No intermediate dimension explosion + +Anti-Pattern: REMOVE then ADD approach + +```pigment +// Inefficient: Remove and add dimensions +'SKU Revenue' [REMOVE: SKU] [ADD: Category] +``` + +Why Inefficient: + +- REMOVE loses scope on SKU dimension +- ADD creates dense combinations +- Two operations instead of one + +Performance Gain: 2-5x faster with property-based BY + +### Pattern 2: Multi-Level Hierarchy Navigation + +Optimal Pattern: Chain properties in single BY operation. + +```pigment +// Efficient: Single operation to aggregate 3 levels up +'SKU Revenue' [BY SUM: SKU.Product.Category.Line, Store, Month] +``` + +Anti-Pattern: Multiple sequential aggregations + +```pigment +// Inefficient: Multiple aggregation steps +Step 1: 'SKU Revenue' [BY SUM: SKU.Product, Store, Month] +Step 2: 'Step 1' [BY SUM: Product.Category, Store, Month] +Step 3: 'Step 2' [BY SUM: Category.Line, Store, Month] +``` + +Why Inefficient: + +- Creates intermediate metrics +- Each step requires full computation +- No optimization across steps + +Performance Gain: 3-10x faster with chained properties + +Exception: If intermediate levels are frequently used, caching them in separate metrics may be beneficial. + +### Pattern 3: Cross-Hierarchy Aggregation + +Optimal Pattern: Combine multiple hierarchy aggregations in single operation. + +```pigment +// Efficient: Aggregate across two hierarchies simultaneously +'Sales' [BY SUM: Product.Category, Store.Region, Month] +``` + +Result: + +- Product aggregated to Category +- Store aggregated to Region +- Month unchanged +- Single efficient operation + +Anti-Pattern: Sequential aggregations + +```pigment +// Inefficient: Two separate aggregations +Step 1: 'Sales' [BY SUM: Product.Category, Store, Month] +Step 2: 'Step 1' [BY SUM: Category, Store.Region, Month] +``` + +Performance Gain: 2-4x faster with combined aggregation + +### Pattern 4: Conditional Hierarchy Aggregation + +Optimal Pattern: Filter before aggregating up hierarchy. + +```pigment +// Efficient: Filter at base level, then aggregate +'SKU Revenue' [FILTER: SKU.Active = TRUE] [BY SUM: SKU.Product.Category, Month] +``` + +Why Efficient: + +- Filters reduce data volume early +- Aggregation operates on smaller dataset +- Scope maintained on filtered dimension + +Anti-Pattern: Aggregate then filter + +```pigment +// Inefficient: Aggregate all data, then filter +'SKU Revenue' [BY SUM: SKU.Product.Category, Month] [FILTER: Category.Type = "Core"] +``` + +Why Inefficient: + +- Aggregates all SKUs (including inactive) +- More data to process +- Cannot leverage base-level filtering + +Performance Gain: 2-10x faster depending on filter selectivity + +### Pattern 5: Hierarchy-Level Caching + +When to Cache: Frequently-accessed hierarchy levels benefit from caching. + +Pattern: Create dedicated metrics for commonly-used aggregation levels. + +```pigment +// Base metric +Revenue (SKU x Store x Month) + +// Cached aggregations (if frequently used) +Revenue by Product = 'Revenue' [BY SUM: SKU.Product, Store, Month] +Revenue by Category = 'Revenue' [BY SUM: SKU.Product.Category, Store, Month] +Revenue by Line = 'Revenue' [BY SUM: SKU.Product.Category.Line, Store, Month] +``` + +When to Use: + +- Aggregation level used in 5+ other metrics +- Real-time dashboards requiring sub-second response +- Complex calculations at parent level +- High-traffic reporting views + +When NOT to Use: + +- Aggregation used only once or twice +- Ad-hoc analysis (use property chains directly) +- Infrequently accessed reports +- Storage constraints + +Trade-off: Storage and maintenance vs. calculation speed + +### Pattern 6: Scope Preservation in Hierarchies + +Key Insight: Property-based aggregation can preserve scope on non-aggregated dimensions. + +Example: + +```pigment +// Input change: SKU "ABC123", Store "NYC", Month "Jan" +// Scope: 3/3 (all dimensions scoped) + +// Aggregation to Product level +'Revenue' [BY SUM: SKU.Product, Store, Month] + +// Scope result: +// - SKU dimension: Scope lost (aggregating across SKUs) +// - Store dimension: Scope preserved (2/2 - only NYC) +// - Month dimension: Scope preserved (2/2 - only Jan) +``` + +Optimization: Order operations to preserve scope as long as possible. + +```pigment +// Good: Preserve scope on Store and Month +'Revenue' [BY SUM: SKU.Product, Store, Month] * 'Growth Rate' + +// Better: Keep scope even longer if Growth Rate is at Product level +'Revenue' * 'Growth Rate' [BY SUM: SKU.Product, Store, Month] +``` + +### Pattern 7: Deep Hierarchy Performance + +Performance by Hierarchy Depth: + +| Depth | Example | Performance | Recommendation | +| --------- | ------------------------------------------ | ----------- | ---------------------------- | +| 2 levels | Product -> Category | Excellent | Use freely | +| 3 levels | SKU -> Product -> Category | Very Good | Use freely | +| 4 levels | SKU -> Product -> Category -> Line | Good | Test with real data | +| 5 levels | SKU -> Product -> Category -> Line -> Division | Acceptable | Consider caching | +| 6+ levels | Deep organizational hierarchies | Variable | Cache frequently-used levels | + +Optimization for Deep Hierarchies: + +```pigment +// If 6-level hierarchy is slow, cache intermediate levels +Level_3_Aggregation = 'Base' [BY SUM: Dim.L1.L2.L3, ...] +Level_6_Aggregation = 'Level_3_Aggregation' [BY SUM: L3.L4.L5.L6, ...] +``` + +Why This Helps: + +- Breaks deep chain into manageable chunks +- Can optimize each level separately +- Easier to troubleshoot performance + +### Pattern 8: Ragged Hierarchy Performance + +Challenge: Ragged hierarchies (variable depth) have special performance considerations. + +Pattern: Use IFDEFINED to handle blanks efficiently. + +```pigment +// Efficient: Skip blanks in property chain +IFDEFINED('Employee'.'Manager'.'Director'.'VP', + 'Salary' [BY SUM: Employee.Manager.Director.VP, Month] +) +``` + +Why Efficient: + +- Only computes for employees with complete hierarchy +- Avoids processing blank property chains +- Maintains sparsity + +Anti-Pattern: Compute all, filter later + +```pigment +// Inefficient: Computes all, including blanks +'Salary' [BY SUM: Employee.Manager.Director.VP, Month] [FILTER: VP != BLANK] +``` + +### Pattern 9: Property-Based Filtering + +Optimal Pattern: Filter using property references. + +```pigment +// Efficient: Filter using property +'Revenue' [FILTER: Product.Category.Active = TRUE] +``` + +Why Efficient: + +- Pigment optimizes property-based filters +- Can leverage indexes on properties +- Scope preserved on filtered dimension + +Comparison with Dimension-Based Filtering: + +```pigment +// If Category was in metric structure +'Revenue' [FILTER: Category.Active = TRUE] + +// Property-based is similar performance but: +// - Metric structure is simpler (fewer dimensions) +// - More flexible (can filter by any property level) +``` + +### Pattern 10: Hierarchy Joins + +Pattern: Joining data across hierarchies using properties. + +```pigment +// Efficient: Join using property relationships +'Product Revenue' * 'Category Margin %' [BY: Product.Category] +``` + +Why Efficient: + +- BY with property handles dimension mismatch +- Single operation +- Pigment optimizes property-based joins + +Anti-Pattern: Manual dimension manipulation + +```pigment +// Inefficient: Remove and add dimensions manually +'Product Revenue' [REMOVE: Product] [ADD: Category] * 'Category Margin %' +``` + +### Performance Benchmarks + +Typical Performance Characteristics: + +| Operation | Property-Based | Dimension-Based | Speedup | +| ---------------------------------- | -------------- | --------------- | ------- | +| Single-level aggregation | 10ms | 25ms | 2.5x | +| Multi-level aggregation (3 levels) | 15ms | 100ms | 6.7x | +| Cross-hierarchy aggregation | 20ms | 150ms | 7.5x | +| Filtered aggregation | 12ms | 40ms | 3.3x | +| Deep hierarchy (5 levels) | 30ms | 300ms | 10x | + +Note: Actual performance depends on data volume, cardinality, and sparsity. + +### Best Practices Summary + +1. Prefer Property-Based Hierarchies: + +- Use dimension-type properties for hierarchies +- Keep metric structures minimal +- Aggregate using BY with property chains + +2. Optimize Aggregation Order: + +- Filter before aggregating +- Combine multiple aggregations in single operation +- Chain properties rather than sequential aggregations + +3. Cache Strategically: + +- Cache frequently-used hierarchy levels +- Don't cache rarely-used aggregations +- Balance storage vs. computation + +4. Preserve Scope: + +- Order operations to maintain scope +- Use property-based BY to preserve scope on non-aggregated dimensions +- Avoid unnecessary REMOVE operations + +5. Handle Ragged Hierarchies: + +- Use IFDEFINED for blanks +- Consider flattening very ragged hierarchies +- Test performance with production data + +6. Monitor Deep Hierarchies: + +- Test performance with 4+ level hierarchies +- Cache intermediate levels if needed +- Consider breaking into smaller chunks + +7. Leverage Property Filters: + +- Filter using property references +- Combine filters with aggregations +- Use early filtering for best performance + +### Troubleshooting Hierarchy Performance + +Issue: Slow aggregation up hierarchy + +Diagnosis: + +- Check hierarchy depth (4+ levels?) +- Verify property mappings are complete +- Look for ragged hierarchy with many blanks + +Solutions: + +- Cache intermediate levels +- Flatten hierarchy if possible +- Use IFDEFINED for ragged hierarchies + +Issue: Cross-hierarchy calculation is slow + +Diagnosis: + +- Multiple sequential aggregations? +- Aggregating before filtering? +- Using REMOVE/ADD instead of BY? + +Solutions: + +- Combine aggregations in single operation +- Filter before aggregating +- Use property-based BY operations + +Issue: Property chain not optimizing + +Diagnosis: + +- Very deep chain (6+ levels)? +- Properties not dimension-type? +- Blanks in property chain? + +Solutions: + +- Break chain into cached intermediate levels +- Verify all properties are dimension-type +- Handle blanks with IFDEFINED + +## Scenario and Version Cardinality + +Version and scenario dimensions multiply the total cell count of every metric that carries them. Growing from 3 to 12 scenarios roughly quadruples computation time for any input that propagates across scenarios. + +Mitigation strategies: + +- Subset inactive scenarios so iterative and AR computations skip them. +- Archive historical versions that are no longer actively planned. +- Use `FILTER` or `SELECT` to limit formulas to the active scenario set when full cross-scenario computation is not needed. + +See `skill:modeling-pigment-applications` for version and scenario architecture. + +--- + +## Common Mistakes + +### Mistake 1: Removing Dimensions Too Early + +```pigment +// Bad: Loses scope immediately, all downstream loses scope +Step 1: 'Revenue'[REMOVE: Product] * 'Growth Rate' +``` + +```pigment +// Good: Defer REMOVE to a separate, late metric +Step 1: 'Revenue' * 'Growth Rate' // scope preserved +Step 2: 'Step 1'[REMOVE: Product] // scope lost only at end +``` + +Note: Moving `[REMOVE: Product]` onto a different metric in the same expression (e.g. `'Revenue' * 'Growth Rate'[REMOVE: Product]`) changes semantics; it removes Product from Growth Rate before multiplication, which is a different calculation. Always defer REMOVE to a separate downstream metric. + +### Mistake 2: Not Using ISDEFINED + +```pigment +// Bad: Computes all cells +'Revenue' * 'Adjustment' + +// Good: Computes only defined cells +IFDEFINED('Revenue', 'Revenue' * 'Adjustment') +``` + +### Mistake 3: Unnecessary ADD Then REMOVE + +```pigment +// Bad: Adds then removes dimension +'Metric'[ADD: Dimension][REMOVE: Dimension] + +// Good: Don't add it in the first place +'Metric' +``` + +## See Also + +- [Performance Profiling](./performance_profiling.md) - profiling tools and output parsing +- [Performance Formula Optimization](./performance_formula_optimization.md) - Formula-level optimization techniques +- [Performance Sparsity Deep Dive](./performance_sparsity_deep_dive.md) - Sparsity and its relationship to scoping diff --git a/plugins/pigment/skills/optimizing-pigment-performance/performance_sparsity_deep_dive.md b/plugins/pigment/skills/optimizing-pigment-performance/performance_sparsity_deep_dive.md new file mode 100644 index 00000000..d6325a9d --- /dev/null +++ b/plugins/pigment/skills/optimizing-pigment-performance/performance_sparsity_deep_dive.md @@ -0,0 +1,627 @@ +# Performance Sparsity Deep Dive + +## Introduction + +Sparsity is a fundamental concept in Pigment that dramatically affects performance, memory usage, and computation efficiency. Understanding the difference between sparse and dense metrics, and knowing which functions preserve sparsity, is critical for building performant applications. + +This guide provides a deep technical dive into sparsity management, common densification anti-patterns, and best practices for preserving sparsity. + +## What is Sparsity? + +### Sparse vs Dense Metrics + +Sparse metric: Only stores cells that have values. Blank cells are not stored. + +Example: + +- Dimension: 1000 products x 12 months = 12,000 possible cells +- Values: Only 500 products sold in any given month +- Stored cells: 500 (sparse) +- Memory savings: 95.8% + +Dense metric: Stores all cells, including those with blank or zero values. + +Example: + +- Same dimensions: 12,000 possible cells +- Stored cells: 12,000 (dense) +- Memory usage: 24x larger than sparse version + +### Why Sparsity Matters + +Performance benefits: + +1. Memory efficiency: Store only meaningful data +2. Computation speed: Compute only cells with values +3. Query performance: Smaller datasets to scan +4. Scalability: Models can handle larger dimensions + +Real-world impact: + +- Sparse metric: 100ms computation time +- Same metric densified: 2,400ms computation time +- Performance degradation: 24x slower + +### How Blanks Interact with Operators + +Pigment's sparse engine has specific rules for how blanks behave with arithmetic and logical operators. Understanding these rules helps predict when formulas preserve sparsity vs. when they might densify. + +Quick summary: + +- Additive operators (`+`, `-`, `OR`): `blank + value = value` +- Multiplicative operators (`*`, `/`, `AND`): `blank x value = blank` +- Constants don't fill blanks: `'Revenue' + 100` keeps blank cells blank + +For detailed behavior including dimension alignment effects, see [How Blanks Behave with Operators](../writing-pigment-formulas/functions_logical.md#how-blanks-behave-with-operators). + +## Functions That Affect Sparsity + +### Functions That Preserve Sparsity + +#### ISDEFINED + +Behavior: Returns TRUE if a value is defined, otherwise returns BLANK (not FALSE). + +```pigment +ISDEFINED('Revenue') +``` + +Output: + +- Where Revenue has a value: TRUE +- Where Revenue is blank: BLANK (not stored) + +Sparsity: GOOD: Preserved - blank cells remain blank. + +#### IFBLANK + +Behavior: Returns the first argument if defined, otherwise returns the second argument. + +```pigment +IFBLANK('Revenue', 'Backup Revenue') +``` + +Output: + +- Where Revenue has a value: Revenue value +- Where Revenue is blank AND Backup Revenue has a value: Backup Revenue value +- Where both are blank: BLANK + +Sparsity: GOOD: Preserved - but only when both arguments have the same dimensions. + +Important note: Despite the name similarity to ISBLANK, IFBLANK does NOT densify when used correctly. + +WARNING: Warning: IFBLANK can densify when arguments have mismatched dimensions (first argument more dimensional than second). See [IFBLANK dimension rules](../writing-pigment-formulas/functions_logical.md#ifblank) for details. + +#### IFDEFINED + +Behavior: Shortcut for IF(ISDEFINED(...), ..., ...). + +```pigment +IFDEFINED('Revenue', 'Revenue' * 1.1) +``` + +Output: + +- Where Revenue has a value: Revenue \* 1.1 +- Where Revenue is blank: BLANK + +Sparsity: GOOD: Preserved - computes only where defined. + +### Functions That Destroy Sparsity (Densification) + +#### ISBLANK + +Behavior: Returns TRUE if blank, FALSE if defined. + +```pigment +ISBLANK('Revenue') +``` + +Output: + +- Where Revenue has a value: FALSE +- Where Revenue is blank: TRUE + +Sparsity: BAD: DESTROYED - all cells now have a value (TRUE or FALSE). + +Impact: A metric with 1% data becomes 100% dense. + +#### ISNOTBLANK + +Behavior: Returns TRUE if defined, FALSE if blank. + +```pigment +ISNOTBLANK('Revenue') +``` + +Output: + +- Where Revenue has a value: TRUE +- Where Revenue is blank: FALSE + +Sparsity: BAD: DESTROYED - all cells now have a value. + +Impact: Same as ISBLANK - complete densification. + +Densification cost over large spaces: ISBLANK/ISNOTBLANK over large dimension spaces (e.g. many products x months, or employees x time) force full evaluation and storage of TRUE/FALSE for every cell. This can cause order-of-magnitude slowdowns and memory growth compared to sparse alternatives. + +Preferred pattern - use dimension-typed metrics in BY to drive sparsity: When a metric is dimension-typed and used in BY (e.g. `Source[BY: DimOrProp, DimensionTypedMetric]`), the engine only computes where that metric is defined; no predicate is needed. This is both cleaner and more performant than guarding with IF(ISBLANK(...), BLANK, ...). Avoid ISBLANK/ISNOTBLANK for sparsity; use BY with dimension-typed metrics or ISDEFINED/IFDEFINED/IFBLANK/EXCLUDE instead. + +## Common Anti-Patterns and Solutions + +### Anti-Pattern 1: Using ISBLANK Instead of ISDEFINED + +Anti-pattern: + +```pigment +IF(ISBLANK('Revenue'), 0, 'Revenue') +``` + +Problem: ISBLANK densifies the metric, creating FALSE values everywhere Revenue is blank. + +Solution: + +```pigment +IFDEFINED('Revenue', 'Revenue') +// or simply +'Revenue' +``` + +Why better: ISDEFINED preserves sparsity. Blank cells remain blank. + +Performance impact: 10-100x faster for sparse metrics. + +### Anti-Pattern 2: Using IF(ISBLANK()) Instead of IFBLANK + +Anti-pattern: + +```pigment +IF(ISBLANK('Revenue'), 'Backup Revenue', 'Revenue') +``` + +Problem: + +1. ISBLANK densifies the metric +2. More complex database computation +3. Harder to read + +Solution: + +```pigment +IFBLANK('Revenue', 'Backup Revenue') +``` + +Why better: + +- Cleaner formula +- Preserves sparsity +- More efficient computation + +Performance impact: 5-20x faster for sparse metrics. + +### Anti-Pattern 3: Creating Dense Boolean Metrics + +Anti-pattern: + +```pigment +// Flag metric +ISNOTBLANK('Revenue') +``` + +Problem: Creates a dense boolean metric (TRUE/FALSE everywhere). + +Solution: + +```pigment +// Flag metric that preserves sparsity +ISDEFINED('Revenue') +``` + +Output: + +- Where Revenue exists: TRUE +- Where Revenue is blank: BLANK (not stored) + +Why better: Only stores TRUE values, not FALSE values. + +### Anti-Pattern 4: Unnecessary Densification for Defaults + +Anti-pattern: + +```pigment +// Set default value of 0 +IF(ISBLANK('Revenue'), 0, 'Revenue') +``` + +Problem: Forces every blank cell to store 0. + +Solution: + +```pigment +// Let blanks remain blank +'Revenue' +``` + +Why better: In Pigment, blank and 0 are different. If you need 0 for calculations, use IFBLANK only where necessary: + +```pigment +// Only densify where actually needed +IFBLANK('Revenue', 0) + 'Fixed Costs' +``` + +### Anti-Pattern 5: Checking for Blank in Conditions + +Anti-pattern: + +```pigment +IF(ISBLANK('Revenue'), 'No Data', 'Revenue' * 1.1) +``` + +Problem: ISBLANK densifies, and "No Data" text creates dense text values. + +Solution: + +```pigment +// Only compute where defined +IFDEFINED('Revenue', 'Revenue' * 1.1) +``` + +Why better: Blank cells remain blank, no text values stored. + +### Anti-Pattern 6: Guarding BY with IF(ISBLANK(dimension_typed_metric), BLANK, ...) + +Anti-pattern: + +```pigment +IF( + ISBLANK('CALC_Employee_Tenure_Month_Index'), + BLANK, + 'ASM_Ramp_Schedule'[BY: Employee.Segment, 'CALC_Employee_Tenure_Month_Index'] +) +``` + +Problem: When a dimension-typed metric is in BY, its sparsity is respected automatically. The IF(ISBLANK(...), BLANK, ...) guard is redundant and harmful (ISBLANK densifies). + +Solution: + +```pigment +'ASM_Ramp_Schedule'[BY: Employee.Segment, 'CALC_Employee_Tenure_Month_Index'] +``` + +Why better: BY alone preserves sparsity; no densifying predicate. See [Sparsity via BY + dimension-typed metrics](../writing-pigment-formulas/formula_modifiers.md#sparsity-via-by--dimension-typed-metrics) in the writing-pigment-formulas skill. + +## Best Practices for Sparsity Management + +### Practice 1: Prefer ISDEFINED Over ISBLANK + +Rule: Use ISDEFINED when you want to check if a value exists. + +Example: + +```pigment +// Bad +IF(ISBLANK('Revenue'), BLANK, 'Revenue' * 1.1) + +// Good +IFDEFINED('Revenue', 'Revenue' * 1.1) +``` + +When to use ISBLANK: Only when you specifically need FALSE as an outcome (rare). + +### Practice 2: Use IFBLANK for Fallback Values + +Rule: Use IFBLANK when you want to provide a fallback value. + +Example: + +```pigment +// Bad +IF(ISBLANK('Forecast'), 'Historical Average', 'Forecast') + +// Good +IFBLANK('Forecast', 'Historical Average') +``` + +Benefit: Cleaner and more efficient. + +### Practice 3: Avoid Storing Zero Unnecessarily + +Rule: Let blank cells remain blank unless zero has a different meaning than blank. + +Example: + +```pigment +// Bad: Forces all cells to have a value +IF(ISBLANK('Revenue'), 0, 'Revenue') + +// Good: Let blanks remain blank +'Revenue' +``` + +When zero is needed: Use IFBLANK only in the specific calculation: + +```pigment +// Calculation that needs zero +IFBLANK('Revenue', 0) / IFBLANK('Units', 1) +``` + +### Practice 4: Chain IFDEFINED for Multiple Conditions + +Rule: Use nested IFDEFINED to check multiple metrics. + +Example: + +```pigment +// Only compute where both metrics exist +IFDEFINED('Revenue', + IFDEFINED('Cost', + 'Revenue' - 'Cost' + ) +) +``` + +Benefit: Computation only happens at the intersection of defined cells. + +### Practice 5: Use ISDEFINED for Early Scoping + +Rule: Start formulas with ISDEFINED to scope to defined cells only. + +Example: + +```pigment +// Without scoping +'Revenue' * 'Growth Rate' + 'Fixed Costs' + +// With scoping +IFDEFINED('Revenue', + 'Revenue' * 'Growth Rate' + 'Fixed Costs' +) +``` + +Benefit: If Revenue is sparse (10% of cells), computation is 10x smaller. + +### Practice 6: Use EXCLUDE Instead of ISBLANK for "Where B Is Blank" + +Rule: When the condition is "A is true and B is blank", use the EXCLUDE modifier instead of AND + ISBLANK. + +Example: + +```pigment +// Bad: ISBLANK densifies B +IF(A AND ISBLANK(B), TRUE) + +// Good: EXCLUDE restricts scope without densifying +IF(A [EXCLUDE: B], TRUE) +``` + +Benefit: EXCLUDE tells the engine to skip cells where B is defined. No intermediate boolean metric is created, so B stays sparse and the formula runs on fewer cells. + +## Understanding the Technical Differences + +### ISBLANK vs ISDEFINED: Database Behavior + +ISBLANK: + +``` +Input: Revenue (sparse, 100 cells with values out of 10,000) +Process: Database generates FALSE for 9,900 blank cells +Output: Dense metric with 10,000 cells (100 FALSE, 9,900 TRUE) +Storage: 10,000 cells +``` + +ISDEFINED: + +``` +Input: Revenue (sparse, 100 cells with values out of 10,000) +Process: Database returns TRUE only for existing cells +Output: Sparse metric with 100 cells (all TRUE) +Storage: 100 cells +``` + +Performance difference: 100x more efficient with ISDEFINED. + +### IFBLANK vs IF(ISBLANK()): Computation Behavior + +IF(ISBLANK()): + +``` +Step 1: ISBLANK densifies the metric (10,000 cells) +Step 2: IF evaluates all 10,000 cells +Step 3: Output may be sparse, but computation was dense +Computation cost: 10,000 cells +``` + +IFBLANK: + +``` +Step 1: Check if first argument is defined +Step 2: If yes, return first argument; if no, return second argument +Step 3: Only compute where at least one argument is defined +Computation cost: Only defined cells +``` + +Performance difference: 10-100x more efficient with IFBLANK. + +## Real-World Sparsity Example + +### Scenario: Sales Forecasting Application + +Dimensions: + +- Products: 10,000 +- Regions: 50 +- Months: 24 +- Total possible cells: 12,000,000 + +Actual data: + +- Only 5,000 products are active +- Active products sold in average 30 regions +- Data exists for 18 months +- Actual cells with data: ~2,700,000 (22.5% sparse) + +### Anti-Pattern Implementation + +```pigment +// Check if forecast exists, otherwise use historical +'Forecast Flag' = ISNOTBLANK('Forecast') +'Final Forecast' = IF(ISBLANK('Forecast'), 'Historical Average', 'Forecast') +``` + +Result: + +- 'Forecast Flag': 12,000,000 cells (100% dense) +- 'Final Forecast': 12,000,000 cells (100% dense) +- Memory usage: 24,000,000 cells stored +- Computation time: 45 seconds per update + +### Optimized Implementation + +```pigment +// Use sparsity-preserving functions +'Final Forecast' = IFBLANK('Forecast', 'Historical Average') +``` + +Result: + +- 'Final Forecast': ~3,500,000 cells (29% sparse - only where either metric exists) +- Memory usage: 3,500,000 cells stored +- Computation time: 2 seconds per update + +Improvement: + +- Memory: 85% reduction +- Performance: 22x faster + +## Sparsity and Aggregation + +### Aggregation Preserves Sparsity + +```pigment +// Sparse input +'Transaction Amount' // 1,000 transactions out of 1,000,000 possible + +// Sparse output +'Customer Total' = 'Transaction Amount'[BY: 'Transaction'.'Customer'] +``` + +Result: Only customers with transactions have values. Sparsity preserved. + +### Densification Through Allocation + +```pigment +// Dense input +'Total Budget' // One value per department + +// Dense output +'Employee Budget' = 'Total Budget'[BY: 'Department'.'Employee'] +``` + +Result: All employees in departments with budgets get values. Partial densification. + +## Monitoring Sparsity + +### Signs of Densification + +1. Metric size increases dramatically without data increase +2. Computation time increases for simple formulas +3. Memory usage spikes in the application +4. Profiler shows long computation for boolean metrics + +### Checking Sparsity + +Agent: Use `tool:get_top_blocks_by_performance` with `CombinedCardinality` on suspect blocks. After a change, `tool:performance_profile_change` shows whether executions stayed scoped (partial compute) vs full recompute. + +User handoff (if needed): Ask for metric cell count vs dimension cardinality product. Sparsity % approx (actual cells / possible cells) x 100. + +Expected sparsity: + +- Transaction data: 0.1% - 5% (very sparse) +- Planning data: 10% - 50% (moderately sparse) +- Configuration data: 80% - 100% (dense, expected) + +## When Densification is Acceptable + +### Scenario 1: Small Dimensions + +If total possible cells < 10,000, densification impact is minimal. + +```pigment +// Acceptable: Only 120 possible cells (10 products x 12 months) +ISBLANK('Revenue') +``` + +### Scenario 2: Already Dense Metrics + +If a metric is already 90%+ dense, densification has little impact. + +```pigment +// Acceptable: Configuration data is already dense +ISBLANK('Default Value') +``` + +### Scenario 3: Required Boolean Logic + +Sometimes you need FALSE, not BLANK. + +```pigment +// Acceptable: Need explicit FALSE for logic +'Has Revenue' = ISNOTBLANK('Revenue') +'Needs Attention' = NOT('Has Revenue') +``` + +Mitigation: Keep these metrics isolated, don't reference them in many places. + +## Sparsity Best Practices Summary + +1. Use ISDEFINED instead of ISBLANK - Preserves sparsity +2. Use IFBLANK instead of IF(ISBLANK()) - Cleaner and more efficient +3. Let blanks remain blank - Don't force zeros unnecessarily +4. Chain IFDEFINED - Scope to intersection of defined cells +5. Use IFDEFINED for early scoping - Restrict computation to defined cells +6. Use EXCLUDE instead of AND + ISBLANK - For "A and B blank", `IF(A [EXCLUDE: B], TRUE)` avoids densifying B +7. Monitor metric sizes - Watch for unexpected densification +8. Profile regularly - Check computation time for boolean metrics +9. Accept densification when necessary - But isolate dense metrics + +## Common Mistakes + +### Mistake 1: Assuming Blank = Zero + +```pigment +// Wrong assumption +IF(ISBLANK('Revenue'), 0, 'Revenue') + +// Correct understanding +// Blank means "no data", not "zero revenue" +'Revenue' +``` + +### Mistake 2: Using ISBLANK for Existence Checks + +```pigment +// Bad: Densifies +IF(ISBLANK('Forecast'), "Missing", "Present") + +// Good: Preserves sparsity +IFDEFINED('Forecast', "Present") +``` + +### Mistake 3: Not Considering Downstream Impact + +```pigment +// Bad: Densifies early in chain +Step 1: 'Has Data' = ISNOTBLANK('Revenue') +Step 2: 'Adjusted' = IF('Has Data', 'Revenue' * 1.1, 0) +// Step 2 is now dense because Step 1 is dense + +// Good: Avoid densification +Step 1: 'Adjusted' = IFDEFINED('Revenue', 'Revenue' * 1.1) +``` + +## See Also + +- [Performance Formula Optimization](./performance_formula_optimization.md) - Formula-level optimization including sparsity +- [Performance Scoping Patterns](./performance_scoping_patterns.md) - Relationship between scoping and sparsity +- [Pigment Formulas & Functions: Logical Functions](../writing-pigment-formulas/functions_logical.md) - Detailed function syntax diff --git a/plugins/pigment/skills/optimizing-pigment-performance/performance_troubleshooting_workflow.md b/plugins/pigment/skills/optimizing-pigment-performance/performance_troubleshooting_workflow.md new file mode 100644 index 00000000..9b5dad40 --- /dev/null +++ b/plugins/pigment/skills/optimizing-pigment-performance/performance_troubleshooting_workflow.md @@ -0,0 +1,85 @@ +# Performance Troubleshooting Workflow (Modeler Agent) + +Agent workflow for compute performance issues. Requires Performance Insights profiling tools unless noted. Work Steps 1-6 in order; after Step 5, loop to Step 1 with new measurements if needed. + +--- + +## Step 1: Establish baseline + +1. Clarify the symptom - slow input, timeout, slow board load, or intermittent slowness; which app, scenario, block, or board. +2. Triage blocks (if hotspot unknown). `tool:get_top_blocks_by_performance` over a recent window (see [./performance_profiling.md](./performance_profiling.md)). +3. Reproduce - repeat the slow action; capture `change_id` from audit trail when possible. +4. Profile compute. `tool:performance_profile_change` with `change_id`; parse per [./performance_profiling.md](./performance_profiling.md). +5. Fork: board-render vs compute - low total execution `Duration` but slow board -> rendering (`skill:designing-pigment-boards`). High `Duration` or `no scope, full computation` -> formula/scope work below. + +Record: wall time estimate, execution count, slowest executions, X/Y at scope-loss origin, block IDs. + +--- + +## Step 2: Identify bottlenecks + +1. Sort executions by `Duration`; flag > 1000 ms (or dominant share of wall time). +2. Trace scope: first `no scope, full computation`; unnecessary scope loss? +3. Map `Blocks:` lines to metrics; read formulas (`tool:search`, formula tools). +4. Check dimensionality and sparsity on suspect blocks. + +Common patterns: early `REMOVE`, `ISBLANK`/`ISNOTBLANK`, long iterative horizons, AR without `IFDEFINED(User)`, high `CombinedCardinality`. + +--- + +## Step 3: Analyze root causes + +| Area | Key questions | +|---|---| +| Scope | Where is scope first lost? Necessary? Defer scope-losing ops? | +| Sparsity | `ISBLANK`/`ISNOTBLANK`? Use `ISDEFINED` / `IFDEFINED` / `EXCLUDE`? | +| Formula shape | Filter early? `REMOVE` deferred? `BY` vs `ADD`? | +| Dimensionality | Too many or high-cardinality dimensions? | +| Iterative | `PREVIOUS`/`CUMULATE` horizon too long? Subset time? | +| Access rights | `IFDEFINED(User)` on AR metrics? AR repeated in chain? | + +--- + +## Step 4: Prioritize and implement + +| Priority | Examples | +|---|---| +| First | `IFDEFINED(User)`, `ISDEFINED` over `ISBLANK`, early `FILTER`, remove unnecessary `REMOVE` | +| Second | Defer aggregations, subset time, simplify AR | +| Third | Redesign dimensions, split metrics, refactor chains | + +One change at a time. After each change, new `change_id` -> `tool:performance_profile_change` -> compare to baseline. + +--- + +## Step 5: Verify + +1. Re-profile the same user action (new `change_id`). +2. Confirm results unchanged (spot-check affected metrics). +3. Check no new slow executions appeared elsewhere. + +--- + +## Step 6: Document + +Summarize for the user: symptom, bottleneck execution(s), root cause, change made, before/after durations and scope. Use product language, not raw API field names (see analysis doc vocabulary rules). + +--- + +## Common scenarios + +| Symptom | Agent actions | +|---|---| +| Slow board load | Profile a typical board interaction; if compute is fast, route to board design skill | +| Slow after input | `performance_profile_change`; scope-loss + formula patterns | +| Timeout | Check iterative metrics, cardinality (`get_top_blocks_by_performance`), full recompute chains | +| Intermittent | AR by user, scenario count, data-dependent sparsity | + +--- + +## See Also + +- [./performance_profiling.md](./performance_profiling.md) - profiling tools and output parsing +- [./performance_scoping_patterns.md](./performance_scoping_patterns.md) - scope fixes +- [./performance_formula_optimization.md](./performance_formula_optimization.md) - formula patterns +- [./performance_sparsity_deep_dive.md](./performance_sparsity_deep_dive.md) - sparsity diff --git a/plugins/pigment/skills/planning-cycles-pigment-applications/SKILL.md b/plugins/pigment/skills/planning-cycles-pigment-applications/SKILL.md new file mode 100644 index 00000000..9f7ee4bb --- /dev/null +++ b/plugins/pigment/skills/planning-cycles-pigment-applications/SKILL.md @@ -0,0 +1,140 @@ +--- +name: planning-cycles-pigment-applications +description: Always use this skill when the user mentions or implies versions, Actual, Actuals, Budget, Budgeting, Forecast, Reforecast, Rolling Forecast, Version, Versioning, Plan, switchover, scenarios, snapshots, planning cycles, Actual/Plan layering, plan vs actual, "create version dimension", "set up versioning", or asks for Actual Budget Forecast best practices - or when they extend realized data into a plan or budget (Actual/Budget/Plan layering, forward forecast from actuals) or need to combine or compare actual and plan versions and periods. Covers Version Dimensions (foundational to all planning applications), Native Scenarios (what-if), and Snapshots (freeze data). +metadata: + skill_path: /planning-cycles-pigment-applications/SKILL.md + base_directory: /planning-cycles-pigment-applications + includes: + - "*.md" +--- + +# Planning Cycles in Pigment + +Three features handle planning cycles, scenarios, and lifecycle in Pigment. They are complementary, not alternatives. Read first to pick the right one. Jump to the linked deep dives for the procedures. + +## When to Use This Skill + +Read this skill whenever the user touches metric structure or mentions: + +- Versions, Budget, Actual, Forecast, Reforecast, Rolling Forecast +- "Create version dimension", "set up versioning", "best practices for Actual Budget Forecast" +- Switchover, lock, layering Actual with Plan, IsActual / IsPlan / IsVersion +- Native Scenarios, what-if, Optimistic / Pessimistic / Stress +- Snapshots, archiving, closing a planning cycle + +--- + +## Mental Model + +Planning lifecycle in Pigment is three orthogonal features at the application level: + +- Version Dimension -- custom structural dim on metrics, modeler built + - Default bootstrap items: `Actual`, `Budget`, `Forecast` (semantic names, no year suffix) + - For rolling forecast / multi-cycle: cycle-explicit names (`Budget FY`, `Reforecast Q FY`) + - Mandatory companion: `Version Type` Dimension (Actual, Budget, Forecast, Reforecast, Rolling Forecast, Long Range Planning) + - Properties: Start / End Month, Switchover Month, Active Version, Lock Version, Version Type + - Boolean Metrics: IsVersion, IsActual, IsPlan (gate Actual vs Plan layering) +- Native Scenarios -- app-level overlay, not a dimension + - Optimistic, Pessimistic, Stress + - Formula Groups for safe formula trials +- Snapshot -- point-in-time copy of the app, used for closed cycles and archives + +Invariants: + +- The Version Dimension is a custom dimension built by the modeler. It sits in metric structures and drives cross-version formulas, locking, and AR. +- Version Type is mandatory. It enables T&D-safe formula references and avoids hard-coding Version items. +- Native Scenarios are not a dimension. They are an application-level overlay for ad-hoc sensitivity on top of an existing plan. +- Snapshots are copies of an app. They never replace a Version. They freeze a state. +- Do not use the calendar's Actual vs Forecast toggle for planning cycles. Use the Version Dimension Switchover pattern. + +--- + +## Three Distinct Features + +| Feature | Use it for | Identity | Read | +|---|---|---|---| +| Version Dimension | Modeling planning cycles (Budget, Forecast, Reforecast, Rolling Forecast). Cross-version formulas, governance, locking. | Regular Dimension created by the modeler. Part of the Metric structure. | [./planning_cycles_versions.md](./planning_cycles_versions.md) | +| Native Scenario | Quick what-if sensitivity on top of an existing model. Safe trialing of formula changes via Formula Groups. | Application-level feature. Not a Dimension. | [./planning_cycles_scenarios.md](./planning_cycles_scenarios.md) | +| Snapshot | Freezing the state of an Application. Closing planning cycles. Archiving. | Point-in-time copy of an Application. | [./planning_cycles_snapshots.md](./planning_cycles_snapshots.md) | + +--- + +## Decisions in Order + +1. Identify the intent. Structured plan with governance and cross-plan formulas -> Version Dimension. Ad-hoc sensitivity -> Native Scenario. Archiving a state -> Snapshot. +2. Build the Version Dimension with its mandatory companion `Version Type` Dimension. Default items: `Actual`, `Budget`, `Forecast`. Use cycle-explicit names (`Budget FY`) only for rolling forecast / multi-cycle setups. +3. Add all mandatory properties in one pass: `Start Month`, `End Month`, `Switchover Month`, `Active Version` (Bool), `Lock Version` (Bool), `Version Type` (Dimension). Populate values immediately using calendar and current date. +4. Build the three Boolean Metrics: IsVersion (inside window), IsActual (inside window up to Switchover Month inclusive), IsPlan (inside window after Switchover Month). +5. Deliver everything atomically. Dimension + companion Version Type + properties populated + boolean metrics. Nothing is "phase 2." +6. Wire Access Rights from `Active Version` and `Lock Version`: locked Versions are read-only, active Versions are open for edit. See `skill:securing-pigment-applications`. +7. Use Native Scenarios only for overlays (Optimistic, Pessimistic, Stress) or for trialing formula changes in a Formula Group. +8. Snapshot at lifecycle boundaries. Closing a Budget cycle, year-end, or before a structural rework. +9. Govern the live set. Regularly review and clean up the Version list. Archive locked or outdated Versions via Snapshots. + +--- + +## Versions vs Native Scenarios: Decision + +Use a Version Dimension for any structured plan, any cross-plan formula reference, and any per-plan access control. + +Use a Native Scenario only for ad-hoc sensitivity (Optimistic, Pessimistic, Stress) on top of an existing plan, or for trialing a new formula in a Formula Group before porting it back to the main model. + +MG12: model planning cycles as a Dimension, never as a Native Scenario. + +--- + +## Calendars vs Versions: Do Not Confuse Them + +Calendars define the time structure of an application: Month, Quarter, Year, fiscal year, date range. They are configured once via the calendar tools. + +Versions are a modeling pattern. A Dimension you create to hold Budget, Actual, Forecast, etc., with Switchover properties and Boolean metrics that gate Actual vs Plan data. + +Before setting up versions, ensure the calendar is configured with `Month`, `Quarter`, and `Year`, with Quarter and Year available as properties on Month. If they are missing, complete the calendar setup first. For calendar setup see [`../modeling-pigment-applications/modeling_time_and_calendars.md`](../modeling-pigment-applications/modeling_time_and_calendars.md). + +Do not use Calendar tools (`calendar_create`, `calendar_expand`, `calendar_add_time_dimension`, etc.) to implement planning cycles. They configure time, not planning cycles. Do not use the calendar's built-in "Actual vs Forecast" toggle for version-level switchover. For calendar setup see [`../modeling-pigment-applications/modeling_time_and_calendars.md`](../modeling-pigment-applications/modeling_time_and_calendars.md). + +--- + +## Glossary + +- Version Dimension: the custom Dimension holding planning cycles. Business-friendly name (`Version`), no `LST_` prefix unless the workspace already uses it. +- Version Item: one row of the Version Dimension. Default bootstrap: `Actual`, `Budget`, `Forecast` (semantic names). Rolling forecast variant: `Budget FY`, `Reforecast Q FY` (cycle-explicit names). +- Version Type: mandatory companion Dimension. Items: `Actual`, `Budget`, `Forecast`, `Reforecast`, `Rolling Forecast`, `Long Range Planning`. Referenced via `VAR_` input Metrics in formulas for T&D / MP02 safety. +- Switchover Month / Year: per-version property marking the last month (or year) of actual data. Plan picks up after it. Only `Month` or `Year`; not Quarter or Date. +- Start Month / End Month: per-version properties defining the planning window of that Version. +- Active Version: Boolean property flagging Versions currently displayed for input or reporting. +- Lock Version: Boolean property flagging Versions locked from edits once approved. Drives the read-only AR rule. +- IsVersion / IsActual / IsPlan: three Boolean metrics over Version x Time. IsVersion = inside the window. IsActual = inside the window up to Switchover Month inclusive. IsPlan = inside the window after Switchover Month. +- Layering: combining Actuals up to Switchover Month with Plan beyond it, inside a single metric. +- Formula Group: a set of formula overrides scoped to a Native Scenario, used to trial alternative logic without touching the base model. +- Shared vs Local Scenario: scope of a Native Scenario (shared across users, or private to one user). +- Live Version: a Version Item currently in active use. Regularly clean up locked or outdated ones. +- Snapshot: point-in-time copy of an Application. Read-only by default. + +--- + +## Critical Rules + +- Always read the matching document before building. Do not rely on this SKILL.md summary alone. +- Never use Calendar tools to implement versioning. +- Never use the calendar Actual vs Forecast toggle for version-level switchover. Use the Version Dimension Switchover pattern. +- Never model Budget, Actual or Forecast as Native Scenarios. Use a Version Dimension. +- Never hard-code Version Items in formulas (MP02). Use IsActual / IsPlan or `VAR_` metrics - see [planning_cycles_versions.md](./planning_cycles_versions.md). +- Never use `REMOVE` on Version. Use `FILTER` or `SELECT`. +- Keep the Version Dimension lean. Only keep active Versions in use and locked Versions needed for reference. Regularly review and clean up; archive older Versions via Snapshots. +- Deliver the full setup atomically. Dimension + Version Type + properties populated + boolean metrics in one pass. + +--- + +## Deeper Dives + +| Need | Doc | +|---|---| +| Version Dimension setup, bootstrap checklist, switchover, boolean metrics, layering, Do Not, multi-app | [./planning_cycles_versions.md](./planning_cycles_versions.md) | +| Native Scenarios: when to use, Shared vs Local, anti-patterns, combining with Versions | [./planning_cycles_scenarios.md](./planning_cycles_scenarios.md) | +| Snapshots and lifecycle: when to snapshot, cycle workflow, performance budget | [./planning_cycles_snapshots.md](./planning_cycles_snapshots.md) | +| Calendar setup (fiscal year, granularity, date range) | [`../modeling-pigment-applications/modeling_time_and_calendars.md`](../modeling-pigment-applications/modeling_time_and_calendars.md) | +| Modeling foundations (mental model, core concepts) | `skill:modeling-pigment-applications` | +| Architecture (Version Dimension in Hub app) | [`../modeling-pigment-applications/modeling_architecture_design.md`](../modeling-pigment-applications/modeling_architecture_design.md) | +| Access Rights on locked versions | `skill:securing-pigment-applications` | +| Formula patterns (cross-version, layering, MP02) | `skill:writing-pigment-formulas` | diff --git a/plugins/pigment/skills/planning-cycles-pigment-applications/planning_cycles_scenarios.md b/plugins/pigment/skills/planning-cycles-pigment-applications/planning_cycles_scenarios.md new file mode 100644 index 00000000..ecdba320 --- /dev/null +++ b/plugins/pigment/skills/planning-cycles-pigment-applications/planning_cycles_scenarios.md @@ -0,0 +1,46 @@ +# Native Scenarios in Pigment + +A Native Scenario is an application-level feature (not a Dimension) that creates an independent calculation environment with its own inputs. Use Scenarios for quick "what-if" sensitivity on top of an existing model, or for safe trialing of formula changes via Formula Groups. + +For structured planning cycles (Budget, Forecast, Reforecast), do NOT use Native Scenarios. Use a Version Dimension instead, see [planning_cycles_versions.md](./planning_cycles_versions.md). + +## 1. When to Use a Native Scenario + +Use a Native Scenario only for: + +- Ad-hoc sensitivity analysis (Optimistic, Pessimistic, Stress test) on top of an existing plan. +- Trialing a new formula in a Formula Group before porting it back to the main model. + +### Example + +- Simple what-if analysis. Compare three independent revenue projections (Base, Optimistic, Pessimistic) with their own assumptions and no need to reference each other. + +## 2. Constraints + +Before proposing a Native Scenario, the agent must know its constraints: + +- Independent calculation environment with its own inputs. +- Cannot reference another Scenario's data in formulas. +- Lists must be consistent across Scenarios. +- Not a Dimension. Cannot enter a Metric structure or pivot on a Page. +- Shared vs Local is set at creation and cannot be reverted. Pick Shared if Shared Blocks must carry Scenario-specific assumptions across Applications. Otherwise Local. + +## 3. Anti-Patterns + +1. Modeling Budget, Actual or Forecast as Native Scenarios. Use a Version Dimension (see [planning_cycles_versions.md](./planning_cycles_versions.md)). +2. Treating Shared vs Local Scenarios as switchable. The choice is irreversible. + +## 4. Combining Scenarios with a Version Dimension + +Versions and Scenarios are complementary, not alternatives: + +- Keep the structured plan in the Version Dimension (Budget, Forecast, Reforecast). +- Layer Native Scenarios on top for sensitivity or formula trials. + +## See Also + +- [planning_cycles_versions.md](./planning_cycles_versions.md): Version Dimension setup for planning cycles. +- [planning_cycles_snapshots.md](./planning_cycles_snapshots.md): Snapshots for freezing application state. +- [`../optimizing-pigment-performance/performance_auditing_application.md`](../optimizing-pigment-performance/performance_auditing_application.md): reviewing live Scenarios. +- [`../designing-pigment-boards/board_design_rules.md`](../designing-pigment-boards/board_design_rules.md): Scenario Planning Board pattern. +- Pigment KB: [Get started with Scenarios](https://kb.pigment.com/docs/get-started-scenarios). diff --git a/plugins/pigment/skills/planning-cycles-pigment-applications/planning_cycles_snapshots.md b/plugins/pigment/skills/planning-cycles-pigment-applications/planning_cycles_snapshots.md new file mode 100644 index 00000000..f2976bba --- /dev/null +++ b/plugins/pigment/skills/planning-cycles-pigment-applications/planning_cycles_snapshots.md @@ -0,0 +1,42 @@ +# Snapshots and Lifecycle in Pigment + +A Snapshot is a point-in-time copy of an entire Pigment Application. It is the only way to freeze state in Pigment because the platform is a live calculation engine. Use Snapshots to close planning cycles, archive past Versions, and keep the live Version count under control. + +## 1. When to Snapshot + +- At the end of a planning cycle. Close the prior Budget Version once the new Budget Version is live. +- At each cycle for Rolling Forecasting. Snapshot the Application monthly (or per cycle cadence) so each cycle has a frozen reference. +- Before a major model change that may affect prior results. + +## 2. Cycle Workflow + +For each new planning cycle: + +1. Snapshot the Application at the end of the previous cycle (or monthly for Rolling Forecasting). +2. Add a new Version Item in the Version Dimension. Use Clone data to to copy assumptions from the previous Version. Update `Switchover Month` on the new Version. + +This pairs naturally with the Version Dimension pattern. See [planning_cycles_versions.md](./planning_cycles_versions.md) for the Version setup. + +## 3. Performance Budget + +- Target ~6-10 live Versions in the Version Dimension; review when above 10. +- Archive older Versions via Snapshots; do not leave them as live Items. +- Live Versions inflate recalculation cost. + +## 4. Lifecycle Guidance + +- Frozen reference -> use a Snapshot. Snapshots cannot be edited. +- Active plan -> use a live Version in the Version Dimension. +- Closing a Version -> Snapshot the App, then optionally remove the Version Item if no longer needed. + +## Anti-Patterns + +1. Keeping more than 10 live Versions without review instead of archiving via Snapshots. +2. Editing data in a frozen cycle by re-adding live Versions instead of restoring from a Snapshot. + +## See Also + +- [planning_cycles_versions.md](./planning_cycles_versions.md): Version Dimension setup. +- [planning_cycles_scenarios.md](./planning_cycles_scenarios.md): Native Scenarios for sensitivity. +- [`../optimizing-pigment-performance/performance_auditing_application.md`](../optimizing-pigment-performance/performance_auditing_application.md): reviewing live Versions. +- Pigment KB: [Compare Data with Data slices](https://kb.pigment.com/docs/compare-versions-with-data-slices). diff --git a/plugins/pigment/skills/planning-cycles-pigment-applications/planning_cycles_versions.md b/plugins/pigment/skills/planning-cycles-pigment-applications/planning_cycles_versions.md new file mode 100644 index 00000000..c2f1e137 --- /dev/null +++ b/plugins/pigment/skills/planning-cycles-pigment-applications/planning_cycles_versions.md @@ -0,0 +1,129 @@ +# Version Dimension: Planning Cycles in Pigment + +A Version Dimension is a regular Pigment Dimension created by the modeler to hold planning cycles (Budget, Forecast, Reforecast, Rolling Forecast). It is part of the Metric structure, supports cross-version formula references, and powers governance and locking. It is the canonical way to model planning cycles in Pigment. + +MG12: model planning cycles as a Dimension, never as a Native Scenario. + +## 1. When to Use a Version Dimension + +Use a Version Dimension for: + +- Any structured plan (Budget, Forecast, Reforecast, Rolling Forecast). +- Any cross-plan formula reference (e.g. a Reforecast references the previous cycle). +- Any per-plan access control (lock past plans). + +### Examples + +- Budget vs Actual with variance. Track Budget and Actual side by side and compute `'Variance' = 'Actual Revenue' - 'Budget Revenue'` per Account and Month. +- Rolling Forecast. Quarterly rolling forecast where each cycle builds on the previous one. Pattern: cycle-explicit Version Items (`Reforecast Q FY`); advance `Switchover Month` by one quarter per cycle; reference the previous Version via input Metrics of type Dimension `VAR_Current Reforecast Version` and `VAR_Previous Reforecast Version`: + +```pigment +IF( + 'Version' = 'VAR_Current Reforecast Version', + 'Revenue'[SELECT: 'Version' = 'VAR_Previous Reforecast Version'] * (1 + 'Growth Factor'), + BLANK +) +``` + +For ad-hoc what-if sensitivity that does not require cross-version references, see [planning_cycles_scenarios.md](./planning_cycles_scenarios.md). + +## 2. Default Bootstrap: Version Dimension Setup Checklist + +When the user asks to set up versions, planning cycles, a Budget, or a Forecast, deliver all of the following in a single pass. Nothing is deferred to a later phase. + +### Checklist (atomic, all items mandatory) + +- [ ] Create `Version` Dimension. Use business-friendly name (no `LST_` prefix unless the workspace already uses it). Default items: `Actual`, `Budget`, `Forecast`. +- [ ] Create companion `Version Type` Dimension. Items: `Actual`, `Budget`, `Forecast`, `Reforecast`, `Rolling Forecast`, `Long Range Planning`. This is mandatory, not optional. +- [ ] Add `Version Type` Property on `Version`, typed `Dimension(Version Type)`. Set each item: Actual -> Actual, Budget -> Budget, Forecast -> Forecast. +- [ ] Add `Switchover Month` Property on `Version`, typed `Dimension(Month)`. Populate immediately using calendar and current date (see defaults below). Use `Switchover Year` only if planning grain is Year. +- [ ] Add `Start Month` Property on `Version`, typed `Dimension(Month)`. Populate immediately. +- [ ] Add `End Month` Property on `Version`, typed `Dimension(Month)`. Populate immediately. +- [ ] Add `Active Version` Property (Boolean) on `Version`. Set TRUE for all initial items. +- [ ] Add `Lock Version` Property (Boolean) on `Version`. Set TRUE for Actual, FALSE for Budget and Forecast. +- [ ] Create IsVersion Boolean Metric at `Version x Month`. Formula: `'Version'.'Start Month' <= 'Month' AND 'Month' <= 'Version'.'End Month'`. +- [ ] Create IsActual Boolean Metric at `Version x Month`. Formula: `IsVersion AND 'Month' <= 'Version'.'Switchover Month'`. +- [ ] Create IsPlan Boolean Metric at `Version x Month`. Formula: `IsVersion AND 'Month' > 'Version'.'Switchover Month'`. + +### Default Property Values (auto-populate from calendar + current date) + +Read the fiscal year starting month from the application calendar. Current FY is the fiscal year that contains today's date. First month of current FY is that starting month in the active year; last month of current FY is the month before the next FY starts. + +| Version Item | Version Type | Start Month | End Month | Switchover Month | +|---|---|---|---|---| +| Actual | Actual | Calendar start | Calendar end | Calendar start (actuals from calendar beginning) | +| Budget | Budget | First month of current FY | Last month of current FY | Last month of previous FY (full FY window is plan) | +| Forecast | Forecast | First month of current FY | Last month of current FY | Current month (actuals up to now, plan after) | + +Derive every Month value from the calendar fiscal year start and today's date. Do not hardcode calendar month names or years. + +Before creating the Version properties, verify that the calendar exposes `Month`, `Quarter`, and `Year`, with Quarter and Year available as properties on Month. If not, complete the calendar setup first. See [`../modeling-pigment-applications/modeling_time_and_calendars.md`](../modeling-pigment-applications/modeling_time_and_calendars.md). + +### Naming Patterns + +Two naming patterns are valid. Pick based on intent: + +Semantic names (default bootstrap): `Actual`, `Budget`, `Forecast`. Year is NOT embedded in the name. Year is managed via data slices or the calendar window. Use this when there is one active Budget and one active Forecast at a time. + +Cycle-explicit names (rolling forecast / multi-cycle): `Budget FY`, `Reforecast Q FY`. Year or cycle window IS in the name. Use this when multiple concurrent cycles coexist and the name must disambiguate them. + +Do not mix patterns within the same Version Dimension. + +## 3. Switchover Semantics + +`Switchover Month` is the last month of actual data for that Version. Months strictly after it are Plan. + +- `Budget`: `Switchover Month` = last month of previous FY -> the full current FY window (Start Month through End Month) is Plan. +- `Forecast`: `Switchover Month` = current month -> Actual from first month of current FY through switchover inclusive; Plan for remaining months in the FY window. + +## 4. Layering Actuals and Plan + +For each measure, create three Metrics at ` x Version x Month`: + +- ` Actual` gated by IsActual. +- ` Plan` from forward-looking assumptions. +- `` (final): `IF(IsActual, ' Actual', ' Plan')`. + +## 5. Optional: Dedicated `Actual` Version + +The default bootstrap includes an `Actual` item. If the model does not need to isolate Actuals independently of any plan cycle, the modeler can remove it; but the default is to include it. + +When present, update its `Switchover Month` each period to bring in the latest Actuals; never create one Item per fiscal year (e.g. separate `Actuals` items per year). The `Actual` Version is always locked for edit (the AR rule on `Lock Version` should reflect that). + +## 6. Optional: Display Actual vs Plan in Views via Mapped Dimension + +For richer reporting, build a `Data Type` Metric typed `Dimension(Data Type)` over `Version x Month` that returns `Actual` or `Plan` based on IsActual / IsPlan. Use it as a Mapped Dimension in the View; the View then shows `Actual` and `Plan` columns sourced from the same underlying Metric. + +## 7. Do Not + +- Period Type on Month or the calendar Actual vs Forecast toggle. One global switchover; cannot vary per Version. Use section 2 instead. +- Hardcoded month IFs for actual/plan split. Use IsActual / IsPlan from section 4. +- Partial setup. Items only, blank properties, or booleans deferred to a second prompt. Deliver section 2 in one pass. +- Native Scenarios for Budget / Actual / Forecast. MG12; use a Version Dimension. +- Hard-coded Version items in formulas. Use `VAR_` metrics or `'Version Type'` (MP02). See `skill:writing-pigment-formulas`. +- `REMOVE` on Version. Use `FILTER` or `SELECT`. +- Version on every metric. Skip pure Actuals and reference data. +- Direct formula edits on live Versions. Use the flag pattern in section 8. +- Stale Version lists. Archive via Snapshots ([planning_cycles_snapshots.md](./planning_cycles_snapshots.md)). + +## 8. Boolean Logic Flag for Live Formula Changes + +When a Version is live, never edit a formula directly. Add a Boolean Property on the Version Dimension (e.g. `Y+1 Logic Changes`): + +``` +IF('Y+1 Logic Changes', New formula, Old formula) +``` + +Set TRUE only on the Versions adopting the new logic. This makes transitions explicit and auditable. + +## 9. Multi-Application + +Create the Version Dimension in the Hub Application and share it with the spoke Apps. Reuse the same Version Dimension across as many Applications as possible. Create a separate one only when a clear business need requires a different cadence (e.g. Sales needing 10x more Versions). See [`../modeling-pigment-applications/modeling_architecture_design.md`](../modeling-pigment-applications/modeling_architecture_design.md). + +## See Also + +- [planning_cycles_scenarios.md](./planning_cycles_scenarios.md): Native Scenarios for ad-hoc sensitivity on top of Versions. +- [planning_cycles_snapshots.md](./planning_cycles_snapshots.md): Snapshots for archiving live Versions. +- [`../modeling-pigment-applications/modeling_principles.md`](../modeling-pigment-applications/modeling_principles.md): MG12 (planning cycles), MP02 (no hard-coding of Dimension Items). +- [`../optimizing-pigment-performance/performance_auditing_application.md`](../optimizing-pigment-performance/performance_auditing_application.md): reviewing live Versions. +- Pigment KB: [Versions and Scenarios](https://kb.pigment.com/docs/versions-scenarios), [Implement Versions and plans](https://kb.pigment.com/docs/managing-versions-in-pigment). diff --git a/plugins/pigment/skills/securing-pigment-applications/SKILL.md b/plugins/pigment/skills/securing-pigment-applications/SKILL.md new file mode 100644 index 00000000..b8d08db3 --- /dev/null +++ b/plugins/pigment/skills/securing-pigment-applications/SKILL.md @@ -0,0 +1,105 @@ +--- +name: securing-pigment-applications +description: Always use this skill when designing, applying, or debugging Access Rights and security in Pigment applications. Provides the AR mental model (User, Role, dimension axis, AR Metric, Apply rule), the canonical decision order, mandatory formula patterns (IFDEFINED guard, BLANK over FALSE), multi-app AR, debugging "why can this user see / not see this data?", and security governance. AR is part of model architecture, not an afterthought. +metadata: + skill_path: /securing-pigment-applications/SKILL.md + base_directory: /securing-pigment-applications + includes: + - "*.md" +--- + +# Securing Pigment Applications + +Access Rights design, application, and debugging in Pigment. Read first to get the mental model and non-negotiable rules. Jump to the deep dive for procedures, formulas, and debugging recipes. + +## When to Use This Skill + +Read this skill when the user asks to: + +- Design Access Rights by Country, Region, Department, Entity, or any dimension +- Decide between Role-based and User-based Access Rights +- Configure Apply vs Ignore rules in Data Access Rights +- Build an AR metric (Boolean by User x Dimension, or Role x Dimension) +- Write or debug AR formulas (`IFDEFINED('Users roles', ...)`, `ACCESSRIGHTS`) +- Share AR across multiple applications +- Debug "why can or cannot this user see this data?" +- Reuse AR patterns from a Hub app + +--- + +## Mental Model + +Access Rights in Pigment is a two-step pipeline. Building the AR metric is not applying it. You must do both. + +1. Identity axes (reserved): `User` (app login identities) and `Role` (security roles). +2. Business axis: any dimension that drives access (Country, Entity, Department, ...). +3. AR Metric (Boolean, sparse): defined as `User x Business Dim` OR `Role x Business Dim`. Use BLANK for no access; never FALSE. +4. Apply rule in Data Access Rights: activates the AR Metric on dependent metrics. Without it, the metric is inert. +5. Effective access: what a user actually sees on a given metric, after all Apply rules evaluate. + +Invariants: + +- User and Role are reserved security dimensions. Never use them as business dimensions. Use `Employee` for business, `User` for app login identities. +- AR Metrics are Boolean over a security axis x business axis. They must be sparse (BLANK, not FALSE). +- AR is not active until the Apply rule exists in Data Access Rights. +- AR is part of architecture, designed alongside the dimensional model, not after. + +--- + +## Decisions in Order + +1. Identify the axis. Which dimension drives access (Country, Entity, Department, ...). +2. Pick the security axis. Role-based by default. User-based only when access is per individual and does not cluster. +3. Build the AR Metric. Boolean over `User x Dim` or `Role x Dim`. Use BLANK for no access; never FALSE. +4. Guard the formula with `IFDEFINED('Users roles', ...)` so it does not break when the User or Role context is missing. +5. Create the Apply rule in Data Access Rights. Without it, the metric is inert. +6. Decide Apply vs Ignore per dependent metric. Apply restricts; Ignore bypasses (rare, intentional). +7. Multi-app. Share AR from a Hub app via Push / Pull; do not re-implement per app. +8. Validate. Impersonate a user, check effective access, profile for AR-heavy formulas. + +--- + +## Non-Negotiable Rules + +1. Use BLANK, never FALSE in AR metrics. BLANK preserves sparsity. FALSE densifies and hurts performance. +2. Always guard AR formulas with `IFDEFINED('Users roles', ...)` so the formula does not break when the User or Role context is missing. +3. Building an AR metric is not applying it. You must create the metric and the Apply rule in Data Access Rights. +4. User and Role dimensions are reserved for security. Never use User as a business dimension (use Employee). Never reuse Role for business roles. +5. AR is part of architecture. Design AR alongside the dimensional model, not after. + +For details, examples, and debugging, read [./securing_access_rights.md](./securing_access_rights.md). + +--- + +## Glossary + +- User dimension: reserved system dimension holding application login identities. +- Role dimension: reserved system dimension holding security roles. Items are grouped via the `Users roles` mapping. +- AR Metric: Boolean metric over a security axis (User or Role) x a business axis. Sparse, BLANK = no access. +- Apply rule: configuration in Data Access Rights that activates an AR Metric on dependent metrics. +- Apply vs Ignore: per-metric decision. Apply restricts data through AR; Ignore bypasses AR (use sparingly). +- Effective access: the resulting set of cells a user can read or write after all Apply rules are evaluated. +- Hub AR: AR built once in a Hub app and pushed to domain apps. The only safe pattern for multi-app security. + +--- + +## Critical Rules (quick check) + +- Always read [./securing_access_rights.md](./securing_access_rights.md) before building or debugging Access Rights. Do not rely on this SKILL.md summary alone. +- Build != Apply. Always create the rule in Data Access Rights after building the AR metric. +- BLANK over FALSE. Always. +- IFDEFINED guard. Always wrap AR formulas referencing `'Users roles'` or `User`. +- User dimension is for application users, not employees. + +--- + +## Deeper Dives + +| Need | Doc | +|---|---| +| AR design, Apply vs Ignore, User/Role/Dimension patterns, mandatory formulas, multi-app AR, debugging, governance | [./securing_access_rights.md](./securing_access_rights.md) | +| Formula syntax for security (ACCESSRIGHTS, IFDEFINED) | [`../writing-pigment-formulas/functions_security.md`](../writing-pigment-formulas/functions_security.md) | +| AR performance (ISDEFINED(User), AR-heavy formulas) | [`../optimizing-pigment-performance/performance_access_rights.md`](../optimizing-pigment-performance/performance_access_rights.md) | +| Modeling foundations (mental model, core concepts) | `skill:modeling-pigment-applications` | +| Modeling principles, T&D safety | [`../modeling-pigment-applications/modeling_principles.md`](../modeling-pigment-applications/modeling_principles.md) | +| Architecture (Hub pattern, AR in Hub) | [`../modeling-pigment-applications/modeling_architecture_design.md`](../modeling-pigment-applications/modeling_architecture_design.md) | diff --git a/plugins/pigment/skills/securing-pigment-applications/securing_access_rights.md b/plugins/pigment/skills/securing-pigment-applications/securing_access_rights.md new file mode 100644 index 00000000..0c3ecbdf --- /dev/null +++ b/plugins/pigment/skills/securing-pigment-applications/securing_access_rights.md @@ -0,0 +1,270 @@ +# Access Rights: Design, Implementation, and Governance + +Purpose: Definitive reference for designing, implementing, applying, debugging, and governing Access Rights (AR) in Pigment. Use with [writing-pigment-formulas](../writing-pigment-formulas/functions_security.md) for formula syntax and [optimizing-pigment-performance](../optimizing-pigment-performance/performance_access_rights.md) for AR performance. + +--- + +## 1. Why Access Rights Matter + +AR is not only security-it is model architecture. Good AR enables clean UX, scales across apps, and avoids brittle workarounds (e.g. hiding data only in Boards). Design AR with the same rigor as data modeling and formulas. + +Principle: Design Access Rights as part of the model's architecture, not as an afterthought. + +--- + +## 2. Core Concepts + +### 2.1 Access Rights vs Permissions vs Roles + +| Concept | Controls | Examples | +| ---------------------- | ----------------------------------- | ---------------------------------------- | +| Access Rights (AR) | Cell-level data read/write | User can write US data, read FR data | +| Permissions | Feature access (actions) | Create metrics, edit boards, import data | +| Roles | Bundles of permissions + default AR | Admin, Modeler, Contributor, Reader | + +Golden rule: AR protects data; Permissions protect actions; Roles define defaults. Fine-grained data security is enforced by AR rules, not by roles alone. + +### 2.2 Where Access is Configured + +- Application Settings -> Roles, permissions & access +- Subsections: Overview, Roles, Board access, Board access configuration, Data access rights (AR metrics and rules), Public Blocks +- Only Primary Owner, Security Admin, or Workspace Admin can manage these settings. + +### 2.3 Default Roles (Summary) + +- Admin: Full control including security. +- Modeler: Build content; cannot configure security. +- Contributor / Reader: Default Read/Write (Contributor) or Read-only (Reader). AR rules typically restrict Contributor and Reader to specific data slices. + +### 2.4 User and Role dimensions: reserved for security only + +The User and Role dimensions are system dimensions that exist in the Security context. They represent who can access the application (users) and what permission bundle they have (roles). Use them only for access rights, permissions, and security configuration. + +Do not use User or Role for business or modeling purposes. For example: + +- Workforce planning - Model employees with a dedicated Employee (or similar) dimension, not with the User dimension. User = application users (who log in); Employee = business entity (people in your workforce data). +- Responsible / owner - If you need "plan owner" or "responsible person" in a metric, use a business dimension (e.g. Employee, Cost Center Manager) and map it to User only where needed for AR. Do not use User as the primary dimension for planning data. +- Role - Reserve for application roles (Admin, Modeler, Contributor, Reader). Do not reuse "Role" for business roles (e.g. "Sales Role", "HR Role") unless they are explicitly aligned with application roles; otherwise create a separate dimension with a distinct name (e.g. Department, Job Function). + +This keeps security (who sees what) clearly separate from the model (what you are planning or reporting on) and avoids confusion between "users of the app" and "entities in the data". + +--- + +## 3. Non-Negotiable Rules + +### 3.1 Always Use BLANK (Never FALSE) + +- In AR metrics, use BLANK for "no access", never FALSE. +- BLANK preserves sparsity and performance; FALSE densifies and hurts performance. +- See [performance_access_rights.md](../optimizing-pigment-performance/performance_access_rights.md) and [functions_security.md](../writing-pigment-formulas/functions_security.md). + +### 3.2 Layering: Deny-by-Default + +- AR rules are restrictive: if any applied rule does not grant access for a cell, the user has no access. +- Every layer must grant access for the user to see or edit. Adding rules can only restrict further (except Ignore overrides-see below). + +### 3.3 Build != Apply + +- Build an AR metric: Define who gets Read/Write (boolean matrix, ACCESSRIGHTS formula). This is just data. +- Apply an AR rule: In Data access rights -> Rules, create an Apply rule that links the AR metric to a target (e.g. all metrics using Country). +- Until you Apply the metric via a rule, it has no effect. Build and Apply are two distinct steps. + +### 3.4 Data vs Presentation + +- AR secures data (cell-level); enforced everywhere (Boards, Explorer, exports, API). +- Board permissions control UI (who sees which boards/sections). They do not protect data. +- Anti-pattern: "Security by Board"-never rely only on Board permissions for sensitive data. Always enforce AR on the data. + +### 3.5 Do not assume: AR semantics vs formula semantics (agents) + +Access Rights do not follow the same logic as general Pigment formulas. Do not infer AR behavior from formula AND/OR/blank rules. + +- In AR, BLANK = no access. A cell where the AR metric is BLANK is not accessible to the user, regardless of other rules or roles. +- Combining rules is intersection. When multiple AR rules apply to the same data, the user has access only where every rule grants access. If one rule returns BLANK (no access) for a cell and another returns TRUE (access), the result is no access for that cell-not access. +- Do not assume that "blank AND true" or "one layer grants, another is blank" yields access; it does not. In formulas, multiplicative/AND semantics can be "blank x value = blank"; in AR, the layering rule is stricter: any layer that does not grant access denies access. +- When mixing Role default access and data access rules, verify behavior from this document and product documentation rather than inferring from general formula semantics. If documentation does not explicitly cover the combination, state the gap and recommend verification instead of proposing a solution as if it will work. + +--- + +## 4. Design Principles + +### 4.1 Build at the Highest Possible Dimension + +- Prefer Role x Region, User x Region, or Role x Department over User x Country or User x Employee x Month. +- Higher-level AR -> smaller metrics, better performance, easier maintenance. Use mapping metrics or dimension replacement to map down to detail. + +### 4.2 Use Mapping Metrics + +- Mapping metrics (e.g. Region -> Country, Role -> User via 'Users roles') translate high-level AR to detailed data. +- Keep them in a Security section. Avoid User dimension in mappings when possible; use Role and map to User at the end. + +### 4.3 Prefer Roles over Individual Users + +- Build AR by Role (e.g. Role x Department), then map to users via 'Users roles' (or custom mapping). +- Benefits: smaller metrics, easier governance, less duplication. Use user-level AR only for true exceptions. + +### 4.4 Dimension Replacement + +- Use a property (e.g. Country.Region) or a mapping metric to replace a dimension in AR (e.g. secure by Region, apply to metrics by Country via Country.Region). Reduces AR size and keeps logic at a higher level. + +--- + +## 5. Mandatory Formula Patterns + +### 5.1 Guard with IFDEFINED('Users roles', ...) + +- Always wrap the ACCESSRIGHTS call in `IFDEFINED('Users roles', ACCESSRIGHTS(ReadFlag, WriteFlag))`. +- Ensures AR is computed only for users who are members of the application. Without this, AR may be evaluated for all workspace users (performance and correctness). See [performance_access_rights.md](../optimizing-pigment-performance/performance_access_rights.md). + +### 5.2 Keep AR Metrics Thin + +- AR metrics should only reference precomputed booleans and call ACCESSRIGHTS(ReadFlag, WriteFlag). No business logic inside the AR formula. +- Precompute conditions (e.g. Can_Read, Can_Write) in separate metrics; the AR metric only applies them. + +### 5.3 No Direct User Inputs in AR Metrics + +- Do not let users type into the AR metric. Use boolean input metrics (e.g. Country_Read_Allowed, Country_Write_Allowed) maintained by admins, and reference them in the AR formula. Separates data entry from formula logic and improves auditability. + +--- + +## 6. Applying Rules: Apply vs Ignore + +### 6.1 Apply Rules + +- Apply rule: makes an AR metric active on a target (e.g. "Apply AR_Country to all metrics using Country"). +- Defines where the AR is enforced. + +### 6.2 Ignore Rules + +- Ignore rule: excludes a scope from an AR metric's effect (e.g. "Ignore AR_Country on metric Total_Company_Sales"). +- Use sparingly for exceptions where a broad Apply is too strict. Do not use Ignore to grant access that would otherwise be denied-fix the AR logic instead. + +### 6.3 Scope of Apply Rules + +- All metrics using dimension X (recommended when possible). +- Specific metric(s) for exceptions or very sensitive metrics. +- List properties (including transaction values) to secure list data. +- List item values: hides item labels in selectors/lists; does not secure metric data. Metrics that use those items need their own AR rules. + +### 6.4 Enabling and Precedence + +- Rules can be toggled on/off. Multiple Apply rules on the same data -> intersection (most restrictive). Ignore excludes that scope from the AR metric. + +--- + +## 7. Decision Frameworks + +### 7.1 Role-Based vs User-Based + +- All users in a role share the same access? -> Role-based AR (Role x dimension, map to users via 'Users roles'). +- Each user needs a unique slice? -> User-specific AR (or Role + user overrides for exceptions). + +### 7.2 Granularity + +- Dimension very large? -> Build security at a higher level and map down. +- Stable hierarchy/property? -> Use property replacement. +- Access varies over time? -> Consider mapping metrics (e.g. User x Quarter -> Region). Only use lowest-level AR when unavoidable. + +--- + +## 8. Common Patterns (Summary) + +| Pattern | When to Use | Key Idea | +| ---------------------------- | ------------------------------------------- | ------------------------------------------------------------------------------------------------------ | +| Country-level (per user) | Each user sees only certain countries | User x Country booleans -> AR metric -> Apply to all metrics with Country | +| Role-driven country | Same access by role | Role x Country booleans -> map to User via 'Users roles' -> AR metric -> Apply to metrics with Country | +| Region->Country | Region managers see all countries in region | AR at Region; apply to metrics with Country using dimension replacement by Country.Region | +| Sensitive data overlay | One metric (e.g. Salary) restricted | Separate AR metric (e.g. AR_Salary) applied only to that metric (and related); layer with base AR | +| Scenario-specific | Restrict a scenario (e.g. Board Budget) | Use Scenarios section in Roles & Access to set Read/Write per role for that scenario | +| List item security | Hide list items from some users | List item rule + property; also apply AR on metrics that use that list for data protection | + +--- + +## 9. Boards vs AR / Public Blocks + +- Board permissions: who sees which boards/sections (UX). Do not rely on them for data security. +- AR: cell-level data; enforced everywhere. Use both: AR for data, Board permissions for presentation. +- Public Blocks: override AR-block is visible to everyone in the app. Use with care. + +--- + +## 10. Multi-Application (Hub & Spoke) + +- AR rules do not cross apps. Each app has its own AR setup. +- Pattern: Build and maintain AR logic (user-role mappings, boolean matrices) in a central Hub. In each spoke app, link that data and create thin AR metrics that reference it; then Apply rules in each app. Shared dimensions (e.g. Country, Role) should be linked or have identical codes across apps. Data links do not carry AR-re-apply AR in the receiving app. + +--- + +## 11. Performance (Summary) + +- Use BLANK never FALSE; IFDEFINED('Users roles', ...); keep AR metrics thin; broad Apply rules (e.g. all metrics with dimension X) can enable row-level filtering and are often more performant than many narrow rules. +- Avoid User x Time in AR when possible. Prefer higher-level dimensions and mapping. +- Full details: [performance_access_rights.md](../optimizing-pigment-performance/performance_access_rights.md). + +--- + +## 12. Debugging + +### 12.1 "User cannot see data they should" + +Check in order: (1) Role default access, (2) IFDEFINED guard in AR formula, (3) AR metric values for that user/item (inputs/mappings), (4) Rule scope (metric/dimension covered?), (5) Ignore rules, (6) Scenario access, (7) Board permissions, (8) Public Blocks, (9) List item vs metric AR (right layer?). + +### 12.2 "User can see data they shouldn't" + +Check: (1) Rule actually applied and enabled? (2) Correct dimension in rule scope? (3) Block set to Public? (4) Test via impersonation and raw data (Explorer), not only Boards. + +--- + +## 13. Governance and Naming + +- Security dashboard: central place for User<->Role, AR matrices, documentation. +- Naming: e.g. AR_Country, Bool_Country_Read_Allowed, Map_User_to_Region. Clear prefixes help modelers and agents. Align with general conventions in [modeling_naming_conventions.md](../modeling-pigment-applications/modeling_naming_conventions.md). +- Refactoring: Prefer role-based AR and dimension replacement when scaling. Test with impersonation before rollout. + +--- + +## 14. Anti-Patterns + +| Anti-pattern | Why Bad | Fix | +| ---------------------------------------- | --------------------------- | ------------------------------------ | +| Security by Board only | No data protection | Enforce AR on data | +| FALSE for no access | Densifies, poor performance | Use BLANK | +| One giant AR metric | Unmaintainable, slow | Split by dimension (MS12) | +| Heavy logic inside ACCESSRIGHTS | Slow, opaque | Precompute booleans, thin AR | +| User x Time in AR | Exploding size | Remove time or use mapping | +| Forgetting IFDEFINED('Users roles', ...) | Wrong users / perf | Always wrap AR | +| AR metric but no Apply rule | No effect | Create Apply rule for each AR metric | +| List item rule only for sensitive data | Data still in metrics | Apply AR on metrics too | + +--- + +## 15. Quick Q&A for Agents + +- "Restrict by Country/Region/Department?" -> Dimension-based AR (User or Role x dimension), Apply to all metrics using that dimension. Prefer Role + 'Users roles' when many users share access. +- "Only HR sees Salary?" -> Sensitive data overlay: separate AR metric for Salary, Apply only to Salary metric (and related); others BLANK. +- "Hide Board instead of AR?" -> Use Board permissions for UX, but always secure the data with AR. +- "Multiple apps?" -> Hub for logic; link into each app; create thin AR metrics and Apply rules per app. +- "User can't see data?" -> Debug checklist section 12.1 (role, IFDEFINED, AR values, rule scope, scenario, board, public, list vs metric). +- "User sees too much?" -> Rule applied? Correct scope? Block Public? Impersonate to confirm. + +--- + +## 16. Configuration Checklist + +1. Gather requirements (what to restrict, by which dimension, which roles/users). +2. Choose highest possible grain (Role/dimension); decide Role vs User. +3. Prepare mappings/properties (e.g. Country.Region, 'Users roles'). +4. Create boolean inputs (Read/Write per dimension); keep in Security section. +5. Build AR metrics (ACCESSRIGHTS(Read, Write), guarded by IFDEFINED('Users roles', ...)); keep formulas thin. +6. Apply rules (Data access rights -> Rules): Apply each AR metric to the right scope (e.g. all metrics with dimension X). +7. Add Board permissions where needed (in addition to AR). +8. Test with impersonation (different roles); check edge cases and write access. +9. Document and maintain (Security dashboard, naming, review when adding new metrics/lists). + +--- + +## See Also + +- [functions_security.md](../writing-pigment-formulas/functions_security.md) - ACCESSRIGHTS, RESETACCESSRIGHTS syntax +- [performance_access_rights.md](../optimizing-pigment-performance/performance_access_rights.md) - AR performance, IFDEFINED(User) +- [modeling_principles.md](../modeling-pigment-applications/modeling_principles.md) - MS12, MP10, Roles, security folder diff --git a/plugins/pigment/skills/solving-specific-use-cases/SKILL.md b/plugins/pigment/skills/solving-specific-use-cases/SKILL.md new file mode 100644 index 00000000..f49c5917 --- /dev/null +++ b/plugins/pigment/skills/solving-specific-use-cases/SKILL.md @@ -0,0 +1,93 @@ +--- +name: solving-specific-use-cases +description: Always use this skill when building or extending models for specific planning domains - FP&A, Workforce Planning, Sales Performance Management, Supply Chain Planning, or Financial Consolidation. Covers proven modeling patterns and domain-specific best practices. This skill includes supporting files in this directory - explore as needed. +metadata: + skill_path: /solving-specific-use-cases/SKILL.md + base_directory: /solving-specific-use-cases + includes: + - "*.md" +--- + + +# Pigment Use Cases - Introduction + +Pigment is a business planning platform used across multiple domains. Each domain has its own modeling patterns, dimensions, and reporting needs, but they all share Pigment's core building blocks: lists, metrics, formulas, tables, and boards. + +This skill introduces the five primary use cases and explains what each one typically involves so the agent can orient itself when helping users build or extend a model. The list of patterns is not complete. + +## When to Use This Skill + +Read this skill when: + +- You are deciding which patterns, dimensions, and structures are appropriate for a specific use case +- You want to understand how different planning domains connect within a single Pigment organization +- You are designing or integrating FX currency conversion (Hub app, rates by version, entity mapping, AVG/END, reporting currency) + +> Planning-cycle topics (Version Dimension, Budget vs Actual, Forecast, Reforecast, switchover, Actual/Plan layering, scenarios, snapshots) are NOT covered here. They are owned by `skill:planning-cycles-pigment-applications`. Most patterns below (Centralized Reporting Metric, OPEX, Workforce, FX) assume a Version Dimension already exists; consult that skill before building the planning-cycle layer of the model. + +--- + +## 1. FP&A - Financial Planning & Analysis + +Pattern #1 - Centralized Financial Reporting Metric (Nexus): The recommended approach is to use a centralizing metric that aggregates upstream calculations, maps them to a reporting structure (e.g. a Chart of Accounts), and serves as the sole source of truth for financial statement boards and tables. It blends multiple data sources into one single metric with an Account dimension. This decouples reporting from model internals and provides a clean security boundary. Only use for financial reporting (P&L, Balance Sheet, Cash Flow), not for operational models. +Required reading for this pattern: [Centralizing Financial Reporting Metric (Nexus Pattern)](./finance_nexus_financial_statements.md) + +Pattern #2 - OPEX Planning Application Architecture: Overall structure of a driver-based OPEX planning app: data foundation (PULL layer from Library), configuration layer (version windows, account scope), output pipeline (OUT_FC -> ACT+FC -> Push), folder structure, and naming conventions (INP_, CALC_, OUT_, PULL_, Push_). Load when building the overall structure of an OPEX planning app or working on its data, configuration, or output layers. For forecasting method formulas and how to add new methods, see Pattern #3. +Required reading for this pattern: [OPEX Planning - Architecture & Patterns](./opex_planning_application_architecture.md) + +Pattern #3 - OPEX Forecasting Methods & Engine: Implementation reference for the OPEX forecasting engine internals. Contains: method formula patterns and parameters, YoY and blank-handling modifiers, Actual vs Plan window interactions, and the step-by-step procedure for adding new methods. Load when adding, modifying, or debugging individual forecasting methods and their parameters. +Required reading for this pattern: [OPEX Planning - Forecasting Methods & Engine](./opex_forecasting_planning_methods_engine.md) + +Pattern #4 - FX Currency Conversion (Hub): Centralized, version-aware FX engine in a dedicated Hub app. Dimensions: Currency, FX Rate Types (AVG for P&L, END for Balance Sheet), Reporting Currency (Local, Group). Layers: FX_01 (raw input by Version) -> FX_02 (fill-forward if needed) -> Push_DH_FX_Entity Currencies (entity -> currency mapping) -> Push_DH_FX_FX Rates (only metric referenced by P&L/BS). Use when building or connecting multi-currency models (Nexus, OPEX, Workforce, consolidation). +Required reading for this pattern: [FX Currency Conversion (Hub pattern)](./fx_currency_conversion.md) + +--- + +## 2. Workforce Planning + +Pattern #1 - Application Architecture & Patterns: Full blueprint for employee-based workforce planning: layered metric architecture (Data -> Card -> Stats -> Comp -> Push/KPI), dimension roles, EE + TBH data flows, override-first staging, naming conventions, and folder structure. Load when building or extending a workforce planning app end-to-end. +Required reading for this pattern: [Workforce Planning - Architecture & Patterns](./workforce_planning_architecture_patterns.md) + +Pattern #2 - Workforce Cards & Mapped Dimensions: The 4.0 layer that unifies Existing Employees and To-Be-Hired into a single Workforce dimension, with card metrics (`WF_Card_Entity`, `WF_Card_Department`, etc) and mapped-dimension reporting via `BY: -> WF_Card_...`. Load when you need to consolidate two populations into one workforce view or report by Entity/Department without adding those as structural dimensions. +Required reading for this pattern: [Workforce Planning - Cards & Mapped Dimensions](./workforce_planning_cards_mapped_dimensions.md) + +Pattern #3 - Changelog to Override Metrics: Models discrete change requests (transfers, salary updates, term dates) as Changelog dimension rows, projects them into override metrics at planning grain, and applies override-first staging. Load when users submit change requests with effective dates and an approval workflow. +Required reading for this pattern: [Workforce Planning - Changelog to Override Metrics](./workforce_planning_changelog_overrides.md) + +Pattern #4 - Snapshot Spread Logic: Bridges snapshot-based source data (e.g. HRIS with sparse load months) to a dense planning grid (Version x Employee x Month). Covers snapshot selection, FILLFORWARD propagation, history-vs-plan toggle, and version windows. Load when the source provides as-of snapshots and planning needs values on every month. +Required reading for this pattern: [Workforce Planning - Snapshot Spread Logic](./workforce_planning_snapshot_spread.md) + +--- + +## 3. Sales Performance Management (SPM) + +Pattern to be added Use your general knowledge to answer questions on this use case + +--- + +## 4. Supply Chain Planning (SCP) + +Pattern to be added Use your general knowledge to answer questions on this use case + +--- + +## 5. Financial Consolidation + +Pattern to be added Use your general knowledge to answer questions on this use case + +--- + +## Cross-References + +- Modeling foundations: `skill:modeling-pigment-applications` (dimensions, folder structure, Push/Pull) +- Planning cycles, Versions, Scenarios, Snapshots: `skill:planning-cycles-pigment-applications` -- always use this when a use case involves a Version Dimension, Budget vs Actual, Forecast, Reforecast, switchover, or Actual/Plan layering (most FP&A, OPEX, and Workforce patterns do) +- Formula implementation: `skill:writing-pigment-formulas` (BY modifier, aggregation functions) +- Performance: `skill:optimizing-pigment-performance` (large aggregation optimization) +- FP&A pattern - Centralized Reporting Metric (Nexus): [finance_nexus_financial_statements.md](./finance_nexus_financial_statements.md) +- Workforce Planning - Architecture & Patterns: [workforce_planning_architecture_patterns.md](./workforce_planning_architecture_patterns.md) +- Workforce Planning - Cards & Mapped Dimensions: [workforce_planning_cards_mapped_dimensions.md](./workforce_planning_cards_mapped_dimensions.md) +- Workforce Planning - Changelog to Override Metrics: [workforce_planning_changelog_overrides.md](./workforce_planning_changelog_overrides.md) +- Workforce Planning - Snapshot Spread Logic: [workforce_planning_snapshot_spread.md](./workforce_planning_snapshot_spread.md) +- OPEX Planning - Architecture & Patterns: [opex_planning_application_architecture.md](./opex_planning_application_architecture.md) +- OPEX Planning - Forecasting Methods & Engine: [opex_forecasting_planning_methods_engine.md](./opex_forecasting_planning_methods_engine.md) +- FP&A pattern - FX Currency Conversion (Hub): [fx_currency_conversion.md](./fx_currency_conversion.md) diff --git a/plugins/pigment/skills/solving-specific-use-cases/finance_nexus_financial_statements.md b/plugins/pigment/skills/solving-specific-use-cases/finance_nexus_financial_statements.md new file mode 100644 index 00000000..4b3f94d6 --- /dev/null +++ b/plugins/pigment/skills/solving-specific-use-cases/finance_nexus_financial_statements.md @@ -0,0 +1,242 @@ +# Core P&L Reporting Module - Nexus Pattern + +## Purpose + +This document explains how to build a P&L reporting hub in Pigment that acts as the central nexus of the workspace: it pulls actual data from ERP and plan data from other planning apps (Revenue, OPEX, Workforce) into a single, consistent structure, then applies FX conversion and feeds reporting metrics and tables. The outcome is a robust Actual + Budget/Plan P&L at monthly grain with built-in reconciliation checks. The architecture is modular so new planning apps can be added by plugging into the Nexus layer. + +When to use this pattern + +- You are building a central P&L reporting application that consolidates ERP actuals, budget, and plan/forecast from one or more planning apps. +- You need a single reporting metric (e.g. in reporting currency) that supports Actual vs Plan views (e.g. via a Data Type dimension). +- You want extensible dimensionality (Entity, Department, plus optional Product, Customer, Project, Cost Center) agreed with the user before building. +- Plan data is supplied by separate planning applications (Revenue, OPEX, Workforce); this skill covers the receiving hub only. + +When not to use + +- You are building only a planning app (e.g. Revenue, OPEX, Workforce) with no central reporting hub. +- You need Balance Sheet or Cash Flow reporting (use a dedicated 3-statements or BS/CF skill for those). +- You have a single data source and no need to combine Actual + Budget + multiple plan streams. + +--- + +## 1. Business scope + +The pattern delivers: + +- Layered architecture: Data (raw inputs) -> Staging (reshape + sign normalization) -> Nexus (Actual, Budget, Plan plugs) -> Unified metric (Actual + Plan by Data Type) -> FX reporting (`Rep_PnL Data`) -> Statement table metrics. +- Single grain: Entity x Version x Month x PnL_Account x Department, plus any extra dimensions the user confirms (Product, Customer, Project, Cost Center, etc.). +- Sign logic via account metadata (Operator on PnL_Account / PnL_Account Category), not hard-coded +/- in formulas. +- Integration: `Pull_*` metrics from a Data Hub (ERP, Budget, FX) and from planning apps (Revenue, OPEX, Workforce); Nexus "plug" metrics receive them at the same grain. +- Reporting: Standard P&L lines (Revenue, COGS, Gross Margin, Operating Expenses, EBITDA drivers-Depreciation, Interest, Tax-Net Other Income, Operating Income, EBT, Net Income) and built-in reconciliation checks between detailed data and the statement, all sourced from `Rep_PnL Data`. + +--- + +## 2. Prerequisites + +Before building: + +- A working calendar with Month (linked to Quarter and Year). +- Access to ERP P&L actuals (e.g. transaction list or Data Hub view such as `PnL_GL_Load_ERP`), Budget P&L (e.g. `PnL_Load_Budget`), and optionally plan data from other apps (Revenue, OPEX, Workforce). + +Discovering blocks in the live application: Names like `PnL_GL_Load_ERP` are illustrative meaning they are examples. In the user's workspace, confirm Company / Entity, Month, StatementAccount or PnL_Account, CostCenter, Version, DataFlavor (or equivalent), and reporting Currency using `tool:search` (optionally with `kind` / `regexp`); results include block summaries and identifiers-use those to align Nexus, staging, and reporting metrics to actual names in that app-do not assume the template's naming. + +If any of these are not available, create placeholder metrics at the same grain and replace them later with `Pull_*` from the Data Hub or planning apps. Use Option B in section 5.1 for placeholder signs. + +--- + +## 3. Core concepts + +| Concept | Description | +| ----------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Base grain | Entity x Version x Month x PnL_Account x Department; extended with user-confirmed extra dimensions (Product, Customer, Project, etc.). | +| Data layer | Raw inputs: ERP P&L actuals, Budget load, optional plan data from other apps. Each kept close to source structure. | +| Staging layer | Metrics that reshape raw data onto the common grain and normalize signs (e.g. `PnL_Data_00_GL`, `PnL_Data_02_Roll-up`). | +| Nexus layer | Plug-and-play hub: separate metrics for Actuals (`Nexus_01`), Budget (`Nexus_02`), and each plan source (`Nexus_03_*`); a combined Plan metric (`Nexus_04`); and a unified Actual + Plan metric (`Nexus_99`) by Data Type. | +| Data Type | Dimension (e.g. Actual, Forecast) used to pivot the unified metric so one block supports both Actual and Plan views. | +| Rep_PnL Data | Final P&L metric in reporting currency (FX applied); single source for all statement line metrics and tables. | +| Operator | Property on PnL_Account or PnL_Account Category (e.g. 1 or -1) to normalize accounting signs; used in staging, not hard-coded in formulas. | +| `Pull_*` | Metrics that bring data from Data Hub (ERP, Budget, FX) or from other apps (Revenue, OPEX, Workforce) into the core app. | + +--- + +## 4. Pipeline (generic) + +1. Confirm extra dimensions with the user + Ask: "Which additional breakdowns do you need on your P&L (e.g. Product, Customer, Project, Cost Center, none)?" Turn the answer into a list of extra dimensions. Add them consistently to staging, Nexus, unified (`PnL_Nexus_99`), FX (`Rep_PnL Data`), and P&L table metrics. Whenever the skill shows a dimension set (e.g. Entity, Version, PnL_Account, Month, Department), mentally extend it with the user's extra dimensions. + +2. Define dimensions + - Time: Month (properties: Start Date, End Date, Start of Next Period, Year, Quarter), Quarter, Year. + - Version: Items e.g. Actuals, Budget, Forecast v1, Forecast v2; properties (Dimension -> Month): Last Actuals Month, Window Start Month, Window End Month. Used by Data Hub pulls to control Actual vs Plan months. + - Other axes: Entity, Department, Reporting Currency; Data Type (items: Actual, Forecast)-used only in Nexus_99 and reporting metrics. + - PnL_Account Category: Name; category flags (e.g. Revenue, Cost of Goods Sold, Operating Expenses, Other Income, Other Expenses); Operator (Integer), e.g. 1 for Revenue/Other Income, -1 otherwise. + - PnL_Account: Name, optional Display Name; PnL_Account Category; Operator = IFBLANK(PnL_Account.'PnL_Account Category'.Operator, 1); optional PnL_EBITDA (Dimension tagging EBITDA components: Depreciations & Amortizations, Interest Expenses, Tax Expenses, etc.). + - Extra dimensions (only if user confirmed): Product, Customer, Project, Cost Center. + +3. Data staging + Map ERP GL into `PnL_Data_00_GL` at base grain (BY from transaction list). Normalize signs in `PnL_Data_02_Roll-up` = `PnL_Data_00_GL` _ PnL_Account.Operator _ -1. No separate elimination layer in this pattern. + +4. Nexus metrics + - `PnL_Nexus_01_Actual Data`: staging actuals with Version added, gated by a view/pull that limits to actual months. + - `PnL_Nexus_02_Budget Data`: budget load with Version added, gated by plan view. + - Plan plugs: `Pull_RE_Revenue Plan Data`, `Pull_OP_OPEX Plan Data`, `Pull_WF_Workforce Plan Data` (same grain; BLANK or PULL from apps). + - `PnL_Nexus_03_Revenue/OPEX/Workforce Plan Data`: IF Version not Actuals/Budget then respective `Pull_*`, else BLANK. + - `PnL_Nexus_04_Plan Data`: IF Version = Budget then `Nexus_02`; else sum of `Nexus_03_...`. + +5. Unified Actual + Plan + `PnL_Nexus_99_Actual + Plan Data`: IF Data Type = Actual then Nexus_01 (ADD Data Type, BY SUM Actual); else Nexus_04 (ADD Data Type, BY SUM Forecast). One metric for both Actual and Plan by pivoting on Data Type. + +6. FX reporting + `Rep_PnL Data` = `PnL_Nexus_99_Actual + Plan Data` \* FX rate (e.g. AVG for P&L), in reporting currency. Dimensions include Reporting Currency, Entity, Version, PnL_Account, Month, Data Type, Department, [extra dims]. + +7. Statement table metrics + Each line = filter on `Rep_PnL Data` by PnL_Account Category (Revenue, COGS, OPEX, etc.). Derived lines (e.g. Gross Margin) = sum of relevant lines \* Category Operator [REMOVE: PnL_Account]. Add `PnL_Tbl_Check` comparing statement total to Nexus/reporting source. + +8. P&L table + Rows: PnL line or category; Pages: Entity, Department, extra dims; Columns: Month, Data Type, Version; Values: the PnL table metrics. + +--- + +## 5. Patterns + +### 5.1 Staging: GL to common grain and sign normalization + +```text +PnL_Data_00_GL = + 'PnL_GL_Load_ERP'.Amt_LC + [BY: Month, Account, Entity, Department + /* + each extra dimension from source */] + +PnL_Data_02_Roll-up = + 'PnL_Data_00_GL' * PnL_Account.Operator * -1 +``` + +Same dimensions for both. Operator on PnL_Account (or Category) drives sign; no hard-coded +/- in formulas. + +Sign conventions: Option A (ERP GL) - source revenue negative, expenses positive -> `x Operator x -1`. Option B (mock, budget, plan) - source all positive -> `x Operator` only. Confirm source convention before staging; do not mix. + +Operator formulas (sign logic via metadata): + +- PnL_Account Category.Operator (Integer): `IF('PnL_Account Category'.IsRevenueCategory OR 'PnL_Account Category'.IsOtherIncomeCategory, 1, -1)` +- PnL_Account.Operator: `IFBLANK(PnL_Account.'PnL_Account Category'.Operator, 1)` + +### 5.2 Nexus Actual and Budget + +```text +PnL_Nexus_01_Actual Data = + 'PnL_Data_02_Roll-up' + [ADD: Version] + ['Pull_DH_View_Load Actual'] + +PnL_Nexus_02_Budget Data = + 'PnL_Load_Budget' + [ADD: Version] + [BY SUM: VAR_Budget_Version] + ['Pull_DH_View_Load Plan'] +``` + +View/pull metrics restrict which months are Actual vs Plan per Version. + +### 5.3 Nexus plan plugs + +```text +PnL_Nexus_03_Revenue Plan Data = + IF( + NOT Version IN (VAR_Actuals_Version, VAR_Budget_Version), + 'Pull_RE_Revenue Plan Data', + BLANK + ) +``` + +Same pattern for OPEX and Workforce (`Pull_OP_OPEX Plan Data`, `Pull_WF_Workforce Plan Data`). If no separate Revenue app, can use Budget filtered to Revenue category instead of `Pull_RE_`. + +### 5.4 Nexus combined Plan + +```text +PnL_Nexus_04_Plan Data = + IF( + Version = VAR_Budget_Version, + 'PnL_Nexus_02_Budget Data', + 'PnL_Nexus_03_Revenue Plan Data' + + 'PnL_Nexus_03_OPEX Plan Data' + + 'PnL_Nexus_03_Workforce Plan Data' + ) +``` + +### 5.5 Unified Actual + Plan by Data Type + +```text +PnL_Nexus_99_Actual + Plan Data = + IF( + IsActual, + 'PnL_Nexus_01_Actual Data' + [ADD: 'Data Type'] + [BY SUM: VAR_Actual_Data_Type], + 'PnL_Nexus_04_Plan Data' + [ADD: 'Data Type'] + [BY SUM: VAR_Forecast_Data_Type] + ) +``` + +Dimensions include Data Type. One metric serves both Actual and Forecast views. + +### 5.6 FX reporting metric + +```text +Rep_PnL Data = + 'PnL_Nexus_99_Actual + Plan Data' + * 'Pull_DH_FX_FX Rates'[SELECT: 'FX Rate Types'."AVG"] +``` + +Dimensions: Reporting Currency, Entity, Version, PnL_Account, Month, Data Type, Department, [extra dims]. Extend for source vs reporting currency if required. + +For full FX engine design (Hub app, dimensions, layers, AVG/END, entity mapping), see [FX currency conversion (Hub pattern)](./fx_currency_conversion.md). + +### 5.7 Statement line from Rep_PnL Data + +```text +PnL_Tbl_01_Revenue = + 'Rep_PnL Data' + [FILTER: PnL_Account.IsRevenueCategory] + +PnL_Tbl_02_Gross Margin = + ( + 'PnL_Tbl_01_Revenue' + + 'PnL_Tbl_01_Cost of Goods Sold' + ) + * PnL_Account.'PnL_Account Category'.Operator + [REMOVE: PnL_Account] +``` + +Other lines (OPEX, EBITDA drivers, Net Income) follow the same idea: filter by category or aggregate with Operator, then REMOVE PnL_Account where needed. + +--- + +## 6. How to apply elsewhere + +1. Before building: Ask the user which extra breakdowns they need (Product, Customer, Project, Cost Center, none). Create those dimensions and add them to every layer (staging, Nexus, unified, Rep_PnL Data, table metrics). +2. Dimensions: Calendar (Month with Start/End Date, Start of Next Period, Year, Quarter); Entity, Department; Version (Actuals, Budget, Forecast v1/v2...; Last Actuals Month, Window Start/End Month); Reporting Currency; Data Type (Actual, Forecast); PnL_Account Category (Operator: 1 for Revenue/Other Income, -1 else); PnL_Account (Operator = IFBLANK(Category.Operator, 1), optional PnL_EBITDA); extra dimensions as confirmed. +3. Staging: One metric from ERP GL (BY to base grain), one roll-up with sign normalization (Operator). No elimination layer unless the user explicitly needs it. +4. Nexus: `Nexus_01` (Actual), `Nexus_02` (Budget), three plan plugs and `Nexus_03_...` (Revenue, OPEX, Workforce), `Nexus_04` (combined Plan), `Nexus_99` (Actual + Plan by Data Type). +5. FX: One Rep_PnL Data metric with AVG rate (or user-specified rate type). +6. Tables: Statement line metrics as filtered/aggregated views of Rep_PnL Data; one check metric; one P&L table with rows = lines, columns = Month / Data Type / Version. +7. Naming: Keep prefixes (`PnL_Data_*`, `PnL_Nexus_*`, `Rep_PnL Data`, `PnL_Tbl_*`) so the flow stays traceable. + +--- + +## 7. Pitfalls and reminders + +- Extra dimensions: Add them everywhere from the start; retrofitting later is error-prone. Confirm with the user once. +- Operator consistency: Operator on PnL_Account (or Category) must be correct for Revenue, COGS, OPEX, etc.; wrong sign breaks all statement totals. Pair input signs with the roll-up option in section 5.1. +- View/pull logic: Nexus_01 and Nexus_02 depend on view or pull metrics that define "actual months" vs "plan months" per Version; align these with Version properties (Last Actuals Month, etc.). +- Plan plugs: If an app (e.g. Revenue) doesn't exist yet, use BLANK or a placeholder; do not build that app inside this skill-this skill is the hub only. +- Single grain: All Nexus and Rep metrics share the same dimension set; mismatched grains cause wrong totals or sparse/blank results. + +--- + +## 8. Illustration (conceptual) + +- Staging: PnL_Data_00_GL from PnL_GL_Load_ERP; PnL_Data_02_Roll-up = sign normalization. +- Nexus: PnL_Nexus_01_Actual Data, PnL_Nexus_02_Budget Data; Pull_RE_Revenue Plan Data, Pull_OP_OPEX Plan Data, Pull_WF_Workforce Plan Data; PnL_Nexus_03_Revenue/OPEX/Workforce Plan Data; PnL_Nexus_04_Plan Data; PnL_Nexus_99_Actual + Plan Data. +- Reporting: Rep_PnL Data (FX AVG); PnL_Tbl_01_Revenue, PnL_Tbl_01_COGS, PnL_Tbl_02_Gross Margin, etc.; PnL_Tbl_Check; [Tbl] Income Statement. + +Template names (e.g. [FIN]01 Core Reporting) map to this structure; the generic pattern is the layered flow and plug-and-play Nexus design. diff --git a/plugins/pigment/skills/solving-specific-use-cases/fx_currency_conversion.md b/plugins/pigment/skills/solving-specific-use-cases/fx_currency_conversion.md new file mode 100644 index 00000000..45d48102 --- /dev/null +++ b/plugins/pigment/skills/solving-specific-use-cases/fx_currency_conversion.md @@ -0,0 +1,117 @@ +Use this document when designing or integrating FX currency conversion in Pigment: Hub app pattern, FX_01/FX_02/Push_DH_FX_* layers, AVG vs END rate types, reporting currency, entity mapping, and optional triangulation. + +# Pigment FX: Currency Conversion Design & Usage Guide + +## Overview + +FX conversion in Pigment is built as a centralized, Version-aware engine living in a dedicated Hub app. All financial metrics (P&L, Balance Sheet, etc.) consume a single canonical FX rate metric - they never implement their own FX logic. + +The engine is responsible for: +- Storing FX rates by currency, rate type, version, and month. +- Filling forward missing rates where source data is incomplete. +- Mapping each entity to its local and reporting currencies. +- Handling triangulation for multi-leg conversions. +- Outputting one consolidated FX rate metric used by all downstream metrics. + +--- + +## 1. FX Dimensions + +### Currency +The universe of currencies in the model. Examples: `USD`, `EUR`, `GBP`, `CNY`, `JPY`, `BRL`. Single property: `Name` (Text, display). + +### FX Rate Types +Distinguishes the context in which a rate is used - same Currency x Month can carry multiple rates. The two standard types are: + +- AVG (Average rate) - used for P&L conversion. Reflects the average exchange rate over the month, appropriate for income statement items that accrue continuously. +- END (End / closing rate) - used for Balance Sheet conversion. Reflects the rate at the last day of the month, appropriate for balance sheet items that are point-in-time. + +Additional rate types (e.g. "Budget FX", "Spot") can be added but are the exception, not the default. + +### Reporting Currency +The target currencies for reporting output. The standard setup is: + +- Local - the functional currency of the entity itself (see note below). +- Group - the consolidation currency of the organization (e.g. USD or EUR). + +Additional reporting currencies (e.g. regional currencies) can be added, but this should be a deliberate choice, not a default. + +> Local currency vs. transactional currency: "Local" in this context means the functional currency of the entity - the primary currency in which it operates. It does not mean the transactional currency of individual line items (which may differ). Transactional currency handling is a more advanced pattern and not the default setup. + +--- + +## 2. Metric Architecture + +The FX engine is built in layers, each with a single responsibility. The final output metric is the only one that P&L, Balance Sheet, and other financial metrics should ever reference. + +``` +FX_01 Raw FX rates input (by Version) + down +FX_02 Fill-forward / cleaning (by Version, only if source data has gaps) + down +Push_DH_FX_Entity Currencies - entity -> currency mapping + down +Push_DH_FX_FX Rates <- only this is referenced by P&L, BS, etc. +``` + +> Triangulation (an optional intermediate step) is only needed if your rate source doesn't cover all required currency pairs directly - e.g. you have CNY -> USD and USD -> EUR but no CNY -> EUR rate. In that case, add a triangulation metric between entity mapping and the final output to build multi-leg rates per entity. Most models don't need this. + +--- + +## 3. Layer-by-Layer Breakdown + +### FX_01_Input_FX Rates +Dimensions: `Currency x FX Rate Types x Version x Month` + +The raw input store for FX rates. Version is required here - rates are entered per Version from the start, so Budget, Reforecast Q1, Reforecast Q2, etc. each carry their own rate series. There is no shared "unversioned" input that gets split later. + +When creating a new Version, use the Clone feature (per application) to copy rates from an existing Version as a starting point, then adjust as needed. See the Versions skill for cloning guidance. + +### FX_02_FX Rates_Spread *(only if needed)* +Dimensions: `Currency x FX Rate Types x Version x Month` + +If the source data feeding FX_01 is complete for every month, this layer is not needed. If there are gaps - e.g. a rate is only provided quarterly, or a new currency starts mid-year - this metric fills those gaps forward so every Currency x Rate Type x Version has a continuous monthly time series. + +Still dimensioned by Version: each Version's series is cleaned independently. + +### Push_DH_FX_Entity Currencies +Dimensions: `Entity x Reporting Currency` -> value is a `Currency` dimension member + +Structural mapping: for a given Entity and Reporting Currency, stores which currency to use as the source. For example, a French entity reporting to Group would map to EUR as its local currency. No Version dimension - this is static entity configuration that applies across all Versions. + +### Push_DH_FX_FX Rates <- *only this leaves the Hub app* +Dimensions: `FX Rate Types x Version x Entity x Month x Reporting Currency` + +The canonical output of the Hub app. Combines the entity-currency mapping with the Version-aware rate series to return a single conversion rate: "to convert this entity's local currency to the selected Reporting Currency, for this Version and Month, using this Rate Type." + +Because this metric is dimensioned by Version, any downstream metric referencing it automatically uses the correct rates for its Version. Switch Version on a board -> all converted values are consistent. + +--- + +## 4. How Financial Metrics Use FX + +The pattern is simple: + +``` +Converted Amount = Local Amount x Push_DH_FX_FX Rates +``` + +The FX rate metric carries Version, Entity, Month, Rate Type, and Reporting Currency - so when a P&L metric is dimensioned by those same axes, Pigment resolves the correct rate automatically. No FX logic lives in the P&L or BS metrics themselves. + +Use AVG rate for P&L items. Use END rate for Balance Sheet items. If a model needs both (e.g. a full financial consolidation), the Rate Type dimension on the FX metric handles this without needing separate FX metrics per statement. + +--- + +## 5. Best Practices + +### Centralize in a Hub app +All FX logic - dimensions, input metrics, cleaning, mapping, triangulation, output - lives in one dedicated Hub app. No other app contains FX calculations. This makes FX auditable, maintainable, and consistent across the model. + +### Rates are input by Version, cloned between Versions +There is no single shared rate series that gets "applied" to Versions later. Each Version owns its rates from FX_01 onward. When a new Version is created, clone the relevant application's data from an existing Version and adjust. This keeps each Version's FX environment independent and explicit. + +### Only reference the final output metric +`Push_DH_FX_FX Rates` is the only FX metric that should be referenced outside the Hub app. Referencing `FX_01` or `FX_02` from a P&L metric bypasses entity mapping and will produce incorrect results. + +### AVG for flow items, END for stock items +This is a standard accounting convention. P&L items (revenues, costs) flow over the month -> use AVG. Balance sheet items are a point-in-time balance -> use END. Establish this convention at model setup and enforce it consistently. diff --git a/plugins/pigment/skills/solving-specific-use-cases/opex_forecasting_planning_methods_engine.md b/plugins/pigment/skills/solving-specific-use-cases/opex_forecasting_planning_methods_engine.md new file mode 100644 index 00000000..aa58e0b6 --- /dev/null +++ b/plugins/pigment/skills/solving-specific-use-cases/opex_forecasting_planning_methods_engine.md @@ -0,0 +1,237 @@ +# OPEX Planning - Forecasting Methods & Engine + +## Purpose + +This use case describes how a driver-based OPEX forecasting engine works when there are several predefined methods (e.g. Prior Year, Last X months average, % of Revenue, $ per Headcount, Manual input) and one method per planning line. It covers: method selection (dimension + input metric + SWITCH), the five calculation methods (concept and formula pattern), global modifiers (YoY and blank handling), row lifecycle and validation (new lines, visibility filter, validation flag), and Actual vs Plan windows so that all methods apply only to plan months and the engine can be extended with new methods without rewriting window logic. + +When to use this skill + +- You are building or extending an OPEX app that uses a central forecast metric (e.g. CALC_Forecast) switching on method ID to pick one of several method-specific metrics (CALC_PYValues, CALC_LastXMonthsAVG, etc.). +- You need to add a new forecasting method or understand how parameters, YoY, and Actual/Plan windows interact with the engine. + +When not to use + +- Purely manual OPEX with no driver-based methods does not need the SWITCH/method pattern. +- For app layout, PULL layer, folder structure, and naming, see OPEX Planning - Application Architecture & Patterns. + +--- + +## 1. How method selection works + +Forecasting Method dimension + +- Encodes each method with: ID (Integer, used in SWITCH), Need Input in Parameter? (Boolean), Parameter Unit (e.g. "months", "%", "$/HC"). +- Example methods: PY Values (1), Last X months Avg (2), % of Revenue (3), $ per Headcount (4), Manual Input (5). + +Input metric: INP_Forecasting Method + +- Type: Dimension -> Forecasting Method. +- Dims: Department, PnL_Account, Version, Entity, Line (or your planning grain). +- Holds the user-selected method per line. +- Prefill (optional): When CHK_HasActuals? and SET_Fill_combo_withactuals? are true, fill from SET_FCMethod_by_PnlAccount[BY: Line."1"] so new lines get a default method from config. + +Numeric code for SWITCH: CDFM_ID + +- CDFM_ID = INP_Forecasting Method.ID so the engine can switch on an integer. + +Central engine: CALC_Forecast + +``` +( + SWITCH( + 'INP_Forecasting Method'.ID, + 1, CALC_PYValues, + 2, CALC_LastXMonthsAVG, + 3, CALC_%ofRevenue, + 4, CALC_$perHeadcount, + 5, CALC_Manual_Input + ) + * CALC_YoY_2_fixblanks +) +[EXCLUDE: 'Push_DH_View_Load Actual'] +``` + +- ID selects which method metric is used. +- CALC_YoY_2_fixblanks is a global modifier (YoY uplift and/or blank handling) applied to all methods. +- EXCLUDE: Push_DH_View_Load Actual ensures forecast is never computed for actual months; methods apply only to plan months. + +--- + +## 2. The five calculation methods (patterns) + +All method metrics share the planning dimensions (e.g. Department, PnL_Account, Version, Entity, Month, Line). Each is wrapped in IF(INP_Forecasting Method = Forecasting Method."MethodName", ...) so only the selected method contributes. + +### 2.1 PY Values (Prior Year) + +Intent: Use prior-year actuals (same month, year-1) as the forecast base; optional YoY uplift via global modifier. + +Pattern: + +``` +IF( + 'INP_Forecasting Method' = 'Forecasting Method'."PY Values", + IFDEFINED( + 'PULL_CR_PnL Data Actual'[ADD: Line][SELECT: Month - 12], + 'PULL_CR_PnL Data Actual'[ADD: Line][SELECT: Month - 12], + PREVIOUS(Month, 12) + ) +) +``` + +- SELECT: Month - 12 = same month last year. IFDEFINED uses shifted actuals; fallback PREVIOUS(Month, 12) for robustness when some months are missing. + +### 2.2 Last X months average + +Intent: Average of the last X actual months as the base for each plan month (e.g. "last 3 months average"). + +Inputs: INP_Parameter = SET_X_forMA when method = Last X months Avg. Helper metrics: PAR_CountLastMonthsActuals (counter over last actual months), PAR_denominator_forAVG (count or adjusted divisor). + +Pattern: + +``` +IF( + 'INP_Forecasting Method' = 'Forecasting Method'."Last X months Avg", + ( + 'PULL_CR_PnL Data Actual'[ADD: Line] + [FILTER: PAR_CountLastMonthsActuals <= INP_Parameter] + [REMOVE: Month] + ) + / + PAR_denominator_forAVG[ADD: Month][FILTER: 'Push_DH_View_Load Plan'] +) +``` + +- Numerator: sum of actuals over last X months (filter by counter), then REMOVE Month. Denominator: aligned to plan months so the ratio is per (Dept, Entity, PnL_Account, Version, Line, Month). + +### 2.3 % of Revenue + +Intent: Forecast = given % of revenue plan. + +Inputs: INP_Parameter = SET_%_forREV (e.g. 5 for 5%). PULL_Rev_Revenue Plan Data at Department x Entity x Version x Month. + +Pattern: + +``` +IF( + 'INP_Forecasting Method' = 'Forecasting Method'."% of Revenue", + INP_Parameter / 100 * 'PULL_Rev_Revenue Plan Data' +)[FILTER: 'Push_DH_View_Load Plan'] +``` + +- Only plan months; no forecast in actual months. + +### 2.4 $ per Headcount + +Intent: Forecast = $ per head x headcount plan. + +Inputs: INP_Parameter = SET_$_forHC. PULL_WF_Headcount Plan Data at Department x Entity x Version x Month. + +Pattern: + +``` +IF( + 'INP_Forecasting Method' = 'Forecasting Method'."$ per Headcount", + INP_Parameter * 'PULL_WF_Headcount Plan Data' +)[FILTER: 'Push_DH_View_Load Plan'] +``` + +### 2.5 Manual Input + +Intent: Users type values by month; engine provides the structure (writable cells) and baseline (e.g. 0). + +Pattern: + +``` +IFDEFINED( + FIL_Rowfilter, + IF( + 'INP_Forecasting Method' = 'Forecasting Method'."Manual Input" + AND 'Push_DH_View_Load Plan', + 0 + ) +) +``` + +- Only when row is visible (FIL_Rowfilter) and method is Manual and month is plan. Baseline 0 creates writable cells; users' edits become the forecast for that line/month. CALC_Forecast then picks CALC_Manual_Input when method = Manual. + +--- + +## 3. Global modifiers: YoY and blank handling + +Inputs + +- INP_YoY% - numeric YoY adjustment per line (default 0). +- INP_YoY_Month - base Month of Year (e.g. IFDEFINED(INP_YoY%, Month."January")). + +CALC_YoY_2_fixblanks + +- Referenced in CALC_Forecast as the multiplier after the SWITCH: (SWITCH result) * CALC_YoY_2_fixblanks. +- Typically implements: (1) YoY uplift/decay (e.g. 1 + INP_YoY% for relevant months), and/or (2) blank handling so that method blanks are not turned into zeros and sparsity is preserved. +- One place for YoY and blank logic: adding a new method does not require duplicating this; it applies to all methods uniformly. + +--- + +## 4. Row lifecycle and validation + +### 4.1 New line creation (Newline -> Line) + +- Users add a new combination (Entity x Department x PnL_Account) via an "Add New Combination" action that writes to a Newline list (properties: Entity, Department, PnL_Account). +- FIL_NewlineAdded maps Newline into the main planning space and flags which Line(s) are new: + +``` +ISDEFINED( + Newline.ID + [BY: Newline.PnL_Account, Newline.Department, Newline.Entity] +)[ADD: Version][BY: Line."1"] +``` + +- So new (Entity, Department, PnL_Account, Version, Line) combinations are identified and can be shown progressively. + +### 4.2 Row visibility: FIL_Rowfilter + +- FIL_Rowfilter is a Boolean that controls which lines appear in the input table (e.g. [TBL] Opex Items Control). +- Logic typically combines: FIL_NewlineAdded (show new lines); SET_Allowactuallineonly (show only lines with actuals if desired); SET_Onlyshownewlineaftervalidation (show a new line only after the previous line is validated); CHK_HasActuals?; INP_Validate[SELECT: Line - 1]; ISDEFINED('INP_Forecasting Method'[SET_OpexAccounts][SELECT: Line - 1]). +- Effect: Lines appear progressively (validation of previous line, or creation via Add), avoiding a huge table of unused rows; optional "actual-only" or "new-only-after-validation" policies. + +### 4.3 Validation: INP_Validate + +- INP_Validate (Boolean per line): IF(SET_Skip_Validation, BLANK, IFDEFINED('INP_Forecasting Method', FALSE)). +- If validation is not skipped, once a method is defined the line defaults to FALSE (pending); users set it to TRUE when the line is reviewed/approved. Used in FIL_Rowfilter and in KPI widgets (e.g. "waiting validation" vs "validated"). + +--- + +## 5. Actual vs Plan windows (impact on methods) + +Window flags (Version-based) + +- Push_DH_View_Load Actual: Month >= Version.'Window Start Month' AND Month <= Version.'Last Actuals Month'. +- Push_DH_View_Load Plan: Month after Last Actuals Month and within Version.'Window End Month' (using month start/end dates). + +Usage in the engine + +- CALC_Forecast: [EXCLUDE: 'Push_DH_View_Load Actual'] -> no forecast in actual months; only plan months get method results. +- Method metrics: % of Revenue and $ per Headcount use [FILTER: 'Push_DH_View_Load Plan'] so they only compute for plan months. CALC_Manual_Input uses AND 'Push_DH_View_Load Plan' so manual cells exist only in plan months. +- Reporting: ACT + FC and REP_FC + Last Actual Month combine actuals and forecast using these flags so the Actual/Forecast boundary is consistent. + +Outcome: Changing Version properties (Last Actuals Month, Window End Month) moves the Actual/Plan boundary without changing any method formula; all methods adapt automatically (e.g. rolling 12+12 forecast). + +--- + +## 6. How to add a new method + +1. Add an item to the Forecasting Method dimension with a new ID (e.g. 6) and metadata (Need Input in Parameter?, Parameter Unit). +2. Create a new method metric (e.g. CALC_NewMethod) with the same dimensions as other CALC_* metrics. Formula: IF(INP_Forecasting Method = Forecasting Method."NewMethod", )[FILTER: 'Push_DH_View_Load Plan'] if it should only run in plan months. +3. Wire parameters if needed: in INP_Parameter (or SET_*), add a branch for the new method (e.g. IF(... = "NewMethod", SET_NewMethodParam)). +4. Add the case to CALC_Forecast: SWITCH(..., 6, CALC_NewMethod, ...). +5. YoY and blank handling stay in CALC_YoY_2_fixblanks; no change needed unless the new method needs special treatment. + +--- + +## 7. Illustration: [FIN]04 OPEX Template + +- Method dimension: Forecasting Method with PY Values (1), Last X months Avg (2), % of Revenue (3), $ per Headcount (4), Manual Input (5). INP_Forecasting Method, CDFM_ID, CALC_Forecast (SWITCH * CALC_YoY_2_fixblanks)[EXCLUDE: Actual]. +- Method metrics: CALC_PYValues, CALC_LastXMonthsAVG, CALC_%ofRevenue, CALC_$perHeadcount, CALC_Manual_Input; PULL_CR_PnL Data Actual, PULL_WF_Headcount Plan Data, PULL_Rev_Revenue Plan Data; INP_Parameter, SET_X_forMA, SET_%_forREV, SET_$_forHC. +- Row lifecycle: Newline, FIL_NewlineAdded, FIL_Rowfilter, INP_Validate; SET_Allowactuallineonly, SET_Onlyshownewlineaftervalidation, CHK_HasActuals?. +- Windows: Push_DH_View_Load Actual, Push_DH_View_Load Plan; Version.'Last Actuals Month', Window Start/End Month. + +For app structure, PULL layer, folders, and naming: OPEX Planning - Application Architecture & Patterns. \ No newline at end of file diff --git a/plugins/pigment/skills/solving-specific-use-cases/opex_planning_application_architecture.md b/plugins/pigment/skills/solving-specific-use-cases/opex_planning_application_architecture.md new file mode 100644 index 00000000..9522bc2e --- /dev/null +++ b/plugins/pigment/skills/solving-specific-use-cases/opex_planning_application_architecture.md @@ -0,0 +1,139 @@ +# OPEX Planning - Application Architecture & Patterns + +## Purpose + +This document is a reusable blueprint for an OPEX (operating expense) planning application built around driver-based forecasting at a fixed planning grain (e.g. Entity x Department x PnL_Account x Version x Month), with user overrides on top of calculated forecasts. It describes the data foundation (PULL from Library), configuration layer, forecasting engine (method per line, SWITCH-based), output and reporting, and naming conventions so another modeler can replicate the approach. + +What to know when building an OPEX app + +- The logic is usually simple: a small set of predefined forecasting methods (Prior Year, Last X months average, % of Revenue, $ per Headcount, Manual input) and a central engine that computes forecast values per line; then users can override those values where needed. +- The main complexity is dimensionality: at which level do you plan? Typically all planning dimensions (Entity, Department, PnL_Account, Version, Month, and often Line) are in the metric structure, unlike Workforce Planning where many axes are mapped dimensions (BY: -> card). In OPEX, you rarely "map" to Entity/Department after the fact; they are structural from the start. + +When to use this blueprint + +- You are building or extending an OPEX planning app that: + - Pulls actuals and drivers (P&L actuals, headcount plan, revenue plan) from other apps via PULL_ metrics. + - Lets users choose a forecasting method per line (per Entity x Department x PnL_Account x Version, with one or more lines per combination). + - Produces forecast then actual + forecast and variance for reporting and sharing (Push_*). + +When not to use + +- Pure manual OPEX input with no driver-based methods does not need the full forecasting engine (see OPEX Planning - Forecasting Methods & Engine for the method layer). +- Apps that get all data from in-app lists (no cross-app PULL) will have a different data foundation but can still reuse the engine and folder patterns. + +--- + +## 1. Business scope (summary) + +- Planning grain: Entity x Department x PnL_Account x Version x Month, with a Line dimension for multiple forecast lines per combination. +- Data foundation: PULL_ metrics from Library (e.g. PULL_CR_PnL Data Actual, PULL_WF_Headcount Plan Data, PULL_Rev_Revenue Plan Data). No in-app transaction lists for OPEX; actuals and plans come from upstream apps. +- Configuration: Version windows (Last Actuals Month, Window Start/End Month); which months are Actual vs Plan (Push_DH_View_Load Actual / Plan); OPEX account scope; default forecasting methods per account; validation and line-prefill options. +- Forecasting engine: One method per line (Prior Year, Last X months avg, % of Revenue, $ per Headcount, Manual). Central CALC_Forecast = SWITCH on method ID -> method-specific metric; apply YoY/blank modifier; exclude actual months. User overrides on top of calculations (especially for Manual method). +- Output: CALC_Forecast per line -> OUT_FC (line removed for sharing) -> ACT + FC, REP_FC + Last Actual Month, Push_OP_OPEX Plan Data for reporting and cross-app sharing. +- UX: Set-up board (config, PULL connection hints); Manager / Controller input boards (method selection, parameters, manual overrides); OPEX Report (actual vs forecast, variance, version comparison). + +--- + +## 2. Core concepts (generalized) + +| Concept | Description | +|--------|-------------| +| Planning dimensionality in structure | Entity, Department, PnL_Account, Version, Month (and Line) are in the structure of most OPEX metrics. There is no "Workforce card" style mapping; reporting slices (e.g. by PnL category) use the same dimensions or their parents (BY: PnL_Account.PnL_Account Category). | +| PULL from Library | Data foundation = PULL_ metrics that reference Push_ metrics from other applications. Set-up board provides formula hints to connect PULL_* to the right Push_* (with REMOVE/FILTER/ADD as needed). | +| Actual vs Plan windows | Version properties (Window Start Month, Last Actuals Month, Window End Month) drive Boolean metrics (e.g. Push_DH_View_Load Actual, Push_DH_View_Load Plan). All forecast logic excludes actual months and filters to plan months where relevant. | +| Method per line | A Forecasting Method dimension (e.g. PY Values, Last X months Avg, % of Revenue, $ per Headcount, Manual Input) with an ID. User selects method per line via INP_Forecasting Method. Central engine SWITCHes on method ID to pick the correct CALC_* metric. | +| Parameters | INP_Parameter (and config SET_* metrics) supply method-specific values (X months, %, $/headcount). One parameter metric, populated by IF/SWITCH based on selected method. | +| Overrides on top | Calculated forecast is the base; users override where needed (especially when method = Manual Input). Override structure is part of the same engine (e.g. CALC_Manual_Input provides writable cells). | +| Naming conventions | INP_ (input/driver), CALC_ (intermediate calculation), OUT_ (primary output), PULL_ / Push_ (cross-app), FIL_ (Boolean filters), REP_ (reporting-only), SET_ (settings/config). | + +--- + +## 3. Pipeline (end-to-end) + +``` +External apps (Push_*) + -> PULL_* (PnL actuals, headcount plan, revenue plan) + window flags (Push_DH_View_Load Actual/Plan) + -> Configuration (SET_*: scope, default methods, validation options) + -> Line creation (Newline -> FIL_NewlineAdded -> Line dimension) + -> Inputs per line: INP_Forecasting Method, INP_Parameter, INP_YoY%, INP_Validate, etc. + -> Method-specific CALC_* (CALC_PYValues, CALC_LastXMonthsAVG, CALC_%ofRevenue, CALC_$perHeadcount, CALC_Manual_Input) + -> CALC_Forecast = SWITCH(method ID, ...) * YoY/blank modifier [EXCLUDE: Actual months] + -> OUT_FC -> REMOVE Line for sharing + -> ACT + FC, REP_FC + Last Actual Month, Push_OP_OPEX Plan Data + -> Boards: Set-up, Manager/Controller Input, OPEX Report +``` + +See OPEX Planning - Forecasting Methods & Engine for method selection, the 5 methods, YoY/blank handling, row lifecycle, and Actual vs Plan window usage. + +--- + +## 4. Folder structure (generic) + +| Folder / area | Role | Typical content | +|---------------|------|------------------| +| Business Dimensions | Org axes | Department, Entity, Segment, placeholders. | +| Chart of Accounts | P&L (and optionally BS) | PnL_Account, PnL_Account Category, PnL_EBITA, Operator for sign. | +| Calendar | Time | Year, Quarter, Month, Month of Year, Quarter of Year. | +| Actual + Plan View Management | Version & windows | Version (Last Actuals Month, Window Start/End), Data Type; metrics for Push_DH_View_Load Actual / Plan. | +| FX Currency Conversion | Multi-currency | Currency, FX Rate Types, Reporting Currency. | +| Output | Primary outputs | OUT_FC, ACT + FC, REP_FC + Last Actual Month, Push_OP_OPEX Plan Data. | +| Admin / Dimensions | Technical dims | Line (for OPEX lines), Forecasting Method, Newline (for Add New Combination). | +| Admin / Library | Cross-app | Push_OP_OPEX Plan Data, formula hints for connecting PULL_* to upstream Push_*. | +| Security | Access | Role, ARM_*, permissions. | +| Boards | UX | Set-up, Manager OPEX Input, Controller OPEX Input, OPEX Report. | + +--- + +## 5. Data foundation + +- No in-app OPEX transaction lists. Data comes from PULL_ metrics that reference Push_ metrics in upstream apps (Hub, Consolidation, etc.). +- Typical PULL_*: + - PULL_CR_PnL Data Actual (Entity x Department x PnL_Account x Version x Month) - P&L actuals. + - PULL_WF_Headcount Plan Data (Department x Entity x Version x Month) - for $ per Headcount method. + - PULL_Rev_Revenue Plan Data (Department x Entity x Version x Month) - for % of Revenue method. +- Window flags: + - Push_DH_View_Load Actual: Month in [Window Start Month, Last Actuals Month]. + - Push_DH_View_Load Plan: Month after Last Actuals Month and within Window End Month. + Forecast engine excludes actual months; method metrics filter to plan months where needed. + +--- + +## 6. Dimension framework + +- Planning axes: Department, Entity, PnL_Account, Version, Month. Line for multiple forecast lines per (Entity, Department, PnL_Account, Version). +- Forecasting: Forecasting Method (ID, Need Input in Parameter?, Parameter Unit). Newline (Entity, Department, PnL_Account) for "Add New Combination" -> mapped into Line via FIL_NewlineAdded. +- Version: Last Actuals Month, Window Start Month, Window End Month, Close Date. Drives actual vs plan and version-specific windows. +- Time: Year, Quarter, Month, Month of Year, Quarter of Year (with date properties and hierarchy). +- Financial: PnL_Account Category, PnL_EBITA; Operator on category/account for sign. Currency, FX, Reporting Currency. +- Security: User, Role. + +--- + +## 7. Metric layers (summary) + +- Input & config: INP_Forecasting Method, INP_Parameter, INP_YoY%, INP_YoY_Month, INP_Description, INP_Validate; SET_* for defaults and scope; CHK_HasActuals?, SET_Fill_combo_withactuals?, SET_FCMethod_by_PnlAccount. +- Row control: FIL_NewlineAdded (Newline -> Line), FIL_Rowfilter (visibility: new lines, actual-only, validation gating); CDFM_ID = INP_Forecasting Method.ID for SWITCH. +- Method-specific CALC_*: CALC_Actuals, CALC_PYValues, CALC_LastXMonthsAVG, CALC_%ofRevenue, CALC_$perHeadcount, CALC_Manual_Input (dims: Department, PnL_Account, Version, Entity, Month, Line). +- Engine: CALC_Forecast = SWITCH(INP_Forecasting Method.ID, 1->PY, 2->LastXAvg, 3->%Rev, 4->$HC, 5->Manual) * CALC_YoY_2_fixblanks [EXCLUDE: Push_DH_View_Load Actual]. +- Output: OUT_FC; ACT + FC = OUT_FC[REMOVE: Line] + PULL_CR_PnL Data Actual; REP_FC + Last Actual Month (chart continuity); Push_OP_OPEX Plan Data = OUT_FC[REMOVE: Line]. + +Details of each method and of YoY/blank, row lifecycle, and window usage: OPEX Planning - Forecasting Methods & Engine. + +--- + +## 8. How to apply this blueprint elsewhere + +1. Define planning grain (Entity x Department x PnL_Account x Version x Month [+ Line]) and keep all these dimensions in structure for core metrics (no "mapped dimensions" pattern like WFP). +2. Set up PULL_ metrics and window flags; connect to upstream Push_* via Set-up board hints. +3. Implement the forecasting engine: Forecasting Method dimension, INP_Forecasting Method, INP_Parameter, one CALC_* per method, CALC_Forecast = SWITCH * modifier [EXCLUDE: Actual]. See OPEX Planning - Forecasting Methods & Engine. +4. Allow overrides (e.g. CALC_Manual_Input for manual method; same structure as forecast for edits). +5. Output: OUT_FC, ACT + FC, REP for reporting, Push_ for sharing; all respect Actual/Plan windows. +6. Adopt naming: INP_, CALC_, OUT_, PULL_, Push_, FIL_, REP_, SET_. + +--- + +## 9. Illustration: [FIN]04 OPEX Planning Template + +In the Pigment [FIN]04 OPEX Planning template: Business Dimensions (Department, Entity); Chart of Accounts (PnL_Account, PnL_Account Category, PnL_EBITA); Calendar; 01. Actual + Plan View Management; 02. FX; 3. Output (OUT_FC, etc.); 0. Admin (0.0 Dimensions: Line, Forecasting Method, Newline; 0.2 Library: Push_OP_OPEX Plan Data, PULL connection hints). Core tables: [TBL] Opex Items Control (method, parameters, validation, FIL_Rowfilter), [TBL] Forecast (manual overrides), [TBL] Act vs FC (reporting). Boards: Set-up, Manager OPEX Input, Controller OPEX Input, OPEX Report. + +Related skill: For forecasting method formulas, method parameters, YoY/blank modifiers, and the step-by-step procedure for adding new methods, read [OPEX Planning - Forecasting Methods & Engine](./opex_forecasting_planning_methods_engine.md). \ No newline at end of file diff --git a/plugins/pigment/skills/solving-specific-use-cases/workforce_planning_architecture_patterns.md b/plugins/pigment/skills/solving-specific-use-cases/workforce_planning_architecture_patterns.md new file mode 100644 index 00000000..25526098 --- /dev/null +++ b/plugins/pigment/skills/solving-specific-use-cases/workforce_planning_architecture_patterns.md @@ -0,0 +1,231 @@ +# Workforce Planning - Application Architecture & Patterns + +## Purpose + +This document is a reusable pattern for an employee-based workforce planning application. It describes the layered metric architecture, dimension roles, data flows (Existing Employees + To-Be-Hired), override and validation patterns, and naming conventions so another modeler can understand the design and replicate it in new applications. + +When to use this pattern + +- You are building or extending a workforce planning app that combines: + - Existing employees (HRIS actuals, spread logic, cards, events, compensation) + - To-Be-Hired (TBH) (request list, FTE/salary/dates, validation, stats, compensation) + - Consolidated workforce (headcount, FTE, transfers, comp at Workforce x Dept x Entity x Version x Month) + - Overrides (Changelog -> override metrics -> staging) + - Governance (validation status, version close, access rights, merit cycles) + - Financial alignment (PnL/BS mapping, FX, merit and tax assumptions) + +When not to use + +- Purely headcount-only models with no compensation or TBH may not need the full layering (e.g. Comp layer, TBH flow). +- Applications that do not use version/scenario dimensions or validation workflows will not need the full scenario and governance patterns described here. + +--- + +## 1. Business scope (summary) + +The application covers end-to-end workforce planning: + +- Master data: Entities, departments, job positions, chart of accounts (PnL, BS). +- Existing employees: HRIS import -> staging (with spread and history/plan logic) -> overrides (Changelog) -> cards -> events (hires, terms, transfers) -> stats at Employee then Workforce. +- To-Be-Hired: TBH request list with FTE, salary, hire/term date, job position -> validation -> TBH stats (headcount, FTE, terminations) -> mapping to Workforce. +- Total workforce: Consolidated Workforce Cards (entity, department, currency, etc.) and stats at Version x Month x Workforce; reporting by Dept x Entity (and other axes) via mapped dimensions (BY: -> `WF_Card_*`), not by adding those dimensions to every metric. See Workforce Planning - Workforce Cards & Mapped Dimensions. +- Compensation: Salary, bonus, benefits, taxes for EE and TBH; merit and tax assumptions; mapping to PnL/BS; FX conversion for multi-currency. +- Scenario & security: Version dimension (windows, close date), Data Type (Actual/Budget/Forecast), validation dimensions, access rights (ARM/ARC) for edit and approval. + +--- + +## 2. Core concepts (generalized) + +| Concept | Description | +| --------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Layered metric architecture | Clear separation: Data (load/staging) -> Card (canonical attributes per entity/month) -> Stats (events: hires, terms, transfers) -> Comp (salary, bonus, benefits, taxes) -> Push/KPI (export, counts) and Security (ARM/ARC). Each layer consumes the layer below; no skipping. | +| Two populations | Existing Employees (EE): from HRIS (and spread logic) + Changelog overrides. To-Be-Hired (TBH): from a request/list with inputs (FTE, dates, salary). Both feed Workforce-level cards and stats. | +| Data -> Card -> Stats | Data resolves raw sources (HRIS, TBH list) to a consistent grain (e.g. Version x Entity x Month). Card fixes attributes at a single "planning load" reference (e.g. one snapshot month per version/month). Stats derive events by comparing cards across months (e.g. entity or department change = transfer). | +| Override-first staging | For EE, staging metrics use IFDEFINED(Override, Override, HRIS_logic). Overrides come from Changelog projection (see skill "Changelog to Override Metrics"). Cards and stats only see the post-override view. | +| Validation dimension | A dimension (e.g. Validation Status: Draft/Submitted/Approved/Rejected) tags rows or versions. Plan metrics and KPIs filter to Approved only; access rights control who can change status. | +| Version & scenario dimensions | Version (or equivalent) holds window start/end, close date, last actuals month. Data Type (Actual/Budget/Forecast) can drive FX or display. Metrics and access rights respect version close and windows. | +| Workforce Cards & mapped dimensions | The 4.0 layer unifies EE + TBH at Version x Month x Workforce. `WF_Card_*` metrics hold each Workforce item's attributes (Entity, Department, Currency). Core stats/comp stay at Workforce only; breakdowns (Dept x Entity, PnL push) use BY: -> `WF_Card_*` so Entity/Department are mapped, not structural. See Workforce Planning - Workforce Cards & Mapped Dimensions. | +| Naming conventions | Prefix by domain: `EE_` (existing employee), `TBH_` (to-be-hired), `WF_` (workforce total), `Asm_` (assumptions), `Push_` (export), `KPI_`, `ARM_`/`ARC_` (access rights). Suffix by layer: `_Data`, `_Card`, `_Stats`, `_Comp`, `_Input`, `_Calc`, `_Req_`, `_Rep_`, `_View`. | + +--- + +## 3. Pipeline (end-to-end) + +Existing Employees + +```text +HRIS load list -> Data layer (EE_Data_*: resolve by snapshot, history/plan toggle, override-first) + -> Spread logic (effective snapshot month, FILLFORWARD dates, version window) + -> Override metrics (EEO_* from Changelog) + -> Card layer (EE_Card_*: canonical attributes at Version x Employee x Month) + -> Stats (EE_Stats_*: new hires, terminations, transfer in/out via card comparison) + -> Workforce aggregation (EE_Stats_* [BY: Employee.Workforce] -> WF_Stats_*) +``` + +To-Be-Hired + +```pigment +TBH request list + TBH_Req_Input_* -> Validation status + -> TBH_Stats_* (headcount, FTE, terminations; PRORATA by hire/term; filter Approved, plan months; exclude hired) + -> Workforce mapping (TBH_Stats_* [BY: TBH Requests.Workforce] -> WF_Card_*, WF_Stats_*) +``` + +Consolidation & output (4.0 Workforce layer) + +```pigment +WF_Card_* (unified attributes at Version x Month x Workforce: EE_Card_* or TBH-side cards) + -> WF_Stats_* at Workforce only (EE_Stats_* [BY: Employee.Workforce] + TBH_Stats_* [BY: TBH Requests.Workforce]) + -> Breakdowns via mapped dimensions: WF_Stats_Headcount [BY: -> WF_Card_Department, WF_Card_Entity], etc. + -> WF_Comp_* -> WF_Comp Plan Data -> Push_* (map to Entity/Dept/PnL via cards) + -> KPI_* (export, filled TBH count, changelog count) +``` + +See Workforce Planning - Workforce Cards & Mapped Dimensions for why core metrics stay at Workforce and how mapping works. + +Governance throughout: Validation Status = Approved; version close excluded; ARM/ARC for edit and approval; merit/tax assumptions and FX in dedicated layers. + +--- + +## 4. Folder structure (generic) + +A typical ordering that supports the pipeline: + +| Folder / area | Role | Typical content | +| --------------------------------- | ---------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Admin / Dimensions | Structural and scenario dimensions | Entity, Department, Workforce, Employee, Job Position, Grade, Changelog, TBH Requests, Version, Data Type, Validation Status, Calendar (Month, Quarter), PnL/BS accounts, Currency. | +| Library | Cross-app mapping | `Push_*` mapping metrics for job position, FTE, etc. | +| Actual + Plan View Management | Version and view control | Version, Data Type, plan window metrics, view load flags. | +| FX Currency Conversion | Multi-currency | Currency, FX rate types, FX rates, triangulation, entity currencies. | +| Data | Load and staging | HRIS load list; EE spread logic; `EE_Data_*` (override-first); `EEO_*` (Changelog overrides). | +| Existing Employee Planning | EE cards and events | `EE_Card_*`; `EE_Stats_*` (new hires, terminations, transfers). | +| TBH Planning | TBH inputs and stats | `TBH_Req_Input_*`, `TBH_Req_Calc_*` (validation, TBH ID); `TBH_Stats_*`; `TBH_Comp_*`. | +| Total Workforce | Consolidation | `WF_Card_*`; `WF_Stats_*`; `WF_Comp_*`; WF_Comp Plan Data. | +| KPI | Counts and governance | KPI_TBH_Rep_Filled Count, KPI_ChangelogCount_emp, completeness flags. | +| Security | Access rights | `ARM_*`, `ARC_*`, `MAP_*` (e.g. merit cycle, TBH approval). | +| Boards | UX | Homepage, Employee Directory, TBH Requests, Merit Management, Settings, Security. | + +--- + +## 5. Metric layers (patterns) + +### 5.1 Data layer (`EE_Data_*`, resolution to Version x Employee x Month) + +Role: Resolve HRIS (and spread) to planning grain; apply history vs plan; apply override-first. + +Pattern (schematic): + +```text +IF(Populate_History?, + Source_list.'Attribute' + [BY LASTNONBLANK: Entity_Mapping, Snapshot_Date] + [ADD: Version][FILTER: Is_Actual] + [BY CONSTANT: Spd_Effective_Snapshot_History], + Source_list.'Attribute' + [BY LASTNONBLANK: Entity_Mapping, Snapshot_Date] + [ADD: Version][FILTER: Is_Actual] + [BY CONSTANT: Spd_Effective_Snapshot_Plan] +) +``` + +Override-first is applied when the attribute can be overridden: IFDEFINED(`EEO_*`, `EEO_*`, ). See "Snapshot Spread Logic" and "Changelog to Override Metrics" skills. + +### 5.2 Card layer (`EE_Card_*`, `WF_Card_*`) + +Role: Canonical attributes at Version x Entity x Month (EE) or Version x Month x Workforce (WF). EE cards fix to one "planning load" snapshot; WF cards unify EE + TBH so every Workforce item has Entity, Department, Currency, etc. without adding those dimensions to core metrics. + +Pattern (EE): Card_Attribute = Data_Attribute [BY CONSTANT: Spd_Effective_Snapshot_Month]. + +Pattern (WF): WF_Card_Attribute = IFDEFINED(EE_Card_Attribute[BY CONSTANT: Workforce.Employee], same, WF_Card_Attribute_TBH). TBH-side cards use TBH_Stats_FTE or list properties mapped via Workforce.TBH. Breakdowns (e.g. headcount by Dept x Entity) use core metric [BY: -> WF_Card_Department, WF_Card_Entity]. Full pattern: Workforce Planning - Workforce Cards & Mapped Dimensions. + +### 5.3 Stats layer (`EE_Stats_*`, `TBH_Stats_*`, `WF_Stats_*`) + +Role: Events (new hire, termination, transfer in/out) and levels (headcount, FTE). + +EE events (schematic): Compare cards across months. Example transfer out: + +```text +IF( + (EE_Card_Entity <> EE_Card_Entity[SELECT: Month + 1]) OR + (EE_Card_Department <> EE_Card_Department[SELECT: Month + 1]), + -1 +) +``` + +New hire / termination: presence in current month but not previous (or reverse). Use +1/-1 flags then aggregate at Workforce. + +TBH stats: `PRORATA(Month, STARTOFMONTH(Hire Date), STARTOFMONTH(Term Date + 1)) * ROUNDUP(FTE, 0) [Set_Plan Month View] [BY: Validation Status -> Approved]`. Exclude rows already linked to a hired employee. + +WF stats: `EE_Stats_... [BY: Employee.Workforce] [FILTER: Validation Status = Approved] + TBH_Stats_... [BY: TBH Requests.Workforce]`. Keep `WF_Stats_...` at Workforce only. Breakdowns: `WF_Stats_Headcount_Dep_Entity = WF_Stats_Headcount [BY: -> WF_Card_Department, WF_Card_Entity]` (mapped dimensions). See Workforce Planning - Workforce Cards & Mapped Dimensions. + +### 5.4 Comp layer (`WF_Comp_*`, `TBH_Comp_*`) + +Role: Salary, bonus, benefits, taxes by Workforce (or TBH) x Version x Month; apply merit and tax assumptions; map to PnL/BS. + +Pattern (PnL export): `(WF_Comp_01_Salary[BY: PnL_Account."6001"] + WF_Comp_01_Bonus[BY: PnL_Account."6002"] + ...) [SELECT: Validation Status."Approved"] [Push_DH_View_Load Plan]`. + +### 5.5 KPI & Security layers + +KPI: Counts (e.g. filled TBH per Dept/Entity; pending changelog per Employee). IFDEFINED(...) or IFBLANK(Changelog[BY: ...][FILTER: ...][REMOVE COUNT: Changelog], 0). + +Security: `ARM_*` (read/write by User, optionally Department/Entity). `ARC_*` (input flags for approval). `MAP_*` (e.g. merit cycle: editable only when cycle month matches and version not closed). ACCESSRIGHTS(TRUE, TRUE) / ACCESSRIGHTS(TRUE, BLANK) with IF(Admin, ..., IF(ARC_Input, ..., BLANK)). + +--- + +## 6. Dimension framework + +- Structural: Entity, Department, Workforce, Employee, Job Position, Grade, State, Country. Used for org and reporting axes. +- Scenario: Version (window start/end, close date, last actuals month), Data Type (Actual/Budget/Forecast). Version drives which months are calculated and whether a version is frozen. +- Time: Month, Quarter, Month of Year, Quarter of Year. Properties: Start Date, End Date, mapping to Year. +- Financial: PnL_Account, PnL_Account Category, BS_Account, BS_Account Class/Category. Operator for sign. Currency, FX Rate Types, Reporting Currency. +- Validation / workflow: Validation Status (Draft/Submitted/Approved/Rejected). Used to filter plan metrics and KPIs and to drive access (who can approve). +- Security: User (Name, Email, ID). Access rights metrics by User (and optionally Department, Entity). + +Metrics rarely use Pigment native Scenarios for business logic; Version and Data Type dimensions model scenarios; application-level Scenarios can be used for sandboxing. + +--- + +## 7. Key modeling patterns + +- Driver-based events: Derive hires, terms, transfers from card comparison across months (SELECT: Month + 1), not from manual event flags. +- PRORATA for FTE: Allocate FTE across months from hire to term using PRORATA(Month, STARTOFMONTH(Hire Date), STARTOFMONTH(Term Date + 1)); use TIMEDIM for date-period alignment. +- Validation filtering: All plan-facing metrics and exports filter to Validation Status = Approved (and exclude after version close where relevant). +- Override-first: Staging = IFDEFINED(Override, Override, Source). Cards and stats never implement override logic; they consume staging. +- Access rights by cycle/version: Merit (or similar) inputs editable only when (e.g.) Month of Year matches merit cycle and Version has no Close Date. Use `MAP_*` metrics with EXCLUDE/ADD to scope ACCESSRIGHTS. +- Workforce Cards & mapped dimensions: Keep core stats/comp at Workforce (plus Version, Month, Validation). Attach Entity, Department, Currency via `WF_Card_*`. Report by Dept x Entity (or PnL, etc.) with BY: -> `WF_Card_*` so those axes are mapped, not structural. See Workforce Planning - Workforce Cards & Mapped Dimensions. +- Naming: Prefix by domain (`EE_`, `TBH_`, `WF_`, `Asm_`, `Push_`, `KPI_`, `ARM_`/`ARC_`); suffix by layer (`_Data`, `_Card`, `_Stats`, `_Comp`, `_Input`, `_Calc`, `_Req_`, `_Rep_`, `_View`). Folder order: Admin/Dimensions -> Data -> EE Planning -> TBH Planning -> Total Workforce -> KPI -> Security -> Boards. + +--- + +## 8. How to apply this pattern elsewhere + +1. Replicate the layered folder structure: Data -> Cards -> Stats -> Comp -> Push/KPI -> Security; keep Admin/Dimensions and Calendar/FX at the top. +2. Define dimension roles clearly: Structural (org, workforce), scenario (Version, Data Type), time (Month, Quarter), financial (PnL/BS, Currency), validation, User. +3. Model events from cards: Use card comparison (SELECT: Month + 1) for transfers and presence for hires/terms; avoid ad-hoc event flags. +4. Isolate assumptions: Keep user inputs in `Asm_*` or `_Input_*` metrics and dedicated lists; reference them in Comp and driver logic. +5. Use validation and access rights: Validation Status dimension; filter to Approved in plan metrics; ARM/ARC for edit and approval; version close and cycle-based access (`MAP_*`). +6. Implement the 4.0 Workforce layer: Unify EE + TBH in a Workforce dimension; build `WF_Card_*` for attributes; keep `WF_Stats_*` and `WF_Comp_*` at Workforce only; use BY: -> `WF_Card_*` for breakdowns (mapped dimensions). See Workforce Planning - Workforce Cards & Mapped Dimensions. +7. Adopt the naming conventions: Prefix by domain, suffix by layer; consistent folder ordering. + +--- + +## 9. Pitfalls and reminders + +- Do not skip layers. Every layer builds on the one below (Data -> Card -> Stats -> Comp -> Push/KPI). Jumping from Data directly to Stats bypasses overrides and card canonicalization. +- Override-first belongs in staging only. Cards and stats consume staging; they never reference Changelog or override metrics directly. +- Keep WF_Stats and WF_Comp at Workforce only. Use `BY: -> WF_Card_*` for breakdowns (mapped dimensions); do not add Entity or Department as structural dimensions to core metrics. +- Naming discipline. Consistent prefixes (`EE_`, `TBH_`, `WF_`, `Asm_`, `Push_`, `KPI_`, `ARM_`/`ARC_`) and suffixes (`_Data`, `_Card`, `_Stats`, `_Comp`) prevent confusion as the model grows. +- Version close. Ensure all relevant metrics and access rights respect version close; a single missing filter can allow edits on a frozen version. + +--- + +## 10. Illustration: Workforce Planning Template + +In the Pigment [FIN]02 Workforce Planning - Employee based application, the pattern above is implemented as follows (names are template-specific). + +Folders: 0. Admin (Dimensions, Library); 01. Actual + Plan View Management; 02. FX Currency Conversion; Calendar; 10. Business Dimensions; 11. Chart of Accounts; 1. Data (EE*Load_HRIS, spread 1.3, overrides 1.4); 2. Existing Employee Planning (`EE_Data*_`, `EE*Card*_`, `EE*Stats*_`); 3. TBH Planning (`TBH*Req*_`, `TBH*Stats*_`, `TBH*Comp*_`); 4. Total Workforce (`WF*Card*_`, `WF*Stats*_`, `WF*Comp*\_`, WF_Comp Plan Data); 5. KPI; Security (`ARM\_\_`, `ARC\_\*`, MAP_Cycle2); Boards. + +Key blocks: `EE_Load_HRIS` (HRIS load list); Changelog (change requests -> `EEO_...`); TBH Requests (TBH list + `TBH_Req_Input_...`, `TBH_Req_Calc_...`); `EE_Spd_...` (spread logic); `EE_Data_...`, `EE_Card_...`, `EE_Stats_...`; `EEO_...`; `TBH_Stats_...`, `TBH_Comp_...`; `WF_Card_...`, `WF_Stats_...`, `WF_Comp_...`; `KPI_ChangelogCount_emp`, `KPI_TBH_Rep_Filled Count`; ARM_Department_Read/Write, ARM_Entity x Department_Read/Write, ARM_Valid_TBH Req Approval_Write, MAP_Cycle2 (merit). + +Flows: HRIS -> `EE_Data_...` (with Populate*History?, EE_Spd_99 / EE_Spd_03) -> `EEO_` override-first -> `EE_Card_...` -> `EE_Stats_...` -> `WF_Stats_...[BY: Employee.Workforce]`. TBH Requests -> TBH_Req_Calc_03_Validation ->`TBH_Stats_...` (PRORATA, Set_Plan Month View, Approved) -> WF_Card_Entity_TBH,`WF_Stats_...`. Changelog -> `EEO_...`->`EE_Data_...` (IFDEFINED). Merit/tax:`Asm_Input_...`, `EE_MI_...`, `TBH_MI_...` -> `WF_Comp_...`, `TBH_Comp_...`; MAP_Cycle2 controls merit edit access. The 4.0 Total Workforce layer (`WF_Card_...`, stats at Workforce only, breakdowns via BY: -> `WF_Card_...`) is detailed in Workforce Planning - Workforce Cards & Mapped Dimensions. + +Related skills: [Workforce Planning - Snapshot Spread Logic](./workforce_planning_snapshot_spread.md); [Workforce Planning - Changelog to Override Metrics](./workforce_planning_changelog_overrides.md); [Workforce Planning - Workforce Cards & Mapped Dimensions](./workforce_planning_cards_mapped_dimensions.md). diff --git a/plugins/pigment/skills/solving-specific-use-cases/workforce_planning_cards_mapped_dimensions.md b/plugins/pigment/skills/solving-specific-use-cases/workforce_planning_cards_mapped_dimensions.md new file mode 100644 index 00000000..1ec44a33 --- /dev/null +++ b/plugins/pigment/skills/solving-specific-use-cases/workforce_planning_cards_mapped_dimensions.md @@ -0,0 +1,135 @@ +# Workforce Planning - Workforce Cards & Mapped Dimensions + +## Purpose + +This use case describes the 4.0 layer of a workforce planning application: Workforce Cards that provide a unified view at Version x Month x Workforce, and the mapped-dimensions pattern that lets you report by Entity, Department, PnL, etc. without adding those dimensions to every core metric. The core model stays on Employee x Month x Version (existing employees) and TBH Requests x Month x Version (to-be-hired); the Workforce layer lifts both into a single dimension and attaches attributes via card metrics; reporting then uses BY: -> Card_Attribute to project metrics along those attributes when needed. + +When to use this pattern + +- You have two populations (e.g. Existing Employees and To-Be-Hired) that must be reported together as one workforce (headcount, FTE, comp, events). +- You want flexible reporting by Entity, Department, or other axes without hard-coding those dimensions into every stats/comp metric. +- You want to keep core metrics on a small set of driver dimensions (Employee, TBH Requests, then Workforce) and only "map" to richer dimensions where needed. + +When not to use + +- Single-population models (e.g. only employees, no TBH) may not need a unified Workforce dimension; a single "card" layer per population can suffice. +- If every report always needs the same fixed axes (e.g. always Dept x Entity), you might embed those in the metric structure-at the cost of sparsity and complexity; the mapped pattern is still usually preferable. + +--- + +## 1. Problem in plain terms + +- Core data lives at: Version x Employee x Month (EE) and Version x TBH Requests x Month (TBH). Adding Entity, Department, PnL, etc. as structural dimensions to every metric would create huge sparsity, complex formulas, and heavy access-rights logic. +- Need: A single "total workforce" view (headcount, FTE, comp, events) that can be sliced by Entity, Department, or other axes on demand, without bloating the core. +- Design: Introduce a Workforce dimension that unifies Employee and TBH (each Workforce item is either employee-backed or TBH-backed). Build Workforce Card metrics (WF_Card_Entity, WF_Card_Department, WF_Card_Currency, ...) that hold the attribute of each Workforce item for that Version x Month. Keep stats and comp at Workforce only. When you need a breakdown, map the metric using BY: -> WF_Card_Department, WF_Card_Entity (and similar). No extra structural dimensions on the core metric. + +--- + +## 2. Core concepts (generalized) + +| Concept | Description | +| ----------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Workforce dimension | A dimension that unifies the two populations. Each item is either employee-backed (linked via a property to Employee) or TBH-backed (linked to TBH Requests). Mapping: Employee -> Workforce via e.g. Workforce[BY FIRSTNONBLANK: Workforce.Employee]; TBH Requests -> Workforce via Workforce[BY FIRSTNONBLANK: Workforce.TBH]. | +| Workforce Cards | Metrics at Version x Month x Workforce that hold one attribute per Workforce item (Entity, Department, Currency, etc.). They are calculated: for employee-backed rows, pull from `EE_Card_*` [BY CONSTANT: Workforce.Employee]; for TBH-backed rows, use a TBH-side card (e.g. WF_Card_Entity_TBH from TBH_Stats_FTE or TBH Requests.Entity). So Workforce is "decorated" by attributes without adding Entity/Department as structural dimensions to the app. | +| Mapped dimensions | Reporting axes (Entity, Department, PnL_Account, etc.) that are not in the structure of core metrics. You get them by mapping: Core_metric [BY: -> WF_Card_Department, WF_Card_Entity]. This redistributes the metric along the card values (each Workforce item contributes to the Dept x Entity implied by its cards). | +| Core stays lean | Core stats and comp metrics are defined only at Workforce (and Version, Month, Validation Status where relevant). Entity, Department, etc. appear only in derived metrics that use BY: -> cards. So the "engine" has minimal dimensions; richness comes at the mapping step. | + +--- + +## 3. What Workforce Cards do (4.0 layer) + +Role: For any (Version, Month, Workforce item), answer: "What is this item's Entity / Department / Currency / ...?" so downstream metrics and reports never need to know whether the row is EE or TBH. + +Pattern for a unified attribute (e.g. Entity): + +- WF_Card_Entity = IFDEFINED(EE_Card_Entity[BY CONSTANT: Workforce.Employee], EE_Card_Entity[BY CONSTANT: Workforce.Employee], WF_Card_Entity_TBH) + -> If this Workforce row is employee-backed, use EE_Card_Entity; else use the TBH-side entity card. + +- WF_Card_Entity_TBH = IFDEFINED(TBH_Stats_FTE[REMOVE: Validation Status][BY: TBH Requests.Workforce], TBH Requests.Entity[BY CONSTANT: Workforce.TBH]) + -> If there is FTE for this TBH row, use it to carry the mapping; else use the list property TBH Requests.Entity mapped to Workforce via Workforce.TBH. + +Pattern for Department: Same idea: IFDEFINED(EE_Card_Department[BY CONSTANT: Workforce.Employee], ..., IFDEFINED(WF_Stats_FTE[REMOVE: Validation Status], TBH Requests.Department[BY CONSTANT: Workforce.TBH])). + +So for any Workforce item (employee or TBH), you have a single set of card metrics (Entity, Department, Currency, ...) that give its attributes for that Version x Month. All "decoration" is computed, not structural. + +--- + +## 4. How this keeps the core on Employee x Month x Version (and TBH x Month x Version) + +- Heavy logic (events, FTE, compensation) stays where it's natural: + - EE: Version x Employee x Month + - TBH: Version x TBH Requests x Month + +- Lift to Workforce with simple aggregation: + - WF_Stats_Headcount = EE_Stats_Headcount[BY: Employee.Workforce][BY: Validation Status."Approved"] + TBH_Stats_Headcount[BY: TBH Requests.Workforce] + - Same idea for FTE, new hires, terminations, transfers, comp. + +- Do not add Entity, Department, etc. as structural dimensions to `WF_Stats_*` or `WF_Comp_*`. Instead: + - Keep them at Workforce (plus Version, Month, Validation Status where needed). + - When you need a breakdown: Metric at Workforce then map via cards: e.g. WF_Stats_Headcount_Dep_Entity = WF_Stats_Headcount [BY: -> WF_Card_Department, WF_Card_Entity]. + +So the core engine has only Employee, TBH Requests, and Workforce as the main driver dimensions; all richer axes come from mapping. + +--- + +## 5. Mapped dimensions in practice + +Headcount by Dept & Entity + +- WF_Stats_Headcount_Dep_Entity = WF_Stats_Headcount [BY: -> WF_Card_Department, WF_Card_Entity] + Source: at Workforce. Target: Department x Entity (and remaining dims). Each Workforce item contributes to the (Department, Entity) given by its cards. + +Push to P&L / financial + +- Push_WF_Workforce Plan Data = WF_Comp Plan Data [BY: Workforce -> WF_Card_Entity, WF_Card_Department] + Compensation at Workforce is mapped to Entity x Department (and then typically to PnL_Account in a later step) via the same card pattern. + +Why not add structural dimensions everywhere + +- If you put Entity x Department (and more) into every stats/comp metric you get: large sparsity, bigger formulas, more complex filters and access rights, and harder refactoring when you add a new axis. With mapped dimensions, you only project when needed. + +--- + +## 6. Why unify Employee & TBH into one Workforce dimension + +- Single set of "total" metrics: One WF_Stats_Headcount (and FTE, new hires, terms, transfers, comp) that already combines EE + TBH. All further analysis (by Dept, Entity, cost, etc.) works on that single number. + +- Consistent events & KPIs: `WF_Stats_*` metrics map `EE_Stats_*` via Employee.Workforce and add `TBH_Stats_*` via TBH Requests.Workforce. From the reporting layer there is only "workforce events," not two parallel trees. + +- Unified UX: Boards can show one grid at Version x Month x Workforce or by mapped dimensions, without separate Employee vs TBH sections. + +- Security & governance: Security mapping metrics (e.g. `SEC_WF_Card_Entity` = WF*Card_Entity, SEC_WF_Card_Department = WF_Card_Department) let you express access rights once at Workforce level; they apply consistently to EE- and TBH-backed rows and to all `WF_Stats*_`, `WF*Comp*_`, and push metrics that use the cards. + +--- + +## 7. How to apply elsewhere + +1. Define a Workforce dimension that links to both populations (e.g. Workforce.Employee, Workforce.TBH). Ensure every Employee and every TBH request maps to exactly one Workforce item (BY FIRSTNONBLANK). +2. Build Workforce Card metrics at Version x Month x Workforce for every attribute you need for reporting (Entity, Department, Currency, ...). Use IFDEFINED(`EE_Card_*`[BY CONSTANT: Workforce.Employee], ..., TBH_side_card) so both populations are covered. +3. Keep core stats and comp at Workforce (plus Version, Month, Validation). Do not add Entity/Department/etc. to their structure. +4. Create "breakdown" metrics only where needed: Core_metric [BY: -> WF_Card_Department, WF_Card_Entity] (and similar for PnL, etc.). +5. Use the same cards for security (e.g. SEC_WF_Card_Entity, SEC_WF_Card_Department) so access rights are defined once and apply to all metrics that depend on Workforce. + +--- + +## 8. Pitfalls and reminders + +- Do not add Entity, Department, or other "rich" axes as structural dimensions to `WF_Stats_*` or `WF_Comp_*`. Use `BY: -> WF_Card_*` for breakdowns; structural dimensions bloat every metric and complicate access rights. +- Every Employee and TBH Request must map to exactly one Workforce item (BY FIRSTNONBLANK). Duplicate or missing mappings silently break totals. +- `WF_Card_*` metrics must cover all Workforce items (both EE- and TBH-backed). If the TBH-side card is blank for some rows, mapped breakdowns will lose those rows. +- Security cards (`SEC_WF_Card_*`) must mirror `WF_Card_*`. If they diverge, access rights will not match the actual data groupings. + +--- + +## 9. Illustration: Workforce Planning Template + +In the Pigment Workforce Planning Template, the 4.0 Total Workforce folder implements this pattern as follows (names are template-specific): + +- Workforce Cards: WF_Card_Entity, WF_Card_Department, WF_Card_Currency at Version x Month x Workforce. WF_Card_Entity = IFDEFINED(EE_Card_Entity[BY CONSTANT: Workforce.Employee], ..., WF_Card_Entity_TBH). WF_Card_Entity_TBH = IFDEFINED(TBH_Stats_FTE[REMOVE: Validation Status][BY: TBH Requests.Workforce], TBH Requests.Entity[BY CONSTANT: Workforce.TBH]). +- Core stats at Workforce only: WF_Stats_Headcount, WF_Stats_FTE, WF_Stats_New Hires, WF_Stats_Terminations, WF_Stats_Transfer In/Out. No Entity or Department on these metrics. +- Mapped breakdowns: WF_Stats_Headcount_Dep_Entity = WF_Stats_Headcount [BY: -> WF_Card_Department, WF_Card_Entity]. Push_WF_Workforce Plan Data = WF_Comp Plan Data [BY: Workforce -> WF_Card_Entity, WF_Card_Department]. +- Security: SEC_WF_Card_Entity, SEC_WF_Card_Department mirror the cards for access rights at Workforce level. + +When extending or replicating this design, use the generic concepts and pipeline above and map them to your own block and dimension names. + +Related skills: [Workforce Planning - Application Architecture & Patterns](./workforce_planning_architecture_patterns.md); [Workforce Planning - Changelog to Override Metrics](./workforce_planning_changelog_overrides.md); [Workforce Planning - Snapshot Spread Logic](./workforce_planning_snapshot_spread.md). diff --git a/plugins/pigment/skills/solving-specific-use-cases/workforce_planning_changelog_overrides.md b/plugins/pigment/skills/solving-specific-use-cases/workforce_planning_changelog_overrides.md new file mode 100644 index 00000000..20ff8868 --- /dev/null +++ b/plugins/pigment/skills/solving-specific-use-cases/workforce_planning_changelog_overrides.md @@ -0,0 +1,164 @@ +# Workforce Planning - Changelog to Override Metrics + +## Purpose + +This use case describes how to model change requests for master data (e.g. employee transfers, salary updates, term dates) as rows in a Changelog dimension, then project those rows into override metrics at planning grain (e.g. Version x Employee x Month). Staging metrics use an override-first pattern (IFDEFINED(Override, Override, Source)), so cards and stats consume a single, consistent view of data without caring whether it came from the changelog or from the primary source (e.g. HRIS). + +When to use this pattern + +- Users request discrete changes to master data (new department, entity, job position, salary, term date) with an effective period (e.g. "active as of month X"). +- You want to separate data collection and workflow (who requested, validation status, audit) from impact on planning (overrides applied in staging, then cards and stats). +- You need governance: only approved changes apply; changes after version close are excluded; inputs can be locked after "send for validation." + +When not to use + +- If all master data is edited directly on the source (e.g. HRIS) with no approval workflow, you may not need a Changelog dimension. +- If changes are bulk imports without per-row effective dates or workflow, a simpler override or replace logic may suffice. + +--- + +## 1. Problem in plain terms + +- Need: Users submit change requests (e.g. "Move Jane to Department X as of March") that must become effective in planning at a given time, often after validation. +- Challenge: The primary source (e.g. HRIS) is snapshot-based; change requests are events (one row per change). Planning metrics are on a grid (Version x Employee x Month). You must turn events into overrides on that grid. +- Design choice: Model each change as a row in a dimension (Changelog) with properties: who is affected, which version, what new value, when effective, workflow status. Then use calculated metrics to project those rows into override metrics at planning grain. Staging uses override-first: if an override exists, use it; else use primary source. Downstream (cards, stats) only see the result. + +### Discover blocks in this workspace + +Template names in this skill are illustrative. Before creating a Changelog dimension, override metrics, or governance formulas, resolve real objects in the customer app: + +- Use `tool:search` to find existing Worker, Employee, Scenario / Version, Department, and any primary-source blocks (e.g. HRIS or staging metrics) you must reference; use `kind` or `regexp` when names are similar. +- Confirm freeze / validation and effective date properties on Scenario and Changelog-property API names differ by app; use `tool:search` (and filters) until returned summaries show the property names you need. +- Prefer extending existing lists and metrics that match the pattern over introducing duplicate blocks with new names. + +--- + +## 2. Core concepts (generalized) + +| Concept | Description | +| -------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Changelog dimension | A dimension where each item = one change request. Properties hold: target entity (e.g. Employee), target version (Version), effective period (e.g. Active as of -> Month), new values (New Department, New Entity, New Salary, New Term Date, etc.), workflow (Send for validation, Validation Status), and audit (Created on, Created by). | +| Effective period | When the change applies. Often "active as of" a given month: the change affects that month and all future months in scope. Some attributes (e.g. term date) are a single date; projection logic may differ. | +| Override metrics | Metrics at planning grain (Version x Entity x Month) that hold the new value from the changelog where a change applies, and are blank elsewhere. Built by projecting Changelog rows: filter to approved, exclude after version close, ADD Month, EXCLUDE months before effective, then BY LASTNONBLANK per entity to get "latest applicable change" per (Version, Entity, Month). | +| Projection | Turning dimension rows (change events) into time-series overrides: extend each row over months from "active as of" onward, then pick the latest change per entity per month. | +| Override-first staging | Staging metrics that feed cards use: IFDEFINED(Override_metric, Override_metric, Primary_source_logic). So the primary source (e.g. HRIS) is only used when there is no override. | +| Governance | Filters and access rights: only approved rows contribute to overrides; rows created after version close are excluded (Changelog_Excludeversion); inputs locked after "send for validation"; only certain roles can set Validation Status. | + +--- + +## 3. Pipeline (generic) + +| Layer | Role | Typical content | +| ------------------------ | ------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| Changelog dimension | Change requests | Properties: target entity, version, effective month, new values (department, entity, job position, salary, term date, etc.), workflow flags, validation status, audit (created on, by). | +| Governance metrics | Filter & security | Which rows are valid (approved, not after version close); who can edit vs approve; lock after submit. | +| Override metrics | Projection to planning grain | One metric per overridable attribute. Formula: Changelog.'New X' [FILTER: Approved] [EXCLUDE: after version close] [ADD: Month] [EXCLUDE: Month < Active as of] [BY LASTNONBLANK: Entity]. Result at Version x Entity x Month. | +| Staging | Override-first | IFDEFINED(Override, Override, Primary_source). Primary source = e.g. HRIS or spread logic. | +| Cards | Canonical attributes | Built from staging (and spread helpers if needed). Already contain overrides. | +| Stats & aggregations | Transfers, headcount, FTE, etc. | Use only cards (and derived metrics). They automatically respect overrides. | + +--- + +## 4. Pattern: three building blocks + +### 4.1 Changelog row structure + +Each changelog item represents one change request. Typical properties: + +- Identity & target: Entity (e.g. Employee), Version. +- Effective timing: Active as of -> Month (and optionally a derived "effective date" from that month's start). +- New values: One property per overridable attribute (New Department, New Entity, New Job Position, New Salary, New Term Date, etc.). +- Workflow: Send for validation (Boolean), Validation Status (e.g. Pending / Approved / Rejected). +- Audit: Created on, Created by (and optionally "current state" at creation time, e.g. current department at Created on, for context). + +This gives every row enough information to know who is affected, which version, what the change is, when it becomes active, and whether it is valid to apply. + +### 4.2 Projecting changes to planning grain (override metrics) + +Goal: For each (Version, Entity, Month), get the latest approved change that is effective in that month (i.e. Active as of <= Month), if any. + +Generic pattern for "from month X onward" attributes (e.g. entity, department, job position, salary): + +```pigment +Changelog.'New [Attribute]' + [FILTER: Changelog.'Validation Status' = Validation_Status."Approved"] + [BY: -> Changelog.Version] + [EXCLUDE: Changelog_Excludeversion] // exclude rows created after version close + [ADD: Month] + [EXCLUDE: Month < Changelog.'Active as of'] // only from effective month onward + [BY LASTNONBLANK ON Changelog.'Created on': Changelog.Entity] // or ON Changelog.ID +``` + +- FILTER: Only approved rows. +- EXCLUDE Changelog_Excludeversion: Implemented as a metric (e.g. TRUE when Changelog.'Created on' > Version.'Close Date'); excludes rows that must not affect a closed version. +- ADD Month then EXCLUDE Month < Active as of: Each change is extended over all months from its effective month onward; before that, the change does not apply. +- BY LASTNONBLANK: For each (Version, Entity, Month), keep the latest change (by Created on or ID). That yields one override value per (Version, Entity, Month) where a change applies. + +Single-date attributes (e.g. term date): The override may represent a single date per employee per version, but is still exposed at (Version x Entity x Month) so downstream can use it per month (e.g. for prorata, headcount). Use LASTNONBLANK by Created on or ID to pick the latest approved term-date change per employee, then map to Month as needed (e.g. BY LASTNONBLANK on TIMEDIM(Created on, Month)) so the result is at planning grain. + +### 4.3 Override-first in staging + +Staging metrics that feed cards should not duplicate workflow or projection logic. They only choose between override and primary source: + +```pigment +IFDEFINED( + Override_metric, // e.g. EEO_Entity + Override_metric, + Primary_source_logic // e.g. HRIS or spread-layer attribute +) +``` + +Cards and stats then reference staging (and spread helpers if needed). They always see the post-override value; no need to reference the Changelog or override metrics directly. + +--- + +## 5. Governance (filters and access rights) + +- Which rows apply: + - Validation Status = Approved. + - Changelog_Excludeversion: exclude rows where Created on > Version.'Close Date' (or equivalent) so closed versions are not affected. + +- Who can do what: + - Lock after submit: When "Send for validation" is TRUE, set write access to BLANK so the row cannot be edited. + - Who can approve: A separate metric (e.g. Allow_validation_write) grants write on Validation Status only to Admins or users with an approval role; other users have BLANK. + +- Monitoring: A KPI per entity (e.g. count of changelog rows "pending" - sent for validation but not yet approved/rejected) helps users see what is awaiting action. + +Implement these once; override and staging formulas only reference the result of governance (e.g. FILTER Approved, EXCLUDE Changelog_Excludeversion), not the full workflow logic. + +--- + +## 6. How to apply this elsewhere + +1. Define the Changelog dimension with properties: target entity, version, effective period (e.g. Active as of -> Month), new values for each overridable attribute, workflow (submit, validation status), audit (created on, by). +2. Add governance metrics: Validation Status = Approved; exclude rows after version close; access rights to lock after submit and to control who can approve. +3. Build one override metric per overridable attribute using the projection pattern: Approved only, EXCLUDE after version close, ADD Month, EXCLUDE Month < Active as of, BY LASTNONBLANK per entity (and adjust for single-date attributes like term date). +4. Implement staging as override-first: IFDEFINED(Override, Override, Primary_source). Do not repeat projection or workflow logic in staging. +5. Drive cards and stats only from staging (and spread logic). They must not reference Changelog or overrides directly; they consume the post-override view. +6. Close the loop: KPIs for pending changes; access rights for edit vs approve; version close respected in Changelog_Excludeversion. + +--- + +## 7. Pitfalls and reminders + +- Do not duplicate governance in every formula. Filter to approved and exclude-after-close once in the override metrics; staging and downstream stay simple. +- Override-first in one place. Staging is the only layer that chooses override vs primary source; cards and stats stay agnostic. +- Single-date vs from-month-onward. Most attributes "apply from month X onward"; term date (and similar) may need a different projection (latest event per entity, then expose at Month grain for downstream). +- Version close. Exclude changelog rows created after version close so frozen versions are not modified by late entries. + +--- + +## 8. Illustration: Workforce Planning Template + +In the Pigment Workforce Planning Template, this pattern appears as follows (names are template-specific): + +- Changelog dimension: Items = change requests. Properties include Employee, Version, Active as of -> Month, New Department, New Entity, New Job Position, New Salary (FY), New Term Date, Send for validation, Validation Status, Created on, Created by, etc. "Current Department (AR)" / "Current Entity" are contextual metrics (card value at Created on) for display. +- Governance: ARM_LockInputs_Changelog (lock write when Send for validation); ARM_Changelog_Allow_validation_write (who can set Validation Status); Changelog_Excludeversion (TRUE when Created on > Version.'Close Date'); KPI_ChangelogCount_emp (pending count per employee). +- Override metrics (`EEO_*`): EEO_Entity, EEO_Department, EEO_JobPosition, EEO_Salary, EEO_Term_Date, EEO_TransferDate. Same projection pattern: approved only, EXCLUDE Changelog_Excludeversion, ADD Month, EXCLUDE Month < Active as of, BY LASTNONBLANK (on Created on or ID) per Employee. EEO_Term_Date uses a variant for single-date (LASTNONBLANK by ID and by TIMEDIM(Created on, Month)). +- Staging (`EE_Data_*`): e.g. EE_Data_Term Date = IFDEFINED(EEO_Term_Date, EEO_Term_Date, HRIS term date logic). Same for Entity, Department, Job Position, Salary. +- Cards (`EE_Card_*`): Built from `EE_Data_*` and spread helpers (e.g. BY CONSTANT: EE_Spd_99_HRIS Load Month). Already include overrides. +- Stats (`EE_Stats_*`, `WF_Stats_*`): Transfer and headcount logic compare `EE_Card_Entity`, `EE_Card_Department` across months; they automatically respect `EEO_*` because cards already do. + +When extending or replicating this design, use the generic concepts and pipeline above and map them to your own block and dimension names. + +Related skills: [Workforce Planning - Application Architecture & Patterns](./workforce_planning_architecture_patterns.md); [Workforce Planning - Workforce Cards & Mapped Dimensions](./workforce_planning_cards_mapped_dimensions.md); [Workforce Planning - Snapshot Spread Logic](./workforce_planning_snapshot_spread.md). diff --git a/plugins/pigment/skills/solving-specific-use-cases/workforce_planning_snapshot_spread.md b/plugins/pigment/skills/solving-specific-use-cases/workforce_planning_snapshot_spread.md new file mode 100644 index 00000000..47fcaf9a --- /dev/null +++ b/plugins/pigment/skills/solving-specific-use-cases/workforce_planning_snapshot_spread.md @@ -0,0 +1,154 @@ +# Workforce Planning - Snapshot Spread Logic + +## Purpose + +This use case describes how to model the bridge between snapshot-based source data (e.g. HRIS loads with a "load month" and sparse events like hire/term) and a regular planning grid (e.g. Version x Employee x Month). The "spread logic" layer centralizes snapshot selection, propagation of point-in-time attributes over time, and history-vs-plan behavior so downstream metrics stay simple and maintainable. + +When to use this pattern + +- The source system provides as-of snapshots (e.g. one row per employee per "HRIS Load Month" with hire date, term date, entity, etc.). +- Planning or reporting requires a dense grid on a regular time dimension (e.g. Month). +- You need a single, consistent rule for "which snapshot applies to each (scenario, entity, period)" and for spreading dates/attributes across periods. + +When not to use + +- Spreading is optional if the client can provide historical data already at the planning grain (e.g. monthly history). In that case you may not need backward/forward snapshot mapping. +- Backward spreading is never 100% correct; it is a best-effort way to visualize history using snapshot data. Do not rely on it for audit or legal accuracy. +- Forward spreading is mandatory to build a forecast or plan: you need a clear rule for which snapshot drives each future month. + +--- + +## 1. Problem in plain terms + +- Source: Snapshot data (e.g. HRIS) with a snapshot date dimension (e.g. "HRIS Load Month"). Each row is "as of" that date: hire date, term date, entity, department, etc. Data is sparse along time (only load months exist). +- Target: A planning grain (e.g. Version x Employee x Month) where every month needs values for attributes and events (hire date, entity, etc.). +- Gap: You must decide, for each (Version, Employee, Month), which snapshot to use and how to propagate point-in-time values (hire, term) so they appear on every relevant month without duplicating logic in dozens of metrics. + +The spread logic layer is the small set of helper metrics that answer "which snapshot?" and "what value for this month?" once, so staging and card metrics only reference these helpers. + +--- + +## 2. Core concepts (generalized) + +| Concept | Description | +| ---------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Snapshot source | The list or metric that holds raw snapshot data (e.g. HRIS load list), with a snapshot date dimension (e.g. Load Month) and an entity mapping (e.g. Employee Mapping). | +| Planning grain | The dimensions you need for planning/reporting (e.g. Version x Employee x Month). | +| Snapshot selection | For each (Version, Entity, Planning Period), which snapshot date to use to pull attributes. This is encoded in one or a few metrics (e.g. "effective load month per (Version, Employee, Month)"). | +| Backward propagation (history) | For past periods: "use the snapshot that was actually valid at that time" (e.g. last load month <= this month). Used to approximate historical view when you only have snapshots. | +| Forward propagation (plan) | For plan/future periods: "use a fixed or forward-carried snapshot" (e.g. latest known load month, or a chosen reference month). Mandatory for building a forecast. | +| Spread layer | The set of metrics that: (1) define effective snapshot date per (Version, Entity, Planning Period) for history and for plan, (2) spread point-in-time attributes (hire, term, etc.) over the planning time dimension, (3) provide an active mask (which (Entity, Period) are in scope, e.g. hired and not terminated), (4) apply version windows (only periods inside each version's range). | +| History vs plan | A global toggle (e.g. "Populate history?") and optionally a cutover month per version. When ON: use backward snapshot selection for historical behavior. When OFF: use forward snapshot selection for a stable plan view. Centralizing this in the spread layer avoids repeating `IF(Populate_History?, ..., ...)` in every staging metric. | + +--- + +## 3. Pipeline (generic) + +| Layer | Role | Typical content | +| ------------------------------- | -------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 1. Snapshot load | Raw import | List/metric with snapshot date dimension and entity mapping; sparse attributes (hire date, term date, entity, department, etc.). | +| 2. Spread logic | Bridge | Helpers: effective snapshot date (history + plan), spreaded dates (e.g. FILLFORWARD of hire/term), active mask, plan-month filter; all with version window applied once here. | +| 3. Data staging | Map attributes to planning grain | For each attribute: pull from snapshot source using BY CONSTANT: [effective snapshot metric] (history or plan depending on toggle). No repeated FILLFORWARD or window logic. | +| 4. Overrides (optional) | User overrides | Override staging values where users can edit (e.g. plan overrides). | +| 5. Cards / final attributes | Output for reporting | IFDEFINED(Override, Staging value), still using spread helpers to anchor to the right snapshot. | +| 6. Stats & aggregations | Headcount, FTE, events | Built on top of cards; no direct snapshot logic. | + +The spread layer (2) is the only place that implements snapshot selection, date spreading, history vs plan, and version windows. All other layers reference its outputs. + +--- + +## 4. Pattern: two building blocks + +### 4.1 Spreading a point-in-time attribute (e.g. hire date) + +Goal: Have the attribute available for every (Version, Entity, Month) where it is relevant (e.g. every month after hire), not only the exact month it was set. + +Idea: Use FILLFORWARD over the planning time dimension so blanks are filled with the last known value. Then restrict to version window. + +Generic pattern: + +```text +IFBLANK( + 'Source_Attribute', // from staging or load, already at planning grain where defined + FILLFORWARD('Source_Attribute', Planning_Time_Dimension) +) +[FILTER: Version.'Window End' >= Planning_Time_Dimension] +``` + +Result: downstream metrics can reference this "spreaded" metric and always get a value (e.g. hire date) for every month in scope, without re-implementing fill or window logic. + +### 4.2 Choosing snapshot for history vs plan + +Goal: For each (Version, Entity, Planning Period), have one metric that represents "the snapshot date to use": backward for history, forward for plan. + +Idea: Build two helper metrics: one backward (e.g. "last load month <= this month"), one forward (e.g. "forward-carried load month for plan"). Combine with an active mask and a plan-month filter. Expose a single "effective snapshot date" used by staging. + +In staging (data) metrics: Use a single branching point, driven by a global toggle: + +```text +IF( + Populate_History?, // global toggle + // History: use backward-propagated snapshot date + Source_List.'Attribute' + [BY LASTNONBLANK: Entity_Mapping, Snapshot_Date_Dimension] + [ADD: Version][FILTER: Is_Actual_Version] + [BY CONSTANT: 'Spd_Effective_Snapshot_Date_History'], + // Plan: use forward-propagated snapshot date + Source_List.'Attribute' + [BY LASTNONBLANK: Entity_Mapping, Snapshot_Date_Dimension] + [ADD: Version][FILTER: Is_Actual_Version] + [BY CONSTANT: 'Spd_Effective_Snapshot_Date_Plan'] +) +``` + +All staging metrics use the same pattern; only the attribute and source change. Snapshot selection and history/plan logic live in the spread layer (`Spd_*` metrics). + +### 4.3 Discover blocks and properties in this application + +Template names below (e.g. `EE_Load_HRIS`, `Populate_History?`) are illustrative. In a real customer app, lists, toggles, and Scenario / Version properties (such as start and end month for the planning window) have local names and API paths. Before implementing the spread layer: + +- Use `tool:search` to locate the snapshot source list, planning dimensions, and any history/plan toggles; read returned summaries and IDs. +- Search again (with `kind` or `regexp` if names collide) for Scenario (or Version) and confirm which properties encode the version window-formulas must reference the actual property names in that workspace. +- After building staging metrics, use `tool:search` to verify that reporting or headcount metrics reference spread-layer metrics only, not the raw snapshot list, if your architecture requires that separation. + +Skipping this discovery step often yields a minimal spread layer without version filters, active flags, or downstream metrics that demonstrate the full pattern. + +--- + +## 5. How to apply this elsewhere + +1. Identify the snapshot source (list or metric), its snapshot date dimension, and the entity mapping (how to align to your planning entity, e.g. Employee)-using `tool:search` as needed so names match the live app. +2. Define the planning grain (e.g. Version x Employee x Month) and version windows (start/end period per version). +3. Build spread-layer helpers in a dedicated folder: + - Effective snapshot date for history (backward) and for plan (forward). + - Spreaded point-in-time attributes (FILLFORWARD + version filter). + - Active mask (e.g. "entity is active in this period"). + - Plan-month filter to cap the calculation horizon. +4. Implement staging metrics that reference these helpers with `[BY CONSTANT: Spd_Effective_Snapshot_...]` and a single `IF(Populate_History?, ...)` branch. +5. Keep history vs plan and version windows only in the spread layer; do not scatter snapshot selection or window logic in downstream formulas. +6. Cards and stats reference staging (and overrides) and spread helpers only; they do not implement snapshot or time propagation. + +--- + +## 6. Pitfalls and reminders + +- Backward spread is approximate: Good for visualizing history from snapshots; not a guarantee of correctness for audit or legal. +- Forward spread is required for planning: Always define a clear rule for which snapshot drives plan/future periods. +- Do not duplicate logic: If every staging metric repeats "which snapshot?" and "history vs plan?", changes become risky and hard to maintain. Centralize in the spread layer. +- Version windows: Apply "only months in version window" once in the spread layer so downstream metrics do not need to repeat version filters. + +--- + +## 7. Illustration: Workforce Planning Template + +In the Pigment Workforce Planning Template application, this pattern appears as follows (names are template-specific; the logic is the one described above): + +- Snapshot load: `EE_Load_HRIS` with HRIS Load Month and Employee Mapping; attributes include Hire Date, Term Date, Entity, Department, Job Position, Country, State, etc. +- Spread logic (folder 1.3): Metrics such as `EE_Spd_01_Hire Date` (FILLFORWARD of hire date + version window), `EE_Spd_02_Active Employee`, `EE_Spd_03_HRIS Load Month_fwd`, `EE_Spd_04_HRIS Load Month_bwd`, `EE_Spd_99_HRIS Load Month` (effective snapshot month for the view), and `Set_Plan Month View`. +- History vs plan: Toggle `Populate_History?` and optional `Populate_History_Month`; staging metrics use `EE_Spd_99_HRIS Load Month` (history) or `EE_Spd_03_HRIS Load Month_fwd` (plan) via `[BY CONSTANT: ...]`. +- Data staging (1.2): `EE_Data_*` metrics (Entity, Department, Hire Date, etc.) pull from `EE_Load_HRIS` using the spread helpers and the single history/plan branch. +- Cards (2.1): `EE_Card_*` use overrides (`EEO_*`) when defined, else staged data, still anchored via spread helpers. + +When extending or replicating this design, use the generic concepts and pipeline above and map them to your own block names and dimensions. + +Related skills: [Workforce Planning - Application Architecture & Patterns](./workforce_planning_architecture_patterns.md); [Workforce Planning - Workforce Cards & Mapped Dimensions](./workforce_planning_cards_mapped_dimensions.md); [Workforce Planning - Changelog to Override Metrics](./workforce_planning_changelog_overrides.md). diff --git a/plugins/pigment/skills/writing-pigment-formulas/SKILL.md b/plugins/pigment/skills/writing-pigment-formulas/SKILL.md new file mode 100644 index 00000000..ae0ac215 --- /dev/null +++ b/plugins/pigment/skills/writing-pigment-formulas/SKILL.md @@ -0,0 +1,331 @@ +--- +name: writing-pigment-formulas +description: Always use this skill when writing, editing, or debugging Pigment formulas - including conditional logic, blank handling, date-range logic, aggregation, prior-period lookups, and dimensional transformations. Pigment uses a proprietary formula language - NEVER assume you know the syntax, and ALWAYS read the documentation before writing any formula. Covers data types, modifiers, functions, calculation patterns, and performance trade-offs (calibrated by formula complexity). This skill includes supporting files in this directory; explore as needed. +metadata: + skill_path: /writing-pigment-formulas/SKILL.md + base_directory: /writing-pigment-formulas + includes: + - "*.md" +--- + +# Writing Pigment Formulas + +This skill provides comprehensive guidance for writing formulas in Pigment's multidimensional formula language, including formula builder tools for validation and generation. +Pigment uses proprietary formula language which should never be confused with other language. +Never mix Pigment formula language with other language, and never assume you know the language before reading this documentation. + +CRITICAL - ABSOLUTE PROHIBITION: Pigment has its own unique formula language. +You MUST NEVER write code or functions using another language being Excel, SQL, Python, JavaScript, MDX, DAX, or ANY other programming or query language. +ONLY Pigment syntax exists when writing formulas. + +## When to Use This Skill + +- Write formulas - Creating calculations for metrics and list properties +- Aggregate data - Rolling up from transaction lists or detailed dimensions +- Perform time-series calculations - YTD, rolling averages, sequential logic +- Use functions - CUMULATE, SHIFT, ITEM, MATCH, TIMEDIM, etc. +- Use modifiers - BY, ADD, REMOVE, SELECT, FILTER, EXCLUDE, TOPARENTLIST, TOSUBSET +- Debug syntax - Troubleshooting formula errors +- Test and validate formulas - Verifying formula correctness and expected behavior +- Transform dimensions - Changing dimensional structure of calculations +- Allocate data - Distributing values across dimensions +- Match and lookup - Finding data across dimensions + +--- + +## Syntax Fundamental + +Quoting Rules - MUST FOLLOW: + +| Element | Syntax | Example | +| --------------- | ----------------------------- | ------------------------------------------------- | +| Metric names | Single quotes | `'Revenue'`, `'Total Sales'` | +| Dimension names | Single quotes | `'Product'`, `'Country'` | +| Property access | Dot notation with quotes, chainable | `'Product'.'Category'`, `City.Country.Currency` | +| Dimension items | Double quotes after dimension - MP02: literal only in `VAR_` default | `Month."Jan 25"` only when setting a `VAR_` metric default | +| String values | Double quotes | `"Active"`, `"Completed"` | + +Cross-app references: +- A block from another application can only be referenced if it has been shared through a Library and that Library is activated in the current application. +- The syntax is `'APPLICATION_NAME'::'BLOCK_NAME'`. +- If the block name is unique across all activated libraries the application prefix may be omitted, but always use the full form for clarity. + +No hard-coding (MP02 - hard constraint): See [modeling_principles section 4](../modeling-pigment-applications/modeling_principles.md). Before member-specific or time-bounded formulas, read [formula_writing_workflow.md](./formula_writing_workflow.md) Step 2 and [formula_modifiers.md](./formula_modifiers.md) (FILTER, SELECT, BY CONSTANT). + +Common Mistakes: + +- BAD: `Revenue` -> GOOD: `'Revenue'` (missing quotes) +- BAD: `Product.Category` -> GOOD: `'Product'.'Category'` (missing quotes) +- BAD: `Month.'Jan 25'` -> GOOD: `Month."Jan 25"` (items use double quotes; in formulas use a `VAR_` metric per MP02) + +--- + +## Performance Patterns + +Apply this checklist proportionally to formula complexity. Simple arithmetic between existing same-dimensioned metrics (e.g. `'A' + 'B'`, `'A' * 'B'`, `'A' / 'B'`) needs no performance wrapping - deliver as-is. Use the checklist as a review gate for formulas that introduce conditionals, dimensional changes, date-range logic, or that target large/sparse metrics. + +Read [formula_performance_patterns.md](./formula_performance_patterns.md) and verify: + +Always check (universal): + +- [ ] Identifiers are correctly quoted (single quotes for names, double quotes for items) +- [ ] Dimensions are aligned - no unintended ADD or dimension mismatch +- [ ] Scoping clauses appear FIRST (FILTER, EXCLUDE, IFDEFINED) +- [ ] Aggregations (REMOVE, BY) appear AFTER calculations + +Check when conditionals are present: + +- [ ] Using IFDEFINED instead of IF(ISBLANK()) for existence checks +- [ ] Using IFBLANK instead of IF(ISBLANK(...), default, ...) for defaults +- [ ] Conditional creation: use IF (not ADD + FILTER); subsetting a computed expression: use FILTER: CurrentValue (not IF(expr, expr, BLANK)) + +Check when date ranges are defined by Start/End: + +- [ ] Avoid multi-conditional IFs (`Date >= Start AND Date < End`) when PRORATA semantics apply +- [ ] Prefer `PRORATA()` to express "active within a date range" and derive booleans or numeric flags from `PRORATA()` using ISDEFINED/IFDEFINED + +Check when prior period lookups are needed: + +- [ ] Using SELECT for prior period lookups (NOT PREVIOUS) + +Check when dimensional changes or mappings are involved: + +- [ ] Using BY instead of ADD where mapping exists +- [ ] If you BY on a dimension-typed metric, do not add IF/ISBLANK guards; BY respects that metric's sparsity + +Check when the metric is large/sparse or involves access rights: + +- [ ] Avoid ISBLANK/ISNOTBLANK on large sparse metrics - use ISDEFINED/IFDEFINED +- [ ] Use BLANK instead of 0 for empty values (see exception below for meaningful zeros) +- [ ] Use BLANK instead of FALSE for boolean flags (FALSE is stored, BLANK is not) +- [ ] Access rights wrapped in IFDEFINED(User, ...) +- [ ] MP02: No `Dimension."Item"` in formulas; no `DATE(...)` for planning bounds; relative metric names only - see [formula_writing_workflow.md](./formula_writing_workflow.md) Step 6 checklist + +For the full date-range presence pattern (PRORATA worked examples, ISDEFINED/IFDEFINED derivation, when simple IF is acceptable), see Pattern 11 in [formula_performance_patterns.md](./formula_performance_patterns.md). + +--- + +## Formula Writing Process + +Key phases: Understand Context -> Search Documentation -> Design -> Build -> Optimize -> Validate -> Deliver + +Follow the complete 8-step workflow: [./formula_writing_workflow.md](./formula_writing_workflow.md) + +- Critical: Always search documentation first before writing +- Governance check (MP02 - required): [modeling_principles section 4](../modeling-pigment-applications/modeling_principles.md); Version patterns: `skill:planning-cycles-pigment-applications`. +- Validation & Delivery: Use Formula Builder Tools to validate and deliver formulas + +--- + +## Formula Validation and Building Tools + +Important: These tools are for validation and implementation when working with real formulas. + +### Quick Validation + +- `tool:validate_formula` - Validate formula syntax WITHOUT applying it to any block + - Use for: Checking syntax before calling `tool:update_list_property_formula` + - Use for: Ensuring formula syntax is correct before including in user messages + - Input: `formula` (the Pigment formula text) + - Returns: Validation result with error highlighting and hints if invalid + - Limitations: + - Do NOT use with formulas containing `Previous` or `PreviousOf` functions + +Recommended Workflow: + +1. Draft formula - Write your formula based on requirements +2. Validate - Use `tool:validate_formula` to check syntax +3. Fix errors - Iterate until formula is valid +4. Apply - Use `tool:create_or_update_formula` or `tool:update_list_property_formula` + +How to apply: After validation, use: + +- Metrics: `tool:create_or_update_formula` with the formula +- List properties: `tool:update_list_property_formula` with the formula + +--- + +## Prerequisites + +This skill focuses on formula implementation. Before writing formulas, understand foundational concepts from the modeling-pigment-applications skill: + +- Core platform knowledge (multidimensional engine, dimensions vs properties, sparsity principles) +- Pigment Modeling Best Practices standards (sparsity preservation, dimension alignment, formatting) +- Dimensional design concepts (source-to-target relationships, transformation cases) +- Modifier concepts + +### Type Considerations + +Formulas produce results that must match the target metric or property type: + +- Number: Arithmetic operations, aggregations, most calculations +- Date: Date functions (DATE, DATEVALUE, EDATE), TIMEDIM conversions +- Text: String operations, concatenation, TEXT() conversion +- Dimension: ITEM, MATCH lookups returning dimension references +- Boolean: Logical operations (AND, OR, comparisons) + +Type conversions: Use TEXT() to convert to text, VALUE() to convert to number, TIMEDIM() to convert dates to calendar dimensions. See [functions_text.md](./functions_text.md) and [functions_lookup.md](./functions_lookup.md). + +Reference: For detailed type selection guidance, see modeling-pigment-applications skill. + +--- + +## Quick Reference + +| Topic | File | +| --------------------------------- | -------------------------------------------------------------------- | +| Formula Writing Process | [formula_writing_workflow.md](./formula_writing_workflow.md) | +| Conditionals style (IFBLANK, FILTER/EXCLUDE vs IF) | [formula_conditionals_style.md](./formula_conditionals_style.md) | +| Modifiers (BY, ADD, FILTER, TOPARENTLIST, TOSUBSET, etc.) | [formula_modifiers.md](./formula_modifiers.md) | +| BY with mapping metrics (->) | [formula_by_mapping_arrow.md](./formula_by_mapping_arrow.md) | +| Lookup Functions | [functions_lookup.md](./functions_lookup.md) | +| Numeric Functions | [functions_numeric.md](./functions_numeric.md) | +| Time and Date Functions | [functions_time_and_date.md](./functions_time_and_date.md) | +| Iterative Calculation (PREVIOUS & PREVIOUSOF) | [functions_iterative_calculation.md](./functions_iterative_calculation.md) | +| Logical Functions | [functions_logical.md](./functions_logical.md) | +| Text Functions | [functions_text.md](./functions_text.md) | +| Performance Patterns | [formula_performance_patterns.md](./formula_performance_patterns.md) | + +--- + +## Function Reference + +### Most Common Functions & Modifiers + +- BY -> [./formula_modifiers.md](./formula_modifiers.md) - Aggregate or allocate; BY with mapping metrics (`->`) -> [./formula_by_mapping_arrow.md](./formula_by_mapping_arrow.md) +- TOPARENTLIST -> [./formula_modifiers.md](./formula_modifiers.md#toparentlist-and-tosubset-list-subsets) - subset dimension -> parent (1:1 remap; parent items outside the subset are blank) +- TOSUBSET -> [./formula_modifiers.md](./formula_modifiers.md#toparentlist-and-tosubset-list-subsets) - parent dimension -> subset (1:1 remap; parent rows outside the subset are dropped) +- CUMULATE -> [./functions_numeric.md](./functions_numeric.md) - Running totals (use instead of PREVIOUSOF + value) +- FILTER -> [./formula_modifiers.md](./formula_modifiers.md) - Include data by condition +- EXCLUDE -> [./formula_modifiers.md](./formula_modifiers.md) - Remove data by condition +- FILLFORWARD -> [./functions_time_and_date.md](./functions_time_and_date.md) - Fill blanks (use instead of IFBLANK + PREVIOUS) +- IF -> [./functions_logical.md](./functions_logical.md) - Conditional logic +- IFDEFINED -> [./functions_logical.md](./functions_logical.md) - Sparsity-preserving conditionals +- ITEM -> [./functions_lookup.md](./functions_lookup.md) - Lookup by unique property +- MATCH -> [./functions_lookup.md](./functions_lookup.md) - Lookup by non-unique property +- MOVINGSUM -> [./functions_numeric.md](./functions_numeric.md) - Rolling sums +- MOVINGAVERAGE -> [./functions_numeric.md](./functions_numeric.md) - Rolling averages +- SELECT with time offset -> [./formula_modifiers.md](./formula_modifiers.md) - Prior period lookup: `'Revenue'[SELECT: Month-1]` +- PREVIOUS/PREVIOUSOF -> [./functions_iterative_calculation.md](./functions_iterative_calculation.md) - Iterative calculations (circular dependencies, configuration, syntax); see also [functions_time_and_date.md](./functions_time_and_date.md) for SELECT vs PREVIOUS +- SHIFT -> [./functions_lookup.md](./functions_lookup.md) - Shift dimension-typed properties +- SWITCH -> [./functions_logical.md](./functions_logical.md) - Multi-way branching +- TIMEDIM -> [./functions_lookup.md](./functions_lookup.md) - Date to time dimension + +### By Category + +Lookup Functions: [./functions_lookup.md](./functions_lookup.md) - ITEM, MATCH, SHIFT, TIMEDIM + +Numeric Functions: [./functions_numeric.md](./functions_numeric.md) - CUMULATE, DECUMULATE, MOVINGSUM, MOVINGAVERAGE, ABS, SIGN, EXP, LN, LOG, SIN, COS, SQRT, MIN, MAX, MOD, QUOTIENT, POWER, ROUND, ROUNDUP, ROUNDDOWN, TRUNC, CEILING, FLOOR, RANK, SPREAD + +Time and Date Functions: [./functions_time_and_date.md](./functions_time_and_date.md) - DATE, DATEVALUE, DAY, MONTH, YEAR, DAYS, NETWORKDAYS, WEEKDAY, STARTOFMONTH, EOMONTH, EDATE, INPERIOD, DAYSINPERIOD, PRORATA, MONTHDIF, FILLFORWARD, YEARTODATE, QUARTERTODATE, MONTHTODATE + +Iterative Calculation: [./functions_iterative_calculation.md](./functions_iterative_calculation.md) - PREVIOUS, PREVIOUSOF (full spec: circular dependencies, configuration, performance, debugging) + +Text Functions: [./functions_text.md](./functions_text.md) - TEXT, VALUE, LEN, LEFT, MID, RIGHT, LOWER, UPPER, PROPER, TRIM, CONTAINS, STARTSWITH, ENDSWITH, FIND, SUBSTITUTE, & (concatenation) + +Logical Functions: [./functions_logical.md](./functions_logical.md) - AND, OR, NOT, TRUE, FALSE, ANYOF, ALLOF, ISBLANK, ISNOTBLANK, ISDEFINED, IFDEFINED, IF, SWITCH, IN, IFBLANK + +Basic Aggregation Functions: [./functions_basic_aggregations.md](./functions_basic_aggregations.md) - AVGOF, COUNTALLOF, COUNTBLANKOF, COUNTUNIQUEOF, SUMOF, MINOF, MAXOF, COUNTOF + +Finance Functions: [./functions_finance.md](./functions_finance.md) - NPV, XNPV, IRR, XIRR + +Forecasting Functions: [./functions_forecasting.md](./functions_forecasting.md) - FORECAST_ETS, FORECAST_LINEAR, SIMPLE_EXPONENTIAL_SMOOTHING, DOUBLE_EXPONENTIAL_SMOOTHING, SEASONAL_LINEAR_REGRESSION, STANDARD_NORMAL_DISTRIBUTION + +Security Functions: [./functions_security.md](./functions_security.md) - ACCESSRIGHTS, RESETACCESSRIGHTS + +List subset <-> parent remap (1:1, no aggregator): [./formula_modifiers.md](./formula_modifiers.md#toparentlist-and-tosubset-list-subsets) - TOPARENTLIST, TOSUBSET + +--- + +## Cross-References + +Before formula writing: modeling-pigment-applications (core concepts, Pigment Modeling Best Practices standards, dimensional design) + +Related skills: optimizing-pigment-performance (formula optimization, sparsity management) + +--- + +## Critical Notes + +- ABSOLUTE: Pigment syntax ONLY: You MUST NEVER write functions in other languages like Excel, SQL, Python, JavaScript, MDX, DAX, or ANY other language. Think ONLY in Pigment terms. +- Search first: Always search documentation to discover functions and patterns before writing +- Follow workflow: Complete the 8-step process in [./formula_writing_workflow.md](./formula_writing_workflow.md) +- Review performance patterns: Formulas with conditionals, dimensional changes, date ranges, or large/sparse targets must pass the checklist in [formula_performance_patterns.md](./formula_performance_patterns.md) before delivery. Simple arithmetic between same-dimensioned metrics does not require this review. +- Prerequisites matter: Understand modeling concepts from modeling-pigment-applications skill first +- Document your work: List which files you consulted for transparency + +--- + +## Formula Commenting Standard + +All generated formulas must include `//` comments for readability and maintainability. + +Top-level comment (required): + +- One `//` comment on its own line(s) immediately above the first line of the formula +- Explains the formula's purpose (what it computes and why) +- Use the same language as the block name + +Part-level comments (for non-trivial formulas only): + +- Add when the formula has multiple logical steps (several operations, functions, modifiers) +- Each comment on its own line, below the formula segment it describes +- One blank line between a part-level comment and the next formula segment +- Skip for one-liners or very obvious formulas + +If comments are already present, try to maintain or enhance them. Replace them completely only if a formula update made them wrong or misleading. + +Example (multi-step): + +```pigment +// Final revenue: actual revenue for active scenarios plus budget adjustments by category + +'Revenue'[FILTER: 'Scenario'.'Active' = TRUE] +// Filter revenue to active scenarios only + ++ 'Budget Adjustment'[BY: 'Product'.'Category'] +// Add budget adjustments mapped by product category +``` + +Example (simple): + +```pigment +// Total cost: sum of fixed and variable costs +'Fixed Cost' + 'Variable Cost' +``` + +Comments must be included in the formula string passed to `tool:create_or_update_formula` or `tool:update_list_property_formula`. + +--- + +## Key Rules Summary + +Syntax: + +- Single quotes for identifiers: `'Revenue'`, `'Product'.'Category'` +- Double quotes for dimension items: literal form only when setting a `VAR_` metric default (MP02 - see [modeling_principles section 4](../modeling-pigment-applications/modeling_principles.md)). +- Double quotes for string values: `"Active"`, `"Completed"` + +Modifiers: + +- BY only changes the dimension you specify - use REMOVE to eliminate dimensions +- In BY, list only dimensions whose grain is changing - do not re-list dimensions that are already on the metric (avoid over-explicit BY). For normalization ratios, use double BY in the denominator with only the changing dimension - see [formula_modifiers.md](./formula_modifiers.md). +- Never chain BY on transaction lists - use single BY with comma-separated expressions +- BY > ADD - BY is sparse (uses mappings), ADD is dense (all combinations) +- SELECT = FILTER + REMOVE - removes dimension after filtering + +Transaction Lists in Metrics: + +- Must aggregate: `'List'.'Property'[BY: ...]` +- Use list column names: `'Orders'.'Customer'` not just `Customer` +- Text to dimension: `ITEM('List'.'TextCol', 'Dimension'.'Property')` +- Date to time: `TIMEDIM('List'.'DateCol', Month)` + +Sparsity: + +- Avoid ISBLANK/ISNOTBLANK on large sparse metrics - use ISDEFINED instead (returns TRUE/BLANK, not TRUE/FALSE). For small already-dense metrics or where explicit TRUE/FALSE output is required (e.g. data-completeness exports), ISBLANK is acceptable; see [functions_logical.md](./functions_logical.md) allow-list. +- If you BY on a dimension-typed metric, its sparsity is respected automatically; do not add IF/ISBLANK guards. +- Use IFBLANK(A, B) instead of IF(ISBLANK(A), B, A) - cleaner and doesn't densify +- Use BLANK for empty values when the cell genuinely has no data. Use 0 (not BLANK) when zero is a meaningful business value (zero variance, zero balance, zero growth) that must be displayed or that participates in downstream multiplication. +- EXCLUDE, not FILTER: NOT - see [formula_conditionals_style.md](./formula_conditionals_style.md) +- See [functions_logical.md](./functions_logical.md) for detailed blank handling guidance diff --git a/plugins/pigment/skills/writing-pigment-formulas/formula_by_mapping_arrow.md b/plugins/pigment/skills/writing-pigment-formulas/formula_by_mapping_arrow.md new file mode 100644 index 00000000..ba1c396b --- /dev/null +++ b/plugins/pigment/skills/writing-pigment-formulas/formula_by_mapping_arrow.md @@ -0,0 +1,221 @@ +# BY with Mapping Metrics (->) - Arrow Syntax + +When to use this doc: Use this when the mapping is a dimension-typed metric (or you need multiple dimensions/properties in the BY mapping). For simple property-based BY (e.g. `Country.Region`, `Product.Category`), see [formula_modifiers.md](./formula_modifiers.md). + +--- + +## 1. PURPOSE + +Explain how to use the BY modifier with mapping metrics via the arrow syntax (`->`): + +- When and why to use it instead of simple attribute-based BY +- How dimension replacement and addition work +- How to avoid silent over-aggregation (e.g. on Version, Time, Scenario) + +This complements the main formula modifiers documentation and assumes you already know basic BY, REMOVE, and dimension alignment. + +Concept in one sentence: "BY with `->` replaces specific source dimensions using one or more dimension-typed mapping metrics and/or properties, adds any extra grouping dimensions, and aggregates away any remaining dimensions you don't explicitly keep." The last part - implicit aggregation of "unhandled" dimensions - is the main risk. + +--- + +## 2. SYNTAX AND ROLES + +### 2.1 General syntax + +``` +Source[BY Method: SourceDim1, SourceDim2 -> MappingOrProperty1, MappingOrProperty2, ExtraDimOrProp1, ExtraDimOrProp2] +``` + +| Element | Role | +|--------|------| +| Source | Metric or list expression you start from. | +| Method | Any valid BY aggregation or allocation method (see below). | +| SourceDim1, SourceDim2 (left of `->`) | Dimensions in Source that you intend to replace via mappings. | +| MappingOrProperty\* (immediately after `->`) | One or more mapping metrics (dimension-typed metrics) or dimension properties (e.g. `Country.Region`) that define how to derive new dimensions from the source dimensions. | +| ExtraDimOrProp\* (after mappings) | Additional dimensions or properties you add for grouping (e.g. `Account.Market`, `Product.Family`). | + +You can have multiple dimensions before `->` and multiple mappings/properties after `->`. + +### 2.2 Aggregation methods (when replacing dimensions, N->1) + +SUM, AVG, MIN, MAX, FIRSTNONBLANK, FIRSTNONZERO, FIRST, LASTNONBLANK, LASTNONZERO, LAST, ANY, ALL, COUNT, COUNTBLANK, COUNTALL, COUNTUNIQUE, TEXTLIST. If omitted when replacing dimensions, the default is SUM. (Supported data types follow Pigment's standard rules; see [formula_modifiers.md](./formula_modifiers.md) for full details.) + +### 2.3 Allocation methods (when expanding 1->N or adding dimensions) + +CONSTANT, SPLIT. If omitted when allocating, the default is CONSTANT. + +--- + +## 3. DIMENSION RULES (CRITICAL BEHAVIOR) + +Given Source has some dimensions, `BY ... ->` follows three rules. + +### Rule 1 - Replace + +Every dimension listed before `->` (SourceDim1, SourceDim2, ...) is: + +- Removed from the result, and +- Replaced by the dimension(s) produced by the mappings (e.g. the type of a mapping metric, or a property's referenced dimension). + +Example: + +```pigment +'Revenue'[BY SUM: Country -> 'Country to Region Map'] +``` + +Country is removed. The dimension from `'Country to Region Map'`'s type (e.g. Region) is added. + +### Rule 2 - Add + +Every dimension or property listed after `->` that is not serving as the "replacement target" is added to the result. + +Examples: + +```pigment +'Revenue'[BY SUM: Country -> 'Country to Region Map', Country.Currency] +``` + +Adds Currency (from `Country.Currency`) as an extra dimension. + +```pigment +'Headcount'[BY CONSTANT: -> 'Employee Team Map'] +``` + +No source dimension before `->`: no replacement. The mapping's type dimension (e.g. Team) is added to the result. + +### Rule 3 - Aggregate away everything else + +Any dimension that: + +- Exists in Source, and +- Is not listed before `->` (to be replaced), and +- Is not explicitly added (e.g. via a property), and +- Is not inherently preserved by the mapping, + +is aggregated away using the BY method. + +Typical "at risk" dimensions: Version, Time dimensions (Month, Year, etc.), Scenario, and optional entity dimensions (e.g. Company, Currency) you forget to mention. If you don't explicitly account for them, they will be collapsed by the aggregation. This rule is the most common source of incorrect results. + +--- + +## 4. WHEN TO USE ARROW (`->`) VS PLAIN BY + +### 4.1 Use arrow (`->`) when... + +- The mapping lives in a dimension-typed metric (e.g. Account x Version -> Segment, Country x Product -> Team). +- You need to transform from certain source dimensions to a target dimension and must control which dimensions are replaced (left of `->`), kept, or added. +- You use mapping metrics or multiple dimensions/properties in the BY clause. + +Examples: + +```pigment +'Account_revenue'[BY AVERAGE: Account -> 'Account Segment Map', Account.Market] +'Sales'[BY SUM: Country, Product -> 'Country Product to Team Map', Month] +``` + +### 4.2 Use plain BY (no arrow) when... + +- The mapping is a simple dimension property, not a metric: `Account.Segment`, `Country.Region`, `Month.Quarter`. +- You're just moving along a straightforward hierarchy. + +Examples: + +```pigment +'Revenue'[BY SUM: Country.Region] +'Account_revenue'[BY AVERAGE: Account.Segment, Account.Market] +``` + +No `->` needed because the mapping and resulting dimensions are unambiguous and live in properties. + +### 4.3 Sparsity + +When a dimension-typed metric is used in BY (including as a mapping or grouping dimension), its sparsity is respected automatically. Do not add IF(ISBLANK(metric), BLANK, ...) guards - they are redundant and densify. See [Sparsity via BY + dimension-typed metrics](./formula_modifiers.md#sparsity-via-by--dimension-typed-metrics) in formula_modifiers.md. + +--- + +## 5. CORE PATTERN EXAMPLES + +### 5.1 Aggregation using a mapping metric + +Goal: Aggregate Sales from Country to Region, keeping Product and Month. + +- Setup: Sales dims = Country, Product, Month. `'Country to Region Map'`: dims = Country, type = Dimension(Region). +- Formula: + +```pigment +'Sales'[BY SUM: Country -> 'Country to Region Map', Product, Month] +``` + +Country is replaced by Region via the mapping metric. Product and Month are explicitly kept. Result dims: Region, Product, Month. + +### 5.2 Allocation using a mapping metric + +Goal: Allocate Region_budget down to Countries via a mapping metric. + +- Setup: Region_budget dims = Region, Year. `'Region to Country Map'`: dims = Region, Country, type = Dimension(Country) or numeric weights. +- Formula: + +```pigment +'Region_budget'[BY CONSTANT: Region -> 'Region to Country Map', Country] +``` + +Region is replaced by Country. Year is preserved. CONSTANT copies region budget across mapped countries. Result dims: Country, Year. + +### 5.3 Adding a dimension without replacing (classification / filtering) + +Goal: Add Team as an extra dimension to a metric defined on Product x Country, without removing existing dimensions. + +- Setup: Volume dims = Product, Country, Month. `'Product Country to Team Map'`: dims = Product, Country, type = Dimension(Team). +- Formula: + +```pigment +'Volume'[BY CONSTANT: -> 'Product Country to Team Map'] +``` + +No source dims before `->`: no replacement. Team is added. Result dims: Product, Country, Team, Month. + +--- + +## 6. COMMON PITFALL: UNINTENDED AGGREGATION (VERSION / TIME / SCENARIO) + +Setup: Metric_X dims = Account, Version. `'Account to Segment Map'`: dims = Account, Version, type = Dimension(Segment). Account.Market property. + +Correct intention: Metrics by Segment, Market, and Version. + +Correct formula: + +```pigment +Metric_X[BY COUNT: Account -> 'Account to Segment Map', Account.Market] +``` + +Account is replaced by Segment. Account.Market adds Market. Version is preserved (shared, not replaced). Result dims: Segment, Market, Version. + +Wrong formula (over-aggregation): + +```pigment +Metric_X[BY COUNT: 'Account to Segment Map', Account.Market] +``` + +Here, `Account` is not listed before `->`, so the replacement rule does not apply as intended. Version appears in the mapping metric but is never mentioned in the BY clause. According to Rule 3, Version is aggregated away. You effectively count (Account, Version) combinations per (Segment, Market), not distinct Accounts per (Segment, Market, Version). If most Accounts exist in multiple Versions, counts are inflated. + +Principle: If a dimension exists in your source or mapping and you do not replace it (left of `->`), add it as a grouping dimension, or ensure it's preserved by shared structure, BY will aggregate over it. + +--- + +## 7. CHECKLIST FOR USING BY ... -> + +When designing or reviewing a formula with `BY ... ->`: + +1. List all source dimensions - Write out dims of Source and all mapping metrics involved. +2. Decide for each source dimension: Replace -> put it before `->`. Keep as axis -> ensure it's present where needed and not listed before `->`. Add new/grouping dimension -> include as property or extra dim after `->`. +3. Watch for "silent" dims (Version, Time, Scenario, Company, Currency). If they must be in the result, ensure they're preserved; if not, confirm you intend to aggregate them away. +4. Choose the BY method - Pick the appropriate aggregation or allocation method from the supported list (see section 2 and [formula_modifiers.md](./formula_modifiers.md)). +5. Quick check: "What are the final dimensions of this expression?" If any expected axis is missing, revisit Rules 1-3. + +--- + +## 8. SEE ALSO + +- [formula_modifiers.md](./formula_modifiers.md) - Core documentation for BY, REMOVE, KEEP, etc., including full method lists and data type support. +- Lookup / mapping-pattern docs - For building and maintaining mapping metrics used with `->`. +- [modeling_principles.md](../modeling-pigment-applications/modeling_principles.md) - Conceptual guidance on mapping metrics and dimension transformations. diff --git a/plugins/pigment/skills/writing-pigment-formulas/formula_conditionals_style.md b/plugins/pigment/skills/writing-pigment-formulas/formula_conditionals_style.md new file mode 100644 index 00000000..fc3f4d71 --- /dev/null +++ b/plugins/pigment/skills/writing-pigment-formulas/formula_conditionals_style.md @@ -0,0 +1,279 @@ +# Formula Conditionals Style + +When to use this doc: Conditional logic with IF/SWITCH or modifiers (FILTER, EXCLUDE), or highly complex formulas that require performance optimization. + +Goal: Readable Pigment-native formulas. Use IF to scope early; IFBLANK for overrides/precedence; `[FILTER: ...]` to subset an existing metric when that reads most naturally. Avoid ISBLANK/ISNOTBLANK and unnecessary densification. + +Related: Syntax and function reference -> [functions_logical.md](./functions_logical.md), [formula_modifiers.md](./formula_modifiers.md). Performance (scoping, densification) -> [formula_performance_patterns.md](./formula_performance_patterns.md). + +--- + +## Quick Reference: When to Use What + +Note: IFBLANK is the right tool for sparsity-driven branching (presence checks, override hierarchies). For value-based branching where both branches return non-blank values, IF and SWITCH are equally idiomatic and often more readable. + +| Situation | Prefer | Avoid | +| --------- | ------ | ----- | +| Choose between two scalar values based on a comparison or classification (value-based branching) | `IF(condition, A, B)` or `SWITCH(...)` | Wrapping in IFBLANK when both values are always defined | +| "First non-blank wins" (override hierarchy) | `IFBLANK(Expr1, IFBLANK(Expr2, ...))` | Nested `IF(cond1, Expr1, IF(cond2, Expr2, ...))` | +| Override when present, else base | `IFBLANK('Override', 'Base')` | `IF(ISBLANK('Override'), 'Base', 'Override')` | +| Branch on "is this input defined?" | `IFDEFINED('Input', 'Input', 'Fallback')` or `IFBLANK(ISDEFINED('Input'), 'Fallback'[EXCLUDE: ...])` (advanced) | Nested IF with AND/OR on flags | +| Subset an existing metric by row | `'Metric'[FILTER: ...]` or `[EXCLUDE: ...]` | Repeating the metric in both IF branches | +| Exclude a flagged subset | `'Metric'[EXCLUDE: 'Entity'.'Is_Archived']` | `'Metric'[FILTER: NOT 'Entity'.'Is_Archived']` | +| Sparse boolean "tag where condition holds" | `IF(condition, TRUE)` or `IF(condition, TRUE, BLANK)` | `condition` alone (densifies to TRUE/FALSE) | +| Both IF branches share same expression (e.g. same metric + modifiers) | Factor out: `Expr * IF(cond, a, b)` or `Expr + IF(cond, x, y)` | Repeating `Expr` in both branches | +| Multiple branches repeat same FILTER/EXCLUDE conditions (3+ similar blocks) | Nested IF that factors common expression; vary only the differing part (see section 2.5) | IFBLANK with repeated FILTER/EXCLUDE on each branch (can degrade performance) | + +--- + +## 1. IFBLANK for presence-driven branching + +IFBLANK is for presence-driven branching: + +- Precedence / case chains ("first non-blank wins") +- Override vs default logic + +For value-based branching (comparison-driven classification, threshold assignment), prefer IF or SWITCH - they are the natural tool when both branches always produce a value. + +### 1.1 Precedence chains ("first non-blank wins") + +Goal: Choose the first expression that returns a value; if blank, try the next. + +Canonical pattern: + +```pigment +// First non-blank among Expr1, Expr2, Expr3, Expr4 +IFBLANK( + Expr1, + IFBLANK( + Expr2, + IFBLANK( + Expr3, + Expr4 + ) + ) +) +``` + +- If Expr1 has a value -> use it; else Expr2; else Expr3; else Expr4. +- Use for: override hierarchy, multi-source chaining (e.g. plan -> actual -> baseline -> constant). +- If any expression is complex, move it into a helper metric and reference it in the chain. + +### 1.2 Override vs base (no explicit ISDEFINED) + +Goal: Use an override when present; otherwise use a base value. + +Canonical pattern: + +```pigment +IFBLANK('Override_Metric', 'Base_Metric') +``` + +Avoid: + +```pigment +IF(ISBLANK('Override_Metric'), 'Base_Metric', 'Override_Metric') +``` + +### 1.3 Sparsity-driven branching on definedness + +Goal: When the driver for branching is "does this input exist?", not its value. + +Default pattern (when you need "use override if present, else fallback"): + +```pigment +IFDEFINED('Input', 'Input', 'Fallback') +``` + +Or equivalently: + +```pigment +IFBLANK('Input', 'Fallback') +``` + +Advanced pattern (when you need a TRUE-or-BLANK signal as the branch trigger, e.g. to scope the fallback with EXCLUDE): + +```pigment +IFBLANK( + ISDEFINED('Context_Specific_Input'), + 'Fallback_Expression'[EXCLUDE: 'Entity'.'Excluded_Flag'] < 'Context'.'Cutoff' +) +``` + +- First argument: test signal (TRUE where context input exists, BLANK where it doesn't). +- Second argument: fallback expression when there is no context-specific input. +- Use EXCLUDE (or FILTER) to scope the fallback by row, not AND/OR inside the expression. + +Use the advanced form only when you need EXCLUDE/FILTER scoping on the fallback branch. For most override-vs-fallback formulas, `IFBLANK('Override', 'Fallback')` or `IFDEFINED('Input', 'Input', 'Fallback')` are clearer. + +### 1.4 When to refactor a nested IF chain into IFBLANK + +Nested IF is the natural form for: + +- Value-based threshold classification (e.g. risk tiers, margin buckets, score bands) +- Exact-match multi-way branching (prefer SWITCH for this) +- Cases where both branches always produce a defined value + +Refactor to IFBLANK when the branching driver is data presence (one or more branches may be BLANK). + +Less idiomatic for presence-driven logic: + +```pigment +IF(cond1, Expr1, IF(cond2, Expr2, IF(cond3, Expr3, Expr4))) +``` + +Prefer one of (when presence is the driver): + +- Precedence chain: `IFBLANK(Expr1, IFBLANK(Expr2, IFBLANK(Expr3, Expr4)))` (when "first defined wins"). +- Override vs base: `IFBLANK('Override', 'Base')`. +- Signal-based: `IFBLANK(ISDEFINED('Input'), 'Fallback'[EXCLUDE: ...])` (advanced). + +For 3+ branches with the same FILTER/EXCLUDE stack repeating, see section 2.5 - nested IF with factored conditions is often faster. + +--- + +## 2. FILTER and EXCLUDE: When and How + +FILTER and EXCLUDE control which dimensional combinations exist in an expression. Use them when the question is "which rows/coordinates should this apply to?", not "which of two scalar expressions do I pick?". + +### 2.1 Use separate FILTERs (and EXCLUDEs) + +Canonical: + +```pigment +'Metric' +[FILTER: 'Entity'.'Status' = "Active"] +[FILTER: 'Entity'.'Region' = "EMEA"] +``` + +Less preferred (one big AND): + +```pigment +'Metric'[FILTER: 'Entity'.'Status' = "Active" AND 'Entity'.'Region' = "EMEA"] +``` + +Separate modifiers are easier to read, debug, and mix with EXCLUDE. + +### 2.2 Filter by expression value: use CURRENTVALUE + +Canonical: + +```pigment +'Revenue'[FILTER: CURRENTVALUE > 0] +'Margin %'[FILTER: CURRENTVALUE >= 0.20][FILTER: 'Entity'.'Status' = "Live"] +``` + +This avoids repeating the expression and keeps value rules and property rules separate. + +### 2.3 Exclude a flagged subset: use EXCLUDE, not FILTER NOT + +Canonical: + +```pigment +'Metric'[EXCLUDE: 'Entity'.'Is_Discontinued'] +'Metric'[EXCLUDE: 'Entity'.'Is_Archived'][EXCLUDE: 'Entity'.'Is_Test_Data'] +``` + +Avoid: + +```pigment +'Metric'[FILTER: NOT 'Entity'.'Is_Archived'] +``` + +EXCLUDE keeps semantics positive ("drop these rows") and preserves sparsity; NOT over a boolean that can be BLANK densifies. See section 3. + +### 2.4 Factorize common subexpressions in IF branches + +When both branches of an IF use the same expression (e.g. the same metric with the same FILTER/EXCLUDE), prefer factoring it out so the IF only chooses the differing part (multiplier, constant, etc.). This keeps expressions small and composable (DRY). + +Less idiomatic (repeated EXCLUDE and base expression in both branches): + +```pigment +IF( + IsForecast + AND ( ... segment test ... ), + 'Revenue'[EXCLUDE: Customers.'Exclude from ARR Report'], + 'Revenue'[EXCLUDE: Customers.'Exclude from ARR Report'] +) +``` + +More idiomatic (one EXCLUDE, IF only on the multiplier): + +```pigment +'Revenue'[EXCLUDE: Customers.'Exclude from ARR Report'] +* IF( + IsForecast + AND ( ... segment test ... ), + 1.10, + 1 + ) +``` + +Apply the same idea when the only difference between branches is an additive constant, a divisor, or another scalar: factor the common expression once and use IF for the varying part. + +### 2.5 When nested IF is acceptable (repeated FILTER/EXCLUDE) + +When multiple branches (e.g. 3+) use the same FILTER/EXCLUDE conditions with only a varying expression or multiplier, the IFBLANK/FILTER pattern can degrade performance. Each branch evaluates its own scoped expression; repeated modifiers across many branches add overhead. + +Prefer a nested IF that factors out the common logic and uses IF only for the varying part: + +```pigment +// Multiple branches with same conditions +// Prefer: factor common conditions, vary only the differing expression +IF( + 'Headcount ID'.Approved? AND ( + 'WFP_Elect to Backfill?' = FALSE + OR ISBLANK('WFP_Elect to Backfill?') + OR ('WFP_Elect to Backfill?' AND Month < TIMEDIM('WFP_Backfill Date', Month)) + ), + 'WFP_Salary' + * IF(Month >= 'WFP_Merit Effective', 1, 1 - 'Global Merit Adjustment'[BY: 'Headcount ID'.Geo]) + * 1/12 * 'FTE in Dept Calc', + BLANK +) +``` + +Guidance: When the same `[FILTER: X][FILTER: Y][EXCLUDE: Z]` would repeat on 3+ IFBLANK branches, consider a nested IF that factors the common conditions instead. Benchmarks have shown nested IF can be meaningfully faster in such cases; verify on your workload before committing to either form. + +Rule of thumb: If you would write the same `[FILTER: X][FILTER: Y][EXCLUDE: Z]` on 3+ branches, consider a nested IF that factors the common conditions instead. + +--- + +## 3. Logical Operators and 3-State Booleans + +### 3.1 Pigment booleans are 3-state + +- TRUE -> dense +- FALSE -> dense +- BLANK -> sparse (no value at that coordinate) + +Prefer patterns that preserve BLANK as a distinct state (IFBLANK, ISDEFINED, FILTER, EXCLUDE). Avoid turning BLANK into TRUE or FALSE unless necessary. + +### 3.2 Avoid NOT in favor of EXCLUDE / positive conditions + +NOT flips TRUE<->FALSE. For BLANK, NOT has no meaningful opposite; applying NOT to a boolean that can be BLANK densifies (BLANKs become TRUE or FALSE). Prefer EXCLUDE or a positive condition. + +Rule: Do not write `[FILTER: NOT 'Flag']` or `[FILTER: NOT 'Entity'.'Is_Archived']`. Use: + +```pigment +[EXCLUDE: 'Flag'] +[EXCLUDE: 'Entity'.'Is_Archived'] +``` + +Same for IF: avoid `IF(NOT 'Entity'.'Flag', 'Metric', BLANK)`. Use `'Metric'[EXCLUDE: 'Entity'.'Flag']`. + +Exception: When the boolean is known to be TRUE/FALSE (no BLANK values, e.g. imported from an ERP system or explicitly set for every item), `FILTER: NOT 'Flag'` is acceptable and may be more readable. EXCLUDE is the safer default when the boolean's BLANK behavior is uncertain or when sparsity matters. + +### 3.3 Sparse boolean masks: IF(condition, TRUE, BLANK) + +A bare comparison like `'Metric' = 'Other'` yields a dense boolean (TRUE/FALSE everywhere). To keep a sparse boolean (TRUE only where the condition holds, BLANK elsewhere): + +Canonical: + +```pigment +IF('Metric' = 'Something Else', TRUE) +// or explicitly +IF('Metric' = 'Something Else', TRUE, BLANK) +``` + +Use this when you need a sparse "tag" of cells that meet a condition, without densifying the rest to FALSE. diff --git a/plugins/pigment/skills/writing-pigment-formulas/formula_modifiers.md b/plugins/pigment/skills/writing-pigment-formulas/formula_modifiers.md new file mode 100644 index 00000000..7cc53da4 --- /dev/null +++ b/plugins/pigment/skills/writing-pigment-formulas/formula_modifiers.md @@ -0,0 +1,578 @@ +# Formula Modifiers + +Dimensional transformations using modifiers for aggregation (N->1 or N->none) and allocation (1->N or none->N). + +Key Concept: Modifiers are used for dimensional transformations and require square brackets with a colon: `[BY: ...]`, `[REMOVE: ...]`, `[KEEP: ...]`, `[ADD: ...]`, `[SELECT: ...]`, `[FILTER: ...]`, `[EXCLUDE: ...]`, `[TOPARENTLIST: ...]`, `[TOSUBSET: ...]`. + +--- + +## Understanding Source and Target + +- Source: The Metric or List you are pulling data FROM (referenced in your formula) +- Target: The Metric or List you are writing the formula IN + +Both have dimensions. When dimensions differ, use modifiers to control how values are calculated or distributed. + +--- + +## Dimension Flow Rules + +To verify dimensional alignment, trace how dimensions change through each operation. + +How Operations Change Dimensions: + +| Operation | Effect | Example | +| ---------------- | ----------------------------------- | ----------------------------------------------- | +| `[REMOVE: X]` | Removes X | `/*Prod,Reg,Mo*/[REMOVE: Prod]` -> `/*Reg,Mo*/` | +| `[BY: Prop]` | Replaces dim with property's parent | `/*Prod*/[BY: Prod.Category]` -> `/*Category*/` | +| `[ADD: X]` | Adds X (densifies!) | `/*Reg*/[ADD: Mo]` -> `/*Reg,Mo*/` | +| `[KEEP: X, Y]` | Keeps only X and Y | `/*Prod,Reg,Mo*/[KEEP: Prod]` -> `/*Prod*/` | +| `[SELECT: cond]` | Filters AND removes dimension | `/*Prod,Month*/[SELECT: Month = VAR_Reference_Month]` -> `/*Prod*/` | +| `[FILTER: cond]` | Filters, keeps all dimensions | `/*Prod,Month*/[FILTER: Month = VAR_Reference_Month]` -> `/*Prod,Month*/` | +| `[TOPARENTLIST: Subset]` | Subset dimension -> parent list (1:1; parent items outside subset -> blank) | `/*Subset,Month*/` -> `/*Parent,Month*/` | +| `[TOSUBSET: Subset]` | Parent dimension -> subset (1:1; filters to subset members only) | `/*Parent,Month*/` -> `/*Subset,Month*/` | + +Combining Expressions (union rule): + +When combining expressions (`A + B`, `IF(cond, A, B)`, `IFBLANK(A, B)`), the result has the union of all dimensions from all operands. + +Note: `[BY: dim.prop]` removes `dim` and adds `prop`'s dimension - no need to REMOVE separately. + +--- + +## Four Relationship Types + +| Source->Target | Type | Modifier | Available Methods | Example | +| ----------------------- | ----------------------- | -------- | ------------------------- | ------------------ | +| N -> 1 | Aggregation | `BY` | SUM, AVG, MIN, MAX, COUNT | Country->Region | +| N -> none | Aggregation | `REMOVE` | SUM, AVG, MIN, MAX, COUNT | Remove Product | +| N -> none (filtered) | Conditional Aggregation | `SELECT` | SUM, AVG, MIN, MAX, COUNT | Filter & aggregate | +| 1 -> N | Allocation | `BY` | CONSTANT, SPLIT | Region->Country | +| none -> N | Allocation | `ADD` | CONSTANT, SPLIT | Add Country | + +Default Aggregation: SUM | Default Allocation: CONSTANT + +--- + +## Understanding Parent Dimensions (Hierarchies) + +Lists can have properties of type "Dimension" that reference other dimension lists, creating hierarchies: + +- Country list has "Region" property -> Country -> Region +- Month list has "Quarter" property -> Month -> Quarter -> Year +- Product list has "Category" property -> Product -> Category + +### BY with Parent Dimensions + +#### Aggregation (N->1): Going Up + +```pigment +'Country Revenue'[BY SUM: Country.Region] // Countries -> Regions +'Monthly Sales'[BY SUM: Month.Quarter] // Months -> Quarters +'Product Revenue'[BY SUM: Product.Category] // Products -> Categories +'Employee Salary'[BY SUM: Employee.Department] // Employees -> Departments +``` + +#### Allocation (1->N): Going Down + +```pigment +// Replicate (CONSTANT - default) +'Region Budget'[BY CONSTANT: Country.Region] // Same value to all countries +'Category Price'[BY CONSTANT: Product.Category] // Same price to all products + +// Split equally +'Region Revenue'[BY SPLIT: Country.Region] // Equally distribute +'Quarterly Target'[BY SPLIT: Month.Quarter] // Equally distribute + +// Seed a specific dimension member (MP02) - VAR_Actual_Version: input metric, type Dimension +'ActualData'[BY CONSTANT: VAR_Actual_Version] // not Version."Actual" +``` + +#### Multi-Level Hierarchies + +```pigment +'Monthly Revenue'[BY: Month.Quarter][BY: Quarter.Year] +'City Revenue'[BY: City.Country][BY: Country.Region] +``` + +--- + +## TOPARENTLIST and TOSUBSET (list subsets) + +Use these when a metric is on a list subset dimension and you need the same values on the parent list (or the reverse). They perform a 1:1 dimensional remap (each subset item maps to its parent item); they do not aggregate or allocate, so you do not specify `SUM`, `BY CONSTANT`, etc. + +- TOPARENTLIST - subset dimension -> parent dimension. Parent items not in the subset become blank on the result. +- TOSUBSET - parent dimension -> subset dimension. Data on parent items outside the subset is dropped from the result. + +Syntax (spaces around `:` are optional in the product docs): + +```pigment +'Metric on Subset'[TOPARENTLIST: 'My Subset'] +'Metric on Parent'[TOSUBSET: 'My Subset'] +``` + +When to prefer these vs `[BY: ...]` on a mapping property + +- Prefer TOPARENTLIST / TOSUBSET for the straight subset <-> parent remap of the same list's items (natural 1:1 identity between subset row and parent row). +- Keep `[BY: Parent.'Subset Mapping']` (or similar) when you need a custom mapping, multiple subsets feeding one structure, or other shapes the subset modifiers do not cover - see [List Subsets](../modeling-pigment-applications/modeling_subsets.md). + +The compiler enforces valid combinations (e.g. the expression must be dimensioned by the subset for TOPARENTLIST with that subset, and structural rules about which dimensions may appear together). If a formula is rejected, read the error and fall back to an explicit mapping + `BY` pattern when appropriate. + +--- + +## Modifier Reference + +### BY Modifier + +Most versatile - aggregates up or allocates down using mapping attributes. + +Syntax: `Block[BY: mapping_attribute]` + +Only list dimensions whose grain is changing. BY only transforms the dimension(s) you specify; it doesn't remove other dimensions. List in BY only the dimensions (or hierarchy steps) you are aggregating or allocating over. Do not re-list dimensions that are already on the metric and unchanged - that is unnecessary and verbose (over-explicit BY). + +If a dimension already exists on the metric and you don't reference it in BY, it stays as-is. To remove dimensions, use REMOVE. + +Example - avoid over-explicit BY: Metric is dimensioned by Employee and Month. To normalize by year total, only the time grain changes (Month -> Year). Put only `Month.'Year'` in BY: + +```pigment +// BAD: Over-explicit: re-listing Employee and Month.'Year' when only year grain changes +'Metric' / 'Metric'[BY SUM: Employee, Month.'Year'][BY CONSTANT: Employee, Month.'Year'] + +// GOOD: Correct: only the dimension whose grain changes +'Metric' / 'Metric'[BY SUM: Month.'Year'][BY CONSTANT: Month.'Year'] +``` + +Aggregation (child -> parent): + +```pigment +'Country Revenue'[BY: Country.Region] // Sum by default +'Product Price'[BY AVERAGE: Product.Category] // Average +``` + +Allocation (parent -> child): + +```pigment +'Region Budget'[BY: Country.Region] // Constant by default +'Region Revenue'[BY SPLIT: Country.Region] // Split equally +``` + +Add dimension: + +```pigment +'Global Price'[BY: Product] // Add Product dimension +``` + +#### Sparsity via BY + dimension-typed metrics + +When one of the BY arguments is a dimension-typed metric (e.g. `'CALC_Employee_Tenure_Month_Index'` typed as a time or index dimension), that metric's sparsity is respected automatically: the result is only defined where the metric is defined. + +Do not add IF(ISBLANK(metric), BLANK, ...) or similar guards - they are redundant and harmful (ISBLANK densifies). + +Anti-pattern (redundant guard, densifies): + +```pigment +IF( + ISBLANK('CALC_Employee_Tenure_Month_Index'), + BLANK, + 'ASM_Ramp_Schedule'[BY: Employee.Segment, 'CALC_Employee_Tenure_Month_Index'] +) +``` + +Correct (BY alone; sparsity preserved): + +```pigment +'ASM_Ramp_Schedule'[BY: Employee.Segment, 'CALC_Employee_Tenure_Month_Index'] +``` + +Methods: + +- Aggregation: SUM (default), AVERAGE, MIN, MAX, COUNT +- Allocation: CONSTANT (default), SPLIT + +BY with mapping metrics (arrow `->`): When the mapping is a dimension-typed metric (e.g. `'Country to Region Map'`) rather than a simple property, use the arrow syntax: `Source[BY Method: SourceDim -> MappingMetric, ...]`. See [formula_by_mapping_arrow.md](./formula_by_mapping_arrow.md) for full syntax, rules, and pitfalls (e.g. unintended aggregation on Version, Time, Scenario). + +--- + +### REMOVE Modifier + +Remove dimension and aggregate to remaining dimensions. + +```pigment +'Revenue'[REMOVE: Product] // Remove Product (sum) +'Sales'[REMOVE SUM: Region] // Explicit SUM +'Salary'[REMOVE AVERAGE: Employee] // Average +``` + +Note: Loses scope (performance impact). Prefer BY when possible. + +For the tiered/banded lookup pattern (assign each item to a band or tier based on thresholds on a dimension, e.g. account segmentation, salary bands), see [formula_segmentation_tiered_lookup.md](./formula_segmentation_tiered_lookup.md) - it uses `IF(...)[REMOVE FIRSTNONBLANK: Dim]` or `LASTNONBLANK`. + +--- + +### KEEP Modifier + +Retain specific dimensions, aggregate away the rest. + +Syntax: `Block[KEEP [METHOD]: Dimension1, Dimension2, ...]` + +```pigment +'Revenue'[KEEP: Country] // Keep only Country, sum over all other dimensions +'Revenue'[KEEP AVG: Month] // Keep Month, average over all other dimensions +'Revenue'[KEEP FIRSTNONBLANK ON RANK(Product): Country, Month] // Keep Country and Month +``` + +Methods: SUM (default), AVG, MIN, MAX, FIRST, FIRSTNONBLANK +Note: FIRST/FIRSTNONBLANK with multiple dimensions require `ON RANK(Dimension)` + +--- + +### SELECT Modifier + +Conditional aggregation - filters by condition AND removes the filtered dimension. + +```pigment +// Sum only items matching condition (removes dimension) +// VAR_Labor_Type: input metric, type Dimension +'Labor_Cost' = 'Cost'[SELECT SUM: Type = VAR_Labor_Type] + +// Multiple conditions - VAR metrics for item-specific members +'Q1_North' = 'Revenue'[SELECT SUM: Month.Quarter = VAR_Selected_Quarter AND Region = VAR_Selected_Region] + +// With parent dimensions - prefer property-based filter when semantic +'Category_Revenue' = 'Revenue'[SELECT SUM: Product.'Include Category'] +``` + +Key Behavior: SELECT with condition = FILTER + REMOVE in one operation + +Methods: SUM (default), AVERAGE, MIN, MAX, COUNT + +Note: SELECT with condition removes the filtered dimension (use FILTER to keep dimension) + +#### SELECT with Dimension Offsets (Different Behavior) + +Offset SELECT is different - it shifts the dimension value but keeps the dimension. + +Syntax: `Block[SELECT: Dimension +/- N]` + +Works with any ordered dimension (time dimensions, ranked lists, etc.). The offset shifts by N positions in the dimension's sort order. + +Examples: + +```pigment +// Time dimension offsets (most common) +'Revenue'[SELECT: Month-1] // Previous month's value +'Revenue' - 'Revenue'[SELECT: Month-1] // Month-over-Month change +'Revenue'[SELECT: Month-12] // Same month last year + +// Other ordered dimensions +'Sales Rank'[SELECT: Product-1] // Previous product in ranking +'Score'[SELECT: Employee+1] // Next employee in order + +// Specific item selection - VAR_Reference_Month: input metric, type Dimension +'Revenue'[SELECT: Month = VAR_Reference_Month] // Value for specific month (keeps Month dimension) +``` + +Time Dimensions: For prior period lookups, SELECT is fast (parallel). PREVIOUS/PREVIOUSOF are slow (iterative) - only use when current value depends on calculating prior value first (e.g., running balances). See [functions_iterative_calculation.md](./functions_iterative_calculation.md) for when and how to use PREVIOUS/PREVIOUSOF. + +See [functions_time_and_date.md](./functions_time_and_date.md) for SELECT vs PREVIOUS and time function guidance. + +--- + +### FILTER Modifier + +Filter data based on a boolean condition. Keeps the dimension (unlike SELECT which removes it). + +Syntax: `Block[FILTER: BooleanCondition]` + +Style: Prefer separate FILTERs for readability (e.g. one FILTER per condition) instead of one FILTER with a long AND. See [formula_conditionals_style.md](./formula_conditionals_style.md). + +MP02 - dimension members in conditions: MUST NOT hard-code `Dimension."Item"`. Create a `VAR_` input metric of type Dimension; the item literal may appear only in its default value. + +Examples: + +```pigment +// Filter by dimension member via VAR metrics +'Revenue'[FILTER: Country = VAR_Selected_Country] +'Revenue'[FILTER: Country = VAR_Selected_Country AND Product = VAR_Selected_Product] + +// Filter multiple members - prefer a boolean property or mapping metric +'Revenue'[FILTER: Country.'Include in Report'] + +// Filter by value threshold +'Revenue'[FILTER: 'Revenue' > 1000] +'Revenue'[FILTER: CurrentValue > 1000] // CurrentValue = same as expression + +// Filter by boolean metric +'Revenue'[FILTER: 'Is Active'] + +// Filter via parent-dimension property (semantic, MP02-safe) +'Revenue'[FILTER: Product.'Include Category'] +``` + +CurrentValue Keyword: Use `CurrentValue` to reference the filtered expression itself, avoiding repetition: + +```pigment +// Instead of repeating the expression: +('Revenue' * 'Margin')[FILTER: ('Revenue' * 'Margin') > 1000] + +// Use CurrentValue: +('Revenue' * 'Margin')[FILTER: CurrentValue > 1000] + +// More complex example - filter after aggregation +'Revenue'[REMOVE: Product][FILTER: CurrentValue > 100000] +// Equivalent to: +'Revenue'[REMOVE: Product][FILTER: 'Revenue'[REMOVE: Product] > 100000] +``` + +When to Use CurrentValue: + +- When filtering on the result of a complex expression +- When filtering after an aggregation (REMOVE, BY) +- To avoid recalculating the same expression in the condition + +When subsetting a computed expression by its value, prefer this pattern over `IF(Expression > threshold, Expression, BLANK)` - it evaluates once per cell and is often faster. See [formula_performance_patterns.md](./formula_performance_patterns.md) (Pattern 4, Case B). + +Key Points: + +- Keeps dimension - Result has same dimensions as source +- Returns values where condition is TRUE, BLANK elsewhere +- Blanks in condition result in BLANK output (not included) + +--- + +### EXCLUDE Modifier + +Exclude data based on a boolean condition. Opposite of FILTER - removes matching rows. + +Syntax: `Block[EXCLUDE: BooleanCondition]` + +Examples: + +```pigment +// Exclude specific dimension members via VAR metric or boolean property +'Revenue'[EXCLUDE: Country = VAR_Excluded_Country] +'Revenue'[EXCLUDE: Country = VAR_Excluded_Country AND Product = VAR_Excluded_Product] + +// Exclude multiple members - prefer boolean property +'Revenue'[EXCLUDE: Country.'Exclude from Report'] + +// Exclude by value threshold +'Revenue'[EXCLUDE: 'Revenue' > 1000] // Keep values <= 1000 or BLANK +'Revenue'[EXCLUDE: CurrentValue > 1000] // CurrentValue = same as expression + +// Exclude by boolean metric +'Revenue'[EXCLUDE: 'Is Inactive'] +``` + +CurrentValue Keyword: Same as FILTER - use to reference the expression being filtered. + +FILTER vs EXCLUDE: + +```pigment +// These are equivalent: +'Revenue'[FILTER: Country = VAR_Selected_Country] +'Revenue'[EXCLUDE: NOT(Country = VAR_Selected_Country)] + +// But they differ on BLANK handling: +// FILTER: condition must be TRUE -> BLANKs excluded +// EXCLUDE: condition must be TRUE to exclude -> BLANKs included +``` + +When to Use EXCLUDE vs FILTER: + +- Use EXCLUDE when you want to keep BLANKs (recommended for readability and performance) +- Use FILTER when you want to remove BLANKs +- Always prefer EXCLUDE for exclusions - do not use `FILTER: NOT(...)`. NOT over a boolean that can be BLANK densifies; EXCLUDE preserves sparsity. See [formula_conditionals_style.md](./formula_conditionals_style.md). + +--- + +### ADD Modifier + +Add dimension where source has none. Creates values for all items in the added dimension. + +Syntax: `Block[ADD: Dimension]` or `Block[ADD SPLIT: Dimension]` + +```pigment +'Global Price'[ADD: Country] // Same value for every country +'Total Budget'[ADD SPLIT: Country] // Equally distribute across countries +``` + +Methods: CONSTANT (default), SPLIT + +WARNING: Performance Warning: ADD creates dense structures (all combinations). Prefer BY when a mapping exists. Make sure to check [formula_performance_patterns.md](./formula_performance_patterns.md) for guidance. + +--- + +## Aggregation & Allocation Methods + +Aggregation Methods (work with BY, REMOVE, KEEP, SELECT): + +| Data Type | Available Methods | +| ------------------ | ---------------------------------------------------------------------------------- | +| Number/Integer | SUM (default), AVG, MIN, MAX, FIRSTNONZERO | +| Date | MIN, MAX | +| Boolean | ANY, ALL | +| Text | TEXTLIST | +| All types | FIRST, LAST, FIRSTNONBLANK, LASTNONBLANK, COUNT, COUNTBLANK, COUNTALL, COUNTUNIQUE | + +Allocation Methods (work with BY, ADD): + +- CONSTANT (default) - Same value for all items +- SPLIT - Equally distribute based on number of items + +--- + +## Implicit Behavior (No Modifiers) + +When dimensions differ without modifiers: + +- Fewer target dims -> Implicitly sums (like `REMOVE SUM`) +- More target dims -> Implicitly replicates (like `ADD CONSTANT`) + +Best Practice: Use explicit modifiers when you need to change grain or alignment. "Explicit" means specifying the transformation you need (e.g. `BY SUM: Month.'Year'`), not repeating dimensions that are already on the metric and unchanged - avoid over-explicit BY. + +--- + +## Common Patterns + +```pigment +// Multi-level aggregation +'Product Revenue'[BY: Product.Category][BY: Category.Division] + +// Weighted average +('Revenue' * 'Margin')[BY SUM: Country.Region] / 'Revenue' + +// Normalization (ratio to total): double BY in denominator - only list dimensions whose grain changes +// Metric is on Employee and Month; normalize to year total -> only Month.'Year' in BY +'Metric' / 'Metric'[BY SUM: Month.'Year'][BY CONSTANT: Month.'Year'] + +// Percentage of total +'Country Revenue' / 'Country Revenue'[REMOVE: Country] + +// Allocation: equally distribute +'Region Revenue'[BY SPLIT: Country.Region] + +// Transaction aggregation with TIMEDIM +'Transactions'.'Amount'[BY: TIMEDIM('Transactions'.'Date', Month)] + +// Transaction aggregation with TIMEDIM and other dimensions +'Orders'.'Amount'[BY: TIMEDIM('Orders'.'Date', Quarter), Customer] + +// Transaction aggregation with filtering +'Transactions'.'Amount'[SELECT: 'Transactions'.'Date' >= DATE(2024,1,1)][BY: TIMEDIM('Transactions'.'Date', Month)] +``` + +--- + +## Critical Rules + +### Transaction List Properties in Metrics + +A Transaction List is not a dimension and cannot be used as a structural dimension of a metric. For the distinction between Dimension list and Transaction list, see [modeling_fundamentals](../modeling-pigment-applications/modeling_fundamentals.md) (section 2.3). To use Transaction List data in a metric, reference the list's properties in the formula and aggregate with BY (e.g. list properties of type Dimension, or TIMEDIM for dates). + +When using a transaction list property in a METRIC formula, you MUST aggregate it. + +Transaction lists are unbounded - they have rows but no dimensions. Metrics have dimensions. To use a list property in a metric, you must aggregate to align dimensions. + +```pigment +// BAD: WRONG: List property without aggregator in a metric +'Orders'.'Amount' // Error! No dimensions to align + +// GOOD: CORRECT: Aggregate list property to metric dimensions +'Orders'.'Amount'[BY: TIMEDIM('Orders'.'Date', Month)] +'Orders'.'Amount'[BY: 'Orders'.'Customer', TIMEDIM('Orders'.'Date', Month)] + +// GOOD: CORRECT: Multiple aggregation dimensions +'Transactions'.'Value'[BY: 'Transactions'.'Product', 'Transactions'.'Region'] +``` + +Note: This rule applies to metrics. Within the same list's property formulas, you can reference other properties directly without aggregation. + +Common Error Pattern: + +```pigment +// BAD: In a metric formula - ERROR +'Transactions'.'Amount' * 'Exchange Rate' + +// GOOD: In a metric formula - CORRECT +'Transactions'.'Amount'[BY: TIMEDIM('Transactions'.'Date', Month), 'Transactions'.'Currency'] * 'Exchange Rate' +``` + +### WARNING: NEVER Chain BY on Transaction Lists + +When aggregating by multiple dimensions, use a SINGLE BY with comma-separated expressions. + +After `list[BY: dim1]`, the result is on `dim1` only - the list's other properties are LOST. + +```pigment +// BAD: WRONG: Chaining BY loses list properties +'Orders'.'Amount'[BY: TIMEDIM('Orders'.'Date', Month)][BY: 'Orders'.'Product'] +// After first BY, result is on Month only - 'Orders'.'Product' is no longer accessible! + +// GOOD: CORRECT: Single BY with multiple dimensions +'Orders'.'Amount'[BY: TIMEDIM('Orders'.'Date', Month), 'Orders'.'Product'] +// Both dimensions applied together, result is on Month AND Product +``` + +Key Rule: Always use the LIST column name (e.g., `'Orders'.'Customer'`) in modifiers, not just the dimension name (`Customer`). + +### Transaction List Column Types + +Dimension-typed columns: Use directly in BY + +```pigment +'Orders'.'Amount'[BY: 'Orders'.'Customer'] // Customer is dimension-typed +``` + +Text columns referencing dimensions: Convert with ITEM() first + +```pigment +// 'Orders'.'ProductCode' is TEXT, but matches 'Products'.'Code' +'Orders'.'Amount'[BY: ITEM('Orders'.'ProductCode', 'Products'.'Code')] +``` + +Date columns: Convert with TIMEDIM() + +```pigment +// 'Orders'.'OrderDate' is DATE type, not a dimension +'Orders'.'Amount'[BY: TIMEDIM('Orders'.'OrderDate', Month)] +``` + +--- + +### Parent Dimensions (Hierarchies) + +- Dimension properties create hierarchies - Country.Region, Month.Quarter +- BY works both ways - Child->Parent aggregates UP, Parent->Child allocates DOWN +- Chain for multi-level - `[BY: Month.Quarter][BY: Quarter.Year]` + +### Modifier Behavior + +- BY - Most versatile (aggregation and allocation with hierarchies) +- REMOVE - Always aggregates, loses scope (prefer BY) +- KEEP - Retain only specified dimensions +- SELECT - Conditional aggregation, filters + removes dimension +- FILTER - Keeps dimension, includes matching rows, excludes BLANKs +- EXCLUDE - Keeps dimension, excludes matching rows, keeps BLANKs +- ADD - Creates dense structure (prefer BY for sparsity) +- Default aggregation: SUM | Default allocation: CONSTANT +- SPLIT - Equally distribute based on number of items + +### Performance (CRITICAL) + +- Prefer BY over ADD - BY is sparse (uses mappings), ADD is dense (all combinations) +- BY CONSTANT over ADD CONSTANT - Same value replication but sparse vs dense +- BY SPLIT over ADD SPLIT - Same splitting but sparse vs dense +- Prefer BY over REMOVE or KEEP - Preserves scope +- Explicit > Implicit - Better control; but in BY list only dimensions whose grain is changing - avoid over-explicit BY (re-listing unchanged dimensions). + +The BY vs ADD Rule: If a mapping property exists, use BY. Only use ADD as last resort when no mapping exists. + +--- + +## See Also + +- [formula_by_mapping_arrow.md](./formula_by_mapping_arrow.md) - BY with mapping metrics (arrow `->`): when and how to use it, dimension rules, pitfalls +- [functions_basic_aggregations.md](./functions_basic_aggregations.md) - OF functions (SUMOF, AVGOF, etc.) +- [formula_writing_workflow.md](./formula_writing_workflow.md) - 8-step formula writing process +- [formula_performance_patterns.md](./formula_performance_patterns.md) - Performance optimization diff --git a/plugins/pigment/skills/writing-pigment-formulas/formula_performance_patterns.md b/plugins/pigment/skills/writing-pigment-formulas/formula_performance_patterns.md new file mode 100644 index 00000000..8589c6c7 --- /dev/null +++ b/plugins/pigment/skills/writing-pigment-formulas/formula_performance_patterns.md @@ -0,0 +1,435 @@ +# Formula Performance Patterns + +Apply this checklist proportionally to formula complexity. Simple arithmetic between existing same-dimensioned metrics (e.g. `'A' + 'B'`, `'A' * 'B'`, `'A' / 'B'`) needs no performance wrapping - deliver as-is. Review these patterns for formulas that involve conditionals, dimensional changes, date-range logic, or that target large/sparse metrics. + +This checklist ensures formulas are performant. For detailed explanations, see the `skill:optimizing-pigment-performance`. + +--- + +## Performance Checklist + +Before delivering a formula, verify the applicable items: + +Always check (universal): + +- [ ] Identifiers are correctly quoted (single quotes for names, double quotes for items) +- [ ] Dimensions are aligned - no unintended ADD or dimension mismatch +- [ ] Scoping clauses appear FIRST (FILTER, EXCLUDE, IFDEFINED) +- [ ] Aggregations appear AFTER calculations + +Check when conditionals are present: + +- [ ] Avoid ISBLANK on large sparse metrics - use IFDEFINED or ISDEFINED +- [ ] Use IFBLANK for defaults, not IF(ISBLANK(...)) +- [ ] Conditional creation: use IF (not ADD + FILTER); subsetting a computed expression: use FILTER: CurrentValue (not IF(expr, expr, BLANK)) + +Check when date ranges are defined by Start/End: + +- [ ] Avoid multi-conditional IFs (`Date >= Start AND Date < End`) when PRORATA semantics apply +- [ ] Prefer `PRORATA()` for "active within a date range" and derive booleans/numeric flags from `PRORATA()` (use ISDEFINED/IFDEFINED, not ISBLANK/ISNOTBLANK) + +Check when prior period lookups are needed: + +- [ ] Using SELECT for prior period lookups, NOT PREVIOUS + +Check when dimensional changes or mappings are involved: + +- [ ] Using BY instead of ADD where mapping exists +- [ ] Do not use ISBLANK/ISNOTBLANK to guard BY when a dimension-typed metric is in BY; BY respects that metric's sparsity (see [formula_modifiers.md](./formula_modifiers.md)) + +Check when the metric is large/sparse or involves access rights: + +- [ ] Use BLANK instead of 0 or FALSE for empty values (see Pattern 9 exception for meaningful zeros) +- [ ] Access rights wrapped in IFDEFINED(User, ...) + +--- + +## Core Principles + +1. Scope First: Start formulas with scoping clauses +2. Preserve Sparsity: Use ISDEFINED, not ISBLANK +3. Reduce Early: Aggregate/filter before complex calculations +4. Understand Execution Order: Structure for minimal computation + +--- + +## Understanding Sparsity and Densification + +### Key Terminology + +BLANK, undefined, and "not defined" are ALL THE SAME THING - they mean no value exists in a cell. + +| Term | Meaning | Stored in Database? | +| ----------- | ---------------------- | ------------------- | +| BLANK | No value exists | No (sparse) | +| undefined | No value exists | No (sparse) | +| not defined | No value exists | No (sparse) | +| FALSE | Explicit boolean value | Yes (dense) | +| TRUE | Explicit boolean value | Yes (dense) | +| 0 | Explicit numeric value | Yes (dense) | + +### What is Densification? + +Densification occurs when cells that should have no value (BLANK/undefined) are given explicit values (TRUE, FALSE, 0, etc.). This forces Pigment to store and compute ALL cells instead of just the ones with actual data. + +Example: A metric with 1,000 products x 12 months = 12,000 possible cells. If only 500 have actual values: + +- Sparse: 500 cells stored (4% of space) - empty cells remain undefined +- Dense: 12,000 cells stored (100% of space, 24x larger) - if we store 0 for empty cells, all 12,000 must be stored + +### Why ISBLANK Densifies + +`ISBLANK` returns explicit boolean values for ALL cells: + +- Where value exists: returns FALSE (stored) +- Where value is blank: returns TRUE (stored) + +Both TRUE and FALSE are stored -> all cells now have values -> dense. + +### Why ISDEFINED Preserves Sparsity + +`ISDEFINED` returns: + +- Where value exists: returns TRUE (stored) +- Where value is blank: returns BLANK (not stored) + +Only TRUE values are stored -> blank cells remain blank -> sparse. + +--- + +## Performance Patterns + +### Pattern 1: Early Scoping + +Why: Scoping at the end forces Pigment to compute ALL data first, then filter. Scoping at the start limits computation to only relevant data. + +Anti-pattern (computes everything, then filters): + +```pigment +('Revenue' * 'Growth' + 'Costs')[FILTER: 'Product'.'Active' = TRUE] +``` + +Optimized (filters first, computes only active products): + +```pigment +'Revenue'[FILTER: 'Product'.'Active' = TRUE] * 'Growth' + 'Costs' +``` + +--- + +### Pattern 2: Sparsity Preservation with IFDEFINED + +Why: ISBLANK returns explicit boolean values (TRUE/FALSE) for ALL cells, causing densification. IFDEFINED returns BLANK for undefined cells, preserving sparsity. + +Less idiomatic on large sparse metrics (densifies - ISBLANK returns TRUE for blank cells, FALSE for defined cells): + +```pigment +IF(ISBLANK('Revenue'), 0, 'Revenue' * 1.1) +``` + +What happens: Every cell gets a value (TRUE, FALSE, 0, or calculated result) -> dense. + +Optimized (preserves sparsity - returns BLANK for undefined cells): + +```pigment +IFDEFINED('Revenue', 'Revenue' * 1.1) +``` + +What happens: Only cells where Revenue is defined get calculated; others remain BLANK -> sparse. + +Context guard: On small already-dense metrics or where explicit TRUE/FALSE output is required (e.g. data-completeness exports), `IF(ISBLANK(...))` is acceptable; see [functions_logical.md](./functions_logical.md) allow-list. + +--- + +### Pattern 3: Use IFBLANK for Default Values + +Why: IFBLANK is simpler, clearer, and optimized for the common pattern of providing a default when a value is blank. + +Less idiomatic (verbose and densifies on large sparse metrics): + +```pigment +IF(ISBLANK('Revenue'), 'Default Revenue', 'Revenue') +``` + +Optimized (cleaner and faster): + +```pigment +IFBLANK('Revenue', 'Default Revenue') +``` + +Context guard: On small already-dense metrics, `IF(ISBLANK(...))` is functionally equivalent and acceptable if readability is the priority; see [functions_logical.md](./functions_logical.md) allow-list. + +--- + +### Pattern 4: IF vs FILTER - Two Distinct Cases + +The rule "use IF, not ADD + FILTER" applies only to conditional creation. When subsetting an expression you are already computing, prefer FILTER: CurrentValue over IF. Do not interpret "use IF" as "IF is always better than FILTER." + +#### Case A: Conditional creation (no existing expression / using ADD) + +Why: ADD creates all possible cells then filters (dense). IF creates only cells where the condition is true (sparse). + +Anti-pattern (dense - creates all Month cells, then filters): + +```pigment +10[ADD: Month][FILTER: Month > VAR_Reference_Month] +``` + +Optimized (sparse - only creates cells where condition is true): + +```pigment +// VAR_Reference_Month: input metric, type Dimension +IF(Month > VAR_Reference_Month, 10) +``` + +#### Case B: Subsetting an expression you're already computing (no ADD) + +Why: When IF repeats the same expensive expression in the condition and result, it will be evaluated twice. [FILTER: CurrentValue] computes it only once, then filters on the result - usually faster and clearer for large or complex calculations. + +For simple metric references or arithmetic on small spaces, IF is equally performant and often more readable. + +Less idiomatic on large spaces with expensive expressions (repeats expression): + +```pigment +IF( + MONTHDIF(Month.'Start Date', Employee.'Hire Date'[ADD: Month]) >= 0, + MONTHDIF(Month.'Start Date', Employee.'Hire Date'[ADD: Month]) + 1, + BLANK +) +``` + +Optimized (single evaluation, filter on result): + +```pigment +( + MONTHDIF(Month.'Start Date', Employee.'Hire Date'[ADD: Month]) + 1 +)[FILTER: CurrentValue > 0] +``` + +When to use each: + +- IF: Conditional creation - adding values to new cells (no ADD in the alternative). Prefer IF over `value[ADD: Dim][FILTER: condition]`. Also acceptable for simple expressions on small spaces where readability matters. +- FILTER: CurrentValue: Subsetting a computed expression - you have one expression and want to keep only cells where its value meets a condition. Prefer `(Expression)[FILTER: CurrentValue > threshold]` over `IF(Expression > threshold, Expression, BLANK)`, especially when Expression is non-trivial and the metric space is large. + +--- + +### Pattern 5: Defer Aggregations + +Why: Aggregating early reduces data then multiplies, which can lose precision or produce wrong results. Aggregating late ensures calculations happen at full granularity. + +Anti-pattern (aggregates first, then multiplies - wrong if Growth varies by Product): + +```pigment +'Revenue'[REMOVE: Product] * 'Growth' +``` + +Optimized (calculates at Product level, then aggregates): + +```pigment +('Revenue' * 'Growth')[REMOVE: Product] +``` + +--- + +### Pattern 6: Prefer BY over ADD + +Why: BY uses a mapping and is sparse (only computes for existing data). ADD creates all possible combinations and is dense. Always prefer BY when a mapping property exists. + +Anti-pattern (dense allocation to all combinations then filter, instead of targeted allocation): + +```pigment +'MyMetric'[ADD: Version][FILTER: Version = MyVersion] +``` + +Optimized (sparse allocation via mapping): + +```pigment +'MyMetric'[BY: MyVersion] +``` + +Note: BY requires a mapping property or a dimension formatted metric. If no mapping exists and you must use ADD, consider if the formula design can be changed. + +--- + +### Pattern 7: SELECT for Prior Period Lookups (NOT PREVIOUS) + +Why: SELECT is parallel (fast). PREVIOUS/PREVIOUSOF are sequential iterative functions (slow). Use SELECT for all simple lookups. + +Anti-pattern (slow - iterative computation): + +```pigment +'Last Month' = PREVIOUS(Month) +'MoM Change' = 'Revenue' - PREVIOUSOF('Revenue') +``` + +Optimized (fast - parallel computation): + +```pigment +'Last Month' = 'Revenue'[SELECT: Month-1] +'MoM Change' = 'Revenue' - 'Revenue'[SELECT: Month-1] +``` + +When PREVIOUS/PREVIOUSOF is OK: Only when current period's calculated result depends on prior period's calculated result (e.g., running balances: `PREVIOUSOF('Balance') + 'Inflow' - 'Outflow'`). See [functions_iterative_calculation.md](./functions_iterative_calculation.md) for full guidance. + +--- + +### Pattern 8: Access Rights with IFDEFINED(User) + +Why: Without IFDEFINED(User), access rights are computed for ALL users in the system. Wrapping in IFDEFINED(User) ensures computation only happens for the current user. + +Anti-pattern (computes for all users): + +```pigment +'Revenue'[AR: 'Rules'] +``` + +Optimized (computes only for current user): + +```pigment +IFDEFINED(User, 'Revenue'[AR: 'Rules']) +``` + +--- + +### Pattern 9: Use BLANK Instead of 0 (When Zero Has No Meaning) + +Why: Using 0 creates dense data with explicit zeros stored. BLANK preserves sparsity - empty cells take no storage or computation. + +Less idiomatic on large sparse metrics (creates explicit zeros - dense): + +```pigment +IF(condition, value, 0) +``` + +Optimized (preserves sparsity): + +```pigment +IF(condition, value, BLANK) +``` + +Or simply omit the else clause (defaults to BLANK): + +```pigment +IF(condition, value) +``` + +Exception - when 0 is a meaningful business value: + +Use 0 (not BLANK) when zero is a meaningful result that the user expects to see, or when 0 participates in downstream multiplication / summation where the additive identity matters. Examples: + +- Zero variance (budget equals actual) +- Zero balance (account fully settled) +- Zero growth rate (flat period) +- A line item is inactive but the total row must show 0, not BLANK + +```pigment +// Budget vs actual variance: show 0 when they match, not BLANK +IF('Actual' > 'Budget', 'Actual' - 'Budget', 0) +``` + +Rule of thumb: BLANK means "not applicable / no data at this coordinate." Zero means "the value is zero." Choose based on semantic intent. + +--- + +### Pattern 10: Use BLANK Instead of FALSE for Boolean Flags + +Why: FALSE is an explicit boolean value that gets stored. BLANK means "not defined" and is not stored. For sparse boolean metrics, use BLANK where the condition is not met. + +Anti-pattern (stores FALSE for every non-matching cell - dense): + +```pigment +// Creates TRUE/FALSE for ALL cells +ISNOTBLANK('Revenue') + +// Or explicitly returning FALSE +IF('Revenue' > 1000, TRUE, FALSE) +``` + +Optimized (only stores TRUE where condition is met - sparse): + +```pigment +// Returns TRUE where defined, BLANK (not FALSE) elsewhere +ISDEFINED('Revenue') + +// Or returning BLANK instead of FALSE +IF('Revenue' > 1000, TRUE, BLANK) +// Or simply: +IF('Revenue' > 1000, TRUE) +``` + +Key insight: `BLANK != FALSE`. BLANK means "no value" (not stored). FALSE means "explicit boolean false" (stored). Use BLANK for sparsity. + +--- + +### Pattern 11: Date Range Presence (Prefer PRORATA over multi-conditional IF) + +Why: Single source of truth for date-range presence, less verbose, correct boundaries (Start included, End+1 for inclusive), sparsity preserved when deriving via ISDEFINED/IFDEFINED. + +Less idiomatic on Day-level dimensions or when boundary handling and reuse matter (multi-conditional IF for presence - verbose, error-prone at boundaries): + +```pigment +IF( + Day >= 'Start Date' + AND Day <= 'End Date', + 1, + BLANK +) +``` + +Optimized (encode once with PRORATA, derive flags with ISDEFINED/IFDEFINED): + +```pigment +// Numeric presence on Day (1 on active days, BLANK outside range) +PRORATA(Day, 'Start Date', 'End Date' + 1) + +// Numeric presence on Month (proportional factor per month) +PRORATA(Month, 'Start Date', 'End Date' + 1) + +// Boolean presence: TRUE when active, BLANK otherwise +ISDEFINED(PRORATA(Day, 'Start Date', 'End Date' + 1)) + +// Numeric 1/BLANK flag +IFDEFINED(PRORATA(Day, 'Start Date', 'End Date' + 1), 1) +``` + +Do not use ISBLANK/ISNOTBLANK for this pattern - they densify. Use ISDEFINED or IFDEFINED on the PRORATA result. + +When simple IF is acceptable: On small planning horizons with Month-level presence flags or for one-off single-date cutover comparisons, `IF(Month >= 'Start Month' AND Month <= 'End Month', TRUE)` is acceptable and clearer. Prefer PRORATA when proration semantics, boundary correctness, or cross-metric reuse matter. + +--- + +## Quick Decision Guide + +Note: These defaults assume large/sparse metric spaces. On small dense spaces, several of the "avoid" options are acceptable - see the per-pattern context guards above. + +| Situation | Use | Avoid | +| -------------------------------- | ------------------------------ | ----------------------------------------- | +| Prior period lookup | `[SELECT: Month-1]` | PREVIOUS, PREVIOUSOF | +| Check if value exists | ISDEFINED (returns TRUE/BLANK) | ISBLANK (returns TRUE/FALSE - densifies!) | +| Conditional with existence check | IFDEFINED | IF(ISBLANK()) | +| Provide default for blank | IFBLANK | IF(ISBLANK(), default, value) | +| Add values conditionally | IF | ADD + FILTER | +| Subset computed expression by value | `(Expression)[FILTER: CurrentValue > threshold]` | IF(Expression > threshold, Expression, BLANK) | +| Empty/no value | BLANK or omit | 0 (unless 0 is a meaningful business value) | +| Boolean flag (sparse) | TRUE or BLANK | TRUE or FALSE (FALSE densifies!) | +| Replicate value to dimension | BY CONSTANT (with mapping) | ADD CONSTANT | +| Aggregate via mapping | BY | ADD | +| Remove dimensions | REMOVE | BY on existing dimension (does nothing) | +| List with multiple dimensions | `[BY: dim1, dim2]` | `[BY: dim1][BY: dim2]` (loses properties) | +| Filter and remove dimension | SELECT | FILTER (when you want to aggregate) | +| Filter and keep dimension | FILTER | SELECT (keeps dimension) | +| Division | Just divide | IF(x<>0, a/b) - Pigment handles natively | +| Aggregate via mapping | BY | ADD | +| Filter existing data | FILTER | IF when subsetting same expression | +| Access rights | IFDEFINED(User, [AR]) | [AR] alone | +| Presence in date range | PRORATA + ISDEFINED/IFDEFINED | Multi-conditional IF, ISBLANK/ISNOTBLANK | + +Remember: BLANK = undefined = not defined (all mean "no value", not stored). FALSE != BLANK (FALSE is an explicit value that IS stored). + +--- + +## See Also + +- [Performance Formula Optimization](../optimizing-pigment-performance/performance_formula_optimization.md) +- [Performance Sparsity Deep Dive](../optimizing-pigment-performance/performance_sparsity_deep_dive.md) +- [Performance Scoping Patterns](../optimizing-pigment-performance/performance_scoping_patterns.md) diff --git a/plugins/pigment/skills/writing-pigment-formulas/formula_segmentation_tiered_lookup.md b/plugins/pigment/skills/writing-pigment-formulas/formula_segmentation_tiered_lookup.md new file mode 100644 index 00000000..e46f5b47 --- /dev/null +++ b/plugins/pigment/skills/writing-pigment-formulas/formula_segmentation_tiered_lookup.md @@ -0,0 +1,115 @@ +# Segmentation: Tiered / Banded Lookup + +Use when: You need to assign each item to a *band* or *tier* based on thresholds stored on a dimension (e.g., account segmentation, salary bands, discount brackets), similar to Excel's XMATCH/XLOOKUP with `match_mode = -1` or `1`. + +--- + +## Concept + +You have: + +- A measure per base item (e.g., revenue per Account) +- A dimension of bands/tiers (e.g., Tier A/B/C/D), with: + - A threshold metric defined *by that dimension* (e.g., floor or ceiling per Tier) + +You want: + +- For each base item, one dimension item (one Tier) that best matches its value according to the thresholds. + +We implement this with: + +1. An `IF` to mark all *eligible* tiers as non-blank +2. `[REMOVE FIRSTNONBLANK: Dim]` or `[REMOVE LASTNONBLANK: Dim]` to pick one tier per base item + +--- + +## Canonical "floor" pattern (largest floor <= value) + +Example: + +- Dimension: `Tier` (ordered from highest to lowest priority: `Tier A`, `Tier B`, `Tier C`, `Tier D`) +- Metric: `'DATA_Value'` (Number x `Account`) +- Metric: `'INPUT_Tier_Threshold'` (Number x `Tier`) - floors per Tier +- Target metric: `'CALC_Account_Tier'` (Dimension-typed = `Tier`, dimensioned by `Account`) + +Formula: + +```pigment +IF( + 'DATA_Value' > 'INPUT_Tier_Threshold', + Tier +)[REMOVE FIRSTNONBLANK: Tier] +``` + +How it works + +1. The comparison `'DATA_Value' > 'INPUT_Tier_Threshold'` is evaluated over the combined dimensionality Account x Tier. It returns TRUE where the account's value is greater than that tier's floor. +2. The `IF('DATA_Value' > 'INPUT_Tier_Threshold', Tier)` creates a Tier-typed block on (Account, Tier): non-blank (the Tier item) where the condition is TRUE, BLANK where the condition is FALSE. +3. The modifier `...[REMOVE FIRSTNONBLANK: Tier]` removes the Tier dimension and, for each Account, returns the first non-blank Tier in the Tier order. With Tier ordered Tier A, Tier B, Tier C, Tier D and increasing floors, this effectively selects the highest-priority Tier whose floor is <= the account's value. + +Excel analogy: This pattern is equivalent to XMATCH/XLOOKUP with match_mode = -1 (exact or next smaller), where the floors are sorted and we want the largest floor <= value. + +--- + +## Why dimension order matters + +The meaning of FIRSTNONBLANK / LASTNONBLANK is defined by the physical order of items in the dimension: + +- If Tier is ordered A -> D and floors increase from D to A: use `[REMOVE FIRSTNONBLANK: Tier]` to pick the highest qualifying tier. +- If Tier is ordered D -> A with the same floors: use `[REMOVE LASTNONBLANK: Tier]` instead to get the same result. + +Rule of thumb: + +- Top-of-list is highest band -> use FIRSTNONBLANK +- Bottom-of-list is highest band -> use LASTNONBLANK + +Changing the dimension order without adjusting FIRST/LAST will change which tier is chosen. + +--- + +## Variations + +### 1. Inclusive thresholds + +For inclusive floors, prefer `>=` instead of `>`: + +```pigment +IF( + 'DATA_Value' >= 'INPUT_Tier_Threshold', + Tier +)[REMOVE FIRSTNONBLANK: Tier] +``` + +### 2. Ceiling pattern (smallest ceiling >= value) + +If thresholds represent ceilings (upper bounds instead of floors), invert the comparison: + +```pigment +IF( + 'DATA_Value' < 'INPUT_Tier_Threshold', + Tier +)[REMOVE FIRSTNONBLANK: Tier] +``` + +Choose FIRST vs LAST based on how tiers are ordered. + +### 3. Fallback handling + +If no tier qualifies (e.g., all floors > value), wrap the expression and manage BLANK explicitly in a second step, or define a "catch-all" tier with a floor of 0. + +--- + +## Best practices + +- Be explicit about dimension order whenever using FIRSTNONBLANK/LASTNONBLANK: document in the dimension description which end is "highest" vs "lowest". +- Keep band thresholds on the band dimension (e.g., `INPUT_Tier_Threshold` by Tier), not on the base metric. +- Use dimension-typed metrics when the output is a classification/category (e.g., `CALC_Account_Tier` typed as Tier), to enable consistent reuse in other metrics and tables. +- When explaining to users, map this to the Excel concept of: + - match_mode = -1 (next smaller) for floor-based bands + - match_mode = 1 (next larger) for ceiling-based bands + +--- + +## Formula syntax reference + +For REMOVE, FIRSTNONBLANK, and LASTNONBLANK syntax, see [formula_modifiers.md](./formula_modifiers.md). diff --git a/plugins/pigment/skills/writing-pigment-formulas/formula_writing_workflow.md b/plugins/pigment/skills/writing-pigment-formulas/formula_writing_workflow.md new file mode 100644 index 00000000..1afdd6d9 --- /dev/null +++ b/plugins/pigment/skills/writing-pigment-formulas/formula_writing_workflow.md @@ -0,0 +1,262 @@ +# Pigment Formula Writing Workflow + +Purpose: Translate business requirements into correct, performant Pigment formulas efficiently. + +--- + +## Workflow Overview + +``` +Step 1: Understand Context -> Step 2: Search Documentation -> Step 3: Design Approach -> Step 4: [OPTIONAL] Define Tests -> Step 5: Build -> Step 6: Optimize -> Step 7: Validate -> Step 8: Deliver +``` + +_Feedback loops: Step 3->Step 1, Step 5->Step 3, Step 6->Step 5, Step 7->appropriate step_ + +--- + +## Step 1: Understand Context + +Gather Information: + +- Clarify business calculation in plain language +- Identify existing blocks (Metrics, Dimensions, Lists, properties) +- Application workspace (not documentation): use `tool:search` to find relevant blocks, confirm exact friendly names, and spot close duplicates (e.g. lists whose names differ by a prefix). Use `kind` / `regexp` on search when you need a precise name match. This step is separate from Step 2, which searches markdown skills only. +- Understand source and target metric: name, dimensions, type (Number/Date/Text/Dimension/Boolean). Note: Formula result must match target type - see type conversion functions if needed +- Check for obvious circular reference risks + +Success means: + +- Formula produces correct results +- Dimensional alignment is correct +- No syntax errors + +If unclear -> Ask user for clarification + +--- + +## Step 2: Search Documentation + +Search Strategy: + +- Query: `"how to [specific calculation from Step 1]"` +- Search key concepts from requirements +- Review ALL returned documentation chunks +- Read discovered files completely +- Note performance patterns mentioned +- If the request involves a specific dimension member (e.g. a month, version, country): read [modeling_principles section 4](../modeling-pigment-applications/modeling_principles.md) (MP02) and [formula_modifiers.md](./formula_modifiers.md) (FILTER, SELECT, BY CONSTANT). + +Why search matters: Discovers functions you don't know exist and verifies proper parameter usage. + +--- + +## Step 3: Design Approach + +Quick Design: + +- Identify operations needed (aggregate, allocate, filter, transform) +- Determine sequence of operations +- Consider sparsity preservation (use BLANK, IFDEFINED, early filtering) + +For complex formulas only: + +- Consider alternative approaches if first seems problematic +- Choose approach based on simplicity and efficiency + +If no clear approach -> Loop back to Step 1 (refine requirements) + +--- + +## Step 4: Define Test Strategy [OPTIONAL] + +Only for complex or critical formulas - Skip for simple calculations + +Quick Test Definition: + +- Identify 1-2 simple test cases to verify logic +- Note expected output mentally or in comments + +For critical formulas: + +- Define simple case, typical case, edge case +- Document expected outputs + +--- + +## Step 5: Build Formula + +Build Efficiently: + +- Start with core calculation (no modifiers) +- Add dimensional transformations (BY, ADD, REMOVE, SELECT, FILTER, KEEP) +- Verify syntax and dimensional alignment +- No excessive nesting - if you're nesting > 3 levels, review and simplify if truly needed +- Write the direct calculation first - only add error handling if truly needed + +Apply Operations & Align Dimensions: + +Compare source vs target dimensions, apply modifiers: + +| Modifier | Purpose | +| ----------- | --------------------------------------------------- | +| BY | Aggregate/allocate via mapping (changes dimensions) | +| ADD | Add dimension (allocation) | +| REMOVE | Remove dimension (aggregation) | +| SELECT | Filter and remove dimension | +| FILTER | Filter without changing dimensions | +| EXCLUDE | Exclude specific data | +| KEEP | Keep only specified dimensions | + +Preserve Sparsity: + +- Use `BLANK` instead of `0` where appropriate +- Use `IFDEFINED(block, value)` instead of `IF(ISBLANK(block), 0, value)` +- Apply early scoping: Filter/scope BEFORE calculations +- Defer aggregations: Aggregate AFTER calculations when possible +- Choose `IF` vs `FILTER`: + - `IF` for adding values to new cells + - `FILTER` for subsetting existing cells + +Chaining Modifiers: + +- When multiple modifiers needed, chain them in the correct order +- Order matters, especially with non-commutative aggregators (AVG, FIRSTNONBLANK) +- Use separate brackets or semicolon within one bracket + Quick Validation: +- Syntax is correct +- Dimensional alignment makes sense +- Logic matches requirements +- MP02: No `Dimension."Item"`; use `VAR_` metrics - see [modeling_principles section 4](../modeling-pigment-applications/modeling_principles.md) +WARNING: REQUIRED - Validate before applying: + +You MUST call `tool:validate_formula` before calling `tool:create_or_update_formula` or `tool:update_list_property_formula`. Applying an invalid formula puts the metric/property into an error state. + +- `tool:validate_formula` checks syntax, entity references, types, and dimensional alignment WITHOUT applying +- Returns error highlighting and hints if invalid - use these to fix the formula before retrying +- Use before including formulas in user messages + +Exception: Do NOT use `tool:validate_formula` with formulas containing `Previous` or `PreviousOf` functions (the validator does not support them). For these, verify syntax manually and confirm iterative calculation is configured on the target metric. + +Note: Formula builder tools are only for actual implementation when you have concrete metric/list IDs. For general formula discussions or learning, write formulas manually following the steps above. + +If validation fails -> Loop back to Step 3 + +--- + +## Step 6: Optimize for Performance + +REQUIRED: Read [formula_performance_patterns.md](./formula_performance_patterns.md) and apply ALL applicable patterns. + +Pre-Delivery Checklist: + +- [ ] Scoping clauses appear FIRST (FILTER, EXCLUDE, IFDEFINED) +- [ ] Using IFDEFINED instead of IF(ISBLANK()) +- [ ] Using IFBLANK instead of IF(ISBLANK(...), default, ...) +- [ ] Using IF for sparse additions, FILTER for subsetting +- [ ] Aggregations (REMOVE, BY) appear AFTER calculations +- [ ] Using BY instead of ADD where a mapping exists +- [ ] Access rights wrapped in IFDEFINED(User, ...) +- [ ] Using BLANK instead of 0 for empty values +- [ ] Using BLANK instead of FALSE for boolean flags (FALSE is stored, BLANK is not) +- [ ] MP02: No `Dimension."Item"` or `Dimension.Property."Item"` in the formula +- [ ] MP02: No `DATE(YYYY, M, D)` for planning bounds - use `VAR_` metrics (Date or Month) +- [ ] MP02: Required `VAR_` metrics exist (create first if missing) +- [ ] MP02: Metric names use relative temporal labels only (e.g. `'Next Period Forecast'`, not `Forecast 2026`) +- [ ] MP02: User has not overridden MP02 without seeing the compliant alternative + +Quick Examples: + +Early scoping: Apply filters/scopes first + +```pigment +// Good +'Revenue'[FILTER: 'Product'.'Active' = TRUE] * 'Growth' + +// Bad +('Revenue' * 'Growth')[FILTER: 'Product'.'Active' = TRUE] +``` + +IF vs FILTER: Use IF for sparse operations + +```pigment +// Good (sparse) - VAR_Reference_Month: input metric, type Dimension +IF(Month > VAR_Reference_Month, 10) + +// Bad (dense) +10[ADD: Month][FILTER: Month > VAR_Reference_Month] +``` + +Access rights with IFDEFINED(User): + +```pigment +IFDEFINED(User, 'Revenue'[AR: 'Rules']) +``` + +Do NOT deliver formulas without completing this checklist. + +If formula seems slow or complex -> Loop back to Step 5 (try simpler approach) + +--- + +## Step 7: Final Validation + +Quick Checklist: + +- Syntax valid (no errors) +- Dimensional alignment correct +- Logic produces expected results +- Sparsity best practices applied where relevant +- No circular references + +WARNING: REQUIRED - Call `tool:validate_formula` before applying: + +Do NOT call `tool:create_or_update_formula` or `tool:update_list_property_formula` without validating first. Applying an invalid formula puts the block into an error state. + +- Call `tool:validate_formula` - it returns detailed error messages with position highlighting +- If valid, proceed to apply using `tool:create_or_update_formula` or `tool:update_list_property_formula` +- If invalid, fix the formula and re-validate before applying + +Exception: Skip `tool:validate_formula` for formulas containing `Previous`/`PreviousOf` (not supported by the validator). + +If validation fails -> Loop back to appropriate step based on failure type + +--- + +## Step 8: Deliver + +Provide: + +1. Formula - Complete Pigment formula with comments following the commenting standard: + - Top-level comment (required): `//` comment on its own line(s) above the formula explaining its purpose (what it computes and why) + - Part-level comments (for multi-step formulas): `//` on their own line below the segment they describe, with a blank line before the next segment. Skip for one-liners or obvious formulas. + - Use the same language as the block name + - Maintain or enhance existing comments; replace them completely only if a formula update made them wrong or misleading + - Comments must be included in the formula string passed to tools (`tool:create_or_update_formula`, `tool:update_list_property_formula`, etc.) +2. Explanation - What the formula does, key operations and dimensional transformations +3. Documentation referenced - Key files/functions consulted +4. Validation suggestions (if relevant) - How user can verify results + +--- + +## Critical Rules + +- Always validate before applying - Call `tool:validate_formula` before `tool:create_or_update_formula` / `tool:update_list_property_formula`. Invalid formulas put blocks into error states. Exception: skip for `Previous`/`PreviousOf` formulas. +- Always search documentation - Discovers functions you don't know about +- Pigment syntax ONLY - You MUST NEVER write code or functions using another language being Excel, SQL, Python, JavaScript, MDX, DAX, or ANY other programming or query language. Think ONLY in Pigment terms. +- Never invent functions - Only use documented Pigment functions +- Modifiers use square brackets - `[BY: ...]`, `[REMOVE: ...]`, `[FILTER: ...]`, etc. +- If unsure about syntax - Search documentation, never assume +- Verify dimensional alignment - Ensure source and target dimensions match +- Verify referenced entities exist - Use `tool:search` to confirm metric names, list names, and property names before writing formulas +- Apply sparsity best practices - Use BLANK, IFDEFINED, early filtering when relevant +- Iterate when needed - Go back and refine if validation fails + +--- + +## Feedback Loops + +| When | Loop Back To | Action | +| -------------------------------- | ---------------- | -------------------- | +| No clear approach in Step 3 | Step 1 | Refine requirements | +| Validation fails in Step 5 | Step 3 | Redesign approach | +| Formula seems slow in Step 6 | Step 5 | Try simpler approach | +| Final validation fails in Step 7 | Appropriate step | Fix specific issue | diff --git a/plugins/pigment/skills/writing-pigment-formulas/functions_basic_aggregations.md b/plugins/pigment/skills/writing-pigment-formulas/functions_basic_aggregations.md new file mode 100644 index 00000000..569430ed --- /dev/null +++ b/plugins/pigment/skills/writing-pigment-formulas/functions_basic_aggregations.md @@ -0,0 +1,221 @@ +# Basic Aggregation Functions + +Functions that aggregate dimension items into single values (no dimensions in result). + +Covers: AVGOF, COUNTALLOF, COUNTBLANKOF, COUNTUNIQUEOF, SUMOF, MINOF, MAXOF, COUNTOF + +--- + +## Quick Reference + +| Function | Purpose | Syntax Example | +| ----------------- | ----------------------------- | --------------------------- | +| SUMOF | Sum of all values | `SUMOF('Sales')` | +| AVGOF | Average of all values | `AVGOF('Revenue')` | +| MINOF | Minimum value | `MINOF('Price')` | +| MAXOF | Maximum value | `MAXOF('Price')` | +| COUNTOF | Count non-blank items | `COUNTOF('Revenue')` | +| COUNTALLOF | Count all items (incl. blank) | `COUNTALLOF('Revenue')` | +| COUNTBLANKOF | Count blank items | `COUNTBLANKOF('Revenue')` | +| COUNTUNIQUEOF | Count unique values | `COUNTUNIQUEOF('Customer')` | + +--- + +## When to Use OF Functions vs Modifiers + +xxxOF Functions (this file): + +- Aggregate ALL dimensions to single value +- Return a scalar (no dimensions) +- Use when you need a single total +- Example: `SUMOF('Revenue')` -> one total across all dimensions + +Modifiers ([formula_modifiers.md](./formula_modifiers.md)): + +- Aggregate specific dimensions +- Keep other dimensions in result +- Use for dimensional aggregation +- Example: `'Revenue'[REMOVE: Product]` -> totals by other dimensions + +Rule of Thumb: Prefer modifiers for dimensional control. Use OF functions only when you need a single aggregate value. + +--- + +## SUMOF + +Returns the sum of all values in a Block. + +Syntax: `SUMOF(Block)` + +Examples: + +```pigment +SUMOF('Revenue') // Total revenue +SUMOF('Quantity') // Total quantity +``` + +--- + +## AVGOF + +Returns the average of all non-blank values in a Block. + +Syntax: `AVGOF(Block)` + +Examples: + +```pigment +AVGOF('Revenue') // Average of all revenue values +AVGOF(Employee.ID) // Average of all Employee IDs +``` + +--- + +## MINOF + +Returns the minimum value in a Block. + +Syntax: `MINOF(Block)` + +Examples: + +```pigment +MINOF('Price') // Lowest price +MINOF('Start Date') // Earliest start date +``` + +--- + +## MAXOF + +Returns the maximum value in a Block. + +Syntax: `MAXOF(Block)` + +Examples: + +```pigment +MAXOF('Price') // Highest price +MAXOF('Salary') // Highest salary +``` + +--- + +## COUNTOF + +Counts the number of non-blank values in a Block. + +Syntax: `COUNTOF(Block)` + +Examples: + +```pigment +COUNTOF('Revenue') // Number of products with revenue +COUNTOF('Sales') // Number of customers with sales +``` + +--- + +## COUNTALLOF + +Counts all items in a Block, including blanks. + +Syntax: `COUNTALLOF(Block)` + +Examples: + +```pigment +COUNTALLOF('Revenue') // Total number of products (even if no revenue) +COUNTALLOF('Price') // Total number of SKUs +``` + +--- + +## COUNTBLANKOF + +Counts the number of blank values in a Block. + +Syntax: `COUNTBLANKOF(Block)` + +Examples: + +```pigment +COUNTBLANKOF('Price') // Number of products missing prices +COUNTBLANKOF('Manager') // Number of employees without managers +``` + +--- + +## COUNTUNIQUEOF + +Counts the number of unique non-blank values in a Block. + +Syntax: `COUNTUNIQUEOF(Block)` + +Examples: + +```pigment +COUNTUNIQUEOF('Customer') // Unique customers +COUNTUNIQUEOF('Product'.'Category') // Unique categories +COUNTUNIQUEOF('Employee'.'Department') // Unique departments +``` + +--- + +## Common Patterns + +### Pattern 1: Average Salary by Department + +```pigment +'Salary'[BY AVG: Employee.Department] +``` + +### Pattern 2: Count of Active Customers + +```pigment +COUNTOF('Revenue'[SELECT: 'Customer'.'IsActive']) +``` + +### Pattern 3: Max Sale by Region + +```pigment +'Transaction'.'Amount'[REMOVE MAX: Transaction][BY: 'Region'] +``` + +### Pattern 4: Missing Data Check + +```pigment +COUNTBLANKOF('Price') / COUNTALLOF('Price') +``` + +### Pattern 5: Unique Customer Count by Month + +```pigment +'Order'.'Customer'[REMOVE COUNTUNIQUE: 'Order'][BY: 'Month'] +``` + +## When to Use Functions vs Modifiers + +| Scenario | Use | Example | +| -------------------------- | ----------- | --------------------------------------- | +| Aggregate all items | OF Function | `SUMOF('Revenue')` | +| Aggregate by dimension | Modifier | `'Revenue'[REMOVE: Product]` | +| Non-SUM aggregation by dim | Modifier | `'Price'[REMOVE AVG: Product]` | +| Min/Max by dimension | Modifier | `'Price'[REMOVE MIN: Product]` | +| Unique count by dimension | Modifier | `'Customer'[REMOVE COUNTUNIQUE: Order]` | + +--- + +## Critical Rules + +- OF functions return single value - No dimensions in result +- Prefer modifiers for dimensional aggregation - Better performance and clarity +- Functions ignore blank values - Except COUNTALLOF and COUNTBLANKOF +- COUNTALLOF counts all items - Even blanks +- COUNTUNIQUEOF on expressions - Can count unique property values + +--- + +## See Also + +- [formula_modifiers.md](./formula_modifiers.md) - BY, REMOVE, KEEP, SELECT for dimensional aggregation diff --git a/plugins/pigment/skills/writing-pigment-formulas/functions_finance.md b/plugins/pigment/skills/writing-pigment-formulas/functions_finance.md new file mode 100644 index 00000000..724e030c --- /dev/null +++ b/plugins/pigment/skills/writing-pigment-formulas/functions_finance.md @@ -0,0 +1,219 @@ +# Financial Functions + +Financial calculations for investment analysis and valuation. + +Covers: NPV, XNPV, IRR, XIRR + +--- + +## Quick Reference + +| Function | Purpose | Syntax Example | +| -------- | ----------------------------------------- | ----------------------------------------------------------------------------------------- | +| NPV | Net present value (periodic) | `NPV(Rate, CashFlows [, ComputeAllCells] [, RankingDimension])` | +| XNPV | Net present value (irregular dates) | `XNPV(Rate, CashFlows [, ComputeAllCells] [, RankingDimension] [, DaysUsed])` | +| IRR | Internal rate of return (periodic) | `IRR(CashFlows [, InitialGuess] [, ComputeAllCells] [, RankingDimension])` | +| XIRR | Internal rate of return (irregular dates) | `XIRR(CashFlows [, InitialGuess] [, ComputeAllCells] [, RankingDimension] [, DaysUsed])` | + +> WARNING: Parameter order differs from Excel. `Rate` is only in NPV/XNPV (1st arg); IRR/XIRR take `InitialGuess` as their 2nd (optional) arg. `RankingDimension` (the time dimension) is the 4th positional arg, after `ComputeAllCells` (a boolean), not the 3rd. + +--- + +## Net Present Value Functions + +### NPV + +Calculate NPV for periodic cash flows. + +Syntax: `NPV(DiscountRate, CashFlows [, ComputeAllCells] [, RankingDimension])` + +Parameters: + +- DiscountRate: Discount rate per period (e.g., 0.1 for 10%). Can be a constant Metric or a Metric dimensioned by RankingDimension for a variable rate. +- CashFlows: Metric with cash flow values (negative for payments, positive for income). +- ComputeAllCells (optional): Boolean, defaults to FALSE. If TRUE, returns the NPV for every item of RankingDimension; if FALSE, returns only for the first non-empty item. +- RankingDimension (optional): Required only when CashFlows is defined on several dimensions. + +Examples: + +```pigment +// Project NPV with 10% discount rate - single time dimension, RankingDimension implicit +NPV(0.10, 'Cash Flows') + +// Monthly cash flows with annual rate +NPV(0.12 / 12, 'Monthly Cash Flows') + +// Multi-dimensional CashFlows: compute every cell along Year +NPV(0.10, 'Cash Flows by Country', TRUE, Year) + +// Compare projects +IF(NPV(0.10, 'Project A Cash Flow') > NPV(0.10, 'Project B Cash Flow'), "Project A", "Project B") +``` + +Key Points: + +- Cash flows must be on a regular time dimension +- The initial investment (typically negative) must be included in the cash flows for the first period if it occurs at the same time +- Discount rate is per period (annual rate / periods per year) +- The Pigment calculation of NPV excludes the initial investment for the first period; include it in the cash flows explicitly if it occurs at the same time. + +--- + +### XNPV + +Calculate NPV for cash flows on irregular dates. + +Syntax: `XNPV(DiscountRate, CashFlows [, ComputeAllCells] [, RankingDimension] [, DaysUsed])` + +Parameters: + +- DiscountRate: Discount rate (constant Metric or Metric on RankingDimension for a variable rate). +- CashFlows: Metric with cash amounts (defined on RankingDimension). +- ComputeAllCells (optional): Boolean, defaults to FALSE. Same semantics as NPV. +- RankingDimension (optional): The Dimension along which payments are discounted. Required when CashFlows is defined on several dimensions. +- DaysUsed (optional): Date Property of RankingDimension; defines the exact day each payment is made. Required when RankingDimension is not a calendar Dimension; defaults to the dimension's Start Date otherwise. + +Examples: + +```pigment +// Compute on every cell along Month (calendar dimension; DaysUsed defaults to Month start) +XNPV(0.10, 'Investment Cashflow', TRUE) + +// Multi-dimensional CashFlows, explicit RankingDimension and DaysUsed +XNPV(0.10, 'Investment Cashflow by Country', TRUE, Month, Month.'Start Date') +``` + +When to Use: Cash flows on irregular dates (not periodic time dimension). + +--- + +## Internal Rate of Return Functions + +### IRR + +Calculate IRR for periodic cash flows. + +Syntax: `IRR(CashFlows [, InitialGuess] [, ComputeAllCells] [, RankingDimension])` + +Parameters: + +- CashFlows: Metric with cash flow values (must include at least one negative and one positive value). +- InitialGuess (optional): Starting guess for the iterative solver. Default 0.1. Must be greater than -1. +- ComputeAllCells (optional): Boolean, defaults to FALSE. Same semantics as NPV. +- RankingDimension (optional): Required only when CashFlows is defined on several dimensions. + +Examples: + +```pigment +// Project IRR (single time dimension, defaults to first non-empty cell) +IRR('Cash Flows') + +// Compute IRR for every cell with explicit Guess +IRR('Cash Flows', 0.5, TRUE) + +// Multi-dimensional CashFlows: rank on Fiscal Year +IRR('Payment Per Country', 0.1, FALSE, 'Fiscal Year') + +// Compare IRR to hurdle rate +IF(IRR('Project Cash Flows', 0.1) > 0.15, "Accept", "Reject") +``` + +Key Points: + +- Returns the discount rate where NPV = 0 +- Cash flows must include at least one negative (investment) and one positive (return) value +- InitialGuess parameter is the starting value for the iterative calculation +- Returns BLANK if no solution is found after 200 iterations + +--- + +### XIRR + +Calculate IRR for cash flows on irregular dates. + +Syntax: `XIRR(CashFlows [, InitialGuess] [, ComputeAllCells] [, RankingDimension] [, DaysUsed])` + +Parameters: + +- CashFlows: Metric with cash amounts (defined on RankingDimension). +- InitialGuess (optional): Starting guess (default: 0.1). +- ComputeAllCells (optional): Boolean, defaults to FALSE. Same semantics as NPV. +- RankingDimension (optional): Required when CashFlows is defined on several dimensions. +- DaysUsed (optional): Date Property of RankingDimension; defines the exact day each payment is made. Required when RankingDimension is not a calendar Dimension. + +Examples: + +```pigment +// Compute on every cell along Month (DaysUsed defaults to Month start) +XIRR('Cash Flow', 0.1, TRUE) + +// Multi-dimensional, custom dates on a non-calendar dimension +XIRR('Portfolio'.'Cash Flow', 0.12, TRUE, 'Transaction', 'Transaction'.'Date') +``` + +When to Use: Cash flows on irregular dates. + +--- + +## Function Comparison + +### NPV vs XNPV + +| Aspect | NPV | XNPV | +| -------------------- | ------------------------------- | ----------------- | +| Cash Flow Timing | Periodic (Month, Quarter, Year) | Irregular dates | +| Use Case | Regular intervals | Transaction-based | +| Performance | Faster | Slower | + +### IRR vs XIRR + +| Aspect | IRR | XIRR | +| -------------------- | ----------------- | ----------------- | +| Cash Flow Timing | Periodic | Irregular dates | +| Use Case | Regular intervals | Transaction-based | +| Performance | Faster | Slower | + +--- + +## Common Patterns + +### Pattern 1: Project Evaluation + +```pigment +// Calculate NPV and compare to threshold +IF(NPV(0.10, 'Project Cash Flows', TRUE, Year) > 0, "Accept", "Reject") +``` + +### Pattern 2: Investment Decision + +```pigment +// Compare IRR to hurdle rate +IF(IRR('Investment Cash Flows', 0.1, TRUE, Year) > 'Hurdle Rate', "Invest", "Pass") +``` + +### Pattern 3: Portfolio Analysis + +```pigment +// NPV of transactions per portfolio +XNPV(0.12, 'Portfolio'.'Cash Flow', TRUE, 'Portfolio', 'Portfolio'.'Transaction Date') +``` + +### Pattern 4: Monthly to Annual Rate Conversion + +```pigment +// Monthly cash flows with annual discount rate (rate per period) +NPV('Annual Discount Rate' / 12, 'Monthly Cash Flows') +``` + +--- + +## Critical Rules + +- First cash flow is typically negative - Initial investment +- NPV > 0 = positive return - Above discount rate +- IRR > hurdle rate = accept - Project meets requirements +- Discount rate is per period - Adjust for time period +- XNPV/XIRR for irregular dates - More flexible but slower +- InitialGuess helps convergence - Use reasonable estimate +- Cash flows must change sign - At least one negative and one positive for IRR +- Blank if no solution - IRR/XIRR return BLANK if can't converge diff --git a/plugins/pigment/skills/writing-pigment-formulas/functions_forecasting.md b/plugins/pigment/skills/writing-pigment-formulas/functions_forecasting.md new file mode 100644 index 00000000..1cf028f0 --- /dev/null +++ b/plugins/pigment/skills/writing-pigment-formulas/functions_forecasting.md @@ -0,0 +1,220 @@ +# Forecasting Functions + +Statistical forecasting and time series analysis functions. + +Covers: Exponential Smoothing (ETS, Simple, Double), Linear Forecasting, Seasonal Regression, Normal Distribution + +--- + +## Quick Reference + +| Function | Method | Best For | +| ------------------------------------------- | ---------------------------- | ------------------------ | +| FORECAST_ETS | Triple exponential smoothing | Trend + Seasonality | +| SIMPLE_EXPONENTIAL_SMOOTHING | Single smoothing | No trend, no seasonality | +| DOUBLE_EXPONENTIAL_SMOOTHING | Double smoothing | Trend, no seasonality | +| FORECAST_LINEAR | Linear regression | Simple linear trend | +| SEASONAL_LINEAR_REGRESSION | Seasonal regression | Seasonality with trend | +| STANDARD_NORMAL_DISTRIBUTION | Z-score probability | Statistical analysis | +| STANDARD_NORMAL_DISTRIBUTION_CUMULATIVE | Cumulative probability | Confidence intervals | + +--- + +## Exponential Smoothing Functions + +### FORECAST_ETS + +Triple exponential smoothing (Holt-Winters). Handles trend and seasonality. + +Syntax: `FORECAST_ETS(Input Block, Seasonality_length [, Ranking Dimension] [, Alpha] [, Beta] [, Gamma])` + +Parameters: + +- Input Block: Metric containing historical values (Number or Integer type) +- Seasonality_length: Integer. Length of the seasonal cycle (e.g., 12 for monthly data with yearly seasonality) +- Ranking Dimension (optional): Time dimension (e.g., Month). Optional if using native calendar dimension; required otherwise or if multiple time dimensions exist +- Alpha (optional): Level smoothing factor (0-1). Default: 0.25 +- Beta (optional): Trend smoothing factor (0-1). Default: 0.1 +- Gamma (optional): Seasonal smoothing factor (0-1). Default: 0.25 + +Examples: + +```pigment +// Monthly forecast with yearly seasonality +FORECAST_ETS('Sales', 12, 'Month') + +// Quarterly forecast with yearly seasonality, custom smoothing +FORECAST_ETS('Revenue', 4, 'Quarter', 0.3, 0.1, 0.1) +``` + +When to Use: Data has both trend and seasonality. + +--- + +### SIMPLE_EXPONENTIAL_SMOOTHING + +Single exponential smoothing. No trend, no seasonality. + +Syntax: `SIMPLE_EXPONENTIAL_SMOOTHING(Input Block [, Ranking Dimension [, Alpha]])` + +Parameters: + +- Input Block: Metric containing historical values +- Ranking Dimension (optional): Time dimension (e.g., Month). Optional if using native calendar dimension; required otherwise or if multiple time dimensions exist +- Alpha (optional): Smoothing factor (0-1). Default: 0.5 + +Examples: + +```pigment +// Smooth monthly data with default alpha +SIMPLE_EXPONENTIAL_SMOOTHING('Price', 'Month') + +// Smooth daily data with custom alpha +SIMPLE_EXPONENTIAL_SMOOTHING('Sensor Reading', 'Day', 0.2) +``` + +When to Use: Data is stable with no clear trend or seasonality. + +--- + +### DOUBLE_EXPONENTIAL_SMOOTHING + +Double exponential smoothing (Holt's method). Handles trend, no seasonality. + +Syntax: `DOUBLE_EXPONENTIAL_SMOOTHING(Input Block [, Ranking Dimension [, Alpha, Beta]])` + +Parameters: + +- Input Block: Metric containing historical values +- Ranking Dimension (optional): Time dimension (e.g., Month). Optional if using native calendar dimension; required otherwise or if multiple time dimensions exist +- Alpha (optional): Level smoothing factor (0-1). Default: 0.25 +- Beta (optional): Trend smoothing factor (0-1). Default: 0.1 + +Examples: + +```pigment +// Forecast with linear trend, default smoothing +DOUBLE_EXPONENTIAL_SMOOTHING('Monthly Growth', 'Month') + +// Smoothed trend forecast with custom parameters +DOUBLE_EXPONENTIAL_SMOOTHING('User Count', 'Week', 0.3, 0.15) +``` + +When to Use: Data has a trend but no seasonality. + +--- + +## Linear Forecasting Functions + +### FORECAST_LINEAR + +Simple linear regression forecast. + +Syntax: `FORECAST_LINEAR(Source Metric [, Ranking Dimension] [, Alternate Metric])` + +Parameters: + +- Source Metric: Metric containing historical values +- Ranking Dimension (optional): Time dimension (e.g., Month). Optional if using native calendar dimension; required otherwise or if multiple time dimensions exist +- Alternate Metric (optional): Use another metric as the independent variable + +Examples: + +```pigment +// Simple linear trend by month +FORECAST_LINEAR('Monthly Revenue', 'Month') + +// Linear regression using another metric as X +FORECAST_LINEAR('Cost of Sales', 'Month', 'Sales per Month') +``` + +When to Use: Simple linear trend, no seasonality. + +--- + +### SEASONAL_LINEAR_REGRESSION + +Linear regression with seasonal adjustment. + +Syntax: `SEASONAL_LINEAR_REGRESSION(Input Block, Seasonality_length [, Ranking Dimension])` + +Parameters: + +- Input Block: Metric containing historical values +- Seasonality_length: Integer. Length of the seasonal cycle (e.g., 12 for monthly data with yearly seasonality) +- Ranking Dimension (optional): Time dimension (e.g., Month). Optional if using native calendar dimension; required otherwise or if multiple time dimensions exist + +Examples: + +```pigment +// Monthly with yearly seasonality +SEASONAL_LINEAR_REGRESSION('Sales', 12, 'Month') + +// Quarterly with yearly seasonality +SEASONAL_LINEAR_REGRESSION('Revenue', 4, 'Quarter') +``` + +When to Use: Linear trend with clear seasonal pattern. + +--- + +## Statistical Functions + +### STANDARD_NORMAL_DISTRIBUTION + +Probability density function (PDF) of standard normal distribution. + +Syntax: `STANDARD_NORMAL_DISTRIBUTION(Z)` + +Example: + +```pigment +STANDARD_NORMAL_DISTRIBUTION(0) // 0.3989 (peak at z=0) +STANDARD_NORMAL_DISTRIBUTION(1.96) // ~0.058 +``` + +--- + +### STANDARD_NORMAL_DISTRIBUTION_CUMULATIVE + +Cumulative distribution function (CDF) of standard normal distribution. + +Syntax: `STANDARD_NORMAL_DISTRIBUTION_CUMULATIVE(Z)` + +Example: + +```pigment +STANDARD_NORMAL_DISTRIBUTION_CUMULATIVE(0) // 0.5 (50th percentile) +STANDARD_NORMAL_DISTRIBUTION_CUMULATIVE(1.96) // 0.975 (97.5th percentile) +``` + +--- + +## Choosing the Right Method + +| Data Characteristics | Recommended Method | +| -------------------------------- | ----------------------------------------------- | +| Stable, no trend, no seasonality | SIMPLE_EXPONENTIAL_SMOOTHING | +| Trend, no seasonality | DOUBLE_EXPONENTIAL_SMOOTHING or FORECAST_LINEAR | +| Seasonality, no trend | SEASONAL_LINEAR_REGRESSION | +| Trend + Seasonality | FORECAST_ETS | +| Simple linear trend | FORECAST_LINEAR | + +--- + +## Parameter Tuning Guide + +- Alpha (Level Smoothing): 0.1-0.2 = heavy smoothing; 0.2-0.3 = moderate (most common); 0.4-0.5 = light smoothing, more responsive +- Beta (Trend Smoothing): 0.1 = smooth trend (most common); 0.2 = more responsive; 0.3+ = very responsive (rare) +- Gamma (Seasonal Smoothing): 0.1 = stable seasonality (most common); 0.2 = moderate; 0.3+ = changing seasonality + +--- + +## Critical Rules + +- More historical data = better forecast (minimum 2 seasonal cycles for ETS) +- Seasonality periods must match data (e.g., 12 for monthly/yearly, 4 for quarterly/yearly) +- Alpha/Beta/Gamma in range [0,1] +- Test different parameters (validate forecast accuracy with holdout data) +- ETS is most sophisticated (use for production forecasting) +- Linear methods are fast (good for simple trends) diff --git a/plugins/pigment/skills/writing-pigment-formulas/functions_iterative_calculation.md b/plugins/pigment/skills/writing-pigment-formulas/functions_iterative_calculation.md new file mode 100644 index 00000000..e955b743 --- /dev/null +++ b/plugins/pigment/skills/writing-pigment-formulas/functions_iterative_calculation.md @@ -0,0 +1,335 @@ +# Iterative Calculation (PREVIOUS & PREVIOUSOF) + +Full technical reference for Pigment's iterative calculation functions: when to use them, syntax, configuration, performance, and debugging. + +Terminology: In this document, "Metric" refers to a Pigment metric (Block content), and "Block" refers to a Pigment Block. + +Deprecation: `PREVIOUSBASE()` is deprecated; use `PREVIOUSOF()` instead. + +See also: +- [Performance - Iterative Calculations](../optimizing-pigment-performance/performance_iterative_calculations.md) - Optimization strategies, subsetting, FILLFORWARD vs PREVIOUS +- [Time and Date Functions](./functions_time_and_date.md) - SELECT vs PREVIOUS/PREVIOUSOF, FILLFORWARD, CUMULATE + +--- + +## 1. What Pigment Considers Circular Dependencies + +Pigment checks for circular dependencies each time a formula is created or edited. If a metric directly or indirectly references itself, Pigment raises a circular dependency error to prevent infinite loops. + +Example of a circular dependency within a single Block: + +```pigment +// BAD: Circular: Metric A references itself via SELECT +Metric A = Metric A[SELECT: Month - 1] + 1 +``` + +This attempts to increase Metric A month-on-month but errors because Metric A cannot reference itself via SELECT in the formula. + +Modeling rules: + +- Only use PREVIOUS or PREVIOUSOF when there is a true circular dependency at the metric level (not cell level). +- Pigment does not handle cell-level circulars the way Excel can (no Excel-style iterative cell loops). + +--- + +## 2. Two Iterative Tools: PREVIOUS vs PREVIOUSOF + +| Scope | Use | When | +|-------|-----|------| +| Single Block | `PREVIOUS()` | Metrics containing circular-referencing formulas exist within one Block. | +| Multiple Blocks | `PREVIOUSOF()` | Metrics containing circular-referencing formulas exist in different Blocks. | + +Rule of thumb: + +- To loop something in a metric itself -> use `PREVIOUS()`. +- To build a loop across multiple metrics -> use `PREVIOUSOF()` (requires an iterative calculation configuration first). + +--- + +## 3. PREVIOUS() - Iterative Calculations Within a Single Block + +Definition: PREVIOUS returns the value of the previous cell of the current Metric in the iteration Dimension. + +Critical syntax rule: PREVIOUS takes the iterating dimension, not the metric. + +- Correct: `PREVIOUS(Month)` +- Incorrect: `PREVIOUS(Metric)` + +### Syntax + +`PREVIOUS(Iteration Dimension [, Offset])` + +Offsets: + +- `PREVIOUS(Month)` returns one item prior by default. +- `PREVIOUS(Month, 2)` returns two items prior. +- The offset parameter defaults to 1 but can be any positive integer. +- Offset can also be provided by a Metric of type Integer defined on the same Dimensions as the Metric. +- A negative offset value is automatically ignored. + +Engine behavior: Because a cell depends on the previous cell's final value, the engine computes the whole formula (including syntax before and after PREVIOUS) before moving to the next cell. Post-processing such as filtering or certain operations should happen in another metric that references the iterative metric. + +### Common PREVIOUS() Examples + +A) Simple increment: `PREVIOUS(Month) + 1` +(Teaching example; often more performant with CUMULATE.) + +B) Dynamic offset: `PREVIOUS(Month, 2) + 1` +Offset must be a positive constant integer or an Integer metric on same dimensions. + +C) Cash flow balance: + +```pigment +Cash = PREVIOUS(Month) + Income - Expense +``` + +D) Inventory - resolving circularity: + +- `Beginning Inventory = End Inventory[SELECT: Month - 1]` -> creates circular dependency when End Inventory uses Beginning Inventory. +- `End Inventory = PREVIOUS(Month) + Incoming Re-order - Outgoing Sales` -> resolves recursion within the same metric. + +--- + +## 4. PREVIOUS() Special Case - Filtering / IFBLANK Interaction + +Some operations are not compatible with PREVIOUS (full list on the PREVIOUS function page). Others are compatible but can have important implications. + +Pitfall: A modeler uses IFBLANK to fill patchy data from a source metric, using `PREVIOUS(Week)` to fill blank cells with the prior item. They then append a boolean filter (e.g. `['Filter boolean']`) in the same formula to keep values only where the boolean is TRUE. + +Outcome: Pigment returns blanks everywhere, including where the boolean is TRUE. + +Why: For earlier items where the boolean is FALSE, the engine emits blanks. When the boolean becomes TRUE, IFBLANK sees blank input and evaluates the `PREVIOUS(...)` branch. PREVIOUS points to a prior cell that is blank (because earlier cells were blank), so BLANK is returned and propagated. + +Fix: Put the PREVIOUS-containing expression in a separate metric first (e.g. "sales data metric"). Then apply the boolean filter in a second metric that references the first: `(SalesDataMetric['Filter boolean'])`. + +--- + +## 5. Performance Tips for PREVIOUS() + +Caution: Always try to avoid iterative calculations; only use them when there is no other option. They impact performance because Pigment must compute each metric x the length of the iterating dimension. + +Tips: + +1. Keep sub-expressions light and preserve sparsity + - Densifying sub-expressions increase computation (e.g. `ISBLANK(Metric_1)` replaces blanks with TRUE). + - Prefer `IFDEFINED(Metric_1, Metric_1, PREVIOUS(Month))` instead of `IF(ISBLANK(Metric_1), PREVIOUS(Month), ...)`. + +2. Reduce the number of times you use PREVIOUS in a formula + - Group expressions and isolate PREVIOUS. + - More performant: `PREVIOUS(Month) * (A + B)` + - Less performant: `PREVIOUS(Month) * A + PREVIOUS(Month) * B` + - In nested IF patterns, bring PREVIOUS up higher (evaluate earlier, not deeply nested). + +3. Minimize number of iterations + - More items in the iterating dimension increases execution time. + - Iterating dimension has a maximum of 10,000 items when using PREVIOUS. + - Use List Subsets to reduce the iterating dimension (e.g. Calendar subset for 2025-2026). + - Even if values are blank, dimension length still matters; choose the iterating dimension as short as possible. + +For more optimization strategies (subsetting, FILLFORWARD, CUMULATE), see [Performance - Iterative Calculations](../optimizing-pigment-performance/performance_iterative_calculations.md). + +--- + +## 6. PREVIOUSOF() - Iterative Calculations Across Multiple Blocks + +Definition: A multi-block iterative calculation configuration consists of: + +- an iteration Dimension +- a list of allowed Metrics + +When the configuration is created, the allowed metrics can reference each other only within PREVIOUSOF(). + +PREVIOUSOF returns the referenced metric value shifted by one period along the iteration dimension. It is equivalent to `Metric[SELECT: IterationDimension - 1]` but does not trigger the circular reference error. + +Implementation rule: Before using PREVIOUSOF(), you must set up a calculation cycle (iterative calculation configuration) and select all metrics that are part of the cycle. + +Programmatic workflow for PREVIOUSOF: + +1. Call `tool:list_cycles` to check if a cycle already exists for the target metrics. +2. If no cycle exists, identify all metrics in the dependency chain and the iteration dimension. +3. Create any metrics that do not exist yet (they must exist before being added to the cycle). +4. Call `tool:create_cycle` with a descriptive name, the iteration dimension ID, and all metric IDs. +5. Only after the cycle is created, write PREVIOUSOF formulas on the participating metrics. +6. If you need to add or remove metrics from an existing cycle, use `tool:update_cycle`. + +Allowed metrics: + +- The allowed metrics list must contain all metrics that are part of the dependency cycle. +- Omit unnecessary metrics from the list for performance. + +Using PREVIOUSOF like PREVIOUS: `PREVIOUSOF('Ending Inventory')` inside the Ending Inventory metric can behave like PREVIOUS within that metric. + +### Limitations + +- Maximum number of allowed metrics: 10 (KB). Some sources mention 20; verify the current product limit in your environment. +- All metrics in the configuration must include the iteration dimension in their structures. +- PREVIOUSOF is not allowed if the iteration dimension has more than 10,000 items (computation error). +- Cycle topology: You cannot combine two cycles into one or link them together. + +--- + +## 7. How to Create an Iterative Calculation Configuration + +### Programmatic (via tools) + +1. Ensure all participating metrics exist (create them first if needed). +2. Identify the iteration dimension UUID (typically a time dimension like Month). +3. Call `tool:create_cycle` with `cycleName`, `iterativeDimensionId`, and `metricIds` (all metrics in the chain). +4. Write PREVIOUSOF formulas on the participating metrics. + +### Manual (via UI) + +1. Go to Application Settings. +2. Click Calculations. +3. Click Add an iterative calculation. +4. Fill in: Cycle Name, Iteration Dimension, and Metrics that can reference each other. +5. Save. +6. Use PREVIOUSOF formulas in the participating metrics. + +Example configuration (Inventory): + +- Cycle Name: `Inventory calculation` +- Iteration Dimension: `Month` +- Allowed Metrics: Beginning Inventory, Incoming Re-order, End Inventory (and any other metrics in the cycle) + +--- + +## 8. Typical Use Case for PREVIOUSOF - Balance / Inventory Roll-forward + +Standard pattern: + +- Opening Balance = `PREVIOUSOF(Ending Balance)` +- Movements (can be multiple lines) +- Ending Balance = Opening + Movements + +Inventory example: + +```pigment +// Beginning Inventory +PREVIOUSOF('Ending Inventory') + +// End Inventory +'Beginning Inventory' + 'Incoming Re-order' - 'Outgoing Sales' +``` + +Incoming Re-order iterative example: + +```pigment +Incoming Re-order = 200[ADD: Month] - PREVIOUSOF('Incoming Re-order') +``` + +--- + +## 9. Workaround - Avoid PREVIOUSOF by Rewriting Opening Balance + +In some scenarios you can avoid PREVIOUSOF and improve performance by rewriting the model to use PREVIOUS and offsets: + +```pigment +Opening Balance = Opening first month + PREVIOUS(Month) + Movements[SELECT: Month - 1] +Movements +Ending Balance = Opening + Movements +``` + +This is often more performant than using PREVIOUSOF. + +--- + +## 10. Reduction Heuristic - When a PREVIOUSOF Cycle Can Collapse into PREVIOUS + +Method: + +1. Write the dependency chain explicitly. + Example: Opening(n) = Ending(n-1), Ending(n) = Opening(n) + Movements(n). +2. Substitute the forward definition of Ending(n-1): + Ending(n-1) = Opening(n-1) + Movements(n-1). +3. If substitution leaves only one recursive metric, implement with PREVIOUS instead of PREVIOUSOF. + +When NOT possible: Multiple metrics independently depend on previous values; multiple backward edges remain. + +Example non-reducible structure: + +- Metric1: `InputMetric + PREVIOUSOF(Metric3)` +- Metric2: `10 + PREVIOUSOF(Metric1)` +- Metric3: `Metric1 + Metric2` + +--- + +## 11. How PREVIOUSOF Cycles Compute + +When a configuration is created, Pigment identifies cyclical dependencies and calculates the whole cycle iteratively. + +For each item of the iterating dimension: + +- Formulas of each metric in the cycle are executed sequentially (with some parallelization under the hood). +- Calculation order depends on dependencies between formulas. + +Temporary dataset behavior: Compute all metrics for the first item (e.g. Jan) and store a temporary dataset. Compute the next item (Feb), referring back to the stored dataset for Jan as needed. Store Jan+Feb dataset, compute Mar, and so on. + +--- + +## 12. Optimization for PREVIOUSOF Configurations + +- Eliminate unused metrics: Metrics in the configuration are calculated differently; omit unnecessary metrics from the allowed list. An Active icon can indicate whether blocks generate a cycle; if inactive, the configuration can be removed. +- Profiling: Use Profiling to see total cycle time and time per metric. Optimizing each metric can reduce overall time (depending on number of metrics and iteration items). +- Performance insights: Use Performance insights to view the impact of iterative calculation on performance. + +See [Performance - Iterative Calculations](../optimizing-pigment-performance/performance_iterative_calculations.md) for more. + +--- + +## 13. Debugging Formula Errors in Iterative Configurations + +When using an iterative calculation configuration, Pigment builds a base formula subject to similar limitations as PREVIOUS. It is possible to write three individually valid formulas that combine into an invalid base formula. + +Example: + +- End Inventory = Beginning inventory + Incoming re-order +- Beginning Inventory = PREVIOUSOF(Ending Inventory) +- Incoming Re-order = Beginning inventory[REMOVE: Month] + +Error: *A dimension modifier using the iterating dimension Month can't be applied to a metric in the iterative calculation.* + +Modifier rules: + +- Modifiers (BY, SELECT, REMOVE, FILTER) are supported as long as they do not reference the iterating dimension. +- All window functions (CUMULATE, MOVINGAVERAGE, MOVINGSUM) are supported within metrics used in an iterative calculation. + +--- + +## 14. Compatibility Matrix + +PREVIOUS(): + +- Supported: Arithmetic; IF/IFDEFINED; many modifiers and operations not involving the iterating dimension; offsets (including dynamic integer metric offsets). +- Not allowed / constrained: Iterating dimension > 10,000 items; certain operations listed on the PREVIOUS function page; problematic patterns where post-processing is embedded in the PREVIOUS expression (see section 4). + +PREVIOUSOF(): + +- Supported: Arithmetic; IF/IFDEFINED; window functions; modifiers not involving the iterating dimension. +- Not allowed / constrained: Iterating dimension > 10,000 items; modifiers involving the iterating dimension; more than the allowed metric limit in the configuration; linking or combining configurations. + +--- + +## 15. Practical Decision Framework + +- Loop in a single metric -> use `PREVIOUS()`. +- Loop across multiple metrics -> use `PREVIOUSOF()` and create an iterative calculation configuration first using `tool:create_cycle`. +- Use PREVIOUS/PREVIOUSOF only for true metric-level circular dependencies; Pigment does not support cell-level circulars. +- Prefer non-iterative logic when possible (performance); use CUMULATE, offset-based patterns, or the Opening Balance rewrite (section 9) where applicable. +- Choose the iterating dimension as short as possible; consider subsets if you only need a portion. + +--- + +## Appendix A - Verbatim Notes + +- "Forget about previousbase, its old and should not be used, previousof is the newest version." +- "If you want to loop something in a metric itself, you use previous()." +- "If you want to build a loop inside multiple metrics, you use previousof()." +- "Before you can use previousof(), you have to set-up a calculation cycle first, and select all metrics of the cycle." +- "You can't combine 2 cycles into 1 or link each other." +- "You always choose an iterating dimension... that dimension is as short as possible... Consider using subsets." +- "You only use previous or previousof if there is a true circular (on metric level, not cell level)... Pigment doesn't handle cell level circulars (like excel)." +- "Always try to avoid it though... It can impact performance because Pigment needs to compute each metric * the length of the iterating dimension." +- "Your cycle of metrics can't be bigger than 20 metrics, take that into account." (Verify current platform limit; KB states 10.) +- "Typical use case: Opening Balance = Previousof(ending balance); Movements; Ending Balance = Opening + Movements." +- "More performant alternative: Opening Balance = Opening first month + previous(month) + movements [select: month-1] ... This will probably be more performant as we're not using previousof." diff --git a/plugins/pigment/skills/writing-pigment-formulas/functions_logical.md b/plugins/pigment/skills/writing-pigment-formulas/functions_logical.md new file mode 100644 index 00000000..0c17c2c1 --- /dev/null +++ b/plugins/pigment/skills/writing-pigment-formulas/functions_logical.md @@ -0,0 +1,660 @@ +# Logical Functions + +Boolean operations, conditional logic, and sparsity-aware functions. + +Covers: Boolean Logic (AND, OR, NOT), Conditional Functions (IF, SWITCH), Blank Handling (ISBLANK, IFDEFINED, IFBLANK), Collection Functions (ANYOF, ALLOF, IN) + +--- + +## Quick Reference + +| Function | Purpose | Syntax Example | +| -------------- | --------------------------- | ----------------------------------------------------- | +| IF | Conditional logic | `IF(Condition, ValueIfTrue, ValueIfFalse)` | +| SWITCH | Multiple conditions | `SWITCH(Expression, Case1, Result1, ..., Default)` | +| AND | All conditions true | `AND(Cond1, Cond2, ...)` | +| OR | Any condition true | `OR(Cond1, Cond2, ...)` | +| NOT | Negate condition | `NOT(Condition)` | +| IFDEFINED | Check if defined (sparse) OK | `IFDEFINED(Block, ValueIfDefined, ValueIfNotDefined)` | +| IFBLANK | Default if blank | `IFBLANK(Block, DefaultValue)` | +| ISBLANK | Check if blank (densifies!) | `ISBLANK(Block)` | +| ISNOTBLANK | Check if not blank | `ISNOTBLANK(Block)` | +| ISDEFINED | Check if defined (sparse) OK | `ISDEFINED(Value)` | +| ANYOF | Any value is TRUE | `ANYOF(BooleanBlock)` | +| ALLOF | All values are TRUE | `ALLOF(BooleanBlock)` | +| IN | Value in set (infix) | `Block IN (Item1, Item2, ...)` | +| TRUE | Boolean true | `TRUE` | +| FALSE | Boolean false | `FALSE` | + +--- + +## Conditional Functions + +### IF + +Execute conditional logic. + +Syntax: `IF(Condition, ValueIfTrue, ValueIfFalse)` + +Examples: + +```pigment +// Basic IF +IF('Revenue' > 1000000, "High", "Low") + +// Nested IF +IF('Score' >= 90, "A", IF('Score' >= 80, "B", "C")) + +// IF with calculations +IF('Actual' > 'Budget', 'Actual' - 'Budget', BLANK) + +// IF with blank handling +IF(IFDEFINED('Price'), 'Price' * 'Quantity', BLANK) +``` + +Key Points: + +- Both ValueIfTrue and ValueIfFalse are always evaluated +- Result dimensions = union of ALL branches and conditions - If condition uses dimensions, result gains them +- For sparsity preservation, use IFDEFINED instead of IF(ISBLANK()) +- Do not densify with 0 when not absolutely needed +- Condition must be Boolean expression + +Dimension Conditions: + +```pigment +// Condition with dimension member - use VAR_ input metric of type Dimension +IF(Country = VAR_Selected_Country, 'Local Rate', 'Default Rate') +IF(Month = VAR_Reference_Month, 'Budget', 'Forecast') + +// Condition with metric value +IF('Revenue' > 1000000, "High", "Low") +``` + +--- + +### SWITCH + +Multi-way conditional (replaces nested IFs). + +Syntax: `SWITCH(Expression, Case1, Result1, Case2, Result2, ..., DefaultResult)` + +Examples: + +```pigment +// Category classification +SWITCH('Score', + 90, "A", + 80, "B", + 70, "C", + "F" +) + +// Text matching +SWITCH('Status', + "Active", 1, + "Inactive", 0, + "Pending", 0.5, + 0 +) + +// Dimension-based logic +SWITCH('Product'.'Category', + "Electronics", 'Price' * 1.2, + "Clothing", 'Price' * 1.1, + 'Price' +) +``` + +Key Points: + +- Cleaner than nested IF statements +- Evaluates all cases (not short-circuit) +- Last argument is default if no match +- More readable for multi-way logic + +--- + +## Boolean Logic + +Note: AND and OR follow the same dimension alignment rules as other combining operations. Both operands should have the same dimensions. See [Dimension Flow Rules](./formula_modifiers.md#dimension-flow-rules). + +### AND + +All conditions must be true. + +Syntax: `AND(Condition1, Condition2, ...)` + +Examples: + +```pigment +AND('Revenue' > 1000, 'Profit' > 0) // Both must be true +AND('Employee'.'IsActive', 'Employee'.'Department' = "Sales") // Active AND Sales +AND('Score' >= 70, 'Attendance' >= 0.8, 'Projects' >= 3) // All three true +``` + +--- + +### OR + +Any condition must be true. + +Syntax: `OR(Condition1, Condition2, ...)` + +Examples: + +```pigment +OR('Revenue' > 1000000, 'Customers' > 100) // Either true +OR('Status' = "Active", 'Status' = "Pending") // Active OR Pending +OR('Score' >= 90, 'Bonus Points' >= 100) // Either qualifies +``` + +--- + +### NOT + +Negate a boolean expression. + +Syntax: `NOT(Condition)` + +Examples: + +```pigment +NOT('Employee'.'IsActive') // Not active +NOT('Product'.'IsDiscontinued') // Not discontinued +NOT(ISBLANK('Price')) // Not blank (use ISNOTBLANK instead) +``` + +--- + +## Blank Handling (Critical for Sparsity) + +Choosing the right function impacts model performance, sparsity, and database computation. + +### Quick Recommendations + +| Instead of | Use | Why | +| ------------------------------------- | --------------- | ----------------------------------------- | +| `ISBLANK(A)` | `ISDEFINED(A)` | Avoids densifying metrics | +| `IF(ISBLANK(A), B, A)` | `IFBLANK(A, B)` | Cleaner formula, simpler computation | +| `IFDEFINED(A, A, B)` when B is sparse | `IFBLANK(A, B)` | Preserves sparsity, optimizes computation | + +Note: `IFDEFINED(A, X, Y)` and `IF(ISDEFINED(A), X, Y)` behave identically - use whichever is clearer. + +ISBLANK and ISNOTBLANK are densifying: they return TRUE or FALSE for every cell and are almost never the right primitive for sparsity maintenance. Prefer ISDEFINED, IFDEFINED, IFBLANK, or EXCLUDE. If in doubt, do not use ISBLANK/ISNOTBLANK; use ISDEFINED, IFDEFINED, IFBLANK, or EXCLUDE instead. + +### Function Comparison + +| Function | Returns | Densifies? | +| ------------------------ | ----------------------------------- | ---------------------- | +| ISBLANK / ISNOTBLANK | TRUE/FALSE for ALL cells | Yes - always | +| ISDEFINED | TRUE where defined, BLANK elsewhere | No | +| IFBLANK(A, B) | A if defined, B otherwise | Only if B is dense | +| IFDEFINED(A, X, Y) | X if A defined, Y otherwise | Only if Y is dense | + +```pigment +// BAD: Avoid - densifies +IF(ISBLANK('Revenue'), 0, 'Revenue') +ISBLANK('Price') + +// GOOD: Prefer - preserves sparsity +IFBLANK('Revenue', 0) +ISDEFINED('Price') +``` + +--- + +### Understanding BLANK vs FALSE + +BLANK and "not defined" mean that no value exists. + +| Term | Meaning | Stored as a value | +| ----------- | ---------------- | ----------------- | +| BLANK | No value | No | +| not defined | No value | No | +| FALSE | Explicit boolean | Yes | +| TRUE | Explicit boolean | Yes | + +Key insight: `BLANK != FALSE`. Using FALSE where you mean "no value" causes densification. + +### How Blanks Behave with Operators + +Pigment's sparse engine treats blanks differently depending on the operator and dimension alignment. + +Operator Groups: + +| Operator Group | Operators | Blank Behavior | +| ------------------ | --------------- | --------------------------------------------------- | +| Additive | `+`, `-`, `OR` | `blank + value = value` (blank treated as identity) | +| Multiplicative | `*`, `/`, `AND` | `blank x value = blank` (blank propagates) | + +Dimension Alignment Effects: + +| Scenario | Additive (`+`, `-`, `OR`) | Multiplicative (`*`, `/`, `AND`) | +| ------------------------ | -------------------------------------- | ---------------------------------- | +| Same dimensions | blank + value = value | blank x value = blank | +| One common dimension | Blanks in higher-dim metric stay blank | Blanks in either metric stay blank | +| No common dimensions | Blanks in either metric stay blank | Blanks in either metric stay blank | +| Metric + constant | Blanks stay blank | Blanks stay blank | + +Examples: + +```pigment +// Same dimensions (Product x Month) +'Revenue' + 'Adjustment' +// If Revenue is blank but Adjustment has value -> result = Adjustment value +// If Revenue has value but Adjustment is blank -> result = Revenue value + +'Revenue' * 'Rate' +// If Revenue is blank -> result = blank (regardless of Rate) +// If Rate is blank -> result = blank (regardless of Revenue) + +// Metric + constant +'Revenue' + 100 // Blank Revenue cells stay blank (not 100) +'Revenue' * 1.1 // Blank Revenue cells stay blank (not 0) +``` + +Key Implications: + +1. Addition doesn't fill blanks with constants - `'Revenue' + 100` keeps blank cells blank +2. Multiplication propagates blanks - Any blank input produces blank output +3. Division by blank returns blank - No need to check for blank denominators +4. Dimension mismatch preserves blanks - When dimensions differ, blanks are not broadcast + +When You Need to Fill Blanks: + +Use `IFBLANK` explicitly when you want blanks replaced: + +```pigment +// Fill blanks with 0 for addition +IFBLANK('Revenue', 0) + 'Adjustment' + +// Fill blanks with 1 for multiplication +'Revenue' * IFBLANK('Rate', 1) +``` + +--- + +### WARNING: ISBLANK - Use Sparingly + +Check if value is blank. Warning: Densifies metrics! + +Syntax: `ISBLANK(Block)` + +Returns: + +- TRUE if blank (explicit boolean - stored) +- FALSE if defined (explicit boolean - stored) + +Examples: + +```pigment +ISBLANK('Price') // TRUE if blank, FALSE if defined +IF(ISBLANK('Revenue'), 0, 'Revenue') // BAD: DENSIFIES! Use IFBLANK +``` + +Why it densifies: ISBLANK returns explicit TRUE/FALSE values for ALL cells. Both TRUE and FALSE are stored, so every cell now has a value. + +Critical Warning: + +- ISBLANK densifies - Returns TRUE/FALSE for all dimension combinations +- Massive performance impact for sparse metrics +- Use ISDEFINED instead in 99% of cases (returns TRUE or BLANK, not TRUE or FALSE) + +Default: Avoid ISBLANK - Use ISDEFINED Instead + +- When sparsity is unknown or uncertain -> always use ISDEFINED +- ISDEFINED is the safe choice: returns TRUE where defined, BLANK elsewhere (never densifies) +- ISBLANK writes TRUE or FALSE in every cell, making the metric 100% dense +- Rule of thumb: If you're unsure whether a metric is sparse or dense, assume it's sparse and use ISDEFINED + +Allow list - when ISBLANK/ISNOTBLANK are acceptable (very limited valid use in Pigment): + +- The input is already dense (e.g. small, non-sparse metric). +- You explicitly need a full TRUE/FALSE for every cell (e.g. exporting or reporting where every cell must be TRUE or FALSE). +- Very small dimension space where densification cost is negligible. + +Anti-patterns (avoid): + +- Guarding a metric before using it in BY - If a dimension-typed metric is in BY, its sparsity is respected automatically; do not wrap in `IF(ISBLANK(metric), BLANK, ...)`. Use the BY expression alone. +- Using IF(ISBLANK(A), BLANK, expr) for sparsity - Use IFDEFINED or BY-driven sparsity instead. +- Using ISNOTBLANK for "exists" checks - Use ISDEFINED (returns TRUE/BLANK, not TRUE/FALSE). +- Using ISBLANK/ISNOTBLANK for date-range presence - Use PRORATA + ISDEFINED/IFDEFINED (see [functions_time_and_date.md](./functions_time_and_date.md)). + +Alternative: Use EXCLUDE Instead of ISBLANK + +When you need "A is true and B is blank", use the EXCLUDE modifier instead of combining AND with ISBLANK. EXCLUDE restricts the formula's scope to cells where the excluded metric is blank, avoiding densification entirely. + +```pigment +// BAD: Densifies: ISBLANK evaluates every cell of B +IF(A AND ISBLANK(B), TRUE) + +// GOOD: Sparse: EXCLUDE restricts scope to where B is blank +IF(A [EXCLUDE: B], TRUE) +``` + +Why EXCLUDE is better: + +- ISBLANK(B) produces TRUE/FALSE for every cell of B, densifying the result +- `[EXCLUDE: B]` tells the engine to skip cells where B is defined - no extra boolean metric is created +- The formula only runs where B is blank, so computation is smaller and output stays sparse + +--- + +### ISNOTBLANK + +Check if value is not blank. Same densification warning as ISBLANK. + +Syntax: `ISNOTBLANK(Block)` + +Returns: + +- TRUE if defined (explicit boolean - stored) +- FALSE if blank (explicit boolean - stored) + +Examples: + +```pigment +ISNOTBLANK('Price') // TRUE if defined, FALSE if blank +``` + +Warning: Densifies like ISBLANK - returns TRUE/FALSE for ALL cells. Use `ISDEFINED` instead (returns TRUE/BLANK). + +--- + +### OK ISDEFINED - Sparse Check + +Check if value is defined without densifying. Returns TRUE or BLANK (not FALSE). + +Syntax: `ISDEFINED(Value)` + +Examples: + +```pigment +ISDEFINED(1) // TRUE (1 is defined) +ISDEFINED(BLANK) // BLANK (not FALSE!) +ISDEFINED('Price') // TRUE where Price has value, BLANK elsewhere +``` + +Key Advantage over ISNOTBLANK: + +- ISDEFINED returns TRUE or BLANK -> sparse result, unpopulated cells not written +- ISNOTBLANK returns TRUE or FALSE -> 100% dense, every cell populated + +When to Use: + +- Checking if a value exists without densifying +- Building boolean conditions that preserve sparsity +- Performance-critical formulas on large dimension combinations + +Note: For conditional logic with fallback values, use IFDEFINED instead. + +Why isNotDefined doesn't exist: It would create ambiguous cases mixing FALSE and BLANK. If you need this behavior, use `IFDEFINED(X, BLANK, TRUE)`. + +--- + +### OK IFDEFINED - Preferred for Sparsity + +Check if value is defined without densifying. + +Syntax: `IFDEFINED(Block, ValueIfDefined, ValueIfNotDefined)` + +Returns: + +- ValueIfDefined where Block has a value +- ValueIfNotDefined where Block is blank (defaults to BLANK if omitted) + +Contrast with ISBLANK: + +- `ISBLANK('X')` -> Returns TRUE/FALSE for ALL cells (dense) +- `ISDEFINED('X')` -> Returns TRUE where defined, BLANK (not FALSE) elsewhere (sparse) + +Examples: + +```pigment +// OK Preserves sparsity - returns BLANK for undefined cells +IFDEFINED('Price', 'Price' * 'Quantity', BLANK) + +// OK Default if not defined +IFDEFINED('Exchange Rate', 'Amount' * 'Exchange Rate', 'Amount') + +// OK Conditional calculation +IFDEFINED('Discount', 'Price' * (1 - 'Discount'), 'Price') + +// OK Access rights pattern +IFDEFINED(User, 'Confidential Data', BLANK) +``` + +Key Points: + +- Preserves sparsity - Returns BLANK (not FALSE) for undefined cells +- Always prefer IFDEFINED over IF(ISBLANK()) +- Essential for performance with sparse metrics +- Common pattern for access rights: `IFDEFINED(User, Data, BLANK)` + +--- + +### IFBLANK + +Provide default value if blank. Does NOT densify (unless DefaultValue is dense). + +Syntax: `IFBLANK(Block, DefaultValue)` + +Behavior: Returns Block if defined, otherwise DefaultValue. + +Examples: + +```pigment +IFBLANK('Price', 0) // 0 if blank +IFBLANK('Discount', 0.1) // Default 10% discount +IFBLANK('Exchange Rate', 1) // Default rate of 1 +``` + +Key Points: + +- Cleaner than `IF(ISBLANK(Block), Default, Block)` and more efficient +- Prefer IFBLANK over IFDEFINED when you just want `A or default B` + +Dimension Check (required before using IFBLANK): + +Use the [Dimension Flow Rules](./formula_modifiers.md#dimension-flow-rules) to trace dimensions through your formula, then compare: + +1. Same dimensions -> IFBLANK is safe (preserves sparsity, output = union of defined cells) +2. First argument has more dimensions than second -> Avoid IFBLANK (causes densification) + +When to Use IFBLANK: + +- When both arguments have the same dimensions (verify from metric definitions) +- When dimensions match, IFBLANK preserves reasonable sparsity + +When to Avoid IFBLANK: + +- When the first argument has higher dimensionality than the second +- When dimension alignment is unclear -> use IFDEFINED with explicit BLANK fallback instead: `IFDEFINED(A, A, BLANK)` + +Why Dimension Mismatch Causes Densification: + +When IFBLANK(A, B) has A with more dimensions than B, Pigment broadcasts B across all of A's dimension combinations, filling every blank cell in A's scope with B's value. + +References: See IFBLANK Use Cases, MS01 (Sparse Engine documentation). + +--- + +## Collection Functions + +### ANYOF + +Check if any value in a Boolean block is TRUE. + +Syntax: `ANYOF(BooleanBlock)` + +Examples: + +```pigment +// Any product in target category - prefer boolean property on Product +ANYOF('Product'.'Include Category') + +// Any employee in Sales department +ANYOF('Employee'.'Department' = "Sales") + +// Any country with revenue > 1M +ANYOF('Revenue' > 1000000) +``` + +Returns: Boolean (TRUE if any item matches) + +--- + +### ALLOF + +Returns TRUE if the input boolean block contains only TRUE values. + +Syntax: `ALLOF(BooleanBlock)` + +Example: + +```pigment +// All products are active +ALLOF('IsActiveProduct') +``` + +Returns: Boolean (TRUE if all items match) + +--- + +### IN + +Check if items in a List/Block belong to a given set, or fall within a numeric/date range. IN is an infix operator, not a function call. + +Syntax (specific items): `Block IN (Item1, Item2, ..., ItemN)` + +Syntax (range, inclusive bounds): `Block IN (lower : upper)` + +Examples: + +```pigment +// Specific members - prefer boolean property or mapping metric (MP02) +Country IN (VAR_Primary_Country, VAR_Secondary_Country) + +// Negation over a subset property +NOT Month IN (VAR_Excluded_Month_1, VAR_Excluded_Month_2, VAR_Excluded_Month_3) + +// Range over a Year property +'Switchover Date'[ADD: Year] IN (Year.'Start Date' : Year.'End Date') +``` + +Returns: Boolean (TRUE if Block matches any item / falls in the range). + +Key Point: Cleaner than multiple OR conditions. MP02: Do not list `Dimension."Item"` literals - use `VAR_` metrics or boolean properties (see examples above). + +--- + +## Critical Sparsity Rules + +### BAD: Never Use These Patterns (Densify) + +```pigment +IF(ISBLANK('Price'), 0, 'Price') // BAD: Densifies +IF(ISNOTBLANK('Revenue'), 'Revenue', 0) // BAD: Densifies +NOT(ISBLANK('Price')) // BAD: Densifies +IF(condition, TRUE, FALSE) // BAD: FALSE is stored - use BLANK instead +``` + +### OK Always Use These Patterns (Sparse) + +```pigment +IFDEFINED('Price', 'Price', 0) // OK Sparse +IFBLANK('Price', 0) // OK Better +IFDEFINED('Revenue', 'Revenue', 0) // OK Sparse +IF(condition, TRUE, BLANK) // OK BLANK instead of FALSE for boolean flags +IF(condition, TRUE) // OK Omitting else defaults to BLANK +ISDEFINED('Revenue') // OK Returns TRUE/BLANK, not TRUE/FALSE +IF(A [EXCLUDE: B], TRUE) // OK "A and B blank" without densifying B +``` + +### Performance Pattern: Early Scoping + +```pigment +// BAD: Slow: Calculates everywhere then filters +IF('Revenue' > 1000, 'Revenue', BLANK) + +// OK Fast: Filters early with IFDEFINED +IFDEFINED('Revenue', IF('Revenue' > 1000, 'Revenue', BLANK), BLANK) +``` + +### Access Rights Pattern + +```pigment +// Always wrap sensitive data with IFDEFINED(User) +IFDEFINED(User, 'Confidential Salary', BLANK) +``` + +--- + +## Common Patterns + +### Pattern 1: Division (No Check Needed) + +Pigment handles division by zero natively - it returns BLANK automatically. No need to check! + +```pigment +// GOOD: CORRECT: Just divide - Pigment returns BLANK if denominator is 0 or BLANK +'Numerator' / 'Denominator' + +// BAD: WRONG: Unnecessary check - Pigment already handles this +IF('Denominator' = 0, BLANK, 'Numerator' / 'Denominator') + +// BAD: WRONG: Also unnecessary +IF('Denominator' <> 0, 'Numerator' / 'Denominator', BLANK) +``` + +Key Point: Division by zero or BLANK automatically returns BLANK in Pigment. Don't wrap division in IF checks for zero - it's redundant and adds computation. + +### Pattern 2: Threshold Classification + +```pigment +IF('Revenue' > 1000000, "Large", IF('Revenue' > 100000, "Medium", "Small")) +``` + +### Pattern 3: Default Values + +```pigment +IFDEFINED('ActualData', 'ActualData', 'ForecastData') +``` + +### Pattern 4: Variance with Conditional Formatting + +```pigment +IF('ActualData' > 'BudgetData', 'ActualData' - 'BudgetData', BLANK) +``` + +### Pattern 5: Multi-Condition Filter + +```pigment +IF(AND('Revenue' > 1000, 'Profit' > 0, 'Growth' > 0.1), "Quality", "Review") +``` + +--- + +## Critical Rules + +- BLANK = undefined = not defined - All mean "no value", not stored +- FALSE != BLANK - FALSE is an explicit boolean value that IS stored +- IFDEFINED > IFBLANK > IF(ISBLANK()) - Always prefer IFDEFINED for sparsity +- ISBLANK/ISNOTBLANK densify - They return TRUE/FALSE (both stored) for ALL cells +- ISDEFINED preserves sparsity - Returns TRUE where defined, BLANK (not FALSE) elsewhere +- Use BLANK instead of FALSE - For sparse boolean flags, return TRUE or BLANK, not TRUE or FALSE +- ISDEFINED > ISNOTBLANK - ISDEFINED returns BLANK (sparse), ISNOTBLANK returns FALSE (dense) +- ISBLANK densifies - Avoid unless absolutely necessary +- IF evaluates both branches - Not short-circuit +- SWITCH is cleaner than nested IF - Use for multi-way logic +- IFDEFINED(User) for access rights - Essential security pattern +- Early scoping - Filter before expensive calculations +- AND/OR evaluate all arguments - Not short-circuit +- For "A and B blank", prefer `IF(A [EXCLUDE: B], TRUE)` over `IF(A AND ISBLANK(B), TRUE)` - EXCLUDE avoids densifying B + +--- + +## See Also + +- [formula_performance_patterns.md](./formula_performance_patterns.md) - Sparsity optimization patterns diff --git a/plugins/pigment/skills/writing-pigment-formulas/functions_lookup.md b/plugins/pigment/skills/writing-pigment-formulas/functions_lookup.md new file mode 100644 index 00000000..d45c0d34 --- /dev/null +++ b/plugins/pigment/skills/writing-pigment-formulas/functions_lookup.md @@ -0,0 +1,108 @@ +# Lookup Functions + +Dimensional transformation functions for lookups, matching, and time shift. + +Covers: ITEM, MATCH, SHIFT, TIMEDIM + +--- + +## Quick Reference + +| Function | Purpose | Syntax Example | +| ----------- | ------------------------------------ | ------------------------------------------------- | +| ITEM | Lookup by unique property | `ITEM("ben@corp.com", 'Employees'.'Email')` | +| MATCH | Lookup by any property (first match) | `MATCH('Order'.'ProductCode', 'Products'.'Code')` | +| SHIFT | Offset dimension items | `SHIFT('Sales', 1)` | +| TIMEDIM | Convert date to calendar element | `TIMEDIM('Transactions'.'Date', Month)` | + +--- + +## ITEM + +Looks up an item in a dimension list based on a unique property. Returns BLANK if no match is found. + +Syntax: `ITEM(ValueToFind, Dimension.'UniqueProperty')` + +Examples: + +```pigment +ITEM("ben@corp.com", 'Employees'.'Email') // Returns the corresponding employee +ITEM('Transaction'.'ProductCode', 'Products'.'Code') // Lookup by product code +``` + +--- + +## MATCH + +Looks up an item in a dimension list based on any property (unique or not). Returns the first match or BLANK. + +Syntax: `MATCH(ValueToMatch, Expression)` + +Example: + +```pigment +MATCH('Order'.'ProductName', 'Products'.'Name') // First matching product +``` + +--- + +## SHIFT + +Offsets dimension items by a number of positions. Returns a dimension item, not a value. Used for dimension-typed blocks. + +Syntax: `SHIFT(Block, Offset)` + +Example: + +```pigment +// Shift a dimension-typed block (returns dimension item) +SHIFT('Employee'.'Start Month', 1) +``` + +--- + +## TIMEDIM + +Converts a date into an element of a calendar dimension (Year, Month, Week, etc.). + +Syntax: `TIMEDIM(Date, TimeDimension)` + +Time Dimensions: Year, Half, Quarter, Month, Week, Day + +Examples: + +```pigment +TIMEDIM('Transactions'.'Date', Month) // Convert to month +TIMEDIM('Employee'.'Start Date', Quarter) // Convert to quarter +TIMEDIM(DATE(2024,6,15), Year) // Convert date to year +TIMEDIM('Orders'.'OrderDate', Week) // Convert to week +``` + +Key Points: + +- Essential for aggregating transaction data to time periods +- Respects fiscal year settings in calendar configuration +- Use with BY modifier to aggregate transaction lists +- See [formula_modifiers.md](./formula_modifiers.md) for transaction aggregation patterns + +--- + +## Critical Rules + +- ITEM works only with unique properties (faster) +- MATCH works with any property (returns first match) +- TIMEDIM respects fiscal year settings - Not calendar year +- TIMEDIM essential for transaction aggregation - Use with BY modifier +- Actual/Forecast properties - Configure in calendar, use in formulas +- Time hierarchy is automatic - Month aggregates to Quarter, Year +- Multiple calendars need specification - Specify which calendar in TIMEDIM +- SHIFT facilitates temporal offsets with dimension positions +- Aggregate transactions early - Use TIMEDIM + BY for performance + +--- + +## See Also + +- [formula_modifiers.md](./formula_modifiers.md) - TIMEDIM usage examples with BY modifier +- [functions_iterative_calculation.md](./functions_iterative_calculation.md) - PREVIOUS, PREVIOUSOF for iterative/sequential calculations +- [functions_numeric.md](./functions_numeric.md) - CUMULATE, MOVINGSUM for time series diff --git a/plugins/pigment/skills/writing-pigment-formulas/functions_numeric.md b/plugins/pigment/skills/writing-pigment-formulas/functions_numeric.md new file mode 100644 index 00000000..4b17576b --- /dev/null +++ b/plugins/pigment/skills/writing-pigment-formulas/functions_numeric.md @@ -0,0 +1,200 @@ +# Numeric Functions + +Mathematical operations, cumulative calculations, window functions, and ranking capabilities. + +Covers: Basic Math, Rounding, Cumulative Functions, Window Functions, Ranking + +--- + +## Quick Reference + +| Category | Functions | +| -------------- | ----------------------------------------------------------------------- | +| Basic Math | ABS, SIGN, EXP, LN, LOG, SIN, COS, SQRT, MIN, MAX, MOD, QUOTIENT, POWER | +| Rounding | ROUND, ROUNDUP, ROUNDDOWN, TRUNC, CEILING, FLOOR | +| Cumulative | CUMULATE, DECUMULATE | +| Window | MOVINGSUM, MOVINGAVERAGE | +| Ranking | RANK, SPREAD | + +--- + +## Basic Math Functions + +| Function | Syntax | Returns | Example | +| ------------ | --------------------------- | ---------------- | --------------------- | +| ABS | `ABS(Number)` | Absolute value | `ABS(-5)` -> 5 | +| SIGN | `SIGN(Number)` | 1, -1, or 0 | `SIGN(-10)` -> -1 | +| EXP | `EXP(Number)` | e^Number | `EXP(1)` -> 2.718 | +| LN | `LN(Number)` | Natural log | `LN(2.718)` -> 1 | +| LOG | `LOG(Number)` | Log base 10 | `LOG(100)` -> 2 | +| SIN | `SIN(Number)` | Sine (radians) | `SIN(0)` -> 0 | +| COS | `COS(Number)` | Cosine (radians) | `COS(0)` -> 1 | +| SQRT | `SQRT(Number)` | Square root | `SQRT(16)` -> 4 | +| MIN | `MIN(Value1, Value2, ...)` | Minimum value | `MIN(5, 10, 3)` -> 3 | +| MAX | `MAX(Value1, Value2, ...)` | Maximum value | `MAX(5, 10, 3)` -> 10 | +| MOD | `MOD(Number, Divisor)` | Remainder | `MOD(10, 3)` -> 1 | +| QUOTIENT | `QUOTIENT(Number, Divisor)` | Integer quotient | `QUOTIENT(10, 3)` -> 3 | +| POWER | `POWER(Number, Power)` | Number^Power | `POWER(2, 3)` -> 8 | + +--- + +## Rounding Functions + +| Function | Behavior | Syntax | Example | +| ------------- | ---------------------- | ------------------------------ | -------------------------- | +| ROUND | Round to N decimals | `ROUND(Number [, Digits])` | `ROUND(3.14159, 2)` -> 3.14 | +| ROUNDUP | Round up (away from 0) | `ROUNDUP(Number [, Digits])` | `ROUNDUP(3.14, 0)` -> 4 | +| ROUNDDOWN | Round down (toward 0) | `ROUNDDOWN(Number [, Digits])` | `ROUNDDOWN(3.99, 0)` -> 3 | +| TRUNC | Truncate decimals | `TRUNC(Number [, Digits])` | `TRUNC(3.99)` -> 3 | +| CEILING | Round up to integer | `CEILING(Number)` | `CEILING(3.2)` -> 4 | +| FLOOR | Round down to integer | `FLOOR(Number)` | `FLOOR(3.9)` -> 3 | + +Note: Negative or out-of-range decimals (< -14 or > 14) return BLANK. + +--- + +## Cumulative Functions + +### CUMULATE + +Accumulate values over a dimension (typically Time). + +Syntax: `CUMULATE(Number, Cumulated Dimension [, Group Dimension] [, Aggregation])` + +Examples: + +```pigment +CUMULATE('Monthly Sales', Month) // Running total by month +CUMULATE('Quantity Sold', Month, Month.Year) // Cumulative by month, reset each year +``` + +--- + +### DECUMULATE + +Reverse cumulative sum (convert cumulative to periodic values). + +Syntax: `DECUMULATE(Number, Decumulated Dimension [, Group Dimension])` + +Examples: + +```pigment +DECUMULATE('YTD Revenue', Month) // Monthly revenue from YTD +DECUMULATE(CUMULATE('Sales', Month), Month) // Returns original Sales +``` + +--- + +## Window Functions + +### MOVINGSUM + +Calculate sum over a moving window. + +Syntax: `MOVINGSUM(Input, Window Size [, End Offset] [, Dimension])` + +Examples: + +```pigment +MOVINGSUM('Sales', 3) // 3-period moving sum (default over Time) +MOVINGSUM('Revenue', 12, -1) // 12-period window, offset by -1 +``` + +--- + +### MOVINGAVERAGE + +Calculate average over a moving window. + +Syntax: `MOVINGAVERAGE(Input, Window Size [, End Offset] [, Dimension])` + +Examples: + +```pigment +MOVINGAVERAGE('Sales', 3) // 3-period moving average +MOVINGAVERAGE('Revenue', 12, -1) // 12-period window, offset by -1 +``` + +--- + +## Ranking Functions + +### RANK + +Assign rank to items based on a metric value. Ranks are 1-based: the lowest possible rank is 1, not 0 (e.g. "first" = 1, "second" = 2). + +Syntax: `RANK(Source Block [, Group] [, Direction] [, Ties])` + +Skipping the Group parameter: If you need to set Direction or Ties while skipping Group, you must pass `""` as a placeholder for Group - do not simply omit it. +Examples: `RANK('MetricA', "", DESC)` and `RANK('MetricA', "", DESC, SEQUENTIAL)`. + +- Source Block: The values to rank (e.g. a metric or list property). +- Group: The dimension(s) *within which* ranks are computed. Ranks are calculated separately for each member of Group. This is the grouping dimension (scope of the ranking), not the dimension of the block you are ranking. +- Direction: ASC (smallest value gets rank 1) or DESC (largest value gets rank 1). In both cases, rank 1 is the "best" position; there is no rank 0. +- Ties: Optional. One of MINIMUM (default), MAXIMUM, SEQUENTIAL, AVERAGE. + +Examples: + +```pigment +RANK('Revenue', Product, DESC) // Rank products by revenue, highest = 1 (one rank per Product) +RANK('Salary', Employee, ASC) // Rank employees by salary, lowest = 1 (one rank per Employee) +RANK(Account.TAM, "", ASC) // Rank accounts by TAM across all accounts - skip Group with placeholder +RANK(Account.TAM, Region, ASC) // Rank accounts by TAM within each Region +``` + +Critical - avoid wrong Group (common agent mistake): + +- Do not use the same dimension as the block's dimension when you want to rank *along* that dimension. Using it as Group resets the rank for each member and gives 1 everywhere. +- Wrong: `RANK(Account.TAM, Account, ASC)` - ranks "within each Account", so one value per Account -> every Account gets rank 1. + +--- + +### SPREAD + +Distribute a value evenly across a specified number of items along a dimension. + +Syntax: `SPREAD(Source Block, Ranking Dimension, Spread Number [, Starting Index])` + +Examples: + +```pigment +SPREAD('Quantity Sold', Month, 3) // Split value over 3 months +SPREAD(10[BY: VAR_Spread_Start_Month], Month, 6) // VAR_Spread_Start_Month: input metric, type Dimension +``` + +--- + +## Common Patterns + +```pigment +// Year-to-Date +CUMULATE('Monthly Revenue', Month) + +// Month-over-Month Change +'Revenue' - 'Revenue'[SELECT: Month-1] + +// 3-Month Moving Average +MOVINGAVERAGE('Sales', 3) + +// Top 10 Products +IF(RANK('Revenue', Product, DESC) <= 10, 'Revenue', BLANK) + +// Even Allocation +SPREAD('Total Budget', Department, 5) +``` + +--- + +## Critical Rules + +- CUMULATE/DECUMULATE: Typically used on time dimensions +- Rounding: Negative or out-of-range decimals return BLANK +- SPREAD: Evenly splits value, not proportional allocation +- Window functions: Truncate at dimension edges +- Performance: Use FILTER to subset time dimension for large datasets + +--- + +## See Also + +- [functions_iterative_calculation.md](./functions_iterative_calculation.md) - PREVIOUS, PREVIOUSOF for iterative/sequential calculations diff --git a/plugins/pigment/skills/writing-pigment-formulas/functions_security.md b/plugins/pigment/skills/writing-pigment-formulas/functions_security.md new file mode 100644 index 00000000..c54c42de --- /dev/null +++ b/plugins/pigment/skills/writing-pigment-formulas/functions_security.md @@ -0,0 +1,102 @@ +# Security Functions + +Access control and security-related operations in formulas. + +Covers: ACCESSRIGHTS, RESETACCESSRIGHTS + +--- + +## Quick Reference + +| Function | Purpose | Syntax Example | +| --------------------- | ------------------------------ | ----------------------------------------- | +| ACCESSRIGHTS | Define read/write access | `ACCESSRIGHTS(ReadBoolean, WriteBoolean)` | +| RESETACCESSRIGHTS | Remove inherited access rights | `RESETACCESSRIGHTS(Expression)` | + +--- + +## ACCESSRIGHTS + +Constructs an Access Rights value based on specified read and write Boolean conditions. Used within Access Rights Metrics to define which users can read or write data. + +Syntax: `ACCESSRIGHTS(ReadAccessBoolean, WriteAccessBoolean)` + +Parameters: + +- ReadAccessBoolean: Boolean. TRUE allows Members to read data, FALSE or BLANK prevents reading +- WriteAccessBoolean: Boolean. TRUE allows Members to write data, FALSE or BLANK prevents writing + +Return Type: Access Rights + +Examples: + +```pigment +ACCESSRIGHTS(TRUE, FALSE) // Grants read-only access +ACCESSRIGHTS(User.Role = "Admin", TRUE) // Grants read access to Admins, write access to all +``` + +Key Points: + +- Use BLANK instead of FALSE for better performance with Pigment's sparse engine +- Does not directly apply access rights to a Block; builds Access Rights values for use in Access Rights Metrics +- For performance: `IFDEFINED(User, 'Revenue'[AR: 'Rules'])` (see [formula_performance_patterns.md](./formula_performance_patterns.md)) + +--- + +## RESETACCESSRIGHTS + +Removes inherited access rights from the referenced Block or expression. Direct access rights rules applied to the Block itself will still apply. + +Syntax: `RESETACCESSRIGHTS(Expression)` + +Parameters: + +- Expression: The formula or Block reference from which to remove inherited access rights + +Return Type: Same as Expression + +Examples: + +```pigment +// Remove inherited access rights from 'Metric A' only +RESETACCESSRIGHTS('Metric A') + 'Metric B' +``` + +Use Cases: + +- Aggregations: Allow users with partial access to see totals or calculated results +- Shared Blocks: Control access rights inheritance when referencing Blocks shared from other Applications +- Performance Optimization: Reduce redundant access rights inheritance in complex models + +Key Points: + +- Only removes _inherited_ access rights (direct access rights rules still apply) +- Use as specifically as possible to avoid exposing more data than intended +- Application settings may need adjustment for shared Blocks + +--- + +## Best Practices + +1. Apply Access Rights at the Right Level: Apply as late as possible in calculation chains +2. Document Security Logic: Clearly document where and why access rights are applied or reset +3. Test Security: Verify access rights work correctly for different user roles +4. Minimize RESETACCESSRIGHTS: Only use when calculations truly require the full dataset + +--- + +## Critical Rules + +- Use BLANK over FALSE - Better performance with sparse engine +- ACCESSRIGHTS builds values - Used in Access Rights Metrics, not directly applied +- RESETACCESSRIGHTS removes inherited only - Direct rules still apply +- Performance pattern: `IFDEFINED(User, Data[AR: 'Rules'])` (see performance patterns) +- Test thoroughly - Verify for different user roles + +--- + +## See Also + +- [functions_logical.md](./functions_logical.md) - IFDEFINED for access rights patterns +- [formula_performance_patterns.md](./formula_performance_patterns.md) - Access rights optimization +- [formula_writing_workflow.md](./formula_writing_workflow.md) - Structured approach to writing formulas diff --git a/plugins/pigment/skills/writing-pigment-formulas/functions_text.md b/plugins/pigment/skills/writing-pigment-formulas/functions_text.md new file mode 100644 index 00000000..1c86b13e --- /dev/null +++ b/plugins/pigment/skills/writing-pigment-formulas/functions_text.md @@ -0,0 +1,387 @@ +# Text Functions + +String manipulation, text transformation, and pattern matching functions. + +Covers: Conversion, Extraction, Case Transformation, Text Matching, Concatenation, Search & Replace + +--- + +## Quick Reference + +| Function | Purpose | Syntax Example | +| -------------- | ------------------ | ----------------------------------------- | +| TEXT | Convert to text | `TEXT(123.45)` -> "123.45" | +| VALUE | Convert to number | `VALUE("123.45")` -> 123.45 | +| LEN | String length | `LEN("Hello")` -> 5 | +| LEFT | Left N characters | `LEFT("Hello", 2)` -> "He" | +| MID | Middle substring | `MID("Hello", 2, 3)` -> "ell" | +| RIGHT | Right N characters | `RIGHT("Hello", 2)` -> "lo" | +| LOWER | Lowercase | `LOWER("Hello")` -> "hello" | +| UPPER | Uppercase | `UPPER("Hello")` -> "HELLO" | +| PROPER | Title case | `PROPER("hello world")` -> "Hello World" | +| TRIM | Remove spaces | `TRIM(" hello ")` -> "hello" | +| CONTAINS | Check substring | `CONTAINS("ell", "Hello")` -> TRUE | +| STARTSWITH | Check prefix | `STARTSWITH("He", "Hello")` -> TRUE | +| ENDSWITH | Check suffix | `ENDSWITH("lo", "Hello")` -> TRUE | +| & | Concatenate | `"Hello" & " " & "World"` -> "Hello World" | +| FIND | Find position | `FIND("ell", "Hello")` -> 2 | +| SUBSTITUTE | Replace text | `SUBSTITUTE("Hello", "l", "L")` -> "HeLLo" | + +--- + +## Conversion Functions + +### TEXT + +Convert a number or integer to text. + +Syntax: `TEXT(Value)` + +Examples: + +```pigment +TEXT(123.456) // "123.456" +TEXT(-44.23) // "-44.23" +``` + +--- + +### VALUE + +Convert text to number. Available as `NUMBER` (interchangeable alias). + +Syntax: `VALUE(Text)` or `NUMBER(Text)` + +Examples: + +```pigment +VALUE("123") // 123 +VALUE("123.45") // 123.45 +VALUE("-99.9") // -99.9 +``` + +Key Point: Returns BLANK if text is not numeric. + +--- + +## Extraction Functions + +### LEN + +Get string length. + +Syntax: `LEN(Text)` + +Examples: + +```pigment +LEN("Hello") // 5 +LEN("Hello World") // 11 +LEN("") // 0 +``` + +--- + +### LEFT + +Extract N characters from left. + +Syntax: `LEFT(Text, Count)` + +Examples: + +```pigment +LEFT("Hello World", 5) // "Hello" +LEFT("ABC123", 3) // "ABC" +LEFT("abc", 5) // "abc" (count exceeds length - all returned) +LEFT('Product'.'SKU', 2) // First 2 characters of SKU +``` + +Key Point: Returns BLANK if Count is a negative integer. + +--- + +### MID + +Extract substring from middle. + +Syntax: `MID(Text, Start, Count)` + +Parameters: + +- Start: Starting position (1-based) +- Count: Number of characters to extract + +Examples: + +```pigment +MID("Hello World", 7, 5) // "World" +MID("ABC123DEF", 4, 3) // "123" +MID('Product'.'SKU', 3, 4) // Characters 3-6 of SKU +``` + +Key Point: Returns BLANK if Start is 0/negative or if Count is negative. + +--- + +### RIGHT + +Extract N characters from right. + +Syntax: `RIGHT(Text, Count)` + +Examples: + +```pigment +RIGHT("Hello World", 5) // "World" +RIGHT("ABC123", 3) // "123" +RIGHT("abc", 5) // "abc" (count exceeds length - all returned) +RIGHT('Product'.'SKU', 2) // Last 2 characters of SKU +``` + +Key Point: Returns BLANK if Count is a negative integer. + +--- + +## Case Transformation + +### LOWER + +Convert to lowercase. + +Syntax: `LOWER(Text)` + +Examples: + +```pigment +LOWER("HELLO") // "hello" +LOWER("Hello World") // "hello world" +LOWER('Product'.'Name') // Lowercase product name +``` + +--- + +### UPPER + +Convert to uppercase. + +Syntax: `UPPER(Text)` + +Examples: + +```pigment +UPPER("hello") // "HELLO" +UPPER("Hello World") // "HELLO WORLD" +UPPER('Customer'.'Email') // Uppercase email +``` + +--- + +### PROPER + +Capitalize the first letter of the string and every letter that follows a non-alphabetic character (spaces, digits, punctuation, symbols). All other letters become lowercase. + +Syntax: `PROPER(Text)` + +Examples: + +```pigment +PROPER("hello world") // "Hello World" +PROPER("JOHN SMITH") // "John Smith" +PROPER("c@ss n1te") // "C@Ss N1Te" (digits/symbols trigger capitalization) +PROPER('Customer'.'Name') // Title case name +``` + +--- + +## Text Cleaning + +### TRIM + +Removes leading and trailing spaces, and replaces multiple internal spaces with a single space. + +Syntax: `TRIM(Text)` + +Examples: + +```pigment +TRIM(" hello ") // "hello" +TRIM("hello world") // "hello world" +TRIM(" this is a test ") // "this is a test" +``` + +--- + +## Text Matching + +### CONTAINS + +Check if text contains a substring (not case sensitive by default). + +Syntax: `CONTAINS(Text to Find, Text to Search [, Starting Position to Search] [, Is Case Sensitive])` + +Examples: + +```pigment +CONTAINS("World", "Hello World") // TRUE +CONTAINS("world", "Hello World") // TRUE (not case sensitive by default) +CONTAINS("premium", 'Product'.'Description') // Check for keyword +CONTAINS("A", "abc", 1, TRUE) // FALSE (case sensitive) +``` + +Common Use: Filtering, keyword search, text matching + +--- + +### STARTSWITH + +Check if text starts with a prefix (not case sensitive by default). + +Syntax: `STARTSWITH(Start Text, Text to Search [, Is Case Sensitive])` + +Examples: + +```pigment +STARTSWITH("Hello", "Hello World") // TRUE +STARTSWITH("world", "Hello World") // FALSE +STARTSWITH("PRD", 'Product'.'SKU') // SKU starts with PRD +STARTSWITH("A", "abc", TRUE) // FALSE (case sensitive) +``` + +--- + +### ENDSWITH + +Check if text ends with a suffix (not case sensitive by default). + +Syntax: `ENDSWITH(End Text, Text to Search [, Is Case Sensitive])` + +Examples: + +```pigment +ENDSWITH("World", "Hello World") // TRUE +ENDSWITH("Hello", "Hello World") // FALSE +ENDSWITH(".pdf", 'File'.'Name') // Check file extension +ENDSWITH("C", "abc", TRUE) // FALSE (case sensitive) +``` + +--- + +## Concatenation + +### & Operator + +Concatenate strings. + +Syntax: `Text1 & Text2 & ...` + +Examples: + +```pigment +"Hello" & " " & "World" // "Hello World" +'First Name' & " " & 'Last Name' // "John Smith" +'Product'.'Code' & "-" & 'Product'.'Version' // "PRD-v1.2" +TEXT('Order'.'Number') & "-" & TEXT('Year') // "1-2024" +``` + +Common Uses: Building composite keys, formatting display strings, creating labels + +--- + +## Search & Replace + +### FIND + +Find position of a substring (not case sensitive by default, 1-based). + +Syntax: `FIND(Text to Find, Text to Search [, Starting Position to Search] [, Is Case Sensitive])` + +Examples: + +```pigment +FIND("World", "Hello World") // 7 +FIND("l", "Hello") // 3 (first occurrence) +FIND("x", "Hello") // BLANK (not found) +FIND("A", "abc", 1, TRUE) // BLANK (case sensitive) +``` + +Returns: Position (1-based), or BLANK if not found OR if the starting position is 0/negative. + +--- + +### SUBSTITUTE + +Replace one or all occurrences of a substring. Case sensitive (unlike CONTAINS/STARTSWITH/ENDSWITH/FIND). + +Syntax: `SUBSTITUTE(Text, OldText, NewText [, OccurrenceNumber])` + +Examples: + +```pigment +SUBSTITUTE("aaa", "a", "b") // "bbb" (all occurrences) +SUBSTITUTE("aaa", "a", "b", 2) // "aba" (only second occurrence) +SUBSTITUTE("abc", "A", "d") // "abc" (case sensitive - "A" not found) +``` + +Key Point: Returns BLANK if OccurrenceNumber is a negative integer. + +--- + +## Common Patterns + +### Pattern 1: Composite Key + +```pigment +'Customer'.'Country' & "-" & 'Customer'.'ID' +``` + +### Pattern 2: Format Display Name + +```pigment +PROPER('First Name' & " " & 'Last Name') +``` + +### Pattern 3: Extract SKU Prefix + +```pigment +LEFT('Product'.'SKU', 3) +``` + +### Pattern 4: Clean and Uppercase Email + +```pigment +UPPER(TRIM('User'.'Email')) +``` + +### Pattern 5: Check Category + +```pigment +IF(CONTAINS("premium", 'Product'.'Name'), "Premium", "Standard") +``` + +### Pattern 6: Build Date Range Display + +```pigment +TEXT('Start Date') & " - " & TEXT('End Date') +``` + +### Pattern 7: Extract Domain from User.Email + +```pigment +MID(User.Email, FIND("@", User.Email) + 1, LEN(User.Email) - FIND("@", User.Email)) +``` + +--- + +## Critical Rules + +- Text matching is NOT case sensitive by default - CONTAINS, STARTSWITH, ENDSWITH, FIND (use optional `Is Case Sensitive` argument for case-sensitive matching) +- SUBSTITUTE is case sensitive - No optional argument to change this +- FIND returns BLANK if not found - Not 0. Also returns BLANK if starting position is 0/negative +- LEFT/RIGHT/MID return BLANK on negative counts - MID also returns BLANK on a 0/negative Start +- SUBSTITUTE can replace specific occurrence - Use optional occurrence argument +- TRIM removes leading/trailing spaces and replaces multiple internal spaces with single space +- FIND/MID positions are 1-based - Not 0-based +- & (ampersand) concatenates anything - Numbers, dates, text (auto-converts) +- Text functions are expensive - Minimize in loops or large calculations +- Pre-calculate in properties - Don't recalculate same text transformation in formulas diff --git a/plugins/pigment/skills/writing-pigment-formulas/functions_time_and_date.md b/plugins/pigment/skills/writing-pigment-formulas/functions_time_and_date.md new file mode 100644 index 00000000..25bd5372 --- /dev/null +++ b/plugins/pigment/skills/writing-pigment-formulas/functions_time_and_date.md @@ -0,0 +1,505 @@ +# Time and Date Functions + +Functions for date manipulation, time period calculations, and temporal operations. + +Covers: Date Creation, Date Extraction, Date Math, Period Functions, Previous/Fill Functions + +--- + +## Quick Reference + +| Category | Functions | +| -------------------- | ----------------------------------------------------------------------- | +| Date Creation | DATE, DATEVALUE, EDATE, EOMONTH, STARTOFMONTH | +| Date Extraction | DAY, WEEKDAY, MONTH, YEAR | +| Date Math | DAYS, MONTHDIF, NETWORKDAYS | +| Period Functions | INPERIOD, DAYSINPERIOD, PRORATA, MONTHTODATE, QUARTERTODATE, YEARTODATE | +| Temporal | PREVIOUS, PREVIOUSOF, FILLFORWARD | + +> Converting a date to a dimension member (Month, Quarter, Year)? Use `TIMEDIM(Date, TimeDimension)` from [functions_lookup.md](./functions_lookup.md). TIMEDIM returns a dimension element (not a Date value), which is required when mapping transaction dates into time dimensions via the BY modifier or when creating Dimension-typed properties. Prefer TIMEDIM over STARTOFMONTH when the result must be a Month dimension member rather than a plain Date. + +MP02 - planning period bounds: Do not use `DATE(YYYY, M, D)` for forecast horizon, seed month, or switchover. Use `VAR_` input metrics (type Date or Dimension Month), e.g. `IF(Month >= VAR_Start_Month AND Month <= VAR_End_Month, 'Revenue')` - not `DATE(2026, 1, 1)`. + +--- + +## Date Functions Reference + +| Function | Syntax | Returns | Example | +| ---------------- | ------------------------------ | ------------------ | ---------------------------------------------------- | +| DATE | `DATE(Year, Month, Day)` | Date | `DATE(2024, 3, 15)` -> 2024-03-15 | +| DATEVALUE | `DATEVALUE(Text, Format)` | Date | `DATEVALUE("2024-03-15", "yyyy-MM-dd")` -> 2024-03-15 | +| DAY | `DAY(Date)` | Day (1-31) | `DAY(DATE(2024,3,15))` -> 15 | +| WEEKDAY | `WEEKDAY(Date)` | Weekday (0-6) | `WEEKDAY(DATE(2024,3,15))` -> 5 (Friday) | +| MONTH | `MONTH(Date)` | Month (1-12) | `MONTH(DATE(2024,3,15))` -> 3 | +| YEAR | `YEAR(Date)` | Year | `YEAR(DATE(2024,3,15))` -> 2024 | +| DAYS | `DAYS(StartDate, EndDate)` | Days between | `DAYS(DATE(2024,3,1), DATE(2024,3,15))` -> 14 | +| MONTHDIF | `MONTHDIF(StartDate, EndDate)` | Months between | `MONTHDIF(DATE(2024,1,15), DATE(2024,3,15))` -> 2 | +| NETWORKDAYS | `NETWORKDAYS(Start, End)` | Business days | `NETWORKDAYS(DATE(2024,3,1), DATE(2024,3,31))` -> ~22 | +| EOMONTH | `EOMONTH(Date)` | Last day of month | `EOMONTH(DATE(2024,3,15))` -> 2024-03-31 | +| STARTOFMONTH | `STARTOFMONTH(Date)` | First day of month | `STARTOFMONTH(DATE(2024,3,15))` -> 2024-03-01 | +| EDATE | `EDATE(Date, Months)` | Date + N months | `EDATE(DATE(2024,3,15), 2)` -> 2024-05-15 | + +WEEKDAY Values: 0=Sunday, 1=Monday, 2=Tuesday, 3=Wednesday, 4=Thursday, 5=Friday, 6=Saturday + +--- + +## Period Functions + +### INPERIOD + +Check if date falls within a time dimension item. + +Syntax: `INPERIOD(Date, TimeDimension)` + +Examples: + +```pigment +// Check if transaction date is in current month +INPERIOD('Transactions'.'Date', Month) + +// Filter transactions to specific quarter +IF(INPERIOD('Orders'.'Date', Quarter), 'Orders'.'Amount', BLANK) + +// Check if date is in current year +INPERIOD('Events'.'Date', Year) +``` + +Returns: Boolean (TRUE if date is in period, FALSE otherwise) + +Common Uses: Filtering transactions, date-based conditionals, period matching + +--- + +### DAYSINPERIOD + +Returns the number of days for each period of time. + +Syntax: `DAYSINPERIOD(Time Dimension [, Start Date] [, End Date] [, Working Days] [, Holidays])` + +Parameters: + +- Time Dimension: Required. Time dimension based on calendar settings (Week, Month, Quarter, Half, Year) +- Start Date: Optional. Start date to count from +- End Date: Optional. End date to count to +- Working Days: Optional. Boolean metric on Day of Week dimension defining working days +- Holidays: Optional. Boolean metric on Day dimension defining holidays + +Examples: + +```pigment +// Days in each month (28-31 depending on month) +DAYSINPERIOD(Month) + +// Days in each quarter +DAYSINPERIOD(Quarter) + +// Days in month within a date range +DAYSINPERIOD(Month, DATE(2024,6,1), DATE(2024,12,31)) + +// Days an employee was present in each month +DAYSINPERIOD(Month, 'Employee'.'Start Date', 'Employee'.'End Date') + +// Working days in month (excluding weekends and holidays) +DAYSINPERIOD(Month, DATE(2024,6,1), DATE(2024,12,31), 'Working Days', 'Holidays') +``` + +Returns: Integer (number of days in period) + +Use Cases: Daily averages, FTE calculations, capacity planning, working day calculations + +--- + +### PRORATA + +Returns the proportion of days over time dimensions. Useful for prorating values for partial periods. + +Syntax: `PRORATA(Time Dimension [, Start Date] [, End Date] [, Working Days] [, Holidays])` + +Parameters: + +- Time Dimension: Required. Time dimension based on calendar settings (Day, Week, Month, Quarter, Half, Year) +- Start Date: Optional. Start date (included in calculation) +- End Date: Optional. End date (excluded from calculation) +- Working Days: Optional. Boolean metric on Day of Week dimension defining working days +- Holidays: Optional. Boolean metric on Day dimension defining holidays + +Examples: + +```pigment +// Returns 1 for each month (full period) +PRORATA(Month) + +// Returns 1 for each month starting June 1st 2020, BLANK prior +PRORATA(Month, DATE(2020,6,1)) + +// Prorata of days: June 2020 (16/30) and July 2020 (13/31) +PRORATA(Month, DATE(2020,6,15), DATE(2020,7,14)) + +// Monthly FTE by employee (end date included, so add +1) +'Employee'.'Salary' * PRORATA(Month, 'Employee'.'Start Date', 'Employee'.'End Date' + 1) + +// Monthly headcount (1 if employee present on last day of month) +PRORATA(Month, STARTOFMONTH('Employee'.'Start Date'), STARTOFMONTH('Employee'.'End Date' + 1)) + +// Open-ended: current employees with no term date (range from start to end of calendar) +PRORATA(Month, 'Employee'.'Start Date') +``` + +Returns: Number (proportion between 0 and 1) + +Use Cases: + +- Prorating salaries or costs for partial periods +- FTE (Full-Time Equivalent) calculations +- Allocating annual amounts to partial periods +- Handling employee start/end dates mid-period + +Key Points: + +- Start Date is included in calculation +- End Date is optional. When omitted (2-argument form), the range is from Start Date to the end of the calendar (open-ended). +- End Date is excluded from calculation (add +1 if end date should be included) +- Respects calendar settings (month length, fiscal year, leap years) + +Guideline: Do not use `IFBLANK('Term Date', DATE(9999,12,31))` (or similar) for PRORATA. When there is no end date (e.g. current employees with no term date), use the 2-argument form: `PRORATA(Time Dimension, Start Date)`. + +#### Pattern: Presence + Boolean from PRORATA + +`PRORATA()` is not only for prorating amounts; it is the canonical pattern for "active within a date range" on time dimensions. + +Numeric presence on Day (1 or BLANK): + +```pigment +// 1 on active days, BLANK outside the date range +PRORATA(Day, 'Start Date', 'End Date' + 1) +``` + +This replaces verbose multi-conditional IFs such as: + +```pigment +// Avoid this: +IF( + Day >= 'Start Date' + AND Day <= 'End Date', + 1, + BLANK +) +``` + +Boolean or numeric presence derived from PRORATA (without densifying): + +To get a boolean flag from the same logic: + +```pigment +// TRUE when the day is within [Start Date, End Date], BLANK otherwise +ISDEFINED( + PRORATA(Day, 'Start Date', 'End Date' + 1) +) +``` + +To get an explicit 1/BLANK numeric flag: + +```pigment +// 1 on active days, BLANK otherwise +IFDEFINED( + PRORATA(Day, 'Start Date', 'End Date' + 1), + 1 +) +``` + +Do not use ISBLANK / ISNOTBLANK on this pattern, as they densify. + +Use this pattern whenever you have: + +- A continuous interval [Start Date, End Date], and +- You need either: + - A numeric factor for allocations / FTE / costs, or + - A clean presence flag (boolean or 1/BLANK) built from that factor. + +--- + +### MONTHTODATE (MTD) + +Cumulates a metric defined on the Day Time Dimension and resets the cumulation each month. + +Syntax: `MONTHTODATE(Metric [, Aggregation])` + +Example: + +```pigment +// Cumulative sales in the current month +MONTHTODATE('Daily Sales') +``` + +Returns: Number (cumulative value per day, resets each month) + +--- + +### QUARTERTODATE (QTD) + +Cumulates a metric defined on a Time Dimension (Day or Month) and resets the cumulation each quarter. + +Syntax: `QUARTERTODATE(Metric [, Aggregation])` + +Example: + +```pigment +// Cumulative sales in the current quarter +QUARTERTODATE('Monthly Sales') +``` + +--- + +### YEARTODATE (YTD) + +Cumulates a metric defined on a Time Dimension (Day, Month, or Quarter) and resets the cumulation each year. + +Syntax: `YEARTODATE(Metric [, Aggregation])` + +Example: + +```pigment +// Cumulative sales in the current year +YEARTODATE('Monthly Sales') +``` + +--- + +## Temporal Functions + +### PREVIOUS and PREVIOUSOF + +- PREVIOUS(Dimension [, Offset]): Iterative calculation within a single Block; returns the previous cell of the current metric in the iteration dimension. Use when the formula references itself along that dimension. +- PREVIOUSOF(Metric [, Offset]): Iterative calculation across Blocks; requires an iterative calculation configuration. Use when multiple metrics reference each other in a cycle. + +Examples: + +```pigment +// Previous month value of the current metric +PREVIOUS(Month) + +// Previous value with offset of 2 +PREVIOUS(Month, 2) +``` + +--- + +### PREVIOUSOF + +Returns the value of the previous cell in the iteration dimension of any metric defined in the iterative calculation configuration. + +Syntax: `PREVIOUSOF(Metric [, Offset])` + +WARNING: PREREQUISITE - Iterative Calculation (PREVIOUSOF) Configuration Required: +PREVIOUSOF can ONLY be used on metrics that have been configured for iterative calculation in the Pigment application settings. Use `tool:create_cycle` when available, or `tool:list_cycles` / `tool:update_cycle` to inspect or adjust an existing cycle. If MCP cycle tools are unavailable, ask the user to configure the cycle in the Pigment UI. + +Before writing any formula with PREVIOUSOF: +1. Ask the user whether iterative calculation is already configured for the target metric(s) +2. If not configured, create or update the cycle with available tools. +3. Do NOT apply a PREVIOUSOF formula to a metric that is not configured for iterative calculation - it will fail with: "PREVIOUSOF can only be used in a Metric used by an iterative calculation" + +Examples: + +```pigment +// Previous month's ending inventory +PREVIOUSOF('Ending Inventory') + +// Previous value with offset of 3 +PREVIOUSOF('Revenue', 3) +``` +Full reference: Syntax, circular dependencies, configuration, performance, and debugging are in [Iterative Calculation (PREVIOUS & PREVIOUSOF)](./functions_iterative_calculation.md). + +--- + +### FILLFORWARD + +Fill blank values with most recent non-blank value (forward fill). + +Syntax: `FILLFORWARD(Block, Dimension)` + +Examples: + +```pigment +// Fill missing monthly prices with last known price +FILLFORWARD('Price', Month) + +// Fill missing headcount data +FILLFORWARD('Headcount', Month) + +// Fill forward on custom dimension +FILLFORWARD('Exchange Rate', Date) +``` + +Key Points: + +- Fills blanks with most recent non-blank value +- Moves forward through dimension (left to right) +- If first value is blank, remains blank until first non-blank +- Common for prices, rates, static values + +--- + +## SELECT vs PREVIOUS/PREVIOUSOF + +WARNING: NEVER use PREVIOUS/PREVIOUSOF for simple prior period lookups. ALWAYS use SELECT. + +Key question: "Does this period's result depend on calculating the previous period's result first?" + +| Answer | Use | Example | +| ---------------------------------------------------- | ------------------- | ----------------------------------------------- | +| No - Just need prior period's value | `[SELECT: Month-N]` | `'Revenue'[SELECT: Month-1]` for MoM comparison | +| Yes - Current depends on prior calculated result | `PREVIOUSOF()` | Running balance where balance builds on itself | + +### Common Mistakes + +```pigment +// BAD: WRONG: Using PREVIOUS for simple lookup - 'Last Month Revenue' +PREVIOUS(Month) + +// GOOD: CORRECT: Using SELECT for lookup - 'Last Month Revenue' +'Revenue'[SELECT: Month-1] + +// BAD: WRONG: Using PREVIOUSOF for comparison - 'MoM Change' +'Revenue' - PREVIOUSOF('Revenue') + +// GOOD: CORRECT: Using SELECT for comparison - 'MoM Change' +'Revenue' - 'Revenue'[SELECT: Month-1] + +// GOOD: CORRECT: PREVIOUSOF for true iterative (balance depends on prior balance) - 'Ending Balance' +PREVIOUSOF('Ending Balance', 0) + 'Inflow' - 'Outflow' +``` + +### When PREVIOUS/PREVIOUSOF is Appropriate + +Only use when the current period's calculated result depends on the prior period's calculated result: + +```pigment +// Running balance - current balance = prior balance + changes - 'Ending Balance' +PREVIOUSOF('Ending Balance', 0) + 'Inflow' - 'Outflow' + +// Compounding - current value depends on prior calculated value - 'Compound Value' +PREVIOUSOF('Compound Value', 'Initial') * (1 + 'Growth Rate') +``` + +WARNING: REMINDER: These formulas will only work if iterative calculation is configured on the metric. There is no AI tool to configure this - the user must do it in the Pigment UI. Always confirm with the user before applying. + +For everything else, use SELECT or specialized functions: + +- Running totals -> `CUMULATE()` (not PREVIOUSOF + value) +- Fill blanks -> `FILLFORWARD()` (not IFBLANK + PREVIOUS) +- Prior period lookups -> `[SELECT: Month-N]` + +--- + +## Common Patterns + +### Date Math Patterns + +```pigment +// Number of days by Month +DAYSINPERIOD(Month) + +// Business days by month +NETWORKDAYS(Month.'Start Date', Month.'End Date') + +// Months between dates +MONTHDIF('Date 1', 'Date 2') + +// Date + number of months +EDATE('Date 1', 6) // 6 months later + +// Month + number of months +Month+1 // Next month + +// Penultimate day of the next month +(Month + 1).'End Date' - 1 + +// Check if the first day of the month is a Sunday +WEEKDAY(Month.'Start Date') = 0 +``` + +### Time Series Patterns + +```pigment +// Month-over-Month Change (use SELECT for simple lookups) +'Revenue' - 'Revenue'[SELECT: Month-1] + +// Month-over-Month % Change (use SELECT for simple lookups) +('Revenue' - 'Revenue'[SELECT: Month-1]) / 'Revenue'[SELECT: Month-1] + +// Year-over-Year Change (use SELECT for simple lookups) +'Revenue' - 'Revenue'[SELECT: Month-12] + +// Moving Average (3-month) +MOVINGAVERAGE('Sales', 3) + +// Cumulative Total (use CUMULATE, not PREVIOUSOF + value or PREVIOUS + value) +CUMULATE('Monthly Revenue', Month) + +// Fill Missing Values (use FILLFORWARD, not IFBLANK + PREVIOUS) +FILLFORWARD('Exchange Rate', Month) + +// True iterative: Ending balance depends on prior balance (use PREVIOUSOF) +PREVIOUSOF('Ending Balance') + 'Inflow' - 'Outflow' +``` + +### Period-to-Date Patterns + +```pigment +// MTD Revenue +MONTHTODATE('Amount') + +// QTD Revenue +QUARTERTODATE('Amount') + +// YTD Revenue +YEARTODATE('Amount') + +// Alternative YTD approach +CUMULATE('Monthly Revenue', Month) +``` + +### Common Use Cases + +```pigment +// Transaction aggregation by period +'Transactions'.'Amount'[BY: TIMEDIM('Transactions'.'Date', Month)] + +// Same period last year +'Revenue'[SELECT: Month-12] + +// Fill missing prices +FILLFORWARD('Product Price', Month) + +// Business days for proration +NETWORKDAYS(Month.'Start Date', Month.'End Date') + +// Check if date in period +IF(INPERIOD('Order'.'Date', Quarter), 'Order'.'Amount', BLANK) +``` + +--- + +## Critical Rules + +- Use `[SELECT: Month-N]` for simple lookups - Not `PREVIOUS`/`PREVIOUSOF`. Reserve iterative functions for true iterative calculations (balances, accumulators) +- PREVIOUS/PREVIOUSOF are iterative - Slow, sequential computation. Only use when current value depends on prior calculated value. Requires iterative calculation to be configured on the metric. Use available cycle tools when possible; otherwise ask the user to configure it in the Pigment UI. Always confirm before applying. +- WEEKDAY starts at 0 (Sunday) - Unlike Excel +- PREVIOUS moves on all time dimensions - Use PREVIOUSOF for single dimension +- PREVIOUSOF offset is positive - Always backward movement +- FILLFORWARD fills forward - From past to future, use instead of IFBLANK + PREVIOUS +- CUMULATE for running totals - Use instead of PREVIOUSOF + value or PREVIOUS() + value +- Period-to-date functions use evaluation date - "Current" = when formula runs +- INPERIOD returns Boolean - Use in IF for filtering +- PRORATA End Date is excluded - Add +1 to include the end date +- DAYSINPERIOD returns Integer - Use for day counts, PRORATA for proportions +- NETWORKDAYS excludes weekends by default - Can specify custom working days +- Date calculations are expensive - Pre-calculate when possible + +--- + +## See Also + +- [functions_iterative_calculation.md](./functions_iterative_calculation.md) - Full PREVIOUS/PREVIOUSOF spec: circular dependencies, configuration, performance, debugging +- [functions_lookup.md](./functions_lookup.md) - TIMEDIM for calendar integration, SHIFT for time offsets +- [functions_numeric.md](./functions_numeric.md) - CUMULATE, MOVINGSUM, MOVINGAVERAGE From e9415812e92a2b8d9212c6e816d58b03e7526836 Mon Sep 17 00:00:00 2001 From: Saoud Rizwan <7799382+saoudrizwan@users.noreply.github.com> Date: Tue, 16 Jun 2026 22:37:42 -0700 Subject: [PATCH 2/3] fix: clarify Pigment MCP metadata safety --- plugins/pigment/README.md | 2 ++ plugins/pigment/index.ts | 1 + .../skills/integrating-external-data/data_import_csv.md | 4 ++-- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/plugins/pigment/README.md b/plugins/pigment/README.md index 91494de2..06a68533 100644 --- a/plugins/pigment/README.md +++ b/plugins/pigment/README.md @@ -62,4 +62,6 @@ List my Pigment applications, find the revenue metrics in the FP&A model, and ex Pigment data and model changes can affect financial planning workflows. The plugin adds a rule that requires explicit confirmation before Cline uses advanced MCP tools for writes, imports, access-right changes, board or view edits, scenario or snapshot changes, and deletions. +Advanced MCP search can expose block metadata and application logic, including names, data types, dimensions, and model structure, even when it does not expose actual metric data. Treat that metadata as sensitive workspace context and avoid putting secrets in Pigment block names or metadata. + The bundled skills are licensed by Pigment for use with Pigment services. See `LICENSE.pigment-skills`. The bundled markdown has been format-normalized for this repository's validation rules and is not represented as an official or endorsed Pigment distribution. diff --git a/plugins/pigment/index.ts b/plugins/pigment/index.ts index b7e48b03..72f9bc62 100644 --- a/plugins/pigment/index.ts +++ b/plugins/pigment/index.ts @@ -54,6 +54,7 @@ const plugin: AgentPlugin = { "When working with Pigment, use the bundled Pigment skills for formulas, modeling, views, boards, imports, planning cycles, performance, and access rights.", "Do not invent Pigment application IDs, block IDs, metric names, dimension names, view IDs, or formula syntax. Read available workspace context through Pigment MCP tools when they are configured, or ask the user for the missing details.", "Treat Pigment Advanced MCP tools as workspace-changing operations. Before creating or editing dimensions, metrics, formulas, calendars, imports, boards, views, access rights, scenarios, snapshots, or deleting anything, explain the intended change and get explicit user confirmation.", + "Treat Advanced MCP search output and block metadata as sensitive workspace context. It can reveal application logic, names, dimensions, and structure even when no actual metric data is returned.", "Never print, store, commit, or ask the user to paste OAuth tokens or credentials. Pigment MCP authentication should happen through Cline's MCP OAuth flow after the user provides their workspace MCP URL.", ].join("\n"), }) diff --git a/plugins/pigment/skills/integrating-external-data/data_import_csv.md b/plugins/pigment/skills/integrating-external-data/data_import_csv.md index 2f507be8..9db25e3c 100644 --- a/plugins/pigment/skills/integrating-external-data/data_import_csv.md +++ b/plugins/pigment/skills/integrating-external-data/data_import_csv.md @@ -19,7 +19,7 @@ Each file has: When multiple files share the same name: 1. Examine the metadata, order and preview to differentiate files -2. If genuinely ambiguous, use the `ask_user` tool to clarify and give it the name, metadata, preview data and order +2. If genuinely ambiguous, ask the user to clarify and give them the name, metadata, preview data, and order ## 1. CSV Analysis @@ -148,4 +148,4 @@ The backend handles reference resolution internally. Do not perform multiple imp ## 6. Post-Import Verification -CRITICAL: After every import, fetch the target dimension or transaction list and verify that the number of items is greater than 0. If the count is 0, the import has failed. Inform the user and suggest using the Pigment UI to perform the import manually. \ No newline at end of file +CRITICAL: After every import, fetch the target dimension or transaction list and verify that the number of items is greater than 0. If the count is 0, the import has failed. Inform the user and suggest using the Pigment UI to perform the import manually. From e188a056e90713f351824bf7b84cda2f61ab42d3 Mon Sep 17 00:00:00 2001 From: Saoud Rizwan <7799382+saoudrizwan@users.noreply.github.com> Date: Wed, 17 Jun 2026 19:19:42 -0700 Subject: [PATCH 3/3] fix: remove extra pigment rule primitive --- plugins/pigment/README.md | 4 ++-- plugins/pigment/index.ts | 14 +------------- plugins/pigment/package.json | 1 - 3 files changed, 3 insertions(+), 16 deletions(-) diff --git a/plugins/pigment/README.md b/plugins/pigment/README.md index 06a68533..be1a8903 100644 --- a/plugins/pigment/README.md +++ b/plugins/pigment/README.md @@ -35,7 +35,7 @@ export CLINE_PIGMENT_MCP_URL="https://pigment.app/api/mcp/public/your-mcp-id" cline plugin install pigment ``` -If `CLINE_PIGMENT_MCP_URL` is not set, the plugin still installs the Pigment skills and safety rule, but it does not create a Pigment MCP settings entry. +If `CLINE_PIGMENT_MCP_URL` is not set, the plugin still installs the Pigment skills, but it does not create a Pigment MCP settings entry. ## Example Usage @@ -60,7 +60,7 @@ List my Pigment applications, find the revenue metrics in the FP&A model, and ex ## Security Notes -Pigment data and model changes can affect financial planning workflows. The plugin adds a rule that requires explicit confirmation before Cline uses advanced MCP tools for writes, imports, access-right changes, board or view edits, scenario or snapshot changes, and deletions. +Pigment data and model changes can affect financial planning workflows. The bundled skills require explicit confirmation before Cline uses advanced MCP tools for writes, imports, access-right changes, board or view edits, scenario or snapshot changes, and deletions. Advanced MCP search can expose block metadata and application logic, including names, data types, dimensions, and model structure, even when it does not expose actual metric data. Treat that metadata as sensitive workspace context and avoid putting secrets in Pigment block names or metadata. diff --git a/plugins/pigment/index.ts b/plugins/pigment/index.ts index 72f9bc62..3bbdd556 100644 --- a/plugins/pigment/index.ts +++ b/plugins/pigment/index.ts @@ -27,7 +27,7 @@ function readPigmentMcpUrl(): PigmentMcpUrlConfig { const plugin: AgentPlugin = { name: "pigment", manifest: { - capabilities: ["mcp", "rules", "skills"], + capabilities: ["mcp", "skills"], }, setup(api) { const pigmentMcpUrl = readPigmentMcpUrl() @@ -46,18 +46,6 @@ const plugin: AgentPlugin = { }, }) } - - api.registerRule({ - id: "pigment-workspace-safety", - source: "pigment", - content: [ - "When working with Pigment, use the bundled Pigment skills for formulas, modeling, views, boards, imports, planning cycles, performance, and access rights.", - "Do not invent Pigment application IDs, block IDs, metric names, dimension names, view IDs, or formula syntax. Read available workspace context through Pigment MCP tools when they are configured, or ask the user for the missing details.", - "Treat Pigment Advanced MCP tools as workspace-changing operations. Before creating or editing dimensions, metrics, formulas, calendars, imports, boards, views, access rights, scenarios, snapshots, or deleting anything, explain the intended change and get explicit user confirmation.", - "Treat Advanced MCP search output and block metadata as sensitive workspace context. It can reveal application logic, names, dimensions, and structure even when no actual metric data is returned.", - "Never print, store, commit, or ask the user to paste OAuth tokens or credentials. Pigment MCP authentication should happen through Cline's MCP OAuth flow after the user provides their workspace MCP URL.", - ].join("\n"), - }) }, } diff --git a/plugins/pigment/package.json b/plugins/pigment/package.json index c0a0f272..a8a7db0e 100644 --- a/plugins/pigment/package.json +++ b/plugins/pigment/package.json @@ -12,7 +12,6 @@ ], "capabilities": [ "mcp", - "rules", "skills" ] }