Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
89 changes: 88 additions & 1 deletion autograder/template_library/static_analysis.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
import logging
import logging
from typing import List, Optional, Dict, Any, Type
from pydantic import BaseModel, Field

from autograder.models.abstract.template import Template
from autograder.models.abstract.ai_test_function import AiTestFunction
from autograder.models.abstract.test_function import TestFunction
from autograder.models.dataclass.param_description import ParamDescription
from autograder.models.dataclass.submission import SubmissionFile
Expand Down Expand Up @@ -330,6 +331,89 @@ def execute(self, files: Optional[List[SubmissionFile]], sandbox: Optional[Sandb
)


class AiAlgorithmConfig(BaseModel):
algorithm_name: str = Field(..., min_length=1)
Comment on lines +334 to +335


class AiAlgorithmTestBase(AiTestFunction):
algorithm_family: str = ""
test_name: str = ""

@property
def name(self) -> str:
return self.test_name
Comment on lines +338 to +344

@property
def description(self) -> str:
return t(f"static_analysis.{self.name}.description")

@property
def parameter_description(self) -> List[ParamDescription]:
return [
ParamDescription(
"algorithm_name",
t(f"static_analysis.{self.name}.params.algorithm_name"),
"string",
)
]

@property
def config_schema(self) -> Type[BaseModel]:
return AiAlgorithmConfig

def build_prompt(
self,
files: Optional[List[SubmissionFile]],
**kwargs,
) -> str:
algorithm_name = (kwargs.get("algorithm_name") or "").strip()
file_names = ", ".join(f.filename for f in files) if files else ""

if file_names:
file_scope = f"Focus only on these files: {file_names}."
else:
file_scope = "No submission files were provided for this test."

Comment on lines +364 to +376
algo_label = algorithm_name or "Unknown algorithm"

return (
f"You are verifying a {self.algorithm_family} algorithm implementation.\n"
f"Requested algorithm: {algo_label}.\n"
Comment on lines +377 to +381
f"{file_scope}\n\n"
"Analyze the provided code and determine whether it is a correct and faithful "
"implementation of the requested algorithm.\n"
"Be strict: only accept if the algorithm is clearly implemented as specified.\n\n"
"Criteria:\n"
"1. The implementation must follow the specific logic and complexity "
"characteristics of the requested algorithm.\n"
"2. It must NOT be a wrapper around a built-in library function or standard "
"library implementation.\n"
"3. If it implements a different algorithm, it is incorrect.\n\n"
"Scoring rules:\n"
"- Score 100 only if the implementation is correct and faithful.\n"
"- Otherwise score 0.\n\n"
"In your feedback, briefly justify the decision and cite relevant code "
"evidence. If the required algorithm is missing or there is no relevant "
"code, score 0.\n"
f"Use subject '{algo_label}'."
)
Comment on lines +398 to +399


class AiSortingAlgorithmTest(AiAlgorithmTestBase):
algorithm_family = "sorting"
test_name = "ai_sorting_algorithm"


class AiSearchAlgorithmTest(AiAlgorithmTestBase):
algorithm_family = "search"
test_name = "ai_search_algorithm"


class AiGraphAlgorithmTest(AiAlgorithmTestBase):
algorithm_family = "graph"
test_name = "ai_graph_algorithm"


class StaticAnalysisTemplate(Template):
"""
A template for static analysis of code submissions.
Expand All @@ -341,6 +425,9 @@ def __init__(self):
self.tests = {
"forbidden_import": ForbiddenImportTest(),
"forbidden_keyword": ForbiddenKeywordTest(),
"ai_sorting_algorithm": AiSortingAlgorithmTest(),
"ai_search_algorithm": AiSearchAlgorithmTest(),
"ai_graph_algorithm": AiGraphAlgorithmTest(),
}

@property
Expand Down
18 changes: 18 additions & 0 deletions autograder/translations/en.json
Original file line number Diff line number Diff line change
Expand Up @@ -506,6 +506,24 @@
"success": "No forbidden language constructs found."
}
},
"ai_sorting_algorithm": {
"description": "Uses AI to verify that the submission implements the requested sorting algorithm rather than a library shortcut.",
"params": {
"algorithm_name": "Name of the sorting algorithm to verify (e.g., 'Quick Sort')."
}
},
"ai_search_algorithm": {
"description": "Uses AI to verify that the submission implements the requested search algorithm rather than a library shortcut.",
"params": {
"algorithm_name": "Name of the search algorithm to verify (e.g., 'Binary Search')."
}
},
"ai_graph_algorithm": {
"description": "Uses AI to verify that the submission implements the requested graph algorithm rather than a library shortcut.",
"params": {
"algorithm_name": "Name of the graph algorithm to verify (e.g., 'Dijkstra')."
}
},
"template": {
"name": "Static Analysis",
"description": "A template for evaluating assignments through static and structural code analysis."
Expand Down
18 changes: 18 additions & 0 deletions autograder/translations/pt_br.json
Original file line number Diff line number Diff line change
Expand Up @@ -506,6 +506,24 @@
"success": "Nenhuma construção de linguagem proibida encontrada."
}
},
"ai_sorting_algorithm": {
"description": "Usa IA para verificar se a submissão implementa o algoritmo de ordenação solicitado, sem atalhos de biblioteca.",
"params": {
"algorithm_name": "Nome do algoritmo de ordenação a verificar (ex: 'Quick Sort')."
}
},
"ai_search_algorithm": {
"description": "Usa IA para verificar se a submissão implementa o algoritmo de busca solicitado, sem atalhos de biblioteca.",
"params": {
"algorithm_name": "Nome do algoritmo de busca a verificar (ex: 'Binary Search')."
}
},
"ai_graph_algorithm": {
"description": "Usa IA para verificar se a submissão implementa o algoritmo de grafos solicitado, sem atalhos de biblioteca.",
"params": {
"algorithm_name": "Nome do algoritmo de grafos a verificar (ex: 'Dijkstra')."
}
},
"template": {
"name": "Análise Estática",
"description": "Um modelo para avaliar tarefas através de análise estática e estrutural de código."
Expand Down
7 changes: 6 additions & 1 deletion docs/API.md
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,7 @@ Content-Type: application/json
| Field | Type | Required | Description |
|-------|------|----------|-------------|
| `external_assignment_id` | string | ✓ | External assignment ID from your LMS/platform |
| `template_name` | string | ✓ | Template to use (`input_output`, `web_dev`, `api_testing`) |
| `template_name` | string | ✓ | Template to use (`input_output`, `web_dev`, `api_testing`, `static_analysis`) |
| `criteria_config` | object | ✓ | Grading criteria tree configuration |
| `languages` | list[string] | ✓ | Supported languages: `python`, `java`, `node`, `cpp` |
| `setup_config` | object | ✗ | Setup configuration for preflight checks |
Expand Down Expand Up @@ -791,6 +791,11 @@ GET /api/v1/templates
"name": "web_dev",
"description": "...",
"test_functions": [ ... ]
},
{
"name": "static_analysis",
"description": "...",
"test_functions": [ ... ]
}
]
}
Expand Down
16 changes: 16 additions & 0 deletions docs/core/template-library.md
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ Current registry keys:
- `input_output`
- `api`
- `webdev`
- `static_analysis`

Each key resolves to a template class with:

Expand All @@ -27,6 +28,20 @@ Each key resolves to a template class with:
- `requires_sandbox`
- `tests` map of available `TestFunction` objects

## Static Analysis Template (`static_analysis`)

Static analysis runs on submission files without executing code (no sandbox required). It includes both rule-based checks and AI-assisted algorithm validation.

Available tests:

- `forbidden_import` — Parameters: `forbidden_imports` (list of strings), `submission_language` (string or Language enum).
- `forbidden_keyword` — Parameters: `forbidden_keywords` (list of strings), `custom_ast_grep_rules` (list of rule dicts).
- `ai_sorting_algorithm` — Parameters: `algorithm_name` (string).
- `ai_search_algorithm` — Parameters: `algorithm_name` (string).
- `ai_graph_algorithm` — Parameters: `algorithm_name` (string).

AI algorithm tests require a valid `OPENAI_API_KEY` to be configured in the API environment.

## How it integrates with the pipeline

1. **Load Template** resolves the chosen template.
Expand Down Expand Up @@ -59,3 +74,4 @@ To add a new template:
- [Input/Output Template](../template-library/input_output.md)
- [API Testing Template](../template-library/api_testing.md)
- [Web Development Template](../template-library/web_dev.md)
- [Static Analysis Template](../template-library/static_analysis.md)
1 change: 1 addition & 0 deletions docs/index.md
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ This documentation is organized to be both practical for first-time users and de
- [Input/Output Template](template-library/input_output.md)
- [API Testing Template](template-library/api_testing.md)
- [Web Development Template](template-library/web_dev.md)
- [Static Analysis Template](template-library/static_analysis.md)

### GitHub Classroom / CI integrator

Expand Down
3 changes: 2 additions & 1 deletion docs/pipeline/01-load-template.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ The Load Template step is the entry point of the pipeline. It loads the grading

The step uses the `TemplateLibraryService` singleton to load a template by name. There are two paths:

1. **Built-in template** — Loaded from the template registry by identifier (e.g., `"input_output"`, `"web_dev"`, `"api_testing"`).
1. **Built-in template** — Loaded from the template registry by identifier (e.g., `"input_output"`, `"web_dev"`, `"api_testing"`, `"static_analysis"`).
2. **Custom template** — User-provided template object. This path is planned but not yet implemented; it will require sandboxed loading for security.

The loaded `Template` object is stored in the step result's `data` field, making it available to all subsequent steps.
Expand Down Expand Up @@ -47,6 +47,7 @@ Available built-in templates:
| `input_output` | Input/Output Testing | Yes | Command-line programs with stdin/stdout |
| `web_dev` | Web Development | No | HTML/CSS/JS file validation |
| `api_testing` | API Testing | Yes | HTTP endpoint validation |
| `static_analysis` | Static Analysis | No | Code-quality checks and AI algorithm validation |

## Failure Scenarios

Expand Down
35 changes: 1 addition & 34 deletions docs/template-library/input_output.md
Original file line number Diff line number Diff line change
Expand Up @@ -81,40 +81,7 @@ Executes a program with a specific input and verifies it completes **without cra

---

### `forbidden_keyword`

Tests that a submission does **not** use specific forbidden keywords or language constructs. This test performs structural analysis using `ast-grep`, which is more reliable than regex because it correctly ignores matches inside comments or string literals.

| Parameter | Type | Required | Description |
|-----------|------|----------|-------------|
| `forbidden_keywords` | list[string] | ✗ | List of high-level keywords to forbid (e.g., `"for_loop"`, `"while_loop"`) |
| `custom_ast_grep_rules` | list[dict] | ✗ | Custom `ast-grep` rules for advanced structural matching |

**Supported Keywords for Predefined Rules:**

| Language | Supported Keywords |
|----------|--------------------|
| **Python** | `for_loop`, `while_loop`, `eval_call`, `exec_call` |
| **Java** | `for_loop`, `while_loop` |
| **Node.js** | `for_loop`, `while_loop`, `eval_call` |
| **C++ / C** | `for_loop`, `while_loop`, `do_while_loop` |

**Scoring:** 100 if no forbidden constructs are found, 0 otherwise.

**Example:**
```json
{
"name": "forbidden_keyword",
"parameters": {
"forbidden_keywords": ["for_loop", "eval_call"]
},
"weight": 100
}
```

---

## Usage Example
## Multi-Language Command Resolution

```json
{
Expand Down
Loading
Loading