Keep data on the backend until needed

CHANGELOG.md

@@ -10,6 +10,12 @@
 handled auto-resizing in the plot pane. We now have a per-output-location path
 in the Jupyter kernel (#360)
 
+### Changed
+
+- Reverted an earlier decision to materialize CTEs and the global query in Rust
+before registering them back to the backend. We now keep the data purely on the
+backend until the layer query as was always intended (#363)
+
 ### Removed
 
 - Removed polars from dependency list along with all its transient dependencies. Rewrote DataFrame struct on top of arrow (#350)

src/execute/cte.rs

@@ -146,28 +146,17 @@ pub fn materialize_ctes(ctes: &[CteDefinition], reader: &dyn Reader) -> Result<H
 
         let temp_table_name = naming::cte_table(&cte.name);
 
-        // Execute the CTE body SQL to get a DataFrame, then register it
-        let mut df = reader.execute_sql(&transformed_body).map_err(|e| {
-            GgsqlError::ReaderError(format!("Failed to materialize CTE '{}': {}", cte.name, e))
-        })?;
-
-        // Apply column aliases if present: WITH t(value, label) AS (...) renames columns
-        if !cte.column_aliases.is_empty() && cte.column_aliases.len() == df.width() {
-            let current_names: Vec<String> = df.get_column_names();
-            for (old, new) in current_names.iter().zip(cte.column_aliases.iter()) {
-                df = df.rename(old, new).map_err(|e| {
-                    GgsqlError::ReaderError(format!(
-                        "Failed to apply column alias '{}' for CTE '{}': {}",
-                        new, cte.name, e
-                    ))
-                })?;
-            }
+        let statements = reader.dialect().create_or_replace_temp_table_sql(
+            &temp_table_name,
+            &cte.column_aliases,
+            &transformed_body,
+        );
+        for stmt in &statements {
+            reader.execute_sql(stmt).map_err(|e| {
+                GgsqlError::ReaderError(format!("Failed to materialize CTE '{}': {}", cte.name, e))
+            })?;
         }
 
-        reader.register(&temp_table_name, df, true).map_err(|e| {
-            GgsqlError::ReaderError(format!("Failed to register CTE '{}': {}", cte.name, e))
-        })?;
-
         materialized.insert(cte.name.clone());
     }
 

src/execute/mod.rs

@@ -968,9 +968,16 @@ pub fn prepare_data_with_reader(query: &str, reader: &dyn Reader) -> Result<Prep
     let mut has_global_table = false;
     if sql_part.is_some() {
         if let Some(transformed_sql) = cte::transform_global_sql(&source_tree, &materialized_ctes) {
-            // Execute global result SQL and register result as a temp table
-            let df = execute_query(&transformed_sql)?;
-            reader.register(&naming::global_table(), df, true)?;
+            // Materialize global result as a temp table directly on the backend
+            // (no roundtrip through Rust).
+            let statements = reader.dialect().create_or_replace_temp_table_sql(
+                &naming::global_table(),
+                &[],
+                &transformed_sql,
+            );
+            for stmt in &statements {
+                execute_query(stmt)?;
+            }
 
             // NOTE: Don't read into data_map yet - defer until after casting is determined
             // The temp table exists and can be used for schema fetching

src/reader/mod.rs

@@ -249,6 +249,46 @@ pub trait SqlDialect {
             "FALSE".to_string()
         }
     }
+
+    /// Build the DDL statement(s) needed to (re)create a temporary table
+    /// that holds the result of `body_sql`.
+    ///
+    /// Column aliases from `WITH t(a, b) AS (...)` are preserved portably by
+    /// wrapping the body in a named CTE with a column alias list, so the
+    /// backend never needs to support `CREATE TABLE t(a, b) AS ...` syntax.
+    ///
+    /// Returned statements must be executed in order via `Reader::execute_sql`.
+    fn create_or_replace_temp_table_sql(
+        &self,
+        name: &str,
+        column_aliases: &[String],
+        body_sql: &str,
+    ) -> Vec<String> {
+        let body = wrap_with_column_aliases(body_sql, column_aliases);
+        vec![format!(
+            "CREATE OR REPLACE TEMP TABLE {} AS {}",
+            naming::quote_ident(name),
+            body
+        )]
+    }
+}
+
+/// Wrap a body SQL in a CTE with a column alias list when aliases are present.
+/// This is a portable way to rename the body's output columns without relying
+/// on `CREATE TABLE t(a, b) AS ...` (which SQLite does not support).
+pub(crate) fn wrap_with_column_aliases(body_sql: &str, column_aliases: &[String]) -> String {
+    if column_aliases.is_empty() {
+        return body_sql.to_string();
+    }
+    let cols = column_aliases
+        .iter()
+        .map(|c| naming::quote_ident(c))
+        .collect::<Vec<_>>()
+        .join(", ");
+    format!(
+        "WITH __ggsql_aliased__({}) AS ({}) SELECT * FROM __ggsql_aliased__",
+        cols, body_sql
+    )
 }
 
 pub struct AnsiDialect;

src/reader/sqlite.rs

@@ -92,6 +92,22 @@ impl super::SqlDialect for SqliteDialect {
             table.replace('\'', "''")
         )
     }
+
+    /// SQLite does not support `CREATE OR REPLACE`, so emit a drop-then-create
+    /// pair. Column aliases are preserved portably via the default CTE wrapper.
+    fn create_or_replace_temp_table_sql(
+        &self,
+        name: &str,
+        column_aliases: &[String],
+        body_sql: &str,
+    ) -> Vec<String> {
+        let qname = naming::quote_ident(name);
+        let body = super::wrap_with_column_aliases(body_sql, column_aliases);
+        vec![
+            format!("DROP TABLE IF EXISTS {}", qname),
+            format!("CREATE TEMP TABLE {} AS {}", qname, body),
+        ]
+    }
 }
 
 /// SQLite database reader