feat: capture daily snapshots of aggregated task completion data

app/api/units_api.rb

@@ -578,6 +578,61 @@ class UnitsApi < Grape::API
     present unit.student_task_completion_stats, with: Grape::Presenters::Presenter
   end
 
+  desc 'Get historical task completion snapshots'
+  params do
+    optional :start_date, type: Date, desc: 'Include snapshots captured on or after this date'
+    optional :end_date, type: Date, desc: 'Include snapshots captured on or before this date'
+    optional :limit, type: Integer, desc: 'Maximum number of snapshots to return', default: 365
+  end
+  get '/units/:id/stats/task_completion_snapshots' do
+    unit = Unit.find(params[:id])
+    unless authorise? current_user, unit, :download_stats
+      error!({ error: "Not authorised to download stats of student tasks in #{unit.code}" }, 403)
+    end
+
+    snapshots = unit.task_completion_snapshots.order(snapshot_timestamp: :desc)
+    if params[:start_date].present?
+      start_timestamp = params[:start_date].in_time_zone.beginning_of_day.to_i
+      snapshots = snapshots.where('CAST(snapshot_timestamp AS UNSIGNED) >= ?', start_timestamp)
+    end
+    if params[:end_date].present?
+      end_timestamp = params[:end_date].in_time_zone.end_of_day.to_i
+      snapshots = snapshots.where('CAST(snapshot_timestamp AS UNSIGNED) <= ?', end_timestamp)
+    end
+    snapshots = snapshots.limit([params[:limit].to_i, 365].min)
+
+    present snapshots.map { |snapshot|
+      stats = snapshot.load_stats
+
+      {
+        snapshot_date: snapshot.snapshot_date,
+        snapshot_timestamp: snapshot.snapshot_timestamp,
+        stats: stats
+      }
+    }, with: Grape::Presenters::Presenter
+  end
+
+  desc 'Capture task completion snapshot immediately for this unit'
+  post '/units/:id/stats/task_completion_snapshots/capture' do
+    unit = Unit.find(params[:id])
+    unless authorise? current_user, unit, :capture_task_completion_snapshot
+      error!({ error: "Not authorised to capture stats of student tasks in #{unit.code}" }, 403)
+    end
+
+    # Check if a snapshot was captured within the past 30 minutes
+    recent_snapshot = unit.task_completion_snapshots.where('CAST(snapshot_timestamp AS UNSIGNED) > ?', 30.minutes.ago.to_i).order(snapshot_timestamp: :desc).first
+    if recent_snapshot.present?
+      recent_snapshot_time = recent_snapshot.snapshot_time
+      remaining_seconds = [(recent_snapshot_time + 30.minutes - Time.zone.now).ceil, 0].max
+      remaining_minutes = [(remaining_seconds / 60.0).ceil, 1].max
+      error!({ error: "A snapshot was captured at #{recent_snapshot_time.strftime('%H:%M')}. Please wait #{remaining_minutes} more minute(s) before capturing another snapshot." }, 429)
+    end
+
+    job_id = AggregateTaskCompletionStatsJob.perform_async(unit.id)
+    job = setup_job(job_id)
+    present job, with: Entities::SidekiqJobEntity
+  end
+
   desc 'Download stats related to the number of tasks assessed by each tutor'
   get '/csv/units/:id/tutor_assessments' do
     unit = Unit.find(params[:id])

app/helpers/file_helper.rb

@@ -276,6 +276,25 @@ def unit_portfolio_dir(unit, create: true, archived: true)
     dst
   end
 
+  def unit_analytics_dir(unit, create: true, archived: true)
+    dst = unit_work_root(unit, archived: archived)
+    dst << 'analytics/'
+
+    FileUtils.mkdir_p(dst) if create
+    dst
+  end
+
+  def unit_task_status_snapshot_path(unit, create: true, archived: true)
+    analytics_dir = unit_analytics_dir(unit, create: create, archived: archived)
+    FileUtils.mkdir_p(analytics_dir) if create
+    File.join(analytics_dir, 'task-status-snapshots.zip')
+  end
+
+  def snapshot_csv_filename(snapshot_timestamp)
+    return nil if snapshot_timestamp.blank?
+    "#{sanitized_filename(snapshot_timestamp.to_s)}.csv"
+  end
+
   #
   # Generates a path for storing student portfolios
   #
@@ -778,6 +797,9 @@ def line_wrap(path, width: 160)
   module_function :unit_dir
   module_function :root_portfolio_dir
   module_function :unit_portfolio_dir
+  module_function :unit_analytics_dir
+  module_function :unit_task_status_snapshot_path
+  module_function :snapshot_csv_filename
   module_function :unit_work_root
   module_function :project_work_root
   module_function :student_portfolio_dir

app/models/task_completion_snapshot.rb

@@ -0,0 +1,176 @@
+# frozen_string_literal: true
+
+require 'csv'
+require 'zip'
+
+class TaskCompletionSnapshot < ApplicationRecord
+  include FileHelper
+
+  belongs_to :unit
+
+  validates :snapshot_timestamp, presence: true
+  validates :snapshot_timestamp, uniqueness: { scope: :unit_id }
+
+  after_destroy :delete_snapshot_file
+
+  def snapshot_file_path
+    return nil if unit.blank?
+    FileHelper.unit_task_status_snapshot_path(unit, create: true)
+  end
+
+  def snapshot_contents
+    file_path = snapshot_file_path
+    return nil if file_path.blank?
+    if File.exist?(file_path)
+      return read_csv_from_zip(file_path, snapshot_timestamp)
+    end
+    nil
+  rescue Zip::Error
+    nil
+  end
+
+  def snapshot_date
+    return nil if snapshot_timestamp.blank?
+
+    snapshot_time.to_date
+  end
+
+  def snapshot_time
+    return nil if snapshot_timestamp.blank?
+
+    Time.zone.at(snapshot_timestamp.to_i)
+  end
+
+  def load_stats
+    snapshot_contents = self.snapshot_contents
+
+    return {} if snapshot_contents.blank?
+
+    parse_csv_stats(snapshot_contents)
+  rescue CSV::MalformedCSVError
+    {}
+  end
+
+  def store_stats!(payload)
+    file_path = snapshot_file_path
+    raise 'Cannot store stats without a unit' if file_path.blank?
+
+    FileUtils.mkdir_p(File.dirname(file_path))
+
+    csv_filename = FileHelper.snapshot_csv_filename(snapshot_timestamp)
+    raise 'Cannot store stats without a valid snapshot timestamp' if csv_filename.blank?
+
+    tmp_path = "#{file_path}.tmp"
+
+    # Read existing zip entries (if file exists)
+    existing_entries = {}
+    if File.exist?(file_path)
+      Zip::File.open(file_path) do |zip_file|
+        zip_file.each do |entry|
+          next if entry.directory?
+          existing_entries[entry.name] = entry.get_input_stream.read
+        end
+      end
+    end
+
+    # Update or add the current snapshot entry
+    existing_entries[csv_filename] = payload.to_s
+
+    # Write the zip file with all entries
+    Zip::OutputStream.open(tmp_path) do |zip|
+      existing_entries.each do |filename, content|
+        zip.put_next_entry(filename)
+        zip.write(content)
+      end
+    end
+
+    FileUtils.mv(tmp_path, file_path)
+  ensure
+    FileUtils.rm_f(tmp_path) if defined?(tmp_path) && tmp_path
+  end
+
+  private
+
+  def parse_csv_stats(csv_text)
+    csv = CSV.parse(csv_text, headers: true)
+    return {} if csv.empty?
+
+    stream_headers = unit.tutorial_streams.pluck(:abbreviation)
+    stream_headers = ['Tutorial'] if stream_headers.empty?
+    task_definitions = unit.task_definitions_by_grade
+
+    stats = Hash.new { |hash, key| hash[key] = Hash.new { |tutorial_hash, tutorial_key| tutorial_hash[tutorial_key] = Hash.new { |task_hash, task_key| task_hash[task_key] = Hash.new(0) } } }
+
+    csv.each do |row|
+      campus_abbreviation = row['Campus'].to_s.strip
+      next if campus_abbreviation.blank?
+
+      campus_name = Campus.find_by(abbreviation: campus_abbreviation)&.name || campus_abbreviation
+
+      stream_headers.each do |stream_header|
+        tutorial_name = row[stream_header].to_s.strip
+        next if tutorial_name.blank?
+
+        task_definitions.each do |task_definition|
+          status_value = row[task_definition.abbreviation].to_s.strip
+          status_key = TaskStatus.id_to_key(status_value.to_i) || :not_started
+          stats[campus_name][tutorial_name][task_definition.abbreviation][status_key.to_s] += 1
+        end
+      end
+    end
+
+    stats
+  end
+
+  def read_csv_from_zip(zip_path, snapshot_timestamp)
+    csv_filename = FileHelper.snapshot_csv_filename(snapshot_timestamp)
+    Zip::File.open(zip_path) do |zip_file|
+      entry = zip_file.find_entry(csv_filename)
+      return nil if entry.nil?
+
+      entry.get_input_stream.read
+    end
+  end
+
+  def delete_snapshot_file
+    return if snapshot_timestamp.blank?
+
+    file_path = snapshot_file_path
+    return if file_path.blank? || !File.exist?(file_path)
+
+    csv_filename = FileHelper.snapshot_csv_filename(snapshot_timestamp)
+    return if csv_filename.blank?
+
+    tmp_path = "#{file_path}.tmp"
+
+    begin
+      # Read existing zip entries excluding the one we want to delete
+      remaining_entries = {}
+      Zip::File.open(file_path) do |zip_file|
+        zip_file.each do |entry|
+          next if entry.directory?
+          next if entry.name == csv_filename
+          remaining_entries[entry.name] = entry.get_input_stream.read
+        end
+      end
+
+      if remaining_entries.empty?
+        # If no entries left, just delete the zip file
+        FileUtils.rm_f(file_path)
+      else
+        # Write the zip file with remaining entries
+        Zip::OutputStream.open(tmp_path) do |zip|
+          remaining_entries.each do |filename, content|
+            zip.put_next_entry(filename)
+            zip.write(content)
+          end
+        end
+        FileUtils.mv(tmp_path, file_path)
+      end
+    rescue StandardError => e
+      # If anything goes wrong with zip operations, just clean up and log
+      logger.error("Error managing snapshot zip file: #{e.message}")
+      FileUtils.rm_f(tmp_path)
+    end
+  end
+end