Implemented score recognition from a photo using OpenAI API

packages/api/.env.example

@@ -11,4 +11,5 @@ SCREENSHOT_AGENT_BASE_FOLDER=~/screenshots
 # allowed patterns: {playerId}, {token}, {nickname}, {email}
 # {dateAdded}, {dateTimeAdded} (of upload to server),
 # {date}, {dateTime} (of screenshot from EXIF)
-SCREENSHOT_FILE_PATH_DB={nickname}/{dateAdded}/{dateTimeAdded}.jpg
+SCREENSHOT_FILE_PATH_DB={nickname}/{dateAdded}/{dateTimeAdded}.jpg
+OPENAI_API_KEY=

packages/api/migrations/20250106000000_add_openai_tokens.ts

@@ -0,0 +1,28 @@
+import { Kysely, sql } from 'kysely';
+
+export async function up(db: Kysely<any>): Promise<void> {
+  await db.schema
+    .alterTable('players')
+    .addColumn('openai_prompt_tokens', 'integer', (col) => col.notNull().defaultTo(0))
+    .execute();
+
+  await db.schema
+    .alterTable('players')
+    .addColumn('openai_completion_tokens', 'integer', (col) => col.notNull().defaultTo(0))
+    .execute();
+
+  // Add generated column for cost calculation
+  // $0.25 per 1M input tokens + $2.00 per 1M output tokens
+  await sql`
+    ALTER TABLE players
+    ADD COLUMN openai_cost DECIMAL(10, 6) AS (
+      openai_prompt_tokens * 0.00000025 + openai_completion_tokens * 0.000002
+    ) STORED
+  `.execute(db);
+}
+
+export async function down(db: Kysely<any>): Promise<void> {
+  await db.schema.alterTable('players').dropColumn('openai_cost').execute();
+  await db.schema.alterTable('players').dropColumn('openai_prompt_tokens').execute();
+  await db.schema.alterTable('players').dropColumn('openai_completion_tokens').execute();
+}

packages/api/package.json

@@ -54,6 +54,7 @@
     "mysql2": "^2.3.3",
     "node-cron": "^3.0.3",
     "nodemon": "^2.0.22",
+    "openai": "^6.15.0",
     "pm2": "^5.3.0",
     "regression": "^2.0.1",
     "rimraf": "^5.0.1",

packages/api/src/services/results/recognizeScore.ts

@@ -0,0 +1,127 @@
+import createDebug from 'debug';
+import fs from 'fs';
+import OpenAI from 'openai';
+import { error } from 'utils';
+
+const debug = createDebug('backend-ts:service:recognizeScore');
+
+const openai = process.env.OPENAI_API_KEY
+  ? new OpenAI({
+      apiKey: process.env.OPENAI_API_KEY,
+    })
+  : null;
+
+export interface RecognizeScoreResult {
+  numbers: number[];
+  promptTokens: number;
+  completionTokens: number;
+}
+
+export const recognizeScore = async (imagePath: string): Promise<RecognizeScoreResult> => {
+  if (!process.env.OPENAI_API_KEY || !openai) {
+    throw error(500, 'OpenAI API key is not configured');
+  }
+
+  // Read the image file and convert to base64
+  const imageBuffer = fs.readFileSync(imagePath);
+  const base64Image = imageBuffer.toString('base64');
+
+  // Detect mime type from file header
+  const mimeType = detectMimeType(imageBuffer);
+
+  debug('Sending image to OpenAI for score recognition, size: %d bytes', imageBuffer.length);
+
+  const response = await openai.chat.completions.create({
+    model: 'gpt-5-mini',
+    reasoning_effort: 'minimal',
+    response_format: {
+      type: 'json_schema',
+      json_schema: {
+        name: 'score_numbers_array',
+        description: 'An array of 8 numbers extracted from the game result screen',
+        strict: true,
+        schema: {
+          type: 'object',
+          additionalProperties: false,
+          required: ['numbers'],
+          properties: {
+            numbers: {
+              type: 'array',
+              items: { type: 'number' },
+              minItems: 8,
+              maxItems: 8,
+            },
+          },
+        },
+      },
+    },
+    messages: [
+      {
+        role: 'user',
+        content: [
+          {
+            type: 'text',
+            text: `Extract the vertically lined up white numbers from the photo according to the schema provided. One number per line, some numbers may have leading zeroes. All zeroes have a dot in the middle.`,
+          },
+          {
+            type: 'image_url',
+            image_url: {
+              url: `data:${mimeType};base64,${base64Image}`,
+              detail: 'high',
+            },
+          },
+        ],
+      },
+    ],
+  });
+
+  const content = response.choices[0]?.message?.content;
+  debug('OpenAI response: %s', JSON.stringify(response));
+
+  if (!content) {
+    throw error(500, 'No response from OpenAI');
+  }
+
+  const promptTokens = response.usage?.prompt_tokens ?? 0;
+  const completionTokens = response.usage?.completion_tokens ?? 0;
+
+  try {
+    const parsed = JSON.parse(content) as { numbers: number[] };
+
+    if (!Array.isArray(parsed.numbers) || parsed.numbers.length !== 8) {
+      throw new Error('Invalid response format: expected 8 numbers');
+    }
+
+    return {
+      numbers: parsed.numbers,
+      promptTokens,
+      completionTokens,
+    };
+  } catch (e) {
+    debug('Failed to parse OpenAI response: %s', e);
+    throw error(
+      500,
+      `Failed to parse score recognition result: ${
+        e instanceof Error ? e.message : 'Unknown error'
+      }`
+    );
+  }
+};
+
+const detectMimeType = (buffer: Buffer): string => {
+  // Check magic bytes for common image formats
+  if (buffer[0] === 0xff && buffer[1] === 0xd8 && buffer[2] === 0xff) {
+    return 'image/jpeg';
+  }
+  if (buffer[0] === 0x89 && buffer[1] === 0x50 && buffer[2] === 0x4e && buffer[3] === 0x47) {
+    return 'image/png';
+  }
+  if (buffer[0] === 0x47 && buffer[1] === 0x49 && buffer[2] === 0x46) {
+    return 'image/gif';
+  }
+  if (buffer[0] === 0x52 && buffer[1] === 0x49 && buffer[2] === 0x46 && buffer[3] === 0x46) {
+    return 'image/webp';
+  }
+  // Default to jpeg
+  return 'image/jpeg';
+};

packages/api/src/trpc/routes/results/index.ts

@@ -1,6 +1,8 @@
 import { addResultMutation } from './addResult';
+import { recognizeScoreMutation } from './recognizeScore';
 import { router } from 'trpc/trpc';
 
 export const results = router({
   addResultMutation,
+  recognizeScoreMutation,
 });

packages/api/src/trpc/routes/results/recognizeScore.ts

@@ -0,0 +1,36 @@
+import { db } from 'db';
+import { sql } from 'kysely';
+import { recognizeScore } from 'services/results/recognizeScore';
+import { publicProcedure } from 'trpc/trpc';
+import { base64 } from 'utils/zod';
+import { z } from 'zod';
+
+export const recognizeScoreMutation = publicProcedure
+  .input(
+    z.object({
+      image: base64,
+    })
+  )
+  .mutation(async ({ ctx, input }) => {
+    if (!ctx.user) {
+      throw new Error('Not logged in');
+    }
+
+    try {
+      const result = await recognizeScore(input.image.filePath);
+
+      // Update the player's OpenAI token usage counters
+      await db
+        .updateTable('players')
+        .set({
+          openai_prompt_tokens: sql`openai_prompt_tokens + ${result.promptTokens}`,
+          openai_completion_tokens: sql`openai_completion_tokens + ${result.completionTokens}`,
+        })
+        .where('id', '=', ctx.user.id)
+        .execute();
+
+      return result.numbers;
+    } finally {
+      await input.image.dispose();
+    }
+  });

packages/api/src/types/database.ts

@@ -157,6 +157,9 @@ export interface Players {
   can_add_results_manually: Generated<number | null>;
   arcade_name: string | null;
   exp: Decimal | null;
+  openai_prompt_tokens: Generated<number>;
+  openai_completion_tokens: Generated<number>;
+  openai_cost: Generated<Decimal>;
 }
 
 export interface PpHistory {

packages/web/package.json

@@ -13,6 +13,7 @@
     "preview": "vite preview"
   },
   "dependencies": {
+    "@bmunozg/react-image-area": "^1.1.0",
     "@mantine/core": "^8.3.10",
     "@mantine/form": "^8.3.10",
     "@mantine/hooks": "^8.3.10",

packages/web/src/components/ConfirmationPopup/useConfirmationPopup.tsx

@@ -1,5 +1,7 @@
-import { useState, useRef } from 'react';
-import { Modal, Button, Group } from '@mantine/core';
+import { Button, Group, Modal } from '@mantine/core';
+import { useRef, useState } from 'react';
+
+import { useLanguage } from 'utils/context/translation';
 
 interface UseConfirmationPopupOptions {
   okText?: string;
@@ -11,6 +13,7 @@ interface RenderPopupProps {
 
 export const useConfirmationPopup = ({ okText = 'OK' }: UseConfirmationPopupOptions = {}) => {
   const [open, setOpen] = useState(false);
+  const lang = useLanguage();
   const closeCallback = useRef<((isConfirmed: boolean) => void) | null>(null);
 
   const close = (isConfirmed: boolean) => {
@@ -36,7 +39,7 @@ export const useConfirmationPopup = ({ okText = 'OK' }: UseConfirmationPopupOpti
           {content}
           <Group justify="space-between" mt="md">
             <Button color="red" onClick={() => close(false)}>
-              Cancel
+              {lang.CANCEL}
             </Button>
             <Button color="green" onClick={() => close(true)}>
               {okText}