From 9448ed1c622af4028d7a3a20c0d8c254731ce54a Mon Sep 17 00:00:00 2001 From: Aleksandre Date: Sun, 24 Aug 2025 23:42:04 +0400 Subject: [PATCH 01/49] feat(domain): Add Username field to JWT Claims struct --- api/internal/domain/token.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/api/internal/domain/token.go b/api/internal/domain/token.go index 162a1ef..0b4e5e4 100644 --- a/api/internal/domain/token.go +++ b/api/internal/domain/token.go @@ -8,9 +8,10 @@ import ( ) type Claims struct { - ID string `json:"id"` - Email string `json:"email"` - Type string `json:"type"` + ID string `json:"id"` + Username string `json:"username"` + Email string `json:"email"` + Type string `json:"type"` jwt.RegisteredClaims } From 5301403cd0db04d7e0c18d851a9ff02674ae6f25 Mon Sep 17 00:00:00 2001 From: Aleksandre Date: Sun, 24 Aug 2025 23:42:04 +0400 Subject: [PATCH 02/49] refactor(token): Update token generation functions to accept *domain.User --- .../server/middleware/token/generate.go | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/api/internal/server/middleware/token/generate.go b/api/internal/server/middleware/token/generate.go index d43cf7c..b635eee 100644 --- a/api/internal/server/middleware/token/generate.go +++ b/api/internal/server/middleware/token/generate.go @@ -11,11 +11,11 @@ import ( "github.com/golang-jwt/jwt/v5" ) -func GenerateTokenPair(ID, email string, config domain.TokenConfig) (*domain.TokenPair, error) { - if ID == "" { +func GenerateTokenPair(user *domain.User, config domain.TokenConfig) (*domain.TokenPair, error) { + if user.ID == "" { return nil, &cmn.Error{Err: errors.New("user ID cannot be empty")} } - if email == "" { + if user.Email == "" { return nil, &cmn.Error{Err: errors.New("email cannot be empty")} } if config.SigningKey == "" { @@ -43,12 +43,12 @@ func GenerateTokenPair(ID, email string, config domain.TokenConfig) (*domain.Tok } // GenerateAccessToken creates a new access token (for refresh scenarios) -func GenerateAccessToken(ID, email string, config domain.TokenConfig) (string, time.Time, error) { - return generateAccessToken(ID, email, config) +func GenerateAccessToken(user *domain.User, config domain.TokenConfig) (string, time.Time, error) { + return generateAccessToken(user, config) } // generateAccessToken creates the actual access token -func generateAccessToken(ID, email string, config domain.TokenConfig) (string, time.Time, error) { +func generateAccessToken(user *domain.User, config domain.TokenConfig) (string, time.Time, error) { now := time.Now() expirationTime := now.Add(config.AccessTTL) @@ -59,11 +59,13 @@ func generateAccessToken(ID, email string, config domain.TokenConfig) (string, t } claims := &domain.Claims{ - ID: ID, - Email: email, + ID: user.ID, + Email: user.Email, + Username: user.Username, + Type: user.Type, RegisteredClaims: jwt.RegisteredClaims{ ID: jti, - Subject: ID, + Subject: user.ID, Audience: jwt.ClaimStrings{config.Audience}, Issuer: config.Issuer, IssuedAt: jwt.NewNumericDate(now), From 7b5847050b123c66a84476299738cf6a03fce312 Mon Sep 17 00:00:00 2001 From: Aleksandre Date: Sun, 24 Aug 2025 23:42:04 +0400 Subject: [PATCH 03/49] refactor(token): Adapt token generation call sites to new API --- api/internal/server/handler/token.go | 2 +- api/internal/server/handler/users.go | 2 +- api/internal/server/middleware/token/generate.go | 4 ++-- api/internal/server/middleware/token/refresh_token.go | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/api/internal/server/handler/token.go b/api/internal/server/handler/token.go index b934a30..9ea2f6a 100644 --- a/api/internal/server/handler/token.go +++ b/api/internal/server/handler/token.go @@ -55,7 +55,7 @@ func RefreshToken(c *gin.Context) { } // Generate new token pair - tokens, err := token_middleware.GenerateTokenPair(user.ID, user.Email, config) + tokens, err := token_middleware.GenerateTokenPair(user, config) if err != nil { c.JSON(500, gin.H{"error": "Token generation failed"}) return diff --git a/api/internal/server/handler/users.go b/api/internal/server/handler/users.go index 89880a0..709a1e2 100644 --- a/api/internal/server/handler/users.go +++ b/api/internal/server/handler/users.go @@ -83,7 +83,7 @@ func LoginUser(c *gin.Context) { config := token.DefaultTokenConfig() - tokens, err := token.GenerateTokenPair(user.ID, user.Email, config) + tokens, err := token.GenerateTokenPair(user, config) if err != nil { c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{ "error": "Error logging in the user: " + err.Error(), diff --git a/api/internal/server/middleware/token/generate.go b/api/internal/server/middleware/token/generate.go index b635eee..872407d 100644 --- a/api/internal/server/middleware/token/generate.go +++ b/api/internal/server/middleware/token/generate.go @@ -23,13 +23,13 @@ func GenerateTokenPair(user *domain.User, config domain.TokenConfig) (*domain.To } // Generate access token - accessToken, expiresAt, err := generateAccessToken(ID, email, config) + accessToken, expiresAt, err := generateAccessToken(user, config) if err != nil { return nil, err } // Generate refresh token - refreshToken, err := generateRefreshToken(ID, config) + refreshToken, err := generateRefreshToken(user.ID, config) if err != nil { return nil, err } diff --git a/api/internal/server/middleware/token/refresh_token.go b/api/internal/server/middleware/token/refresh_token.go index e5af331..95cfa42 100644 --- a/api/internal/server/middleware/token/refresh_token.go +++ b/api/internal/server/middleware/token/refresh_token.go @@ -112,6 +112,6 @@ func refreshAccessTokenFromString(refreshTokenString, expectedUserID string, cac } // Generate new access token - newAccessToken, _, err := generateAccessToken(user.ID, user.Email, config) + newAccessToken, _, err := generateAccessToken(user, config) return newAccessToken, err } From 483bcb961fe0aba6f3c1607e3a3cc1d2d80b8e6e Mon Sep 17 00:00:00 2001 From: Aleksandre Date: Sun, 24 Aug 2025 23:42:04 +0400 Subject: [PATCH 04/49] feat(handler): Enhance ValidateToken response with username and role, improve error messages --- api/internal/server/handler/token.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/api/internal/server/handler/token.go b/api/internal/server/handler/token.go index 9ea2f6a..a3e0488 100644 --- a/api/internal/server/handler/token.go +++ b/api/internal/server/handler/token.go @@ -77,7 +77,7 @@ func ValidateToken(c *gin.Context) { result_claims, ok := c.Get("claims") if !ok { c.AbortWithStatusJSON(http.StatusNotFound, gin.H{ - "error": "User not found", + "error": "User claims not found", }) return } @@ -85,13 +85,15 @@ func ValidateToken(c *gin.Context) { claims, ok := result_claims.(*domain.Claims) if !ok { c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{ - "error": "Invalid user structure", + "error": "Invalid claims structure", }) return } c.JSON(http.StatusOK, gin.H{ - "id": claims.ID, - "email": claims.Email, + "id": claims.ID, + "email": claims.Email, + "username": claims.Username, + "role": claims.Type, }) } From 1db021686c3237d0fadd86d6804d98ec54f61407 Mon Sep 17 00:00:00 2001 From: Aleksandre Date: Tue, 2 Sep 2025 21:11:48 +0400 Subject: [PATCH 05/49] feat(common): Add robust domain parsing for env variables --- api/common/env.go | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/api/common/env.go b/api/common/env.go index 9c56cd0..d00cbb2 100644 --- a/api/common/env.go +++ b/api/common/env.go @@ -32,7 +32,20 @@ func GetDomains(v string) []string { log.Fatal(&Error{Err: errors.New("Environmental Variable " + v + " Not Found"), Status: http.StatusNotFound}) } + domains := []string{} + result := strings.Split(env_variable, ",") - return result + if len(result) == 0 { + return []string{"*"} + } + + for _, domain := range result { + cleaned := strings.TrimSpace(domain) + if cleaned != "" { + domains = append(domains, cleaned) + } + } + + return domains } From b841aa00bf947c5f7ec5e90a995c8653842bf507 Mon Sep 17 00:00:00 2001 From: Aleksandre Date: Tue, 2 Sep 2025 21:11:48 +0400 Subject: [PATCH 06/49] feat(db): Implement exponential backoff for database connection --- api/common/river/client.go | 59 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 56 insertions(+), 3 deletions(-) diff --git a/api/common/river/client.go b/api/common/river/client.go index 20a7db8..0947feb 100644 --- a/api/common/river/client.go +++ b/api/common/river/client.go @@ -3,10 +3,13 @@ package queue import ( "Codex-Backend/api/internal/service/worker" "context" + "fmt" "log" "log/slog" + "math" "os" "sync" + "time" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" @@ -19,10 +22,60 @@ var ( riverOnce sync.Once ) +// func InitializeRiverClient(ctx context.Context, workers *river.Workers) *river.Client[pgx.Tx] { +// dbPool, err := pgxpool.New(ctx, os.Getenv("DATABASE_URL")) +// if err != nil { +// panic(err) +// } + +// logger := slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{ +// Level: slog.LevelInfo, +// })) + +// riverClient, err := river.NewClient(riverpgxv5.New(dbPool), &river.Config{ +// Logger: logger, +// Queues: map[string]river.QueueConfig{ +// river.QueueDefault: {MaxWorkers: 10}, +// }, +// MaxAttempts: 3, +// Workers: workers, +// }) +// if err != nil { +// panic(err) +// } + +// return riverClient +// } + func InitializeRiverClient(ctx context.Context, workers *river.Workers) *river.Client[pgx.Tx] { - dbPool, err := pgxpool.New(ctx, os.Getenv("DATABASE_URL")) - if err != nil { - panic(err) + var dbPool *pgxpool.Pool + var err error + + // Retry connection with exponential backoff + maxRetries := 10 + for i := 0; i < maxRetries; i++ { + dbPool, err = pgxpool.New(ctx, os.Getenv("DATABASE_URL")) + if err == nil { + // Test the connection + if pingErr := dbPool.Ping(ctx); pingErr == nil { + log.Printf("Successfully connected to database on attempt %d", i+1) + break + } else { + log.Printf("Database ping failed on attempt %d: %v", i+1, pingErr) + err = pingErr + } + } else { + log.Printf("Failed to create connection pool on attempt %d: %v", i+1, err) + } + + if i == maxRetries-1 { + panic(fmt.Sprintf("Failed to connect to database after %d attempts: %v", maxRetries, err)) + } + + // Wait before retrying (exponential backoff) + waitTime := time.Duration(math.Pow(2, float64(i))) * time.Second + log.Printf("Retrying database connection in %v...", waitTime) + time.Sleep(waitTime) } logger := slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{ From ad46ed0b2dddba7132f8041f1845869ca885351d Mon Sep 17 00:00:00 2001 From: Aleksandre Date: Tue, 2 Sep 2025 21:11:49 +0400 Subject: [PATCH 07/49] refactor(gin): Centralize GIN_MODE setup in init functions --- api/cmd/web/main.go | 9 +++++---- api/cmd/worker/main.go | 9 ++++++--- api/internal/server/server.go | 4 ---- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/api/cmd/web/main.go b/api/cmd/web/main.go index 5f45fb4..ca220f6 100644 --- a/api/cmd/web/main.go +++ b/api/cmd/web/main.go @@ -3,15 +3,16 @@ package main import ( cmn "Codex-Backend/api/common" firestore_server "Codex-Backend/api/internal/server" - "os" + "github.com/gin-gonic/gin" _ "github.com/heroku/x/hmetrics/onload" ) func init() { - if mode := os.Getenv("GIN_MODE"); mode == "debug" { - cmn.LoadEnvVariables() - } + cmn.LoadEnvVariables() + + mode := cmn.GetEnvVariable("GIN_MODE") + gin.SetMode(mode) } func main() { diff --git a/api/cmd/worker/main.go b/api/cmd/worker/main.go index f8d8ab6..de55da5 100644 --- a/api/cmd/worker/main.go +++ b/api/cmd/worker/main.go @@ -9,12 +9,15 @@ import ( "os/signal" "syscall" "time" + + "github.com/gin-gonic/gin" ) func init() { - if mode := os.Getenv("GIN_MODE"); mode == "debug" { - cmn.LoadEnvVariables() - } + cmn.LoadEnvVariables() + + mode := cmn.GetEnvVariable("GIN_MODE") + gin.SetMode(mode) } func main() { diff --git a/api/internal/server/server.go b/api/internal/server/server.go index b0e6940..063f172 100644 --- a/api/internal/server/server.go +++ b/api/internal/server/server.go @@ -1,7 +1,6 @@ package server import ( - cmn "Codex-Backend/api/common" "context" "log" "net/http" @@ -14,9 +13,6 @@ import ( ) func Server() { - mode := cmn.GetEnvVariable("GIN_MODE") - gin.SetMode(mode) - r := gin.Default() RegisteredRoutes(r) From cc314238036a1d20b6b53ac7c717440ca9e237ef Mon Sep 17 00:00:00 2001 From: Aleksandre Date: Tue, 2 Sep 2025 21:11:49 +0400 Subject: [PATCH 08/49] feat(middleware): Introduce granular token middleware components --- .../server/middleware/token/claims.go | 30 ++++++++++++++ .../server/middleware/token/extract.go | 28 +++++++++++++ api/internal/server/middleware/token/parse.go | 40 +++++++++++++++++++ 3 files changed, 98 insertions(+) create mode 100644 api/internal/server/middleware/token/claims.go create mode 100644 api/internal/server/middleware/token/extract.go create mode 100644 api/internal/server/middleware/token/parse.go diff --git a/api/internal/server/middleware/token/claims.go b/api/internal/server/middleware/token/claims.go new file mode 100644 index 0000000..37dee5a --- /dev/null +++ b/api/internal/server/middleware/token/claims.go @@ -0,0 +1,30 @@ +package token + +import ( + "net/http" + + "github.com/gin-gonic/gin" +) + +func SetClaimsFromToken() gin.HandlerFunc { + return func(c *gin.Context) { + tokenString, err := ExtractToken("access_token", c) + if err != nil { + c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{ + "error": "Missing or invalid authorization token", + }) + return + } + + // Parse and validate JWT + claims, err := ParseAndValidateJWT(tokenString) + if err != nil { + c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{ + "error": "Invalid token: " + err.Error(), + }) + return + } + + c.Set("claims", claims) + } +} diff --git a/api/internal/server/middleware/token/extract.go b/api/internal/server/middleware/token/extract.go new file mode 100644 index 0000000..3921085 --- /dev/null +++ b/api/internal/server/middleware/token/extract.go @@ -0,0 +1,28 @@ +package token + +import ( + "fmt" + + "github.com/gin-gonic/gin" +) + +// Get JWT token from cookie or Authorization header +func ExtractToken(token_name string, c *gin.Context) (string, error) { + // Try cookie first + if tokenString, err := c.Cookie(token_name); err == nil { + return tokenString, nil + } + + // Try Authorization header as fallback + authHeader := c.GetHeader(token_name) + if authHeader == "" { + return "", fmt.Errorf("no authorization token provided") + } + + // Handle "Bearer " format + if len(authHeader) > 7 && authHeader[:7] == "Bearer " { + return authHeader[7:], nil + } + + return authHeader, nil +} diff --git a/api/internal/server/middleware/token/parse.go b/api/internal/server/middleware/token/parse.go new file mode 100644 index 0000000..deb16cb --- /dev/null +++ b/api/internal/server/middleware/token/parse.go @@ -0,0 +1,40 @@ +package token + +import ( + cmn "Codex-Backend/api/common" + "Codex-Backend/api/internal/domain" + "fmt" + + "github.com/golang-jwt/jwt/v5" +) + +// Parses and validates the JWT token +func ParseAndValidateJWT(tokenString string) (*domain.Claims, error) { + token, err := jwt.ParseWithClaims(tokenString, &domain.Claims{}, func(token *jwt.Token) (any, error) { + if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok { + return nil, fmt.Errorf("unexpected signing method: %v", token.Header["alg"]) + } + + key := cmn.GetEnvVariable("JWT_SIGN_KEY") + return []byte(key), nil + }) + + if err != nil { + return nil, err + } + + claims, ok := token.Claims.(*domain.Claims) + if !ok || !token.Valid { + return nil, fmt.Errorf("invalid token claims") + } + + // Validate required fields + if claims.ID == "" { + return nil, fmt.Errorf("user ID not found in token") + } + if claims.Email == "" { + return nil, fmt.Errorf("email not found in token") + } + + return claims, nil +} From 5aff0200f85eee29e8a491d7c241267ef1038f0e Mon Sep 17 00:00:00 2001 From: Aleksandre Date: Tue, 2 Sep 2025 21:11:49 +0400 Subject: [PATCH 09/49] feat(middleware): Add user loading and access token refresh --- .../server/middleware/token/load_user.go | 73 +++++++++++++++++++ .../server/middleware/token/refresh.go | 52 +++++++++++++ .../server/middleware/token/update.go | 54 ++++++++++++++ 3 files changed, 179 insertions(+) create mode 100644 api/internal/server/middleware/token/load_user.go create mode 100644 api/internal/server/middleware/token/refresh.go create mode 100644 api/internal/server/middleware/token/update.go diff --git a/api/internal/server/middleware/token/load_user.go b/api/internal/server/middleware/token/load_user.go new file mode 100644 index 0000000..ca42145 --- /dev/null +++ b/api/internal/server/middleware/token/load_user.go @@ -0,0 +1,73 @@ +package token + +import ( + "Codex-Backend/api/internal/domain" + "Codex-Backend/api/internal/service" + "fmt" + "net/http" + "time" + + "github.com/gin-gonic/gin" +) + +func (mf *IMTokenCache) LoadUser() gin.HandlerFunc { + return LookupUser(domain.LookupUser{ + Cache: mf.cache, + CacheDuration: 1 * time.Hour, + }) +} + +func LookupUser(config domain.LookupUser) gin.HandlerFunc { + return func(c *gin.Context) { + claims, exists := c.Get("claims") + if !exists { + c.Next() + return + } + + userClaims, ok := claims.(*domain.Claims) + if !ok { + c.Next() + return + } + + // Check cache first + var user *domain.User + cacheKey := fmt.Sprintf("user:%s", userClaims.ID) + + if config.Cache != nil { + if cached, found := config.Cache.Get(cacheKey); found { + if cachedUser, ok := cached.(*domain.User); ok { + user = cachedUser + } + } + } + + // Fetch user if not in cache + if user == nil { + user, err := service.GetUserByID(userClaims.ID, c.Request.Context()) + if err != nil { + c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{ + "error": "User verification failed", + }) + return + } + + if user == nil { + c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{ + "error": "User not found", + }) + return + } + + // Cache user if cache is available + if config.Cache != nil { + config.Cache.Set(cacheKey, user, config.CacheDuration) + } + } + + // Set user in context + c.Set("user", user) + c.Next() + } +} diff --git a/api/internal/server/middleware/token/refresh.go b/api/internal/server/middleware/token/refresh.go new file mode 100644 index 0000000..92b4fb6 --- /dev/null +++ b/api/internal/server/middleware/token/refresh.go @@ -0,0 +1,52 @@ +package token + +import ( + "Codex-Backend/api/internal/domain" + "Codex-Backend/api/internal/service" + "context" + "errors" + "fmt" + + "github.com/golang-jwt/jwt/v5" +) + +func refreshAccessTokenFromString(refreshTokenString, expectedUserID string, cacheConfig domain.LookupUser, ctx context.Context) (string, error) { + config := DefaultTokenConfig() + + // Parse refresh token + token, err := jwt.ParseWithClaims(refreshTokenString, &jwt.RegisteredClaims{}, func(token *jwt.Token) (any, error) { + if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok { + return nil, fmt.Errorf("unexpected signing method: %v", token.Header["alg"]) + } + return []byte(config.SigningKey), nil + }) + + if err != nil { + return "", err + } + + claims, ok := token.Claims.(*jwt.RegisteredClaims) + if !ok || !token.Valid { + return "", errors.New("invalid refresh token claims") + } + + if claims.Subject != expectedUserID { + return "", errors.New("refresh token user mismatch") + } + + // Get user info (from cache or database) + user, err := service.GetUserByID(claims.ID, ctx) + if err != nil { + return "", err + } + + // Cache user if cache is available + cacheKey := fmt.Sprintf("user:%s", claims.ID) + if cacheConfig.Cache != nil { + cacheConfig.Cache.Set(cacheKey, user, cacheConfig.CacheDuration) + } + + // Generate new access token + newAccessToken, _, err := generateAccessToken(user, config) + return newAccessToken, err +} diff --git a/api/internal/server/middleware/token/update.go b/api/internal/server/middleware/token/update.go new file mode 100644 index 0000000..9044595 --- /dev/null +++ b/api/internal/server/middleware/token/update.go @@ -0,0 +1,54 @@ +package token + +import ( + "Codex-Backend/api/internal/domain" + "net/http" + "time" + + "github.com/gin-gonic/gin" +) + +func (mf *IMTokenCache) UpdateAccessToken() gin.HandlerFunc { + return func(c *gin.Context) { + ctx := c.Request.Context() + defer ctx.Done() + + claims, exists := c.Get("claims") + if !exists { + c.Next() + return + } + + userClaims, ok := claims.(*domain.Claims) + if !ok { + c.Next() + return + } + + refreshToken, err := c.Cookie("refresh_token") + if err != nil { + c.AbortWithStatusJSON(401, gin.H{ + "error": "Missing or invalid refresh token", + }) + return + } + + config := DefaultTokenConfig() + + newAccessToken, err := refreshAccessTokenFromString(refreshToken, userClaims.ID, domain.LookupUser{ + Cache: mf.cache, + CacheDuration: 1 * time.Hour, + }, ctx) + if err != nil { + c.Next() + return + } + + c.SetSameSite(http.SameSiteStrictMode) + c.SetCookie("access_token", newAccessToken, int(config.AccessTTL.Seconds()), "/", "", true, true) + + c.Header("X-Token-Refreshed", "true") + + c.Next() + } +} From 66d133702a9630d84604c82216782996d85576f0 Mon Sep 17 00:00:00 2001 From: Aleksandre Date: Tue, 2 Sep 2025 21:11:49 +0400 Subject: [PATCH 10/49] refactor(middleware): Remove deprecated token middleware files --- .../server/middleware/token/refresh_token.go | 117 ------------- api/internal/server/middleware/token/token.go | 163 ------------------ 2 files changed, 280 deletions(-) delete mode 100644 api/internal/server/middleware/token/refresh_token.go delete mode 100644 api/internal/server/middleware/token/token.go diff --git a/api/internal/server/middleware/token/refresh_token.go b/api/internal/server/middleware/token/refresh_token.go deleted file mode 100644 index 95cfa42..0000000 --- a/api/internal/server/middleware/token/refresh_token.go +++ /dev/null @@ -1,117 +0,0 @@ -package token - -import ( - "Codex-Backend/api/internal/domain" - "Codex-Backend/api/internal/service" - "context" - "errors" - "fmt" - "net/http" - "time" - - "github.com/gin-gonic/gin" - "github.com/golang-jwt/jwt/v5" -) - -func (mf *IMTokenCache) AutoRefreshTokenMiddleware() gin.HandlerFunc { - return func(c *gin.Context) { - ctx := c.Request.Context() - defer ctx.Done() - - path := c.Request.URL.Path - if path == "/user/refresh" || path == "/user/logout" { - c.Next() - return - } - - // Check if access token is close to expiring - claims, exists := c.Get("claims") - if !exists { - c.Next() - return - } - - userClaims, ok := claims.(*domain.Claims) - if !ok { - c.Next() - return - } - - // Refresh if less than 5 minutes remaining on access token - timeUntilExpiry := time.Until(userClaims.ExpiresAt.Time) - if timeUntilExpiry > 5*time.Minute { - c.Next() - return - } - - // Access token expires soon, try to refresh using refresh token - refreshToken, err := c.Cookie("refresh_token") - if err != nil { - // No refresh token available, let it expire naturally - c.Next() - return - } - - config := DefaultTokenConfig() - - // Generate new access token using refresh token - newAccessToken, err := refreshAccessTokenFromString(refreshToken, userClaims.ID, domain.LookupUser{ - Cache: mf.cache, - CacheDuration: 1 * time.Hour, - }, ctx) - if err != nil { - c.Next() - return - } - - // Set new access token cookie - c.SetSameSite(http.SameSiteStrictMode) - c.SetCookie("access_token", newAccessToken, int(config.AccessTTL.Seconds()), "/", "", true, true) - - // Let frontend know token was refreshed - c.Header("X-Token-Refreshed", "true") - - c.Next() - } -} - -func refreshAccessTokenFromString(refreshTokenString, expectedUserID string, cacheConfig domain.LookupUser, ctx context.Context) (string, error) { - config := DefaultTokenConfig() - - // Parse refresh token - token, err := jwt.ParseWithClaims(refreshTokenString, &jwt.RegisteredClaims{}, func(token *jwt.Token) (any, error) { - if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok { - return nil, fmt.Errorf("unexpected signing method: %v", token.Header["alg"]) - } - return []byte(config.SigningKey), nil - }) - - if err != nil { - return "", err - } - - claims, ok := token.Claims.(*jwt.RegisteredClaims) - if !ok || !token.Valid { - return "", errors.New("invalid refresh token claims") - } - - if claims.Subject != expectedUserID { - return "", errors.New("refresh token user mismatch") - } - - // Get user info (from cache or database) - user, err := service.GetUserByID(claims.ID, ctx) - if err != nil { - return "", err - } - - // Cache user if cache is available - cacheKey := fmt.Sprintf("user:%s", claims.ID) - if cacheConfig.Cache != nil { - cacheConfig.Cache.Set(cacheKey, user, cacheConfig.CacheDuration) - } - - // Generate new access token - newAccessToken, _, err := generateAccessToken(user, config) - return newAccessToken, err -} diff --git a/api/internal/server/middleware/token/token.go b/api/internal/server/middleware/token/token.go deleted file mode 100644 index 5063884..0000000 --- a/api/internal/server/middleware/token/token.go +++ /dev/null @@ -1,163 +0,0 @@ -package token - -import ( - cmn "Codex-Backend/api/common" - "Codex-Backend/api/internal/domain" - "Codex-Backend/api/internal/service" - "fmt" - "net/http" - "time" - - "cloud.google.com/go/firestore" - "github.com/gin-gonic/gin" - "github.com/golang-jwt/jwt/v5" -) - -func SetClaimsFromToken() gin.HandlerFunc { - return func(c *gin.Context) { - path := c.Request.URL.Path - if path == "/api/user/refresh" || path == "/api/user/logout" || path == "/api/user/login" { - c.Next() - return - } - - tokenString, err := ExtractToken(c) - if err != nil { - c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{ - "error": "Missing or invalid authorization token", - }) - return - } - - // Parse and validate JWT - claims, err := ParseAndValidateJWT(tokenString) - if err != nil { - c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{ - "error": "Invalid token: " + err.Error(), - }) - return - } - - // Set claims in context (always available) - c.Set("claims", claims) - } -} - -// ValidateToken creates a JWT validation middleware with configurable options -func LookupUser(config domain.LookupUser) gin.HandlerFunc { - return func(c *gin.Context) { - claims, exists := c.Get("claims") - if !exists { - c.Next() - return - } - - userClaims, ok := claims.(*domain.Claims) - if !ok { - c.Next() - return - } - - // Check cache first - var user *domain.User - cacheKey := fmt.Sprintf("user:%s", userClaims.ID) - - if config.Cache != nil { - if cached, found := config.Cache.Get(cacheKey); found { - if cachedUser, ok := cached.(*domain.User); ok { - user = cachedUser - } - } - } - - // Fetch user if not in cache - if user == nil { - user, err := service.GetUserByID(userClaims.ID, c.Request.Context()) - if err != nil { - c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{ - "error": "User verification failed", - }) - return - } - - if user == nil { - c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{ - "error": "User not found", - }) - return - } - - // Cache user if cache is available - if config.Cache != nil { - config.Cache.Set(cacheKey, user, config.CacheDuration) - } - } - - // Set user in context - c.Set("user", user) - c.Next() - } -} - -// extractToken extracts JWT token from cookie or Authorization header -func ExtractToken(c *gin.Context) (string, error) { - // Try cookie first - if tokenString, err := c.Cookie("access_token"); err == nil && tokenString != "" { - return tokenString, nil - } - - // Try Authorization header as fallback - authHeader := c.GetHeader("access_token") - if authHeader == "" { - return "", fmt.Errorf("no authorization token provided") - } - - // Handle "Bearer " format - if len(authHeader) > 7 && authHeader[:7] == "Bearer " { - return authHeader[7:], nil - } - - return authHeader, nil -} - -// parseAndValidateJWT parses and validates the JWT token -func ParseAndValidateJWT(tokenString string) (*domain.Claims, error) { - token, err := jwt.ParseWithClaims(tokenString, &domain.Claims{}, func(token *jwt.Token) (any, error) { - if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok { - return nil, fmt.Errorf("unexpected signing method: %v", token.Header["alg"]) - } - - key := cmn.GetEnvVariable("JWT_SIGN_KEY") - return []byte(key), nil - }) - - if err != nil { - return nil, err - } - - claims, ok := token.Claims.(*domain.Claims) - if !ok || !token.Valid { - return nil, fmt.Errorf("invalid token claims") - } - - // Validate required fields - if claims.ID == "" { - return nil, fmt.Errorf("user ID not found in token") - } - if claims.Email == "" { - return nil, fmt.Errorf("email not found in token") - } - - return claims, nil -} - -type FirestoreUserService struct { - client *firestore.Client -} - -func (mf *IMTokenCache) LoadUser() gin.HandlerFunc { - return LookupUser(domain.LookupUser{ - Cache: mf.cache, - CacheDuration: 1 * time.Hour, - }) -} From d470631eedc866f649fc86dd81c231385f474282 Mon Sep 17 00:00:00 2001 From: Aleksandre Date: Tue, 2 Sep 2025 21:11:49 +0400 Subject: [PATCH 11/49] refactor(api): Update token handler and route definitions --- api/internal/server/handler/token.go | 71 ++-------------------------- api/internal/server/routes.go | 38 +++++++++------ 2 files changed, 27 insertions(+), 82 deletions(-) diff --git a/api/internal/server/handler/token.go b/api/internal/server/handler/token.go index a3e0488..aba10dc 100644 --- a/api/internal/server/handler/token.go +++ b/api/internal/server/handler/token.go @@ -1,83 +1,18 @@ package handler import ( - cmn "Codex-Backend/api/common" "Codex-Backend/api/internal/domain" - token_middleware "Codex-Backend/api/internal/server/middleware/token" - "Codex-Backend/api/internal/service" - "errors" "net/http" "github.com/gin-gonic/gin" - "github.com/golang-jwt/jwt/v5" ) -func RefreshToken(c *gin.Context) { - ctx := c.Request.Context() - defer ctx.Done() - - refreshToken, err := c.Cookie("refresh_token") - if err != nil { - c.JSON(401, gin.H{"error": "No refresh token provided"}) - return - } - - config := token_middleware.DefaultTokenConfig() - token, err := jwt.ParseWithClaims(refreshToken, &jwt.RegisteredClaims{}, func(token *jwt.Token) (any, error) { - if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok { - return nil, errors.New("invalid signing method") - } - return []byte(config.SigningKey), nil - }) - - if err != nil { - c.JSON(401, gin.H{"error": "Invalid refresh token"}) - return - } - - claims, ok := token.Claims.(*jwt.RegisteredClaims) - if !ok || !token.Valid { - c.JSON(401, gin.H{"error": "Invalid refresh token claims"}) - return - } - - user, err := service.GetUserByID(claims.Subject, ctx) - if e, ok := err.(*cmn.Error); ok { - c.AbortWithStatusJSON(e.StatusCode(), gin.H{ - "error": "User not found: " + e.Error(), - }) - return - } else if err != nil { - c.AbortWithStatusJSON(401, gin.H{ - "error": "User not found: " + err.Error(), - }) - return - } - - // Generate new token pair - tokens, err := token_middleware.GenerateTokenPair(user, config) - if err != nil { - c.JSON(500, gin.H{"error": "Token generation failed"}) - return - } - - c.SetSameSite(http.SameSiteStrictMode) - c.SetCookie("access_token", tokens.AccessToken, int(config.AccessTTL.Seconds()), "/", "", true, true) - c.SetCookie("refresh_token", tokens.RefreshToken, int(config.RefreshTTL.Seconds()), "/", "", true, true) - - c.JSON(200, gin.H{ - "message": "Tokens refreshed successfully", - "expires_at": tokens.ExpiresAt, - "expires_in": int(config.AccessTTL.Seconds()), - }) - -} - func ValidateToken(c *gin.Context) { result_claims, ok := c.Get("claims") if !ok { - c.AbortWithStatusJSON(http.StatusNotFound, gin.H{ - "error": "User claims not found", + c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{ + "error": "User claims not found", + "orignal_claims": result_claims, }) return } diff --git a/api/internal/server/routes.go b/api/internal/server/routes.go index aad9d1a..3cda0a3 100644 --- a/api/internal/server/routes.go +++ b/api/internal/server/routes.go @@ -53,10 +53,13 @@ func RegisteredRoutes(r *gin.Engine) { token.InitIMTokenCache() // Add mandatory token check - r.Use(token.SetClaimsFromToken(), token.GlobalToken.AutoRefreshTokenMiddleware()) - client := r.Group("/api/") + client := r.Group("/") { + client.Use(token.SetClaimsFromToken(), token.GlobalToken.UpdateAccessToken(), token.GlobalToken.LoadUser()) + + // Potentially add user public profile view here as well. + client.GET("/all", handler.FindAllNovels) client.GET("/:novel", handler.FindNovel) client.GET("/:novel/all", handler.FindAllChapters) @@ -64,30 +67,37 @@ func RegisteredRoutes(r *gin.Engine) { client.GET("/:novel/chapters", handler.GetPaginatedChapters) } - manage := r.Group("/api/manage") + manage := r.Group("/manage") { - manage.Use(token.GlobalToken.LoadUser()) + manage.Use(token.SetClaimsFromToken(), token.GlobalToken.UpdateAccessToken(), token.GlobalToken.LoadUser()) - // Create - manage.POST("/novel", handler.CreateNovel) - manage.POST("/:novel/chapter", handler.CreateChapter) + // Create Novel/Chapters from epub file. manage.POST("/epub", handler.EPUBNovel) + // Create + manage.POST("/create/novel", handler.CreateNovel) + manage.POST("/create/:novel/chapter", handler.CreateChapter) + // Update - manage.PUT("/:novel", handler.UpdateNovel) - manage.PUT("/:novel/:chapter", handler.UpdateChapter) + manage.PUT("/update/:novel", handler.UpdateNovel) + manage.PUT("/update/:novel/:chapter", handler.UpdateChapter) // Delete - manage.DELETE("/:novel", handler.DeleteNovel) - manage.DELETE("/:novel/:chapter", handler.DeleteChapter) + manage.DELETE("/delete/:novel", handler.DeleteNovel) + manage.DELETE("/delete/:novel/:chapter", handler.DeleteChapter) } - user := r.Group("/api/user") + user := r.Group("/user") { - user.GET("/validate", handler.ValidateToken) user.POST("/login", handler.LoginUser) user.POST("/logout", handler.LogoutUser) user.POST("/register", handler.RegisterUser) - user.GET("/refresh", handler.RefreshToken) + } + + validate := r.Group("/validate") + { + validate.Use(token.SetClaimsFromToken(), token.GlobalToken.UpdateAccessToken(), token.GlobalToken.LoadUser()) + + validate.GET("/", handler.ValidateToken) } } From d3ed64243f657f0e7aa1d21c4150ceb5d51c83db Mon Sep 17 00:00:00 2001 From: Aleksandre Date: Wed, 3 Sep 2025 15:58:49 +0400 Subject: [PATCH 12/49] chore: Remove Procfile --- Procfile | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 Procfile diff --git a/Procfile b/Procfile deleted file mode 100644 index cbaefa8..0000000 --- a/Procfile +++ /dev/null @@ -1,2 +0,0 @@ -web: ./bin/web -worker: ./bin/worker From d0e63a8671a12d122fdd124895995891af977db3 Mon Sep 17 00:00:00 2001 From: Aleksandre Date: Wed, 3 Sep 2025 15:58:49 +0400 Subject: [PATCH 13/49] feat: Add .dockerignore for optimized Docker builds --- .dockerignore | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..33d5c26 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,12 @@ +.git +.gitignore + +.github + +Dockerfile.web +Dockerfile.worker +README.md + +.air.toml + +tmp From c15bf617e71ba81bfa0b5677b4b86feb5345da15 Mon Sep 17 00:00:00 2001 From: Aleksandre Date: Wed, 3 Sep 2025 15:58:49 +0400 Subject: [PATCH 14/49] feat: Implement Dockerfile for web service --- Dockerfile.web | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 Dockerfile.web diff --git a/Dockerfile.web b/Dockerfile.web new file mode 100644 index 0000000..40387cf --- /dev/null +++ b/Dockerfile.web @@ -0,0 +1,35 @@ +FROM golang:1.23-alpine AS builder + +WORKDIR /app +COPY . . +RUN go build -o web ./api/cmd/web + +FROM alpine:latest + +# Install ca-certificates for HTTPS requests and security updates +RUN apk --no-cache add ca-certificates && \ + apk upgrade + +# Create a non-root user for security +RUN addgroup -g 1001 -S appgroup && \ + adduser -u 1001 -S appuser -G appgroup + +# Create app directory +WORKDIR /app + +# Copy binary with proper ownership and permissions +COPY --from=builder --chown=appuser:appgroup /app/web ./web + +COPY .env .env + +RUN chmod +x ./web + +# Switch to non-root user +USER appuser + +# Health check for monitoring +HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \ + CMD wget --no-verbose --tries=1 --spider http://localhost:8080/health || exit 1 + +# Run the application +CMD ["./web"] From 06495d0fc8a00ec07a59f5a517e2f98f2741ca91 Mon Sep 17 00:00:00 2001 From: Aleksandre Date: Wed, 3 Sep 2025 15:58:49 +0400 Subject: [PATCH 15/49] feat: Implement Dockerfile for worker service --- Dockerfile.worker | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 Dockerfile.worker diff --git a/Dockerfile.worker b/Dockerfile.worker new file mode 100644 index 0000000..feb369b --- /dev/null +++ b/Dockerfile.worker @@ -0,0 +1,31 @@ +FROM golang:1.23-alpine AS builder + +WORKDIR /app +COPY . . +RUN go build -o worker ./api/cmd/worker + +FROM alpine:latest + +# Install ca-certificates for HTTPS requests and security updates +RUN apk --no-cache add ca-certificates && \ + apk upgrade + +# Create a non-root user for security +RUN addgroup -g 1001 -S appgroup && \ + adduser -u 1001 -S appuser -G appgroup + +# Create app directory +WORKDIR /app + +# Copy binary with proper ownership and permissions +COPY --from=builder --chown=appuser:appgroup /app/worker ./worker + +COPY .env .env + +RUN chmod +x ./worker + +# Switch to non-root user +USER appuser + +# Run the application +CMD ["./worker"] From d1e6a17e0578380e05a3abc493eae59a4a075a75 Mon Sep 17 00:00:00 2001 From: Aleksandre Date: Fri, 12 Sep 2025 17:07:01 +0400 Subject: [PATCH 16/49] refactor: Prefix all API route groups with `/api/` --- api/internal/server/routes.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/api/internal/server/routes.go b/api/internal/server/routes.go index 3cda0a3..6c935f8 100644 --- a/api/internal/server/routes.go +++ b/api/internal/server/routes.go @@ -54,7 +54,7 @@ func RegisteredRoutes(r *gin.Engine) { // Add mandatory token check - client := r.Group("/") + client := r.Group("/api/") { client.Use(token.SetClaimsFromToken(), token.GlobalToken.UpdateAccessToken(), token.GlobalToken.LoadUser()) @@ -67,7 +67,7 @@ func RegisteredRoutes(r *gin.Engine) { client.GET("/:novel/chapters", handler.GetPaginatedChapters) } - manage := r.Group("/manage") + manage := r.Group("/api/manage") { manage.Use(token.SetClaimsFromToken(), token.GlobalToken.UpdateAccessToken(), token.GlobalToken.LoadUser()) @@ -87,14 +87,14 @@ func RegisteredRoutes(r *gin.Engine) { manage.DELETE("/delete/:novel/:chapter", handler.DeleteChapter) } - user := r.Group("/user") + user := r.Group("/api/user") { user.POST("/login", handler.LoginUser) user.POST("/logout", handler.LogoutUser) user.POST("/register", handler.RegisterUser) } - validate := r.Group("/validate") + validate := r.Group("/api/validate") { validate.Use(token.SetClaimsFromToken(), token.GlobalToken.UpdateAccessToken(), token.GlobalToken.LoadUser()) From f7805e4f6e357650937833bca23f5cfb6f51fe03 Mon Sep 17 00:00:00 2001 From: Aleksandre Date: Fri, 12 Sep 2025 17:56:53 +0400 Subject: [PATCH 17/49] feat: Add basic health check endpoint --- api/internal/server/handler/health.go | 13 +++++++++++++ api/internal/server/routes.go | 6 ++++++ 2 files changed, 19 insertions(+) create mode 100644 api/internal/server/handler/health.go diff --git a/api/internal/server/handler/health.go b/api/internal/server/handler/health.go new file mode 100644 index 0000000..cc937ec --- /dev/null +++ b/api/internal/server/handler/health.go @@ -0,0 +1,13 @@ +package handler + +import ( + "net/http" + + "github.com/gin-gonic/gin" +) + +func HealthCheck(c *gin.Context) { + // TODO: Add better health check for future update. + // Implement resource and status monitoring. + c.JSON(http.StatusOK, gin.H{"status": "OK"}) +} diff --git a/api/internal/server/routes.go b/api/internal/server/routes.go index 6c935f8..ec45c5a 100644 --- a/api/internal/server/routes.go +++ b/api/internal/server/routes.go @@ -100,4 +100,10 @@ func RegisteredRoutes(r *gin.Engine) { validate.GET("/", handler.ValidateToken) } + + // For docker health check + health := r.Group("/health") + { + health.GET("/", handler.HealthCheck) + } } From 7b35501207fadd92e5b25f0ef31844a70ec277bc Mon Sep 17 00:00:00 2001 From: Aleksandre Date: Fri, 19 Sep 2025 22:57:35 +0400 Subject: [PATCH 18/49] Upgrades Go version and refines module dependencies Elevates the Go language version from 1.23.0 to 1.25.0, benefiting from the latest language features and performance enhancements. Removes several direct and indirect dependencies that are no longer needed, such as Firebase and Heroku-related modules, streamlining the module graph. Adjusts various indirect dependencies, including promoting `google.golang.org/api` and `google.golang.org/grpc` to indirect status and integrating new OpenTelemetry SDK components. These changes reflect an overall cleanup and update of the module graph. --- go.mod | 17 +++++------------ go.sum | 19 ------------------- 2 files changed, 5 insertions(+), 31 deletions(-) diff --git a/go.mod b/go.mod index fef7985..f2aec48 100644 --- a/go.mod +++ b/go.mod @@ -1,18 +1,14 @@ module Codex-Backend -go 1.23.0 - -toolchain go1.24.0 +go 1.25.0 require ( cloud.google.com/go/firestore v1.18.0 - firebase.google.com/go v3.13.0+incompatible github.com/JohannesKaufmann/html-to-markdown/v2 v2.3.3 github.com/PuerkitoBio/goquery v1.10.3 github.com/gin-contrib/cors v1.7.3 github.com/gin-gonic/gin v1.10.0 github.com/golang-jwt/jwt/v5 v5.2.2 - github.com/heroku/x v0.5.2 github.com/jackc/pgx/v5 v5.7.5 github.com/joho/godotenv v1.5.1 github.com/oklog/ulid/v2 v2.1.1 @@ -21,8 +17,6 @@ require ( github.com/timsims/pamphlet v0.1.6 golang.org/x/crypto v0.41.0 golang.org/x/time v0.10.0 - google.golang.org/api v0.214.0 - google.golang.org/grpc v1.73.0 ) require ( @@ -30,9 +24,7 @@ require ( cloud.google.com/go/auth v0.13.0 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.6 // indirect cloud.google.com/go/compute/metadata v0.6.0 // indirect - cloud.google.com/go/iam v1.2.2 // indirect cloud.google.com/go/longrunning v0.6.2 // indirect - cloud.google.com/go/storage v1.43.0 // indirect github.com/JohannesKaufmann/dom v0.2.0 // indirect github.com/andybalholm/cascadia v1.3.3 // indirect github.com/bytedance/sonic v1.12.9 // indirect @@ -48,9 +40,7 @@ require ( github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.25.0 // indirect github.com/goccy/go-json v0.10.5 // indirect - github.com/golang/protobuf v1.5.4 // indirect github.com/google/s2a-go v0.1.8 // indirect - github.com/google/uuid v1.6.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect github.com/googleapis/gax-go/v2 v2.14.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect @@ -79,6 +69,8 @@ require ( go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 // indirect go.opentelemetry.io/otel v1.36.0 // indirect go.opentelemetry.io/otel/metric v1.36.0 // indirect + go.opentelemetry.io/otel/sdk v1.36.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.36.0 // indirect go.opentelemetry.io/otel/trace v1.36.0 // indirect go.uber.org/goleak v1.3.0 // indirect golang.org/x/arch v0.14.0 // indirect @@ -87,10 +79,11 @@ require ( golang.org/x/sync v0.16.0 // indirect golang.org/x/sys v0.35.0 // indirect golang.org/x/text v0.28.0 // indirect - google.golang.org/appengine v1.6.8 // indirect + google.golang.org/api v0.214.0 // indirect google.golang.org/genproto v0.0.0-20241118233622-e639e219e697 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20250324211829-b45e905df463 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20250324211829-b45e905df463 // indirect + google.golang.org/grpc v1.73.0 // indirect google.golang.org/protobuf v1.36.6 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 9397196..4c1e7b2 100644 --- a/go.sum +++ b/go.sum @@ -8,14 +8,8 @@ cloud.google.com/go/compute/metadata v0.6.0 h1:A6hENjEsCDtC1k8byVsgwvVcioamEHvZ4 cloud.google.com/go/compute/metadata v0.6.0/go.mod h1:FjyFAW1MW0C203CEOMDTu3Dk1FlqW3Rga40jzHL4hfg= cloud.google.com/go/firestore v1.18.0 h1:cuydCaLS7Vl2SatAeivXyhbhDEIR8BDmtn4egDhIn2s= cloud.google.com/go/firestore v1.18.0/go.mod h1:5ye0v48PhseZBdcl0qbl3uttu7FIEwEYVaWm0UIEOEU= -cloud.google.com/go/iam v1.2.2 h1:ozUSofHUGf/F4tCNy/mu9tHLTaxZFLOUiKzjcgWHGIA= -cloud.google.com/go/iam v1.2.2/go.mod h1:0Ys8ccaZHdI1dEUilwzqng/6ps2YB6vRsjIe00/+6JY= cloud.google.com/go/longrunning v0.6.2 h1:xjDfh1pQcWPEvnfjZmwjKQEcHnpz6lHjfy7Fo0MK+hc= cloud.google.com/go/longrunning v0.6.2/go.mod h1:k/vIs83RN4bE3YCswdXC5PFfWVILjm3hpEUlSko4PiI= -cloud.google.com/go/storage v1.43.0 h1:CcxnSohZwizt4LCzQHWvBf1/kvtHUn7gk9QERXPyXFs= -cloud.google.com/go/storage v1.43.0/go.mod h1:ajvxEa7WmZS1PxvKRq4bq0tFT3vMd502JwstCcYv0Q0= -firebase.google.com/go v3.13.0+incompatible h1:3TdYC3DDi6aHn20qoRkxwGqNgdjtblwVAyRLQwGn/+4= -firebase.google.com/go v3.13.0+incompatible/go.mod h1:xlah6XbEyW6tbfSklcfe5FHJIwjt8toICdV5Wh9ptHs= github.com/JohannesKaufmann/dom v0.2.0 h1:1bragmEb19K8lHAqgFgqCpiPCFEZMTXzOIEjuxkUfLQ= github.com/JohannesKaufmann/dom v0.2.0/go.mod h1:57iSUl5RKric4bUkgos4zu6Xt5LMHUnw3TF1l5CbGZo= github.com/JohannesKaufmann/html-to-markdown/v2 v2.3.3 h1:r3fokGFRDk/8pHmwLwJ8zsX4qiqfS1/1TZm2BH8ueY8= @@ -64,17 +58,12 @@ github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeD github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= -github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= github.com/google/s2a-go v0.1.8 h1:zZDs9gcbt9ZPLV0ndSyQk6Kacx2g/X+SKYovpnz3SMM= github.com/google/s2a-go v0.1.8/go.mod h1:6iNWHTpQ+nfNRN5E00MSdfDwVesa8hhS32PhPO8deJA= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -83,8 +72,6 @@ github.com/googleapis/enterprise-certificate-proxy v0.3.4 h1:XYIDZApgAnrN1c855gT github.com/googleapis/enterprise-certificate-proxy v0.3.4/go.mod h1:YKe7cfqYXjKGpGvmSg28/fFvhNzinZQm8DGnaburhGA= github.com/googleapis/gax-go/v2 v2.14.0 h1:f+jMrjBPl+DL9nI4IQzLUxMq7XrAqFYB7hBPqMNIe8o= github.com/googleapis/gax-go/v2 v2.14.0/go.mod h1:lhBCnjdLrWRaPvLWhmc8IS24m9mr07qSYnHncrgo+zk= -github.com/heroku/x v0.5.2 h1:B3g+m78yQk70Mhe1MsrHICgDJwJMZNVDkaQtLhsSA/U= -github.com/heroku/x v0.5.2/go.mod h1:B025iaZU9I0gPJSsBYmipizkS5TKgX9NWEx8IQQYTuI= github.com/jackc/pgerrcode v0.0.0-20240316143900-6e2875d9b438 h1:Dj0L5fhJ9F82ZJyVOmBx6msDp/kfd1t9GRfny/mfJA0= github.com/jackc/pgerrcode v0.0.0-20240316143900-6e2875d9b438/go.mod h1:a/s9Lp5W7n/DD0VrVoyJ00FbP2ytTPDVOivvn2bMlds= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= @@ -255,7 +242,6 @@ golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= @@ -273,11 +259,8 @@ golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.214.0 h1:h2Gkq07OYi6kusGOaT/9rnNljuXmqPnaig7WGPmKbwA= google.golang.org/api v0.214.0/go.mod h1:bYPpLG8AyeMWwDU6NXoB00xC0DFkikVvd5MfwoxjLqE= -google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= -google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20241118233622-e639e219e697 h1:ToEetK57OidYuqD4Q5w+vfEnPvPpuTwedCNVohYJfNk= google.golang.org/genproto v0.0.0-20241118233622-e639e219e697/go.mod h1:JJrvXBWRZaFMxBufik1a4RpFw4HhgVtBBWQeQgUj2cc= google.golang.org/genproto/googleapis/api v0.0.0-20250324211829-b45e905df463 h1:hE3bRWtU6uceqlh4fhrSnUyjKHMKB9KrTLLG+bc0ddM= @@ -286,8 +269,6 @@ google.golang.org/genproto/googleapis/rpc v0.0.0-20250324211829-b45e905df463 h1: google.golang.org/genproto/googleapis/rpc v0.0.0-20250324211829-b45e905df463/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= google.golang.org/grpc v1.73.0 h1:VIWSmpI2MegBtTuFt5/JWy2oXxtjJ/e89Z70ImfD2ok= google.golang.org/grpc v1.73.0/go.mod h1:50sbHOUqWoCQGI8V2HQLJM0B+LMlIUjNSZmow7EVBQc= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 7c84a2e6638a714ba5dc0071b81a542975e4828f Mon Sep 17 00:00:00 2001 From: Aleksandre Date: Fri, 19 Sep 2025 22:58:07 +0400 Subject: [PATCH 19/49] Converts timestamp fields to time.Time Refactors CreatedAt and UpdatedAt fields in domain models (Chapter, Novel) from string to time.Time. This improves type safety and enables proper date and time operations within the application. --- api/internal/domain/chapter.go | 32 ++++++++++++++++++-------------- api/internal/domain/novel.go | 28 +++++++++++++++------------- 2 files changed, 33 insertions(+), 27 deletions(-) diff --git a/api/internal/domain/chapter.go b/api/internal/domain/chapter.go index c534b4f..72b6b5f 100644 --- a/api/internal/domain/chapter.go +++ b/api/internal/domain/chapter.go @@ -1,6 +1,10 @@ package domain -import "cloud.google.com/go/firestore" +import ( + "time" + + "cloud.google.com/go/firestore" +) type CursorOptions struct { NovelID string `json:"novel_id"` @@ -16,21 +20,21 @@ type CursorResponse struct { // Chapter struct used on backend type Chapter struct { - ID string `json:"id"` - Title string `json:"title"` - Author string `json:"author"` - Description string `json:"description"` - CreatedAt string `json:"creation_date"` - UpdatedAt string `json:"update_date"` - Content string `json:"content"` - Index int `json:"index"` - Deleted bool `json:"deleted"` + ID string `json:"id"` + Title string `json:"title"` + Author string `json:"author"` + Description string `json:"description"` + CreatedAt time.Time `json:"creation_date"` + UpdatedAt time.Time `json:"update_date"` + Content string `json:"content"` + Index int `json:"index"` + Deleted bool `json:"deleted"` } // Chapter struct used on frontend type FrontendChapter struct { - ID string `json:"id"` - Title string `json:"title"` - UpdatedAt string `json:"update_date"` - Content string `json:"content"` + ID string `json:"id"` + Title string `json:"title"` + UpdatedAt time.Time `json:"update_date"` + Content string `json:"content"` } diff --git a/api/internal/domain/novel.go b/api/internal/domain/novel.go index 833da46..f277756 100644 --- a/api/internal/domain/novel.go +++ b/api/internal/domain/novel.go @@ -1,22 +1,24 @@ package domain +import "time" + // Novel struct used on backend type Novel struct { - ID string `json:"id"` - Title string `json:"title"` - Author string `json:"author"` - Description string `json:"description"` - CreatedAt string `json:"creation_date"` - UpdatedAt string `json:"update_date"` - Deleted bool `json:"deleted"` + ID string `json:"id"` + Title string `json:"title"` + Author string `json:"author"` + Description string `json:"description"` + CreatedAt time.Time `json:"creation_date"` + UpdatedAt time.Time `json:"update_date"` + Deleted bool `json:"deleted"` } // Novel struct used on frontend type FrontendNovel struct { - ID string `json:"id"` - Title string `json:"title"` - Author string `json:"author"` - Description string `json:"description"` - CreatedAt string `json:"creation_date"` - UpdatedAt string `json:"update_date"` + ID string `json:"id"` + Title string `json:"title"` + Author string `json:"author"` + Description string `json:"description"` + CreatedAt time.Time `json:"creation_date"` + UpdatedAt time.Time `json:"update_date"` } From 8ae466acd4545d3fdfb27e28cf58671fa6ef5914 Mon Sep 17 00:00:00 2001 From: Aleksandre Date: Fri, 19 Sep 2025 23:05:43 +0400 Subject: [PATCH 20/49] Migrates database to PostgreSQL Introduces a new PostgreSQL client with connection pooling and schema initialization on application startup. Defines relational schemas for novels and chapters, incorporating hash partitioning for chapters and an index for efficient seek pagination. Implements data access patterns for novels and chapters, including atomic chapter indexing and cursor-based pagination. All previous Firestore client and collection logic has been removed. --- api/cmd/web/main.go | 14 +- api/internal/database/client/client.go | 130 ++++++- api/internal/database/collections/chapters.go | 317 ++++++++++-------- api/internal/database/collections/helper.go | 92 +++++ api/internal/database/collections/novels.go | 154 ++++----- api/internal/database/collections/users.go | 2 +- 6 files changed, 476 insertions(+), 233 deletions(-) create mode 100644 api/internal/database/collections/helper.go diff --git a/api/cmd/web/main.go b/api/cmd/web/main.go index ca220f6..8d663bc 100644 --- a/api/cmd/web/main.go +++ b/api/cmd/web/main.go @@ -2,10 +2,12 @@ package main import ( cmn "Codex-Backend/api/common" + db_client "Codex-Backend/api/internal/database/client" firestore_server "Codex-Backend/api/internal/server" + "context" + "fmt" "github.com/gin-gonic/gin" - _ "github.com/heroku/x/hmetrics/onload" ) func init() { @@ -13,6 +15,16 @@ func init() { mode := cmn.GetEnvVariable("GIN_MODE") gin.SetMode(mode) + + ctx := context.Background() + connStr := cmn.GetEnvVariable("DATABASE_URL") + client, err := db_client.GetClient(connStr) + if err != nil { + panic(fmt.Sprintf("db new client: %v", err)) + } + if err := client.EnsureSchema(ctx); err != nil { + panic(fmt.Sprintf("schema ensure failed: %v", err)) + } } func main() { diff --git a/api/internal/database/client/client.go b/api/internal/database/client/client.go index 5241dd1..333d5c9 100644 --- a/api/internal/database/client/client.go +++ b/api/internal/database/client/client.go @@ -1,36 +1,136 @@ -package firestore_client +package db_client import ( - cmn "Codex-Backend/api/common" "context" "errors" + "fmt" "net/http" + "sync" + "time" + + cmn "Codex-Backend/api/common" - "cloud.google.com/go/firestore" - firebase "firebase.google.com/go" - "google.golang.org/api/option" + "github.com/jackc/pgx/v5/pgxpool" ) type Client struct { - *firestore.Client + Pool *pgxpool.Pool +} + +var ( + instance *Client + initErr error + once sync.Once +) + +func NewClient(ctx context.Context, connString string) (*Client, error) { + pool, err := pgxpool.New(ctx, connString) + if err != nil { + return nil, err + } + return &Client{Pool: pool}, nil } -func FirestoreClient() (*firestore.Client, error) { - ctx := context.Background() +func NewClientWithConfig(ctx context.Context, connString string, maxConns int32) (*Client, error) { + cfg, err := pgxpool.ParseConfig(connString) + if err != nil { + return nil, err + } - credentials_json := cmn.GetEnvVariable("GOOGLE_CREDENTIALS") + if maxConns <= 0 { + maxConns = 20 + } - sa := option.WithCredentialsJSON([]byte(credentials_json)) + cfg.MaxConns = maxConns + cfg.MinConns = 1 + cfg.MaxConnLifetime = time.Hour + cfg.HealthCheckPeriod = 30 * time.Second - app, err := firebase.NewApp(ctx, nil, sa) + pool, err := pgxpool.NewWithConfig(ctx, cfg) if err != nil { - return nil, &cmn.Error{Err: errors.New("Firestore Client Error - Firebase App: " + err.Error()), Status: http.StatusInternalServerError} + return nil, err } - client, err := app.Firestore(ctx) + return &Client{Pool: pool}, nil +} + +func GetClient(connString string) (*Client, error) { + once.Do(func() { + instance, initErr = NewClient(context.Background(), connString) + }) + return instance, initErr +} + +func (c *Client) Close() { + if c != nil && c.Pool != nil { + c.Pool.Close() + } +} + +func (c *Client) EnsureSchema(ctx context.Context) error { + if c == nil || c.Pool == nil { + return &cmn.Error{Err: errors.New("postgres client not initialized"), Status: http.StatusInternalServerError} + } + + tx, err := c.Pool.Begin(ctx) if err != nil { - return nil, &cmn.Error{Err: errors.New("Firestore Client Error - Firestore Client: " + err.Error()), Status: http.StatusInternalServerError} + return &cmn.Error{Err: fmt.Errorf("begin tx for schema: %w", err), Status: http.StatusInternalServerError} + } + defer tx.Rollback(ctx) + + stmts := []string{ + // extension for gen_random_uuid + `CREATE EXTENSION IF NOT EXISTS pgcrypto;`, + + // novels with chapter_count for atomic index allocation + `CREATE TABLE IF NOT EXISTS novels ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + title text NOT NULL, + author text NOT NULL, + description text NOT NULL, + chapter_count bigint NOT NULL DEFAULT 0, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now() + );`, + + // parent partitioned chapters table (hash partition on novel_id) + `CREATE TABLE IF NOT EXISTS chapters ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + novel_id uuid NOT NULL, + title text NOT NULL, + author text NOT NULL, + description text NOT NULL, + content text NOT NULL, + chapter_index bigint DEFAULT 0, + deleted boolean DEFAULT false, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now() + ) PARTITION BY HASH (novel_id);`, + // an index that supports seek pagination: novel_id, chapter_index, id + `CREATE INDEX IF NOT EXISTS idx_chapters_novel_index_id ON chapters (novel_id, chapter_index, id);`, + } + + for _, s := range stmts { + if _, err := tx.Exec(ctx, s); err != nil { + return &cmn.Error{Err: fmt.Errorf("schema creation exec: %w", err), Status: http.StatusInternalServerError} + } + } + + // Create partitions (idempotent) + const partitionsCount = 16 + for i := range partitionsCount { + stmt := fmt.Sprintf( + `CREATE TABLE IF NOT EXISTS chapters_p%d PARTITION OF chapters FOR VALUES WITH (MODULUS %d, REMAINDER %d);`, + i, partitionsCount, i, + ) + if _, err := tx.Exec(ctx, stmt); err != nil { + return &cmn.Error{Err: fmt.Errorf("creating partition %d: %w", i, err), Status: http.StatusInternalServerError} + } + } + + if err := tx.Commit(ctx); err != nil { + return &cmn.Error{Err: fmt.Errorf("commit schema creation: %w", err), Status: http.StatusInternalServerError} } - return client, nil + return nil } diff --git a/api/internal/database/collections/chapters.go b/api/internal/database/collections/chapters.go index 11b7351..113995f 100644 --- a/api/internal/database/collections/chapters.go +++ b/api/internal/database/collections/chapters.go @@ -1,4 +1,4 @@ -package firestore_collections +package db import ( cmn "Codex-Backend/api/common" @@ -9,195 +9,234 @@ import ( "net/http" "time" - "cloud.google.com/go/firestore" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" ) -func (c *Client) CursorPagination(options domain.CursorOptions, ctx context.Context) (*domain.CursorResponse, error) { - coll := c.Client.Collection("novels").Doc(options.NovelID).Collection("chapters") - query := coll.OrderBy("Index", options.SortBy) - - limit := min(max(options.Limit, 1), 100) +// SQL query constants +const ( + listChaptersAscSQL = ` + SELECT id, title, author, description, content, chapter_index, deleted, created_at, updated_at + FROM chapters + WHERE novel_id = $1 AND (chapter_index, id) > ($2, $3) + ORDER BY chapter_index ASC, id ASC + LIMIT $4` + + listChaptersAscFirstSQL = ` + SELECT id, title, author, description, content, chapter_index, deleted, created_at, updated_at + FROM chapters + WHERE novel_id = $1 + ORDER BY chapter_index ASC, id ASC + LIMIT $2` + + listChaptersDescSQL = ` + SELECT id, title, author, description, content, chapter_index, deleted, created_at, updated_at + FROM chapters + WHERE novel_id = $1 AND (chapter_index, id) < ($2, $3) + ORDER BY chapter_index DESC, id DESC + LIMIT $4` + + listChaptersDescFirstSQL = ` + SELECT id, title, author, description, content, chapter_index, deleted, created_at, updated_at + FROM chapters + WHERE novel_id = $1 + ORDER BY chapter_index DESC, id DESC + LIMIT $2` +) - snapshots := []*firestore.DocumentSnapshot{} - var err error +/* +ListChaptersSeek returns up to `limit` chapters for a novel using seek-pagination. - if options.Cursor == 0 { - snapshots, err = query.Limit(limit + 1).Documents(ctx).GetAll() - } else { - snapshots, err = query.StartAt(options.Cursor).Limit(limit + 1).Documents(ctx).GetAll() - } + - cursor: encoded cursor string from previous page (or empty for first page) - if err != nil { - return nil, err - } + - limit: max rows to return - if len(snapshots) == 0 { - return nil, &cmn.Error{ - Err: fmt.Errorf("Firestore Client Error - Get Paginated Chapters - No Chapters Found for Novel: %s", options.NovelID), - Status: http.StatusNotFound, - } - } + - asc: if true order by chapter_index ASC, id ASC (older -> newer); if false, DESC Returns: - actualLimit := min(len(snapshots), limit) - chapters := make([]domain.FrontendChapter, 0, actualLimit) + - slice of chapters - for _, snapshot := range snapshots[:actualLimit] { - var chapter domain.Chapter - if err := snapshot.DataTo(&chapter); err != nil { - return nil, err - } - chapters = append(chapters, domain.FrontendChapter{ - ID: chapter.ID, - Title: chapter.Title, - UpdatedAt: chapter.UpdatedAt, - Content: chapter.Content, - }) + - nextCursor: encoded cursor to use for the next page (empty if no more rows) +*/ +func (c *Client) ListChaptersSeek(novelId string, limit int, cursor string, asc bool, ctx context.Context) ([]domain.Chapter, string, error) { + if limit <= 0 { + limit = 100 } - nextCursor := 0 - if len(snapshots) > limit { - var lastChapter domain.Chapter - if err := snapshots[limit].DataTo(&lastChapter); err != nil { - return nil, err - } - nextCursor = lastChapter.Index + // decode cursor + sc, err := decodeCursor(cursor) + if err != nil { + return nil, "", &cmn.Error{Err: fmt.Errorf("invalid cursor: %w", err), Status: http.StatusBadRequest} } - return &domain.CursorResponse{ - Chapters: chapters, - NextCursor: nextCursor, - }, nil -} + var results []domain.Chapter + if err := c.WithConn(ctx, func(conn *pgxpool.Conn) error { + var rows pgx.Rows -func (c *Client) BatchUploadChapters(novelId string, chapters []domain.Chapter, ctx context.Context) error { - coll := c.Client.Collection("novels").Doc(novelId).Collection("chapters") - const chunkSize = 500 + fetchLimit := limit + 1 - for i := 0; i < len(chapters); i += chunkSize { - subset := chapters[i:min(i+chunkSize, len(chapters))] + if asc { + if sc.Index == -1 { // First page + rows, err = conn.Query(ctx, listChaptersAscFirstSQL, novelId, fetchLimit) + } else { + rows, err = conn.Query(ctx, listChaptersAscSQL, novelId, sc.Index, sc.ID, fetchLimit) + } + } else { + if sc.Index == -1 { // First page + rows, err = conn.Query(ctx, listChaptersDescFirstSQL, novelId, fetchLimit) + } else { + rows, err = conn.Query(ctx, listChaptersDescSQL, novelId, sc.Index, sc.ID, fetchLimit) + } + } - batchCtx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) - defer cancel() + if err := rows.Err(); err != nil { + return &cmn.Error{Err: fmt.Errorf("rows error: %w", err), Status: http.StatusInternalServerError} + } + defer rows.Close() - bw := c.Client.BulkWriter(batchCtx) - jobs := make([]*firestore.BulkWriterJob, 0, len(subset)) + results, err = pgx.CollectRows(rows, func(row pgx.CollectableRow) (domain.Chapter, error) { + var chapter domain.Chapter - for _, chap := range subset { - job, err := bw.Set(coll.Doc(chap.ID), chap) + err := row.Scan(&chapter.Title, &chapter.Author, &chapter.Description, + &chapter.Content, &chapter.Index, &chapter.Deleted, &chapter.CreatedAt, &chapter.UpdatedAt) if err != nil { - cancel() - return &cmn.Error{ - Err: fmt.Errorf("Firestore Client Error - Batch Upload Chapters - Enqueue failed for chapter %s: %w", chap.ID, err), - Status: http.StatusInternalServerError, - } + return domain.Chapter{}, &cmn.Error{Err: fmt.Errorf("scan ListChaptersSeek: %w", err), Status: http.StatusInternalServerError} } - jobs = append(jobs, job) - } - bw.Flush() - bw.End() - - // Check each job’s result to catch silent failures - for j, job := range jobs { - if _, err := job.Results(); err != nil { - chap := subset[j] - cancel() - return &cmn.Error{ - Err: fmt.Errorf("Firestore Client Error - Batch Upload Chapters - Write failed for chapter %s: %w", chap.ID, err), - Status: http.StatusInternalServerError, - } - } + return chapter, nil + }) + if err != nil { + return &cmn.Error{Err: fmt.Errorf("collect rows: %w", err), Status: http.StatusInternalServerError} } - cancel() - if i+chunkSize < len(chapters) { - time.Sleep(200 * time.Millisecond) - } + return nil + }); err != nil { + return nil, "", err } - return nil -} + var nextCursor string + hasMore := len(results) > limit + if hasMore { + results = results[:limit] + } -func (c *Client) CreateChapter(novelId string, chapter domain.Chapter, ctx context.Context) error { - _, err := c.Client.Collection("novels").Doc(novelId).Collection("chapters").Doc(chapter.ID).Set(ctx, chapter) - if err != nil { - return &cmn.Error{Err: errors.New("Firestore Client Error - Create Chapter: " + err.Error()), Status: http.StatusInternalServerError} + if len(results) > 0 && hasMore { + lastResult := results[len(results)-1] + lastCursor := seekCursor{Index: int64(lastResult.Index), ID: lastResult.ID} + nextCursor, err = encodeCursor(lastCursor) + if err != nil { + return nil, "", &cmn.Error{Err: fmt.Errorf("encode cursor: %w", err), Status: http.StatusInternalServerError} + } } - return nil + return results, nextCursor, nil } -func (c *Client) GetChapterById(novelId string, chapterId string, ctx context.Context) (*domain.Chapter, error) { - doc, err := c.Client.Collection("novels").Doc(novelId).Collection("chapters").Doc(chapterId).Get(ctx) - if err != nil { - return nil, &cmn.Error{Err: errors.New("Firestore Client Error - Get Chapter By Id: " + err.Error()), Status: http.StatusInternalServerError} - } +func (c *Client) CreateChapter(novelId string, chapter domain.Chapter, ctx context.Context) error { + var newIndex int64 - chapter := domain.Chapter{} - if err = doc.DataTo(&chapter); err != nil { - if status.Convert(err).Code() == codes.NotFound { - return nil, &cmn.Error{Err: errors.New("Firestore Client Error - Get Chapter By Id - Chapter Not Found"), Status: http.StatusNotFound} + if err := c.WithConn(ctx, func(conn *pgxpool.Conn) error { + err := c.Pool.QueryRow(ctx, `UPDATE novels SET chapter_count = chapter_count + 1, updated_at = now() WHERE id = $1 RETURNING chapter_count`, novelId).Scan(&newIndex) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return &cmn.Error{Err: fmt.Errorf("novel not found: %w", err), Status: http.StatusNotFound} + } + return &cmn.Error{Err: fmt.Errorf("Update novels chapter_count: %w", err), Status: http.StatusInternalServerError} + } + + // Insert chapter using newIndex + const insertSQL = ` + INSERT INTO chapters (novel_id, title, author, description, content, chapter_index) + VALUES ($1, $2, $3, $4, $5, $6); + ` + + if _, err = c.Pool.Exec(ctx, insertSQL, + novelId, + chapter.Title, + chapter.Author, + chapter.Description, + chapter.Content, + newIndex, + ); err != nil { + return &cmn.Error{Err: fmt.Errorf("insert chapter: %w", err), Status: http.StatusInternalServerError} } - return nil, &cmn.Error{Err: errors.New("Firestore Client Error - Get Chapter By Id: " + err.Error()), Status: http.StatusInternalServerError} + + return nil + }); err != nil { + return &cmn.Error{Err: fmt.Errorf("create chapter: %w", err), Status: http.StatusInternalServerError} } - return &chapter, nil + return nil } -func (c *Client) GetAllChapters(novelId string, ctx context.Context) (*[]domain.Chapter, error) { - doc, err := c.Client.Collection("novels").Doc(novelId).Collection("chapters").Documents(ctx).GetAll() - if err != nil { - return nil, &cmn.Error{Err: errors.New("Firestore Client Error - Get All Chapters: " + err.Error()), Status: http.StatusInternalServerError} - } +func (c *Client) GetChapterById(novelId string, chapterId string, ctx context.Context) (domain.Chapter, error) { + chapter := domain.Chapter{} - chapters := []domain.Chapter{} - for _, d := range doc { - chapter := domain.Chapter{} - if err = d.DataTo(&chapter); err != nil { - return nil, &cmn.Error{Err: errors.New("Firestore Client Error - Get All Chapters: " + err.Error()), Status: http.StatusInternalServerError} + if err := c.WithConn(ctx, func(conn *pgxpool.Conn) error { + if err := conn.QueryRow(ctx, "SELECT id, novel_id, title, author, description, content, chapter_index, deleted, created_at, updated_at FROM chapters WHERE id = $1 AND novel_id = $2 LIMIT 1", chapterId, novelId).Scan( + &chapter.ID, novelId, &chapter.Title, &chapter.Author, &chapter.Description, &chapter.Content, &chapter.Index, &chapter.Deleted, &chapter.CreatedAt, &chapter.UpdatedAt, + ); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return &cmn.Error{Err: errors.New("chapter not found"), Status: http.StatusNotFound} + } + return &cmn.Error{Err: fmt.Errorf("postgres client error - get chapter by id: %w", err), Status: http.StatusInternalServerError} } - chapters = append(chapters, chapter) + return nil + }); err != nil { + return domain.Chapter{}, err } - return &chapters, nil + return chapter, nil } -func (c *Client) UpdateChapter(novelId string, chapter domain.Chapter, ctx context.Context) error { - updates := make(map[string]any) - - if chapter.Title != "" { - updates["Title"] = chapter.Title +// Use seek pagination to get chapters in batches +func (c *Client) GetAllChapters(novelId string, pageSize int, asc bool, ctx context.Context) ([]domain.Chapter, error) { + if c == nil || c.Pool == nil { + return nil, &cmn.Error{Err: errors.New("postgres client not initialized"), Status: http.StatusInternalServerError} } - - if chapter.Description != "" { - updates["Description"] = chapter.Description + if pageSize <= 0 { + pageSize = 500 } - if chapter.Content != "" { - updates["Content"] = chapter.Content + var all []domain.Chapter + cursor := "" + for { + chs, nextCursor, err := c.ListChaptersSeek(novelId, pageSize, cursor, asc, ctx) + if err != nil { + return nil, err + } + all = append(all, chs...) + if nextCursor == "" { + break + } + cursor = nextCursor } + return all, nil +} - if len(updates) == 0 { +func (c *Client) UpdateChapter(novelId string, chapter domain.Chapter, ctx context.Context) error { + if err := c.WithConn(ctx, func(conn *pgxpool.Conn) error { + query := fmt.Sprintf("UPDATE chapters SET title = $1, description = $2, content = $3, updated_at = $4 WHERE id = $5") + _, err := conn.Exec(ctx, query, chapter.Title, chapter.Description, chapter.Content, time.Now(), chapter.ID) + if err != nil { + return &cmn.Error{Err: fmt.Errorf("update chapter: %w", err), Status: http.StatusInternalServerError} + } return nil + }); err != nil { + return err } - - updates["updatedAt"] = time.Now().Format("2006-01-02 15:04:05") - - _, err := c.Client.Collection("novels").Doc(novelId).Collection("chapters").Doc(chapter.ID).Set(ctx, updates, firestore.MergeAll) - if err != nil { - return &cmn.Error{Err: errors.New("Firestore Client Error - Update Chapter: " + err.Error()), Status: http.StatusInternalServerError} - } - return nil } -func (c *Client) DeleteChapter(novelId string, chapterId string, ctx context.Context) error { - _, err := c.Client.Collection("novels").Doc(novelId).Collection("chapters").Doc(chapterId).Delete(ctx) - if err != nil { - return &cmn.Error{Err: errors.New("Firestore Client Error - Delete Chapter: " + err.Error()), Status: http.StatusInternalServerError} +func (c *Client) DeleteChapter(chapterId string, ctx context.Context) error { + if err := c.WithConn(ctx, func(conn *pgxpool.Conn) error { + query := fmt.Sprintf("UPDATE chapters SET deleted = $1 WHERE id = $2") + _, err := conn.Exec(ctx, query, true, chapterId) + if err != nil { + return &cmn.Error{Err: fmt.Errorf("delete chapter: %w", err), Status: http.StatusInternalServerError} + } + return nil + }); err != nil { + return err } - return nil } diff --git a/api/internal/database/collections/helper.go b/api/internal/database/collections/helper.go new file mode 100644 index 0000000..2a69ba9 --- /dev/null +++ b/api/internal/database/collections/helper.go @@ -0,0 +1,92 @@ +package db + +import ( + cmn "Codex-Backend/api/common" + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "net/http" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +type Client struct { + Pool *pgxpool.Pool +} + +type seekCursor struct { + Index int64 `json:"idx"` + ID string `json:"id"` +} + +// WithConn acquires a connection from the pool, runs fn(conn) and releases it. +// fn receives *pgxpool.Conn (you can call .Exec/.QueryRow on it). +func (c *Client) WithConn(ctx context.Context, fn func(conn *pgxpool.Conn) error) error { + if c == nil || c.Pool == nil { + return &cmn.Error{Err: errors.New("postgres client not initialized"), Status: http.StatusInternalServerError} + } + acq, err := c.Pool.Acquire(ctx) + if err != nil { + return &cmn.Error{Err: fmt.Errorf("failed to acquire conn: %w", err), Status: http.StatusInternalServerError} + } + defer acq.Release() + return fn(acq) +} + +// WithTx runs fn inside a transaction. It ensures proper rollback on error/panic and commits on success. +func (c *Client) WithTx(ctx context.Context, fn func(tx pgx.Tx) error) error { + if c == nil || c.Pool == nil { + return &cmn.Error{Err: errors.New("postgres client not initialized"), Status: http.StatusInternalServerError} + } + + acq, err := c.Pool.Acquire(ctx) + if err != nil { + return &cmn.Error{Err: fmt.Errorf("acquire conn for tx: %w", err), Status: http.StatusInternalServerError} + } + defer acq.Release() + + tx, err := acq.Begin(ctx) + if err != nil { + return &cmn.Error{Err: fmt.Errorf("begin tx: %w", err), Status: http.StatusInternalServerError} + } + + // ensure rollback if fn fails or panic happens + defer func() { + _ = tx.Rollback(ctx) + }() + + if err := fn(tx); err != nil { + return err + } + + if err := tx.Commit(ctx); err != nil { + return &cmn.Error{Err: fmt.Errorf("commit tx: %w", err), Status: http.StatusInternalServerError} + } + return nil +} + +func encodeCursor(c seekCursor) (string, error) { + b, err := json.Marshal(c) + if err != nil { + return "", err + } + return base64.URLEncoding.EncodeToString(b), nil +} + +func decodeCursor(encoded string) (seekCursor, error) { + if encoded == "" { + return seekCursor{Index: -1, ID: ""}, nil // special empty cursor + } + b, err := base64.URLEncoding.DecodeString(encoded) + if err != nil { + return seekCursor{}, err + } + var sc seekCursor + if err := json.Unmarshal(b, &sc); err != nil { + return seekCursor{}, err + } + return sc, nil +} diff --git a/api/internal/database/collections/novels.go b/api/internal/database/collections/novels.go index af6a1de..b812a37 100644 --- a/api/internal/database/collections/novels.go +++ b/api/internal/database/collections/novels.go @@ -1,115 +1,115 @@ -package firestore_collections +package db import ( cmn "Codex-Backend/api/common" "Codex-Backend/api/internal/domain" "context" "errors" + "fmt" "net/http" "time" - "cloud.google.com/go/firestore" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" ) -type Client struct { - *firestore.Client -} - func (c *Client) CreateNovel(novel domain.Novel, ctx context.Context) error { - _, err := c.Client.Collection("novels").Doc(novel.ID).Set(ctx, novel) - if err != nil { - return &cmn.Error{Err: errors.New("Firestore Client Error - Create Novel: " + err.Error()), Status: http.StatusInternalServerError} - } - - return nil + return c.WithConn(ctx, func(conn *pgxpool.Conn) error { + const insertSQL = `INSERT INTO novels (title, author, description) VALUES ($1,$2,$3)` + if _, err := conn.Exec(ctx, insertSQL, novel.Title, novel.Author, novel.Description); err != nil { + return &cmn.Error{Err: fmt.Errorf("insert novel: %w", err), Status: http.StatusInternalServerError} + } + return nil + }) } -func (c *Client) GetNovelById(id string, ctx context.Context) (*domain.Novel, error) { - doc, err := c.Client.Collection("novels").Doc(id).Get(ctx) - if err != nil { - return nil, &cmn.Error{Err: errors.New("Firestore Client Error - Get Novel by ID: " + err.Error()), Status: http.StatusInternalServerError} - } - +func (c *Client) GetNovelById(id string, ctx context.Context) (domain.Novel, error) { novel := domain.Novel{} - if err := doc.DataTo(&novel); err != nil { - if status.Convert(err).Code() == codes.NotFound { - return nil, &cmn.Error{Err: errors.New("Firestore Client Error - Get Novel by ID - Novel not found"), Status: http.StatusNotFound} + + if err := c.WithConn(ctx, func(conn *pgxpool.Conn) error { + if err := conn.QueryRow(ctx, "SELECT id, title, author, description FROM novels WHERE id = $1", id).Scan(&novel.ID, &novel.Title, &novel.Author, &novel.Description); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return &cmn.Error{Err: fmt.Errorf("novel not found: %w", err), Status: http.StatusNotFound} + } + return &cmn.Error{Err: fmt.Errorf("get novel by id: %w", err), Status: http.StatusInternalServerError} } - return nil, &cmn.Error{Err: errors.New("Firestore Client Error - Get Novel by ID: " + err.Error()), Status: http.StatusInternalServerError} + return nil + }); err != nil { + return domain.Novel{}, err } - - return &novel, nil + return novel, nil } -func (c *Client) GetAllNovels(ctx context.Context) (*[]domain.Novel, error) { - doc, err := c.Client.Collection("novels").Documents(ctx).GetAll() - if err != nil { - return nil, &cmn.Error{Err: errors.New("Firestore Client Error - Get All Novels: " + err.Error()), Status: http.StatusInternalServerError} - } - +func (c *Client) GetAllNovels(ctx context.Context) ([]domain.Novel, error) { novels := []domain.Novel{} - for _, d := range doc { - novel := domain.Novel{} - if err := d.DataTo(&novel); err != nil { - return nil, &cmn.Error{Err: errors.New("Firestore Client Error - Get All Novels: " + err.Error()), Status: http.StatusInternalServerError} + + if err := c.WithConn(ctx, func(conn *pgxpool.Conn) error { + rows, err := conn.Query(ctx, "SELECT id, title, author, description FROM novels") + if err != nil { + return &cmn.Error{Err: fmt.Errorf("get all novels: %w", err), Status: http.StatusInternalServerError} } - novels = append(novels, novel) + defer rows.Close() + + for rows.Next() { + novel := domain.Novel{} + if err := rows.Scan(&novel.ID, &novel.Title, &novel.Author, &novel.Description); err != nil { + return &cmn.Error{Err: fmt.Errorf("scan novel row: %w", err), Status: http.StatusInternalServerError} + } + novels = append(novels, novel) + } + if err := rows.Err(); err != nil { + return &cmn.Error{Err: fmt.Errorf("scan novel rows: %w", err), Status: http.StatusInternalServerError} + } + return nil + }); err != nil { + return nil, err } - - return &novels, nil + return novels, nil } func (c *Client) UpdateNovel(novel domain.Novel, ctx context.Context) error { - updates := make(map[string]any) - - if novel.Title != "" { - updates["Title"] = novel.Title - } - - if novel.Description != "" { - updates["Description"] = novel.Description - } - - if len(updates) == 0 { + if err := c.WithConn(ctx, func(conn *pgxpool.Conn) error { + query := fmt.Sprintf("UPDATE novels SET title = $1, description = $2, updated_at = $3 WHERE id = $4") + _, err := conn.Exec(ctx, query, novel.Title, novel.Description, time.Now(), novel.ID) + if err != nil { + return &cmn.Error{Err: fmt.Errorf("update novel: %w", err), Status: http.StatusInternalServerError} + } return nil + }); err != nil { + return err } - - updates["UpdatedAt"] = time.Now().Format("2006-01-02 15:04:05") - - _, err := c.Client.Collection("novels").Doc(novel.ID).Set(ctx, updates, firestore.MergeAll) - if err != nil { - return &cmn.Error{Err: errors.New("Firestore Client Error - Update Novel: " + err.Error()), Status: http.StatusInternalServerError} - } - return nil } func (c *Client) DeleteNovel(novelId string, ctx context.Context) error { - _, err := c.Client.Collection("novels").Doc(novelId).Delete(ctx) - if err != nil { - return &cmn.Error{Err: errors.New("Firestore Client Error - Delete Novel: " + err.Error()), Status: http.StatusInternalServerError} + if err := c.WithConn(ctx, func(conn *pgxpool.Conn) error { + query := fmt.Sprintf("UPDATE novels SET deleted = $1 WHERE id = $2") + _, err := conn.Exec(ctx, query, true, novelId) + if err != nil { + return &cmn.Error{Err: fmt.Errorf("delete novel: %w", err), Status: http.StatusInternalServerError} + } + return nil + }); err != nil { + return err } - return nil } -func (c *Client) GetNovelByTitle(title string, ctx context.Context) (*domain.Novel, error) { - query := c.Client.Collection("novels").Where("Title", "==", title).Limit(1) - docs, err := query.Documents(ctx).GetAll() - if err != nil { - return nil, &cmn.Error{Err: errors.New("Firestore Client Error - Get Novel by Title: " + err.Error()), Status: http.StatusInternalServerError} - } - - if len(docs) == 0 { - return nil, &cmn.Error{Err: errors.New("Novel not found"), Status: http.StatusNotFound} - } - +func (c *Client) GetNovelByTitle(title string, ctx context.Context) (domain.Novel, error) { novel := domain.Novel{} - if err := docs[0].DataTo(&novel); err != nil { - return nil, &cmn.Error{Err: errors.New("Firestore Client Error - Get Novel by Title: " + err.Error()), Status: http.StatusInternalServerError} - } - return &novel, nil + if err := c.WithConn(ctx, func(conn *pgxpool.Conn) error { + query := fmt.Sprintf("SELECT id, title, author, description FROM novels WHERE title = $1 AND deleted = $2") + row := conn.QueryRow(ctx, query, title, false) + if err := row.Scan(&novel.ID, &novel.Title, &novel.Author, &novel.Description); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return &cmn.Error{Err: errors.New("Novel not found"), Status: http.StatusNotFound} + } + return &cmn.Error{Err: fmt.Errorf("get novel by title: %w", err), Status: http.StatusInternalServerError} + } + return nil + }); err != nil { + return domain.Novel{}, err + } + return novel, nil } diff --git a/api/internal/database/collections/users.go b/api/internal/database/collections/users.go index 0b33301..c113402 100644 --- a/api/internal/database/collections/users.go +++ b/api/internal/database/collections/users.go @@ -1,4 +1,4 @@ -package firestore_collections +package db import ( cmn "Codex-Backend/api/common" From 718eead30675f0419bead59faca7605af56c12ce Mon Sep 17 00:00:00 2001 From: Aleksandre Date: Sat, 20 Sep 2025 00:48:17 +0400 Subject: [PATCH 21/49] feat(db): Add users table and essential indexes to schema --- api/internal/database/client/client.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/api/internal/database/client/client.go b/api/internal/database/client/client.go index 333d5c9..6324838 100644 --- a/api/internal/database/client/client.go +++ b/api/internal/database/client/client.go @@ -82,6 +82,16 @@ func (c *Client) EnsureSchema(ctx context.Context) error { // extension for gen_random_uuid `CREATE EXTENSION IF NOT EXISTS pgcrypto;`, + `CREATE TABLE IF NOT EXISTS users ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + username text NOT NULL, + type text NOT NULL, + email text NOT NULL, + password text NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now() + );`, + // novels with chapter_count for atomic index allocation `CREATE TABLE IF NOT EXISTS novels ( id uuid PRIMARY KEY DEFAULT gen_random_uuid(), @@ -108,6 +118,9 @@ func (c *Client) EnsureSchema(ctx context.Context) error { ) PARTITION BY HASH (novel_id);`, // an index that supports seek pagination: novel_id, chapter_index, id `CREATE INDEX IF NOT EXISTS idx_chapters_novel_index_id ON chapters (novel_id, chapter_index, id);`, + + `CREATE INDEX IF NOT EXISTS idx_novels_title_id ON novels (title, id);`, + `CREATE INDEX IF NOT EXISTS idx_users_email_id ON users (email, id);`, } for _, s := range stmts { From 583e847c4b1607361fdcb310a8b675284e5d9b41 Mon Sep 17 00:00:00 2001 From: Aleksandre Date: Sat, 20 Sep 2025 00:48:18 +0400 Subject: [PATCH 22/49] refactor(domain): Use time.Time for User timestamps --- api/internal/domain/user.go | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/api/internal/domain/user.go b/api/internal/domain/user.go index 611dfdc..bd365ea 100644 --- a/api/internal/domain/user.go +++ b/api/internal/domain/user.go @@ -1,13 +1,15 @@ package domain +import "time" + type User struct { - ID string `json:"id"` - Username string `json:"username"` - Password string `json:"password"` - Email string `json:"email"` - Type string `json:"type"` - CreatedAt string `json:"created_at"` - UpdatedAt string `json:"updated_at"` + ID string `json:"id"` + Username string `json:"username"` + Password string `json:"password"` + Email string `json:"email"` + Type string `json:"type"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` } type Credentials struct { From 6782ac667fe1255b391ec8bfa1a839061ff4b4fe Mon Sep 17 00:00:00 2001 From: Aleksandre Date: Sat, 20 Sep 2025 00:48:18 +0400 Subject: [PATCH 23/49] feat(users): Migrate user data access from Firestore to PostgreSQL --- api/internal/database/collections/users.go | 142 ++++++++++----------- 1 file changed, 67 insertions(+), 75 deletions(-) diff --git a/api/internal/database/collections/users.go b/api/internal/database/collections/users.go index c113402..b2e2ea7 100644 --- a/api/internal/database/collections/users.go +++ b/api/internal/database/collections/users.go @@ -4,107 +4,99 @@ import ( cmn "Codex-Backend/api/common" "Codex-Backend/api/internal/domain" "context" - "errors" + "fmt" "net/http" - "cloud.google.com/go/firestore" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" ) func (c *Client) CreateUser(user domain.User, ctx context.Context) error { - _, err := c.Client.Collection("users").Doc(user.ID).Set(ctx, user) - if err != nil { - return &cmn.Error{Err: errors.New("Firestore Client Error - Creating User: " + err.Error()), Status: http.StatusInternalServerError} - } - - return nil + return c.WithConn(ctx, func(conn *pgxpool.Conn) error { + const insertSQL = `INSERT INTO users (id, email, username, type, password) VALUES ($1,$2,$3,$4,$5)` + if _, err := conn.Exec(ctx, insertSQL, user.ID, user.Email, user.Username, user.Type, user.Password); err != nil { + return &cmn.Error{Err: fmt.Errorf("insert user: %w", err), Status: http.StatusInternalServerError} + } + return nil + }) } -func (c *Client) GetUserByEmail(email string, ctx context.Context) (*domain.User, error) { - users, err := c.GetAllUsers(ctx) - if err != nil { - return nil, err - } - - for _, user := range *users { - if user.Email == email { - return &user, nil +func (c *Client) GetUserByEmail(email string, ctx context.Context) (domain.User, error) { + var user domain.User + if err := c.WithConn(ctx, func(conn *pgxpool.Conn) error { + const selectSQL = `SELECT id, email, username, type, password FROM users WHERE email = $1` + if err := conn.QueryRow(ctx, selectSQL, email).Scan(&user.ID, &user.Email, &user.Username, &user.Type, &user.Password); err != nil { + return &cmn.Error{Err: fmt.Errorf("select user by email: %w", err), Status: http.StatusInternalServerError} } + return nil + }); err != nil { + return domain.User{}, err } - - return nil, &cmn.Error{Err: errors.New("User not found"), Status: http.StatusNotFound} + return user, nil } -func (c *Client) GetUserById(userId string, ctx context.Context) (*domain.User, error) { - doc, err := c.Client.Collection("users").Doc(userId).Get(ctx) - if err != nil { - return nil, &cmn.Error{Err: errors.New("Firestore Client Error - Getting User by ID: " + err.Error()), Status: http.StatusInternalServerError} - } - - user := domain.User{} - if err = doc.DataTo(&user); err != nil { - return nil, &cmn.Error{Err: errors.New("Firestore Client Error - Getting User by ID: " + err.Error()), Status: http.StatusInternalServerError} +func (c *Client) GetUserById(userId string, ctx context.Context) (domain.User, error) { + var user domain.User + if err := c.WithConn(ctx, func(conn *pgxpool.Conn) error { + const selectSQL = `SELECT id, email, username, type, password FROM users WHERE id = $1` + if err := conn.QueryRow(ctx, selectSQL, userId).Scan(&user.ID, &user.Email, &user.Username, &user.Type, &user.Password); err != nil { + return &cmn.Error{Err: fmt.Errorf("select user by id: %w", err), Status: http.StatusInternalServerError} + } + return nil + }); err != nil { + return domain.User{}, err } - - return &user, nil + return user, nil } func (c *Client) GetAllUsers(ctx context.Context) (*[]domain.User, error) { - doc, err := c.Client.Collection("users").Documents(ctx).GetAll() - if err != nil { - return nil, &cmn.Error{Err: errors.New("Firestore Client Error - Getting All Users: " + err.Error()), Status: http.StatusInternalServerError} - } + var users []domain.User + if err := c.WithConn(ctx, func(conn *pgxpool.Conn) error { + const selectSQL = `SELECT id, email, username, type, password FROM users` + rows, err := conn.Query(ctx, selectSQL) + if err != nil { + return &cmn.Error{Err: fmt.Errorf("select all users: %w", err), Status: http.StatusInternalServerError} + } + defer rows.Close() + + users, err = pgx.CollectRows(rows, func(row pgx.CollectableRow) (domain.User, error) { + var user domain.User - users := []domain.User{} - for _, d := range doc { - var user domain.User - err = d.DataTo(&user) + err := row.Scan(&user.ID, &user.Email, &user.Username, &user.Type, &user.Password) + if err != nil { + return domain.User{}, err + } + return user, nil + }) if err != nil { - return nil, &cmn.Error{Err: errors.New("Firestore Client Error - Getting All Users: " + err.Error()), Status: http.StatusInternalServerError} + return err } - users = append(users, user) + return nil + }); err != nil { + return nil, err } return &users, nil } func (c *Client) UpdateUser(user domain.User, ctx context.Context) error { - updates := make(map[string]any) - - if user.Email != "" { - updates["Email"] = user.Email - } - - if user.Username != "" { - updates["Username"] = user.Username - } - - if user.Password != "" { - updates["Password"] = user.Password - } - - if user.Type != "" { - updates["Type"] = user.Type - } - - if len(updates) == 0 { + return c.WithConn(ctx, func(conn *pgxpool.Conn) error { + const updateSQL = `UPDATE users SET email = $1, username = $2, type = $3, password = $4 WHERE id = $5` + _, err := conn.Exec(ctx, updateSQL, user.Email, user.Username, user.Type, user.Password, user.ID) + if err != nil { + return &cmn.Error{Err: fmt.Errorf("update user: %w", err), Status: http.StatusInternalServerError} + } return nil - } - - updates["UpdatedAt"] = user.UpdatedAt - - _, err := c.Client.Collection("users").Doc(user.ID).Set(ctx, updates, firestore.MergeAll) - if err != nil { - return &cmn.Error{Err: errors.New("Firestore Client Error - Updating User: " + err.Error()), Status: http.StatusInternalServerError} - } - - return nil + }) } func (c *Client) DeleteUser(id string, ctx context.Context) error { - _, err := c.Client.Collection("users").Doc(id).Delete(ctx) - if err != nil { - return &cmn.Error{Err: errors.New("Firestore Client Error - Deleting User: " + err.Error()), Status: http.StatusInternalServerError} - } - - return nil + return c.WithConn(ctx, func(conn *pgxpool.Conn) error { + const deleteSQL = `UPDATE users SET deleted = $1 WHERE id = $2` + _, err := conn.Exec(ctx, deleteSQL, true, id) + if err != nil { + return &cmn.Error{Err: fmt.Errorf("delete user: %w", err), Status: http.StatusInternalServerError} + } + return nil + }) } From f13d6d0d70b4f084336775720abc1647a08c57d8 Mon Sep 17 00:00:00 2001 From: Aleksandre Date: Sat, 20 Sep 2025 00:48:18 +0400 Subject: [PATCH 24/49] refactor(sql): Simplify static SQL queries using string literals --- api/internal/database/collections/chapters.go | 4 ++-- api/internal/database/collections/novels.go | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/api/internal/database/collections/chapters.go b/api/internal/database/collections/chapters.go index 113995f..6adb0bb 100644 --- a/api/internal/database/collections/chapters.go +++ b/api/internal/database/collections/chapters.go @@ -215,7 +215,7 @@ func (c *Client) GetAllChapters(novelId string, pageSize int, asc bool, ctx cont func (c *Client) UpdateChapter(novelId string, chapter domain.Chapter, ctx context.Context) error { if err := c.WithConn(ctx, func(conn *pgxpool.Conn) error { - query := fmt.Sprintf("UPDATE chapters SET title = $1, description = $2, content = $3, updated_at = $4 WHERE id = $5") + query := "UPDATE chapters SET title = $1, description = $2, content = $3, updated_at = $4 WHERE id = $5" _, err := conn.Exec(ctx, query, chapter.Title, chapter.Description, chapter.Content, time.Now(), chapter.ID) if err != nil { return &cmn.Error{Err: fmt.Errorf("update chapter: %w", err), Status: http.StatusInternalServerError} @@ -229,7 +229,7 @@ func (c *Client) UpdateChapter(novelId string, chapter domain.Chapter, ctx conte func (c *Client) DeleteChapter(chapterId string, ctx context.Context) error { if err := c.WithConn(ctx, func(conn *pgxpool.Conn) error { - query := fmt.Sprintf("UPDATE chapters SET deleted = $1 WHERE id = $2") + query := "UPDATE chapters SET deleted = $1 WHERE id = $2" _, err := conn.Exec(ctx, query, true, chapterId) if err != nil { return &cmn.Error{Err: fmt.Errorf("delete chapter: %w", err), Status: http.StatusInternalServerError} diff --git a/api/internal/database/collections/novels.go b/api/internal/database/collections/novels.go index b812a37..0f3a878 100644 --- a/api/internal/database/collections/novels.go +++ b/api/internal/database/collections/novels.go @@ -69,7 +69,7 @@ func (c *Client) GetAllNovels(ctx context.Context) ([]domain.Novel, error) { func (c *Client) UpdateNovel(novel domain.Novel, ctx context.Context) error { if err := c.WithConn(ctx, func(conn *pgxpool.Conn) error { - query := fmt.Sprintf("UPDATE novels SET title = $1, description = $2, updated_at = $3 WHERE id = $4") + query := "UPDATE novels SET title = $1, description = $2, updated_at = $3 WHERE id = $4" _, err := conn.Exec(ctx, query, novel.Title, novel.Description, time.Now(), novel.ID) if err != nil { return &cmn.Error{Err: fmt.Errorf("update novel: %w", err), Status: http.StatusInternalServerError} @@ -83,7 +83,7 @@ func (c *Client) UpdateNovel(novel domain.Novel, ctx context.Context) error { func (c *Client) DeleteNovel(novelId string, ctx context.Context) error { if err := c.WithConn(ctx, func(conn *pgxpool.Conn) error { - query := fmt.Sprintf("UPDATE novels SET deleted = $1 WHERE id = $2") + query := "UPDATE novels SET deleted = $1 WHERE id = $2" _, err := conn.Exec(ctx, query, true, novelId) if err != nil { return &cmn.Error{Err: fmt.Errorf("delete novel: %w", err), Status: http.StatusInternalServerError} @@ -99,7 +99,7 @@ func (c *Client) GetNovelByTitle(title string, ctx context.Context) (domain.Nove novel := domain.Novel{} if err := c.WithConn(ctx, func(conn *pgxpool.Conn) error { - query := fmt.Sprintf("SELECT id, title, author, description FROM novels WHERE title = $1 AND deleted = $2") + query := "SELECT id, title, author, description FROM novels WHERE title = $1 AND deleted = $2" row := conn.QueryRow(ctx, query, title, false) if err := row.Scan(&novel.ID, &novel.Title, &novel.Author, &novel.Description); err != nil { if errors.Is(err, pgx.ErrNoRows) { From 8427540412d4a6994250673ba972604b2b65f0ce Mon Sep 17 00:00:00 2001 From: Aleksandre Date: Tue, 23 Sep 2025 23:38:47 +0400 Subject: [PATCH 25/49] Simplifies database client initialization Updates the database client factory to no longer require an explicit connection string argument. The connection string is now handled internally by the client, streamlining initialization logic. --- api/cmd/web/main.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/api/cmd/web/main.go b/api/cmd/web/main.go index 8d663bc..cfa32b2 100644 --- a/api/cmd/web/main.go +++ b/api/cmd/web/main.go @@ -17,8 +17,7 @@ func init() { gin.SetMode(mode) ctx := context.Background() - connStr := cmn.GetEnvVariable("DATABASE_URL") - client, err := db_client.GetClient(connStr) + client, err := db_client.GetClient(ctx) if err != nil { panic(fmt.Sprintf("db new client: %v", err)) } From eaf993e70d926c32d4380ddbede67b9ff1575e61 Mon Sep 17 00:00:00 2001 From: Aleksandre Date: Tue, 23 Sep 2025 23:40:13 +0400 Subject: [PATCH 26/49] Improves database client configuration and structure Introduces a `ClientConfig` struct for explicit database connection pool settings, enhancing configurability. Updates `NewClientWithConfig` to accept the `ClientConfig` struct, including basic validation for connection limits. Modifies `GetClient` to retrieve the connection string from environment variables and apply production-ready default connection pool settings. Ensures the global database client singleton is properly reset and reinitialized after `Close()`. Refactors the database package by moving collection files out of the `collections` subdirectory. Changes the database client package name to `db`. --- .../database/{collections => }/chapters.go | 0 api/internal/database/{client => }/client.go | 74 ++++++++++++++++--- .../database/{collections => }/helper.go | 4 - .../database/{collections => }/novels.go | 0 .../database/{collections => }/users.go | 0 5 files changed, 62 insertions(+), 16 deletions(-) rename api/internal/database/{collections => }/chapters.go (100%) rename api/internal/database/{client => }/client.go (70%) rename api/internal/database/{collections => }/helper.go (98%) rename api/internal/database/{collections => }/novels.go (100%) rename api/internal/database/{collections => }/users.go (100%) diff --git a/api/internal/database/collections/chapters.go b/api/internal/database/chapters.go similarity index 100% rename from api/internal/database/collections/chapters.go rename to api/internal/database/chapters.go diff --git a/api/internal/database/client/client.go b/api/internal/database/client.go similarity index 70% rename from api/internal/database/client/client.go rename to api/internal/database/client.go index 6324838..5c453a9 100644 --- a/api/internal/database/client/client.go +++ b/api/internal/database/client.go @@ -1,4 +1,4 @@ -package db_client +package db import ( "context" @@ -23,6 +23,22 @@ var ( once sync.Once ) +type ClientConfig struct { + MaxConns int32 + MinConns int32 + MaxConnLifetime time.Duration + HealthCheckPeriod time.Duration +} + +func DefaultClientConfig() ClientConfig { + return ClientConfig{ + MaxConns: 20, + MinConns: 2, + MaxConnLifetime: time.Hour, + HealthCheckPeriod: 30 * time.Second, + } +} + func NewClient(ctx context.Context, connString string) (*Client, error) { pool, err := pgxpool.New(ctx, connString) if err != nil { @@ -31,20 +47,31 @@ func NewClient(ctx context.Context, connString string) (*Client, error) { return &Client{Pool: pool}, nil } -func NewClientWithConfig(ctx context.Context, connString string, maxConns int32) (*Client, error) { +// func NewClient(ctx context.Context, connString string) (*Client, error) { +// return NewClientWithConfig(ctx, connString, DefaultClientConfig()) +// } + +func NewClientWithConfig(ctx context.Context, connString string, config ClientConfig) (*Client, error) { cfg, err := pgxpool.ParseConfig(connString) if err != nil { return nil, err } - if maxConns <= 0 { - maxConns = 20 + // Apply configuration with validation + if config.MaxConns <= 0 { + config.MaxConns = 20 + } + if config.MinConns <= 0 { + config.MinConns = 2 + } + if config.MinConns > config.MaxConns { + config.MinConns = config.MaxConns } - cfg.MaxConns = maxConns - cfg.MinConns = 1 - cfg.MaxConnLifetime = time.Hour - cfg.HealthCheckPeriod = 30 * time.Second + cfg.MaxConns = config.MaxConns + cfg.MinConns = config.MinConns + cfg.MaxConnLifetime = config.MaxConnLifetime + cfg.HealthCheckPeriod = config.HealthCheckPeriod pool, err := pgxpool.NewWithConfig(ctx, cfg) if err != nil { @@ -54,16 +81,39 @@ func NewClientWithConfig(ctx context.Context, connString string, maxConns int32) return &Client{Pool: pool}, nil } -func GetClient(connString string) (*Client, error) { +func GetClient(ctx context.Context) (*Client, error) { once.Do(func() { - instance, initErr = NewClient(context.Background(), connString) + connString := cmn.GetEnvVariable("DATABASE_URL") + + cfg, err := pgxpool.ParseConfig(connString) + if err != nil { + initErr = err + return + } + + // Production-ready defaults + cfg.MaxConns = 20 + cfg.MinConns = 2 + cfg.MaxConnLifetime = time.Hour + cfg.HealthCheckPeriod = 30 * time.Second + + pool, err := pgxpool.NewWithConfig(ctx, cfg) + if err != nil { + initErr = err + return + } + + instance = &Client{Pool: pool} }) + return instance, initErr } func (c *Client) Close() { - if c != nil && c.Pool != nil { - c.Pool.Close() + if instance != nil && instance.Pool != nil { + instance.Pool.Close() + instance = nil + once = sync.Once{} } } diff --git a/api/internal/database/collections/helper.go b/api/internal/database/helper.go similarity index 98% rename from api/internal/database/collections/helper.go rename to api/internal/database/helper.go index 2a69ba9..9598234 100644 --- a/api/internal/database/collections/helper.go +++ b/api/internal/database/helper.go @@ -13,10 +13,6 @@ import ( "github.com/jackc/pgx/v5/pgxpool" ) -type Client struct { - Pool *pgxpool.Pool -} - type seekCursor struct { Index int64 `json:"idx"` ID string `json:"id"` diff --git a/api/internal/database/collections/novels.go b/api/internal/database/novels.go similarity index 100% rename from api/internal/database/collections/novels.go rename to api/internal/database/novels.go diff --git a/api/internal/database/collections/users.go b/api/internal/database/users.go similarity index 100% rename from api/internal/database/collections/users.go rename to api/internal/database/users.go From 10a807ea0e862fdad1b7a8feb724f7d995e25d59 Mon Sep 17 00:00:00 2001 From: Aleksandre Date: Tue, 23 Sep 2025 23:40:49 +0400 Subject: [PATCH 27/49] Refactor service layer and database interaction Introduces a unified `db` abstraction for database client access, removing direct Firestore dependencies from the service layer. Streamlines service methods by delegating ID generation, timestamp management, and soft-delete logic to the new database abstraction. This simplifies service logic and removes boilerplate. Converts service method return types from pointers to values (`*domain.Type` to `domain.Type`), standardizing data handling and error propagation (e.g., explicit nil checks for "not found" cases are replaced with direct error returns). Enhances the `FindAllChapters` API endpoint to support pagination (`size`) and sorting order (`ascending`) via query parameters. Updates middleware token generation and user loading to align with the new value-based `domain.User` handling. --- api/internal/server/handler/chapters.go | 27 +++- api/internal/server/handler/novels.go | 3 +- .../server/middleware/token/generate.go | 6 +- .../server/middleware/token/load_user.go | 7 -- api/internal/service/chapters.go | 83 ++++--------- api/internal/service/novels.go | 116 +++++------------- api/internal/service/user.go | 88 ++++--------- 7 files changed, 104 insertions(+), 226 deletions(-) diff --git a/api/internal/server/handler/chapters.go b/api/internal/server/handler/chapters.go index 85d83c0..7747eee 100644 --- a/api/internal/server/handler/chapters.go +++ b/api/internal/server/handler/chapters.go @@ -119,7 +119,30 @@ func FindAllChapters(c *gin.Context) { return } - chapters, err := service.GetAllChapters(novelId, ctx) + var err error + + pageSize := 100 + p, exists := c.GetQuery("size") + if exists { + if pageSize, err = strconv.Atoi(p); err != nil { + c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{ + "error": "Invalid page size", + }) + return + } + } + + ascending := false + asc, exists := c.GetQuery("ascending") + if exists { + if asc == "true" { + ascending = true + } else if asc == "false" { + ascending = false + } + } + + chapters, err := service.GetAllChapters(novelId, pageSize, ascending, ctx) if e, ok := err.(*cmn.Error); ok { c.AbortWithStatusJSON(e.StatusCode(), gin.H{ "error": "Failed to retrieve chapters: " + e.Error(), @@ -185,7 +208,7 @@ func UpdateChapter(c *gin.Context) { return } - err := service.UpdateChapter(novelId, &chapter, ctx) + err := service.UpdateChapter(novelId, chapter, ctx) if e, ok := err.(*cmn.Error); ok { c.AbortWithStatusJSON(e.StatusCode(), gin.H{ "error": "Failed to update chapter: " + e.Error(), diff --git a/api/internal/server/handler/novels.go b/api/internal/server/handler/novels.go index 5532b77..f3b5699 100644 --- a/api/internal/server/handler/novels.go +++ b/api/internal/server/handler/novels.go @@ -156,7 +156,7 @@ func CreateNovel(c *gin.Context) { return } - err, id := service.CreateNovel(novel, ctx) + err := service.CreateNovel(novel, ctx) if e, ok := err.(*cmn.Error); ok { c.AbortWithStatusJSON(e.StatusCode(), gin.H{ "error": "Failed to create novel: " + e.Error(), @@ -171,7 +171,6 @@ func CreateNovel(c *gin.Context) { c.JSON(http.StatusOK, gin.H{ "message": "Novel created successfully", - "id": id, }) } diff --git a/api/internal/server/middleware/token/generate.go b/api/internal/server/middleware/token/generate.go index 872407d..573764e 100644 --- a/api/internal/server/middleware/token/generate.go +++ b/api/internal/server/middleware/token/generate.go @@ -11,7 +11,7 @@ import ( "github.com/golang-jwt/jwt/v5" ) -func GenerateTokenPair(user *domain.User, config domain.TokenConfig) (*domain.TokenPair, error) { +func GenerateTokenPair(user domain.User, config domain.TokenConfig) (*domain.TokenPair, error) { if user.ID == "" { return nil, &cmn.Error{Err: errors.New("user ID cannot be empty")} } @@ -43,12 +43,12 @@ func GenerateTokenPair(user *domain.User, config domain.TokenConfig) (*domain.To } // GenerateAccessToken creates a new access token (for refresh scenarios) -func GenerateAccessToken(user *domain.User, config domain.TokenConfig) (string, time.Time, error) { +func GenerateAccessToken(user domain.User, config domain.TokenConfig) (string, time.Time, error) { return generateAccessToken(user, config) } // generateAccessToken creates the actual access token -func generateAccessToken(user *domain.User, config domain.TokenConfig) (string, time.Time, error) { +func generateAccessToken(user domain.User, config domain.TokenConfig) (string, time.Time, error) { now := time.Now() expirationTime := now.Add(config.AccessTTL) diff --git a/api/internal/server/middleware/token/load_user.go b/api/internal/server/middleware/token/load_user.go index ca42145..c04fef2 100644 --- a/api/internal/server/middleware/token/load_user.go +++ b/api/internal/server/middleware/token/load_user.go @@ -53,13 +53,6 @@ func LookupUser(config domain.LookupUser) gin.HandlerFunc { return } - if user == nil { - c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{ - "error": "User not found", - }) - return - } - // Cache user if cache is available if config.Cache != nil { config.Cache.Set(cacheKey, user, config.CacheDuration) diff --git a/api/internal/service/chapters.go b/api/internal/service/chapters.go index 7a77d79..acc6084 100644 --- a/api/internal/service/chapters.go +++ b/api/internal/service/chapters.go @@ -2,29 +2,24 @@ package service import ( cmn "Codex-Backend/api/common" - firestore_client "Codex-Backend/api/internal/database/client" - firestore_collections "Codex-Backend/api/internal/database/collections" + db "Codex-Backend/api/internal/database" "Codex-Backend/api/internal/domain" "context" "errors" "net/http" - "time" ) func GetCursorPaginatedChapters(options domain.CursorOptions, ctx context.Context) (*domain.CursorResponse, error) { - client, err := firestore_client.FirestoreClient() + client, err := db.GetClient(ctx) if err != nil { return nil, err } - defer client.Close() - - c := firestore_collections.Client{Client: client} if options.Limit > 100 || options.Limit <= 0 { options.Limit = 100 } - response, err := c.CursorPagination(options, ctx) + response, err := client.CursorPagination(options, ctx) if err != nil { return nil, err } @@ -33,13 +28,10 @@ func GetCursorPaginatedChapters(options domain.CursorOptions, ctx context.Contex } func BatchUploadChapters(novelId string, chapters []domain.Chapter, ctx context.Context) error { - client, err := firestore_client.FirestoreClient() + client, err := db.GetClient(ctx) if err != nil { return err } - defer client.Close() - - c := firestore_collections.Client{Client: client} if len(chapters) == 0 { return &cmn.Error{ @@ -48,8 +40,7 @@ func BatchUploadChapters(novelId string, chapters []domain.Chapter, ctx context. } } - err = c.BatchUploadChapters(novelId, chapters, ctx) - if err != nil { + if err = client.BatchUploadChapters(novelId, chapters, ctx); err != nil { return err } @@ -57,85 +48,57 @@ func BatchUploadChapters(novelId string, chapters []domain.Chapter, ctx context. } func CreateChapter(novelId string, chapter domain.Chapter, ctx context.Context) error { - client, err := firestore_client.FirestoreClient() + client, err := db.GetClient(ctx) if err != nil { return err } - defer client.Close() - c := firestore_collections.Client{Client: client} - - id, err := cmn.GenerateID("chapter") - if err != nil { - return err - } - - chapter.ID = id - chapter.CreatedAt = time.Now().Format("2006-01-02 15:04:05") - chapter.UpdatedAt = time.Now().Format("2006-01-02 15:04:05") - chapter.Deleted = false - - err = c.CreateChapter(novelId, chapter, ctx) - if err != nil { + if err = client.CreateChapter(novelId, chapter, ctx); err != nil { return err } return nil } -func GetChapter(novelId, chapterId string, ctx context.Context) (*domain.Chapter, error) { - client, err := firestore_client.FirestoreClient() +func GetChapter(novelId, chapterId string, ctx context.Context) (domain.Chapter, error) { + client, err := db.GetClient(ctx) if err != nil { - return nil, err + return domain.Chapter{}, err } - defer client.Close() - c := firestore_collections.Client{Client: client} - - chapter, err := c.GetChapterById(novelId, chapterId, ctx) + chapter, err := client.GetChapterById(novelId, chapterId, ctx) if err != nil { - return nil, err - } - - if chapter == nil { - return nil, &cmn.Error{Err: errors.New("Chapter Service Error - Get Chapter - Chapter With ID " + chapterId + " In Novel With ID " + novelId + " Not Found"), Status: http.StatusNotFound} + return domain.Chapter{}, err } return chapter, nil } -func GetAllChapters(novelId string, ctx context.Context) (*[]domain.Chapter, error) { - client, err := firestore_client.FirestoreClient() +func GetAllChapters(novelId string, pageSize int, asc bool, ctx context.Context) ([]domain.Chapter, error) { + client, err := db.GetClient(ctx) if err != nil { return nil, err } - defer client.Close() - - c := firestore_collections.Client{Client: client} - chapters, err := c.GetAllChapters(novelId, ctx) + chapters, err := client.GetAllChapters(novelId, pageSize, asc, ctx) if err != nil { return nil, err } - if len(*chapters) == 0 { + if len(chapters) == 0 { return nil, &cmn.Error{Err: errors.New("Chapter Service Error - Get All Chapters - Chapters In Novel With ID " + novelId + " Not Found"), Status: http.StatusNotFound} } return chapters, nil } -func UpdateChapter(novelId string, chapter *domain.Chapter, ctx context.Context) error { - client, err := firestore_client.FirestoreClient() +func UpdateChapter(novelId string, chapter domain.Chapter, ctx context.Context) error { + client, err := db.GetClient(ctx) if err != nil { return err } - defer client.Close() - - c := firestore_collections.Client{Client: client} - err = c.UpdateChapter(novelId, *chapter, ctx) - if err != nil { + if err = client.UpdateChapter(novelId, chapter, ctx); err != nil { return err } @@ -143,16 +106,12 @@ func UpdateChapter(novelId string, chapter *domain.Chapter, ctx context.Context) } func DeleteChapter(novelId, chapterId string, ctx context.Context) error { - client, err := firestore_client.FirestoreClient() + client, err := db.GetClient(ctx) if err != nil { return err } - defer client.Close() - c := firestore_collections.Client{Client: client} - - err = c.DeleteChapter(novelId, chapterId, ctx) - if err != nil { + if err = client.DeleteChapter(chapterId, ctx); err != nil { return err } diff --git a/api/internal/service/novels.go b/api/internal/service/novels.go index f6bc3b9..e1d186a 100644 --- a/api/internal/service/novels.go +++ b/api/internal/service/novels.go @@ -2,8 +2,7 @@ package service import ( cmn "Codex-Backend/api/common" - firestore_client "Codex-Backend/api/internal/database/client" - firestore_collections "Codex-Backend/api/internal/database/collections" + db "Codex-Backend/api/internal/database" "Codex-Backend/api/internal/domain" "context" "errors" @@ -44,27 +43,24 @@ func CreateNovelFromEPUB(data []byte, ctx context.Context) error { // Create Novel - id, err := cmn.GenerateID("novel") + description, err := cleanHtml(book.Description) if err != nil { return err } - description, err := cleanHtml(book.Description) + createdAt, err := time.Parse(time.RFC3339, book.Date) if err != nil { return err } - novel := &domain.Novel{ - ID: id, + novel := domain.Novel{ Title: book.Title, Author: book.Author, Description: description, - CreatedAt: cmn.TimeStamp(book.Date), - UpdatedAt: cmn.TimeStamp(""), - Deleted: false, + CreatedAt: createdAt, } - err, id = CreateNovel(*novel, ctx) + err = CreateNovel(novel, ctx) if err != nil { return err } @@ -108,11 +104,6 @@ func CreateNovelFromEPUB(data []byte, ctx context.Context) error { } func processChap(chapter pamphlet.Chapter, index int, author string) (*domain.Chapter, error) { - c_id, err := cmn.GenerateID("chapter") - if err != nil { - return nil, err - } - rawContent, err := chapter.GetContent() if err != nil { return nil, err @@ -132,102 +123,66 @@ func processChap(chapter pamphlet.Chapter, index int, author string) (*domain.Ch } return &domain.Chapter{ - ID: c_id, Title: chapter.Title, Author: author, Description: "", - CreatedAt: cmn.TimeStamp(""), - UpdatedAt: cmn.TimeStamp(""), Content: content, - Index: index, - Deleted: false, }, nil } -func CreateNovel(novel domain.Novel, ctx context.Context) (error, string) { - client, err := firestore_client.FirestoreClient() - if err != nil { - return err, "" - } - defer client.Close() - - c := firestore_collections.Client{Client: client} - - id, err := cmn.GenerateID("novel") +func CreateNovel(novel domain.Novel, ctx context.Context) error { + client, err := db.GetClient(ctx) if err != nil { - return err, "" + return err } - novel.ID = id - novel.CreatedAt = time.Now().Format("2006-01-02 15:04:05") - novel.UpdatedAt = time.Now().Format("2006-01-02 15:04:05") - novel.Deleted = false - - err = c.CreateNovel(novel, ctx) - if err != nil { - return err, "" + if err = client.CreateNovel(novel, ctx); err != nil { + return err } - return nil, id + return nil } -func GetNovelById(id string, ctx context.Context) (*domain.Novel, error) { - client, err := firestore_client.FirestoreClient() +func GetNovelById(id string, ctx context.Context) (domain.Novel, error) { + client, err := db.GetClient(ctx) if err != nil { - return nil, err + return domain.Novel{}, err } - defer client.Close() - c := firestore_collections.Client{Client: client} - - novel, err := c.GetNovelById(id, ctx) + novel, err := client.GetNovelById(id, ctx) if err != nil { - return nil, err - } - - if novel == nil { - return nil, &cmn.Error{Err: errors.New("Novel Service Error - Get Novel - Novel with ID " + id + " not found"), Status: http.StatusNotFound} + return domain.Novel{}, err } return novel, nil } -func GetNovelByTitle(title string, ctx context.Context) (*domain.Novel, error) { - client, err := firestore_client.FirestoreClient() +func GetNovelByTitle(title string, ctx context.Context) (domain.Novel, error) { + client, err := db.GetClient(ctx) if err != nil { - return nil, err + return domain.Novel{}, err } - defer client.Close() - - c := firestore_collections.Client{Client: client} - novel, err := c.GetNovelByTitle(title, ctx) + novel, err := client.GetNovelByTitle(title, ctx) if err != nil { - return nil, err - } - - if novel == nil { - return nil, &cmn.Error{Err: errors.New("Novel Service Error - Get Novel - Novel with title " + title + " not found"), Status: http.StatusNotFound} + return domain.Novel{}, err } return novel, nil } -func GetAllNovels(ctx context.Context) (*[]domain.Novel, error) { - client, err := firestore_client.FirestoreClient() +func GetAllNovels(ctx context.Context) ([]domain.Novel, error) { + client, err := db.GetClient(ctx) if err != nil { return nil, err } - defer client.Close() - - c := firestore_collections.Client{Client: client} - novels, err := c.GetAllNovels(ctx) + novels, err := client.GetAllNovels(ctx) if err != nil { return nil, err } - if len(*novels) == 0 { + if len(novels) == 0 { return nil, &cmn.Error{Err: errors.New("Novel Service Error - Get All Novels - No novels found"), Status: http.StatusNotFound} } @@ -235,19 +190,12 @@ func GetAllNovels(ctx context.Context) (*[]domain.Novel, error) { } func UpdateNovel(id string, novel domain.Novel, ctx context.Context) error { - client, err := firestore_client.FirestoreClient() + client, err := db.GetClient(ctx) if err != nil { return err } - defer client.Close() - - c := firestore_collections.Client{Client: client} - - novel.ID = id - novel.UpdatedAt = time.Now().Format("2006-01-02 15:04:05") - err = c.UpdateNovel(novel, ctx) - if err != nil { + if err = client.UpdateNovel(novel, ctx); err != nil { return err } @@ -255,16 +203,12 @@ func UpdateNovel(id string, novel domain.Novel, ctx context.Context) error { } func DeleteNovel(id string, ctx context.Context) error { - client, err := firestore_client.FirestoreClient() + client, err := db.GetClient(ctx) if err != nil { return err } - defer client.Close() - - c := firestore_collections.Client{Client: client} - err = c.DeleteNovel(id, ctx) - if err != nil { + if err = client.DeleteNovel(id, ctx); err != nil { return err } diff --git a/api/internal/service/user.go b/api/internal/service/user.go index 9c561f1..0482342 100644 --- a/api/internal/service/user.go +++ b/api/internal/service/user.go @@ -2,102 +2,70 @@ package service import ( cmn "Codex-Backend/api/common" - firestore_client "Codex-Backend/api/internal/database/client" - firestore_collections "Codex-Backend/api/internal/database/collections" + db "Codex-Backend/api/internal/database" "Codex-Backend/api/internal/domain" "context" "errors" "net/http" - "time" ) -func LoginUser(credentials domain.Credentials, ctx context.Context) (*domain.User, error) { - client, err := firestore_client.FirestoreClient() +func LoginUser(credentials domain.Credentials, ctx context.Context) (domain.User, error) { + client, err := db.GetClient(ctx) if err != nil { - return nil, err + return domain.User{}, err } - defer client.Close() - c := firestore_collections.Client{Client: client} - - user, err := c.GetUserByEmail(credentials.Email, ctx) + user, err := client.GetUserByEmail(credentials.Email, ctx) if err != nil { - return nil, err - } - - if user == nil { - return nil, &cmn.Error{Err: errors.New("Login Service Error - User not found"), Status: http.StatusNotFound} + return domain.User{}, err } err = cmn.VerifyPassword(user.Password, credentials.Password) if err != nil { - return nil, &cmn.Error{Err: errors.New("Login Service Error - Invalid password"), Status: http.StatusUnauthorized} + return domain.User{}, &cmn.Error{Err: errors.New("Login Service Error - Invalid password"), Status: http.StatusUnauthorized} } return user, nil } -func GetUserByID(id string, ctx context.Context) (*domain.User, error) { - client, err := firestore_client.FirestoreClient() +func GetUserByID(id string, ctx context.Context) (domain.User, error) { + client, err := db.GetClient(ctx) if err != nil { - return nil, err + return domain.User{}, err } - defer client.Close() - c := firestore_collections.Client{Client: client} - - user, err := c.GetUserById(id, ctx) + user, err := client.GetUserById(id, ctx) if err != nil { - return nil, err - } - - if user == nil { - return nil, &cmn.Error{Err: errors.New("Get User By ID Service Error - User not found"), Status: http.StatusNotFound} + return domain.User{}, err } return user, nil } func RegisterUser(newUser domain.NewUser, ctx context.Context) error { - client, err := firestore_client.FirestoreClient() + client, err := db.GetClient(ctx) if err != nil { return err } - defer client.Close() - - c := firestore_collections.Client{Client: client} - user, err := c.GetUserByEmail(newUser.Email, ctx) + _, err = client.GetUserByEmail(newUser.Email, ctx) if e, ok := err.(*cmn.Error); ok { if e.StatusCode() != http.StatusNotFound { return &cmn.Error{Err: errors.New("Register Service Error - Getting User By Email: " + err.Error()), Status: http.StatusInternalServerError} } } - if user != nil { - return &cmn.Error{Err: errors.New("Register Service Error - User With Email " + newUser.Email + " Already Exists"), Status: http.StatusConflict} - } - - id, err := cmn.GenerateID("user") - if err != nil { - return err - } - hashedPassword, err := cmn.HashPassword(newUser.Password) if err != nil { return err } - err = c.CreateUser(domain.User{ - ID: id, - Username: newUser.Username, - Password: string(hashedPassword), - Email: newUser.Email, - Type: "User", - CreatedAt: time.Now().Format("2006-01-02 15:04:05"), - UpdatedAt: time.Now().Format("2006-01-02 15:04:05"), - }, ctx) - if err != nil { + if err = client.CreateUser(domain.User{ + Username: newUser.Username, + Password: string(hashedPassword), + Email: newUser.Email, + Type: "User", + }, ctx); err != nil { return err } @@ -113,16 +81,12 @@ func LogoutUser(tokenString string) error { } func UpdateUser(updatedUser domain.User, ctx context.Context) error { - client, err := firestore_client.FirestoreClient() + client, err := db.GetClient(ctx) if err != nil { return err } - defer client.Close() - c := firestore_collections.Client{Client: client} - - err = c.UpdateUser(updatedUser, ctx) - if err != nil { + if err = client.UpdateUser(updatedUser, ctx); err != nil { return err } @@ -130,16 +94,12 @@ func UpdateUser(updatedUser domain.User, ctx context.Context) error { } func DeleteUser(id string, ctx context.Context) error { - client, err := firestore_client.FirestoreClient() + client, err := db.GetClient(ctx) if err != nil { return err } - defer client.Close() - c := firestore_collections.Client{Client: client} - - err = c.DeleteUser(id, ctx) - if err != nil { + if err = client.DeleteUser(id, ctx); err != nil { return err } From dfa72dc735e06a64ded382edd203f0dd53840bed Mon Sep 17 00:00:00 2001 From: Aleksandre Date: Wed, 24 Sep 2025 15:08:46 +0400 Subject: [PATCH 28/49] feat: Remove Firestore dependencies --- api/internal/domain/chapter.go | 14 +++--- api/internal/server/handler/chapters.go | 1 - go.mod | 29 ++--------- go.sum | 64 ++----------------------- 4 files changed, 12 insertions(+), 96 deletions(-) diff --git a/api/internal/domain/chapter.go b/api/internal/domain/chapter.go index 72b6b5f..0a17b34 100644 --- a/api/internal/domain/chapter.go +++ b/api/internal/domain/chapter.go @@ -2,20 +2,18 @@ package domain import ( "time" - - "cloud.google.com/go/firestore" ) type CursorOptions struct { - NovelID string `json:"novel_id"` - Cursor int `json:"cursor"` - Limit int `json:"limit"` - SortBy firestore.Direction `json:"sort_by"` + NovelID string `json:"novel_id"` + Cursor string `json:"cursor"` + Limit int `json:"limit"` + Ascending bool `json:"sort_by"` } type CursorResponse struct { - Chapters []FrontendChapter `json:"chapters"` - NextCursor int `json:"next_cursor"` + Chapters []Chapter `json:"chapters"` + NextCursor string `json:"next_cursor"` } // Chapter struct used on backend diff --git a/api/internal/server/handler/chapters.go b/api/internal/server/handler/chapters.go index 7747eee..0fe1e39 100644 --- a/api/internal/server/handler/chapters.go +++ b/api/internal/server/handler/chapters.go @@ -7,7 +7,6 @@ import ( "net/http" "strconv" - "cloud.google.com/go/firestore" "github.com/gin-gonic/gin" ) diff --git a/go.mod b/go.mod index f2aec48..ac9516c 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,6 @@ module Codex-Backend go 1.25.0 require ( - cloud.google.com/go/firestore v1.18.0 github.com/JohannesKaufmann/html-to-markdown/v2 v2.3.3 github.com/PuerkitoBio/goquery v1.10.3 github.com/gin-contrib/cors v1.7.3 @@ -20,34 +19,25 @@ require ( ) require ( - cloud.google.com/go v0.117.0 // indirect - cloud.google.com/go/auth v0.13.0 // indirect - cloud.google.com/go/auth/oauth2adapt v0.2.6 // indirect - cloud.google.com/go/compute/metadata v0.6.0 // indirect - cloud.google.com/go/longrunning v0.6.2 // indirect github.com/JohannesKaufmann/dom v0.2.0 // indirect github.com/andybalholm/cascadia v1.3.3 // indirect github.com/bytedance/sonic v1.12.9 // indirect github.com/bytedance/sonic/loader v0.2.3 // indirect github.com/cloudwego/base64x v0.1.5 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/felixge/httpsnoop v1.0.4 // indirect github.com/gabriel-vasile/mimetype v1.4.8 // indirect github.com/gin-contrib/sse v1.0.0 // indirect - github.com/go-logr/logr v1.4.2 // indirect - github.com/go-logr/stdr v1.2.2 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.25.0 // indirect github.com/goccy/go-json v0.10.5 // indirect - github.com/google/s2a-go v0.1.8 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect - github.com/googleapis/gax-go/v2 v2.14.0 // indirect + github.com/google/go-cmp v0.7.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/cpuid/v2 v2.2.10 // indirect + github.com/kr/pretty v0.3.1 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect @@ -57,6 +47,7 @@ require ( github.com/riverqueue/river/riverdriver v0.24.0 // indirect github.com/riverqueue/river/rivershared v0.24.0 // indirect github.com/riverqueue/river/rivertype v0.24.0 // indirect + github.com/rogpeppe/go-internal v1.13.1 // indirect github.com/stretchr/testify v1.10.0 // indirect github.com/tidwall/gjson v1.18.0 // indirect github.com/tidwall/match v1.1.1 // indirect @@ -64,26 +55,12 @@ require ( github.com/tidwall/sjson v1.2.5 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.12 // indirect - go.opentelemetry.io/auto/sdk v1.1.0 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 // indirect - go.opentelemetry.io/otel v1.36.0 // indirect - go.opentelemetry.io/otel/metric v1.36.0 // indirect - go.opentelemetry.io/otel/sdk v1.36.0 // indirect - go.opentelemetry.io/otel/sdk/metric v1.36.0 // indirect - go.opentelemetry.io/otel/trace v1.36.0 // indirect go.uber.org/goleak v1.3.0 // indirect golang.org/x/arch v0.14.0 // indirect golang.org/x/net v0.43.0 // indirect - golang.org/x/oauth2 v0.28.0 // indirect golang.org/x/sync v0.16.0 // indirect golang.org/x/sys v0.35.0 // indirect golang.org/x/text v0.28.0 // indirect - google.golang.org/api v0.214.0 // indirect - google.golang.org/genproto v0.0.0-20241118233622-e639e219e697 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20250324211829-b45e905df463 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20250324211829-b45e905df463 // indirect - google.golang.org/grpc v1.73.0 // indirect google.golang.org/protobuf v1.36.6 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 4c1e7b2..ce7977b 100644 --- a/go.sum +++ b/go.sum @@ -1,15 +1,3 @@ -cloud.google.com/go v0.117.0 h1:Z5TNFfQxj7WG2FgOGX1ekC5RiXrYgms6QscOm32M/4s= -cloud.google.com/go v0.117.0/go.mod h1:ZbwhVTb1DBGt2Iwb3tNO6SEK4q+cplHZmLWH+DelYYc= -cloud.google.com/go/auth v0.13.0 h1:8Fu8TZy167JkW8Tj3q7dIkr2v4cndv41ouecJx0PAHs= -cloud.google.com/go/auth v0.13.0/go.mod h1:COOjD9gwfKNKz+IIduatIhYJQIc0mG3H102r/EMxX6Q= -cloud.google.com/go/auth/oauth2adapt v0.2.6 h1:V6a6XDu2lTwPZWOawrAa9HUK+DB2zfJyTuciBG5hFkU= -cloud.google.com/go/auth/oauth2adapt v0.2.6/go.mod h1:AlmsELtlEBnaNTL7jCj8VQFLy6mbZv0s4Q7NGBeQ5E8= -cloud.google.com/go/compute/metadata v0.6.0 h1:A6hENjEsCDtC1k8byVsgwvVcioamEHvZ4j01OwKxG9I= -cloud.google.com/go/compute/metadata v0.6.0/go.mod h1:FjyFAW1MW0C203CEOMDTu3Dk1FlqW3Rga40jzHL4hfg= -cloud.google.com/go/firestore v1.18.0 h1:cuydCaLS7Vl2SatAeivXyhbhDEIR8BDmtn4egDhIn2s= -cloud.google.com/go/firestore v1.18.0/go.mod h1:5ye0v48PhseZBdcl0qbl3uttu7FIEwEYVaWm0UIEOEU= -cloud.google.com/go/longrunning v0.6.2 h1:xjDfh1pQcWPEvnfjZmwjKQEcHnpz6lHjfy7Fo0MK+hc= -cloud.google.com/go/longrunning v0.6.2/go.mod h1:k/vIs83RN4bE3YCswdXC5PFfWVILjm3hpEUlSko4PiI= github.com/JohannesKaufmann/dom v0.2.0 h1:1bragmEb19K8lHAqgFgqCpiPCFEZMTXzOIEjuxkUfLQ= github.com/JohannesKaufmann/dom v0.2.0/go.mod h1:57iSUl5RKric4bUkgos4zu6Xt5LMHUnw3TF1l5CbGZo= github.com/JohannesKaufmann/html-to-markdown/v2 v2.3.3 h1:r3fokGFRDk/8pHmwLwJ8zsX4qiqfS1/1TZm2BH8ueY8= @@ -26,11 +14,10 @@ github.com/bytedance/sonic/loader v0.2.3/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFos github.com/cloudwego/base64x v0.1.5 h1:XPciSp1xaq2VCSt6lF0phncD4koWyULpl5bUxbfCyP4= github.com/cloudwego/base64x v0.1.5/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= -github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/gabriel-vasile/mimetype v1.4.8 h1:FfZ3gj38NjllZIeJAmMhr+qKL8Wu+nOoI3GqacKw1NM= github.com/gabriel-vasile/mimetype v1.4.8/go.mod h1:ByKUIKGjh1ODkGM1asKUbQZOLGrPjydw3hYPU2YU9t8= github.com/gin-contrib/cors v1.7.3 h1:hV+a5xp8hwJoTw7OY+a70FsL8JkVVFTXw9EcfrYUdns= @@ -39,11 +26,6 @@ github.com/gin-contrib/sse v1.0.0 h1:y3bT1mUWUxDpW4JLQg/HnTqV4rozuW4tC9eFKTxYI9E github.com/gin-contrib/sse v1.0.0/go.mod h1:zNuFdwarAygJBht0NTKiSi3jRf6RbqeILZ9Sp6Slhe0= github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU= github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= -github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= -github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= -github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= -github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= @@ -56,22 +38,10 @@ github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8= github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= -github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/s2a-go v0.1.8 h1:zZDs9gcbt9ZPLV0ndSyQk6Kacx2g/X+SKYovpnz3SMM= -github.com/google/s2a-go v0.1.8/go.mod h1:6iNWHTpQ+nfNRN5E00MSdfDwVesa8hhS32PhPO8deJA= -github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= -github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.3.4 h1:XYIDZApgAnrN1c855gTgghdIA6Stxb52D5RnLI1SLyw= -github.com/googleapis/enterprise-certificate-proxy v0.3.4/go.mod h1:YKe7cfqYXjKGpGvmSg28/fFvhNzinZQm8DGnaburhGA= -github.com/googleapis/gax-go/v2 v2.14.0 h1:f+jMrjBPl+DL9nI4IQzLUxMq7XrAqFYB7hBPqMNIe8o= -github.com/googleapis/gax-go/v2 v2.14.0/go.mod h1:lhBCnjdLrWRaPvLWhmc8IS24m9mr07qSYnHncrgo+zk= github.com/jackc/pgerrcode v0.0.0-20240316143900-6e2875d9b438 h1:Dj0L5fhJ9F82ZJyVOmBx6msDp/kfd1t9GRfny/mfJA0= github.com/jackc/pgerrcode v0.0.0-20240316143900-6e2875d9b438/go.mod h1:a/s9Lp5W7n/DD0VrVoyJ00FbP2ytTPDVOivvn2bMlds= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= @@ -108,6 +78,7 @@ github.com/oklog/ulid/v2 v2.1.1/go.mod h1:rcEKHmBBKfef9DhnvX7y1HZBYxjXb0cP5ExxNs github.com/pborman/getopt v0.0.0-20170112200414-7148bc3a4c30/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o= github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/riverqueue/river v0.24.0 h1:CesL6vymWgz0d+zNwtnSGRWaB+E8Dax+o9cxD7sUmKc= @@ -122,6 +93,7 @@ github.com/riverqueue/river/rivertype v0.24.0 h1:xrQZm/h6U8TBPyTsQPYD5leOapuoBAc github.com/riverqueue/river/rivertype v0.24.0/go.mod h1:lmdl3vLNDfchDWbYdW2uAocIuwIN+ZaXqAukdSCFqWs= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/sebdah/goldie/v2 v2.5.5 h1:rx1mwF95RxZ3/83sdS4Yp7t2C5TCokvWP4TBRbAyEWY= @@ -159,24 +131,6 @@ github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZ github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yuin/goldmark v1.7.11 h1:ZCxLyDMtz0nT2HFfsYG8WZ47Trip2+JyLysKcMYE5bo= github.com/yuin/goldmark v1.7.11/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg= -go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= -go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= -go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= -go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 h1:r6I7RJCN86bpD/FQwedZ0vSixDpwuWREjW9oRMsmqDc= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0/go.mod h1:B9yO6b04uB80CzjedvewuqDhxJxi11s7/GtiGa8bAjI= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 h1:TT4fX+nBOA/+LUkobKGW1ydGcn+G3vRw9+g5HwCphpk= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0/go.mod h1:L7UH0GbB0p47T4Rri3uHjbpCFYrVrwc1I25QhNPiGK8= -go.opentelemetry.io/otel v1.36.0 h1:UumtzIklRBY6cI/lllNZlALOF5nNIzJVb16APdvgTXg= -go.opentelemetry.io/otel v1.36.0/go.mod h1:/TcFMXYjyRNh8khOAO9ybYkqaDBb/70aVwkNML4pP8E= -go.opentelemetry.io/otel/metric v1.36.0 h1:MoWPKVhQvJ+eeXWHFBOPoBOi20jh6Iq2CcCREuTYufE= -go.opentelemetry.io/otel/metric v1.36.0/go.mod h1:zC7Ks+yeyJt4xig9DEw9kuUFe5C3zLbVjV2PzT6qzbs= -go.opentelemetry.io/otel/sdk v1.36.0 h1:b6SYIuLRs88ztox4EyrvRti80uXIFy+Sqzoh9kFULbs= -go.opentelemetry.io/otel/sdk v1.36.0/go.mod h1:+lC+mTgD+MUWfjJubi2vvXWcVxyr9rmlshZni72pXeY= -go.opentelemetry.io/otel/sdk/metric v1.36.0 h1:r0ntwwGosWGaa0CrSt8cuNuTcccMXERFwHX4dThiPis= -go.opentelemetry.io/otel/sdk/metric v1.36.0/go.mod h1:qTNOhFDfKRwX0yXOqJYegL5WRaW376QbB7P4Pb0qva4= -go.opentelemetry.io/otel/trace v1.36.0 h1:ahxWNuqZjpdiFAyrIoQ4GIiAIhxAunQR6MUoKrsNd4w= -go.opentelemetry.io/otel/trace v1.36.0/go.mod h1:gQ+OnDZzrybY4k4seLzPAWNwVBBVlF2szhehOBB/tGA= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= golang.org/x/arch v0.14.0 h1:z9JUEZWr8x4rR0OU6c4/4t6E6jOZ8/QBS2bBYBm4tx4= @@ -205,8 +159,6 @@ golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= -golang.org/x/oauth2 v0.28.0 h1:CrgCKl8PPAVtLnU3c+EDw6x11699EWlsDeWNWKdIOkc= -golang.org/x/oauth2 v0.28.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -259,16 +211,6 @@ golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/api v0.214.0 h1:h2Gkq07OYi6kusGOaT/9rnNljuXmqPnaig7WGPmKbwA= -google.golang.org/api v0.214.0/go.mod h1:bYPpLG8AyeMWwDU6NXoB00xC0DFkikVvd5MfwoxjLqE= -google.golang.org/genproto v0.0.0-20241118233622-e639e219e697 h1:ToEetK57OidYuqD4Q5w+vfEnPvPpuTwedCNVohYJfNk= -google.golang.org/genproto v0.0.0-20241118233622-e639e219e697/go.mod h1:JJrvXBWRZaFMxBufik1a4RpFw4HhgVtBBWQeQgUj2cc= -google.golang.org/genproto/googleapis/api v0.0.0-20250324211829-b45e905df463 h1:hE3bRWtU6uceqlh4fhrSnUyjKHMKB9KrTLLG+bc0ddM= -google.golang.org/genproto/googleapis/api v0.0.0-20250324211829-b45e905df463/go.mod h1:U90ffi8eUL9MwPcrJylN5+Mk2v3vuPDptd5yyNUiRR8= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250324211829-b45e905df463 h1:e0AIkUUhxyBKh6ssZNrAMeqhA7RKUj42346d1y02i2g= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250324211829-b45e905df463/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= -google.golang.org/grpc v1.73.0 h1:VIWSmpI2MegBtTuFt5/JWy2oXxtjJ/e89Z70ImfD2ok= -google.golang.org/grpc v1.73.0/go.mod h1:50sbHOUqWoCQGI8V2HQLJM0B+LMlIUjNSZmow7EVBQc= google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 0eff12114a9e3bf8e3174f81568d3c7fc6934bf3 Mon Sep 17 00:00:00 2001 From: Aleksandre Date: Wed, 24 Sep 2025 15:08:46 +0400 Subject: [PATCH 29/49] refactor(database): Update client import path --- api/cmd/web/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/cmd/web/main.go b/api/cmd/web/main.go index cfa32b2..0ef256a 100644 --- a/api/cmd/web/main.go +++ b/api/cmd/web/main.go @@ -2,7 +2,7 @@ package main import ( cmn "Codex-Backend/api/common" - db_client "Codex-Backend/api/internal/database/client" + db "Codex-Backend/api/internal/database" firestore_server "Codex-Backend/api/internal/server" "context" "fmt" @@ -17,7 +17,7 @@ func init() { gin.SetMode(mode) ctx := context.Background() - client, err := db_client.GetClient(ctx) + client, err := db.GetClient(ctx) if err != nil { panic(fmt.Sprintf("db new client: %v", err)) } From 00d848883f2057dc3e09e42332736b60df6ad889 Mon Sep 17 00:00:00 2001 From: Aleksandre Date: Wed, 24 Sep 2025 15:08:46 +0400 Subject: [PATCH 30/49] refactor(database): Enhance PostgreSQL partitioned table schema for chapters --- api/internal/database/client.go | 83 +++++++++++++++++++-------------- 1 file changed, 48 insertions(+), 35 deletions(-) diff --git a/api/internal/database/client.go b/api/internal/database/client.go index 5c453a9..e31686e 100644 --- a/api/internal/database/client.go +++ b/api/internal/database/client.go @@ -122,6 +122,8 @@ func (c *Client) EnsureSchema(ctx context.Context) error { return &cmn.Error{Err: errors.New("postgres client not initialized"), Status: http.StatusInternalServerError} } + // Note: CREATE EXTENSION sometimes requires superuser privileges. + // If you can't create it inside a transaction in your environment, run it separately. tx, err := c.Pool.Begin(ctx) if err != nil { return &cmn.Error{Err: fmt.Errorf("begin tx for schema: %w", err), Status: http.StatusInternalServerError} @@ -133,40 +135,41 @@ func (c *Client) EnsureSchema(ctx context.Context) error { `CREATE EXTENSION IF NOT EXISTS pgcrypto;`, `CREATE TABLE IF NOT EXISTS users ( - id uuid PRIMARY KEY DEFAULT gen_random_uuid(), - username text NOT NULL, - type text NOT NULL, - email text NOT NULL, - password text NOT NULL, - created_at timestamptz NOT NULL DEFAULT now(), - updated_at timestamptz NOT NULL DEFAULT now() - );`, - - // novels with chapter_count for atomic index allocation + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + username text NOT NULL, + type text NOT NULL, + email text NOT NULL, + password text NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now() + );`, + `CREATE TABLE IF NOT EXISTS novels ( - id uuid PRIMARY KEY DEFAULT gen_random_uuid(), - title text NOT NULL, - author text NOT NULL, - description text NOT NULL, - chapter_count bigint NOT NULL DEFAULT 0, - created_at timestamptz NOT NULL DEFAULT now(), - updated_at timestamptz NOT NULL DEFAULT now() - );`, - - // parent partitioned chapters table (hash partition on novel_id) + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + title text NOT NULL, + author text NOT NULL, + description text NOT NULL, + chapter_count bigint NOT NULL DEFAULT 0, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now() + );`, + + // IMPORTANT: Primary key must include partition key (novel_id) for partitioned tables. `CREATE TABLE IF NOT EXISTS chapters ( - id uuid PRIMARY KEY DEFAULT gen_random_uuid(), - novel_id uuid NOT NULL, - title text NOT NULL, - author text NOT NULL, - description text NOT NULL, - content text NOT NULL, - chapter_index bigint DEFAULT 0, - deleted boolean DEFAULT false, - created_at timestamptz NOT NULL DEFAULT now(), - updated_at timestamptz NOT NULL DEFAULT now() - ) PARTITION BY HASH (novel_id);`, - // an index that supports seek pagination: novel_id, chapter_index, id + id uuid NOT NULL DEFAULT gen_random_uuid(), + novel_id uuid NOT NULL, + title text NOT NULL, + author text NOT NULL, + description text NOT NULL, + content text NOT NULL, + chapter_index bigint DEFAULT 0, + deleted boolean DEFAULT false, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (novel_id, id) + ) PARTITION BY HASH (novel_id);`, + + // Index for seek/pagination that includes the partition key (novel_id) `CREATE INDEX IF NOT EXISTS idx_chapters_novel_index_id ON chapters (novel_id, chapter_index, id);`, `CREATE INDEX IF NOT EXISTS idx_novels_title_id ON novels (title, id);`, @@ -181,14 +184,24 @@ func (c *Client) EnsureSchema(ctx context.Context) error { // Create partitions (idempotent) const partitionsCount = 16 - for i := range partitionsCount { + for i := 0; i < partitionsCount; i++ { + partitionName := fmt.Sprintf("chapters_p%d", i) stmt := fmt.Sprintf( - `CREATE TABLE IF NOT EXISTS chapters_p%d PARTITION OF chapters FOR VALUES WITH (MODULUS %d, REMAINDER %d);`, - i, partitionsCount, i, + `CREATE TABLE IF NOT EXISTS %s PARTITION OF chapters + FOR VALUES WITH (MODULUS %d, REMAINDER %d);`, + partitionName, partitionsCount, i, ) if _, err := tx.Exec(ctx, stmt); err != nil { return &cmn.Error{Err: fmt.Errorf("creating partition %d: %w", i, err), Status: http.StatusInternalServerError} } + + idxStmt := fmt.Sprintf( + `CREATE UNIQUE INDEX IF NOT EXISTS idx_%s_id ON %s (id);`, + partitionName, partitionName, + ) + if _, err := tx.Exec(ctx, idxStmt); err != nil { + return &cmn.Error{Err: fmt.Errorf("creating id index for partition %d: %w", i, err), Status: http.StatusInternalServerError} + } } if err := tx.Commit(ctx); err != nil { From f3ed2f76c54064091211ec32bb9b5c0bc2bd5fc5 Mon Sep 17 00:00:00 2001 From: Aleksandre Date: Wed, 24 Sep 2025 15:08:46 +0400 Subject: [PATCH 31/49] feat(database): Add batch chapter upload functionality --- api/internal/database/chapters.go | 37 +++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/api/internal/database/chapters.go b/api/internal/database/chapters.go index 6adb0bb..0a045e6 100644 --- a/api/internal/database/chapters.go +++ b/api/internal/database/chapters.go @@ -131,6 +131,43 @@ func (c *Client) ListChaptersSeek(novelId string, limit int, cursor string, asc return results, nextCursor, nil } +func (c *Client) BatchUploadChapters(novelID string, chapters []domain.Chapter, ctx context.Context) error { + + chunkSize := 500 + + for i := 0; i < len(chapters); i += chunkSize { + end := min(i+chunkSize, len(chapters)) + chunk := chapters[i:end] + + if err := c.WithTx(ctx, func(tx pgx.Tx) error { + b := &pgx.Batch{} + insertSQL := `INSERT INTO chapters (novel_id, title, author, description, content, chapter_index, deleted) VALUES ($1,$2,$3,$4,$5,$6,$7)` + + for _, ch := range chunk { + b.Queue(insertSQL, novelID, ch.Title, ch.Author, ch.Description, ch.Content, ch.Index, ch.Deleted) + } + + br := tx.SendBatch(ctx, b) + defer br.Close() + + for range chunk { + if _, err := br.Exec(); err != nil { + return fmt.Errorf("batch exec: %w", err) + } + } + if _, err := tx.Exec(ctx, `UPDATE novels SET chapter_count = chapter_count + $1 WHERE id = $2`, len(chunk), novelID); err != nil { + return fmt.Errorf("update chapter_count: %w", err) + } + + return nil + }); err != nil { + return err + } + } + + return nil +} + func (c *Client) CreateChapter(novelId string, chapter domain.Chapter, ctx context.Context) error { var newIndex int64 From c54533f0f69468ba6f2d3b03ea664bfa3990b092 Mon Sep 17 00:00:00 2001 From: Aleksandre Date: Wed, 24 Sep 2025 15:08:46 +0400 Subject: [PATCH 32/49] feat(chapters): Migrate to PostgreSQL seek-based pagination --- api/internal/server/handler/chapters.go | 20 +++++++++----------- api/internal/service/chapters.go | 25 +++++-------------------- 2 files changed, 14 insertions(+), 31 deletions(-) diff --git a/api/internal/server/handler/chapters.go b/api/internal/server/handler/chapters.go index 0fe1e39..27695c0 100644 --- a/api/internal/server/handler/chapters.go +++ b/api/internal/server/handler/chapters.go @@ -23,17 +23,15 @@ func GetPaginatedChapters(c *gin.Context) { } options := domain.CursorOptions{ - NovelID: novelId, - Cursor: 0, - Limit: 100, - SortBy: firestore.Desc, + NovelID: novelId, + Cursor: "", + Limit: 100, + Ascending: false, } if cursor, exists := c.GetQuery("cursor"); exists { - curs, err := strconv.Atoi(cursor) - if err == nil { - options.Cursor = curs - } + options.Cursor = cursor + } if limit, exists := c.GetQuery("limit"); exists { @@ -46,11 +44,11 @@ func GetPaginatedChapters(c *gin.Context) { if sortBy, exists := c.GetQuery("sort"); exists { switch sortBy { case "asc": - options.SortBy = firestore.Asc + options.Ascending = true case "desc": - options.SortBy = firestore.Desc + options.Ascending = false default: - options.SortBy = firestore.Desc + options.Ascending = false } } diff --git a/api/internal/service/chapters.go b/api/internal/service/chapters.go index acc6084..6bdf1f0 100644 --- a/api/internal/service/chapters.go +++ b/api/internal/service/chapters.go @@ -19,32 +19,17 @@ func GetCursorPaginatedChapters(options domain.CursorOptions, ctx context.Contex options.Limit = 100 } - response, err := client.CursorPagination(options, ctx) + chapters, nextCursor, err := client.ListChaptersSeek(options.NovelID, options.Limit, options.Cursor, options.Ascending, ctx) if err != nil { return nil, err } - return response, nil -} - -func BatchUploadChapters(novelId string, chapters []domain.Chapter, ctx context.Context) error { - client, err := db.GetClient(ctx) - if err != nil { - return err - } - - if len(chapters) == 0 { - return &cmn.Error{ - Err: errors.New("Nothing to upload"), - Status: http.StatusInternalServerError, - } + response := &domain.CursorResponse{ + Chapters: chapters, + NextCursor: nextCursor, } - if err = client.BatchUploadChapters(novelId, chapters, ctx); err != nil { - return err - } - - return nil + return response, nil } func CreateChapter(novelId string, chapter domain.Chapter, ctx context.Context) error { From 13630651d667b7d452a3aee4b8e54a9d15b6ebb0 Mon Sep 17 00:00:00 2001 From: Aleksandre Date: Wed, 24 Sep 2025 15:08:46 +0400 Subject: [PATCH 33/49] feat(novels): Integrate batch chapter upload into EPUB processing --- api/internal/service/novels.go | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/api/internal/service/novels.go b/api/internal/service/novels.go index e1d186a..0cfd034 100644 --- a/api/internal/service/novels.go +++ b/api/internal/service/novels.go @@ -87,7 +87,7 @@ func CreateNovelFromEPUB(data []byte, ctx context.Context) error { chapters := make([]domain.Chapter, len(orderedChapters)) for i, chapter := range orderedChapters { - chap, err := processChap(chapter, i, book.Author) + chap, err := processChap(chapter, book.Author) if err != nil { return err } @@ -95,7 +95,21 @@ func CreateNovelFromEPUB(data []byte, ctx context.Context) error { chapters[i] = *chap } - err = BatchUploadChapters(id, chapters, ctx) + client, err := db.GetClient(ctx) + if err != nil { + return err + } + + novel, err = GetNovelByTitle(novel.Title, ctx) + if err != nil { + return err + } + + if novel.ID == "" { + return &cmn.Error{Err: errors.New("Novel Service Error - Create Novel From EPUB - Novel Not Created/Found"), Status: http.StatusNotFound} + } + + err = client.BatchUploadChapters(novel.ID, chapters, ctx) if err != nil { return err } @@ -103,7 +117,7 @@ func CreateNovelFromEPUB(data []byte, ctx context.Context) error { return nil } -func processChap(chapter pamphlet.Chapter, index int, author string) (*domain.Chapter, error) { +func processChap(chapter pamphlet.Chapter, author string) (*domain.Chapter, error) { rawContent, err := chapter.GetContent() if err != nil { return nil, err From 16e79e479548778265ec1c785f2b8c05040748ea Mon Sep 17 00:00:00 2001 From: Aleksandre Date: Wed, 24 Sep 2025 15:08:46 +0400 Subject: [PATCH 34/49] chore(go): Update for-loop idiom --- api/common/river/client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/common/river/client.go b/api/common/river/client.go index 0947feb..67ca0e6 100644 --- a/api/common/river/client.go +++ b/api/common/river/client.go @@ -53,7 +53,7 @@ func InitializeRiverClient(ctx context.Context, workers *river.Workers) *river.C // Retry connection with exponential backoff maxRetries := 10 - for i := 0; i < maxRetries; i++ { + for i := range maxRetries { dbPool, err = pgxpool.New(ctx, os.Getenv("DATABASE_URL")) if err == nil { // Test the connection From e9afc43757dd56c6c86cfdc34666581b881d9205 Mon Sep 17 00:00:00 2001 From: Aleksandre Date: Wed, 24 Sep 2025 15:08:46 +0400 Subject: [PATCH 35/49] chore(build): Remove .air.toml hot-reloading configuration --- .air.toml | 61 ------------------------------------------------------- 1 file changed, 61 deletions(-) delete mode 100644 .air.toml diff --git a/.air.toml b/.air.toml deleted file mode 100644 index 6a48cc0..0000000 --- a/.air.toml +++ /dev/null @@ -1,61 +0,0 @@ -root = "." -testdata_dir = "testdata" -tmp_dir = "tmp" - -[build] -args_bin = [] -bin = "./tmp/web" -cmd = "go build -o ./tmp/web ./api/cmd/web" -delay = 1000 -exclude_dir = [ - ".git", - "tmp", - "vendor", - "testdata", - "node_modules", - ".vscode", - ".idea", - ".github", -] -exclude_file = [] -exclude_regex = ["_test\\.go$", "\\.git", "\\.DS_Store"] -exclude_unchanged = false -follow_symlink = false -full_bin = "" -include_dir = ["api"] -include_ext = ["go", "mod", "sum", "env"] -include_file = [] -kill_delay = "0s" -log = "build-errors.log" -poll = false -poll_interval = 0 -post_cmd = [] -pre_cmd = [] -rerun = false -rerun_delay = 500 -send_interrupt = false -stop_on_error = true - -[color] -app = "" -build = "yellow" -main = "magenta" -runner = "green" -watcher = "cyan" - -[log] -main_only = false -silent = false -time = false - -[misc] -clean_on_exit = false - -[proxy] -app_port = 0 -enabled = false -proxy_port = 0 - -[screen] -clear_on_rebuild = false -keep_scroll = true From a4c1145c9c5baa826615a2207447f57a8498daae Mon Sep 17 00:00:00 2001 From: Aleksandre Date: Wed, 24 Sep 2025 23:33:24 +0400 Subject: [PATCH 36/49] Establishes initial database schema Sets up the schema migration tracking table. Creates core tables for users, novels, and chapters, including essential indexes. Implements hash partitioning for the chapters table to optimize performance and scalability. Introduces a soft-delete mechanism for users and novels by adding a `deleted` column. --- migrations/v0_migration_tracker.sql | 4 +++ migrations/v1_initial_schema.sql | 39 +++++++++++++++++++++++++++ migrations/v2_partitions.sql | 17 ++++++++++++ migrations/v3_add_deleted_columns.sql | 3 +++ 4 files changed, 63 insertions(+) create mode 100644 migrations/v0_migration_tracker.sql create mode 100644 migrations/v1_initial_schema.sql create mode 100644 migrations/v2_partitions.sql create mode 100644 migrations/v3_add_deleted_columns.sql diff --git a/migrations/v0_migration_tracker.sql b/migrations/v0_migration_tracker.sql new file mode 100644 index 0000000..458d749 --- /dev/null +++ b/migrations/v0_migration_tracker.sql @@ -0,0 +1,4 @@ +CREATE TABLE IF NOT EXISTS schema_migrations ( + version VARCHAR(255) PRIMARY KEY, + applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); diff --git a/migrations/v1_initial_schema.sql b/migrations/v1_initial_schema.sql new file mode 100644 index 0000000..6b02841 --- /dev/null +++ b/migrations/v1_initial_schema.sql @@ -0,0 +1,39 @@ +CREATE EXTENSION IF NOT EXISTS pgcrypto; + +CREATE TABLE chapters ( + id uuid NOT NULL DEFAULT gen_random_uuid(), + novel_id uuid NOT NULL, + title text NOT NULL, + author text NOT NULL, + description text NOT NULL, + content text NOT NULL, + chapter_index bigint DEFAULT 0, + deleted boolean DEFAULT false, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (novel_id, id) +) PARTITION BY HASH (novel_id); + +CREATE TABLE users ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + username text NOT NULL, + type text NOT NULL, + email text NOT NULL, + password text NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now() +); + +CREATE TABLE novels ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + title text NOT NULL, + author text NOT NULL, + description text NOT NULL, + chapter_count bigint NOT NULL DEFAULT 0, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now() +); + +CREATE INDEX idx_chapters_novel_index_id ON chapters (novel_id, chapter_index, id); +CREATE INDEX idx_users_email_id ON users (email, id); +CREATE INDEX idx_novels_title_id ON novels (title, id); diff --git a/migrations/v2_partitions.sql b/migrations/v2_partitions.sql new file mode 100644 index 0000000..e824792 --- /dev/null +++ b/migrations/v2_partitions.sql @@ -0,0 +1,17 @@ +DO $$ +DECLARE + i INTEGER; + partition_name TEXT; + partitions_count INTEGER := 16; +BEGIN + FOR i IN 0..partitions_count-1 LOOP + partition_name := 'chapters_p' || i; + + EXECUTE format('CREATE TABLE IF NOT EXISTS %I PARTITION OF chapters + FOR VALUES WITH (MODULUS %s, REMAINDER %s)', + partition_name, partitions_count, i); + + EXECUTE format('CREATE UNIQUE INDEX IF NOT EXISTS idx_%I_id ON %I (id)', + partition_name, partition_name); + END LOOP; +END $$; diff --git a/migrations/v3_add_deleted_columns.sql b/migrations/v3_add_deleted_columns.sql new file mode 100644 index 0000000..fbef18a --- /dev/null +++ b/migrations/v3_add_deleted_columns.sql @@ -0,0 +1,3 @@ +ALTER TABLE novels ADD COLUMN IF NOT EXISTS deleted BOOLEAN DEFAULT false; + +ALTER TABLE users ADD COLUMN IF NOT EXISTS deleted boolean DEFAULT false; From fe236ec7cdd2be1a773eb70e3d069bf702ec98e4 Mon Sep 17 00:00:00 2001 From: Aleksandre Date: Wed, 24 Sep 2025 23:34:11 +0400 Subject: [PATCH 37/49] Adds SQL migrations and refines chapter/novel APIs Introduces a dedicated migration runner to manage database schema evolution using SQL script files, replacing the ad-hoc `EnsureSchema` logic. Refactors `ListChaptersSeek` to accept a `CursorOptions` struct, enhancing parameter clarity. Updates `DeleteChapter` to require `novel_id`, improving deletion data integrity. Adds `CreateNovelFromEpub` for atomic creation of a novel with batch chapter insertion. Enriches novel retrieval queries to fetch full metadata including timestamps and deletion status. Simplifies `CreateUser` by allowing the database to handle ID generation. --- api/internal/database/chapters.go | 82 +++++---------- api/internal/database/client.go | 97 ------------------ api/internal/database/migration.go | 158 +++++++++++++++++++++++++++++ api/internal/database/novels.go | 47 ++++++++- api/internal/database/users.go | 4 +- 5 files changed, 226 insertions(+), 162 deletions(-) create mode 100644 api/internal/database/migration.go diff --git a/api/internal/database/chapters.go b/api/internal/database/chapters.go index 0a045e6..9050169 100644 --- a/api/internal/database/chapters.go +++ b/api/internal/database/chapters.go @@ -57,13 +57,9 @@ ListChaptersSeek returns up to `limit` chapters for a novel using seek-paginatio - nextCursor: encoded cursor to use for the next page (empty if no more rows) */ -func (c *Client) ListChaptersSeek(novelId string, limit int, cursor string, asc bool, ctx context.Context) ([]domain.Chapter, string, error) { - if limit <= 0 { - limit = 100 - } - +func (c *Client) ListChaptersSeek(options domain.CursorOptions, ctx context.Context) ([]domain.Chapter, string, error) { // decode cursor - sc, err := decodeCursor(cursor) + sc, err := decodeCursor(options.Cursor) if err != nil { return nil, "", &cmn.Error{Err: fmt.Errorf("invalid cursor: %w", err), Status: http.StatusBadRequest} } @@ -72,19 +68,19 @@ func (c *Client) ListChaptersSeek(novelId string, limit int, cursor string, asc if err := c.WithConn(ctx, func(conn *pgxpool.Conn) error { var rows pgx.Rows - fetchLimit := limit + 1 + fetchLimit := options.Limit + 1 - if asc { + if options.Ascending { if sc.Index == -1 { // First page - rows, err = conn.Query(ctx, listChaptersAscFirstSQL, novelId, fetchLimit) + rows, err = conn.Query(ctx, listChaptersAscFirstSQL, options.NovelID, fetchLimit) } else { - rows, err = conn.Query(ctx, listChaptersAscSQL, novelId, sc.Index, sc.ID, fetchLimit) + rows, err = conn.Query(ctx, listChaptersAscSQL, options.NovelID, sc.Index, sc.ID, fetchLimit) } } else { if sc.Index == -1 { // First page - rows, err = conn.Query(ctx, listChaptersDescFirstSQL, novelId, fetchLimit) + rows, err = conn.Query(ctx, listChaptersDescFirstSQL, options.NovelID, fetchLimit) } else { - rows, err = conn.Query(ctx, listChaptersDescSQL, novelId, sc.Index, sc.ID, fetchLimit) + rows, err = conn.Query(ctx, listChaptersDescSQL, options.NovelID, sc.Index, sc.ID, fetchLimit) } } @@ -96,7 +92,7 @@ func (c *Client) ListChaptersSeek(novelId string, limit int, cursor string, asc results, err = pgx.CollectRows(rows, func(row pgx.CollectableRow) (domain.Chapter, error) { var chapter domain.Chapter - err := row.Scan(&chapter.Title, &chapter.Author, &chapter.Description, + err := row.Scan(&chapter.ID, &chapter.Title, &chapter.Author, &chapter.Description, &chapter.Content, &chapter.Index, &chapter.Deleted, &chapter.CreatedAt, &chapter.UpdatedAt) if err != nil { return domain.Chapter{}, &cmn.Error{Err: fmt.Errorf("scan ListChaptersSeek: %w", err), Status: http.StatusInternalServerError} @@ -114,9 +110,9 @@ func (c *Client) ListChaptersSeek(novelId string, limit int, cursor string, asc } var nextCursor string - hasMore := len(results) > limit + hasMore := len(results) > options.Limit if hasMore { - results = results[:limit] + results = results[:options.Limit] } if len(results) > 0 && hasMore { @@ -131,48 +127,11 @@ func (c *Client) ListChaptersSeek(novelId string, limit int, cursor string, asc return results, nextCursor, nil } -func (c *Client) BatchUploadChapters(novelID string, chapters []domain.Chapter, ctx context.Context) error { - - chunkSize := 500 - - for i := 0; i < len(chapters); i += chunkSize { - end := min(i+chunkSize, len(chapters)) - chunk := chapters[i:end] - - if err := c.WithTx(ctx, func(tx pgx.Tx) error { - b := &pgx.Batch{} - insertSQL := `INSERT INTO chapters (novel_id, title, author, description, content, chapter_index, deleted) VALUES ($1,$2,$3,$4,$5,$6,$7)` - - for _, ch := range chunk { - b.Queue(insertSQL, novelID, ch.Title, ch.Author, ch.Description, ch.Content, ch.Index, ch.Deleted) - } - - br := tx.SendBatch(ctx, b) - defer br.Close() - - for range chunk { - if _, err := br.Exec(); err != nil { - return fmt.Errorf("batch exec: %w", err) - } - } - if _, err := tx.Exec(ctx, `UPDATE novels SET chapter_count = chapter_count + $1 WHERE id = $2`, len(chunk), novelID); err != nil { - return fmt.Errorf("update chapter_count: %w", err) - } - - return nil - }); err != nil { - return err - } - } - - return nil -} - -func (c *Client) CreateChapter(novelId string, chapter domain.Chapter, ctx context.Context) error { +func (c *Client) CreateChapter(chapter domain.CreateChapter, ctx context.Context) error { var newIndex int64 if err := c.WithConn(ctx, func(conn *pgxpool.Conn) error { - err := c.Pool.QueryRow(ctx, `UPDATE novels SET chapter_count = chapter_count + 1, updated_at = now() WHERE id = $1 RETURNING chapter_count`, novelId).Scan(&newIndex) + err := c.Pool.QueryRow(ctx, `UPDATE novels SET chapter_count = chapter_count + 1, updated_at = now() WHERE id = $1 RETURNING chapter_count`, chapter.NovelID).Scan(&newIndex) if err != nil { if errors.Is(err, pgx.ErrNoRows) { return &cmn.Error{Err: fmt.Errorf("novel not found: %w", err), Status: http.StatusNotFound} @@ -187,7 +146,7 @@ func (c *Client) CreateChapter(novelId string, chapter domain.Chapter, ctx conte ` if _, err = c.Pool.Exec(ctx, insertSQL, - novelId, + chapter.NovelID, chapter.Title, chapter.Author, chapter.Description, @@ -237,7 +196,12 @@ func (c *Client) GetAllChapters(novelId string, pageSize int, asc bool, ctx cont var all []domain.Chapter cursor := "" for { - chs, nextCursor, err := c.ListChaptersSeek(novelId, pageSize, cursor, asc, ctx) + chs, nextCursor, err := c.ListChaptersSeek(domain.CursorOptions{ + NovelID: novelId, + Limit: pageSize, + Cursor: cursor, + Ascending: asc, + }, ctx) if err != nil { return nil, err } @@ -264,10 +228,10 @@ func (c *Client) UpdateChapter(novelId string, chapter domain.Chapter, ctx conte return nil } -func (c *Client) DeleteChapter(chapterId string, ctx context.Context) error { +func (c *Client) DeleteChapter(novelId, chapterId string, ctx context.Context) error { if err := c.WithConn(ctx, func(conn *pgxpool.Conn) error { - query := "UPDATE chapters SET deleted = $1 WHERE id = $2" - _, err := conn.Exec(ctx, query, true, chapterId) + query := "UPDATE chapters SET deleted = $1 WHERE novel_id = $2 AND id = $3" + _, err := conn.Exec(ctx, query, true, novelId, chapterId) if err != nil { return &cmn.Error{Err: fmt.Errorf("delete chapter: %w", err), Status: http.StatusInternalServerError} } diff --git a/api/internal/database/client.go b/api/internal/database/client.go index e31686e..540dc8f 100644 --- a/api/internal/database/client.go +++ b/api/internal/database/client.go @@ -2,9 +2,6 @@ package db import ( "context" - "errors" - "fmt" - "net/http" "sync" "time" @@ -116,97 +113,3 @@ func (c *Client) Close() { once = sync.Once{} } } - -func (c *Client) EnsureSchema(ctx context.Context) error { - if c == nil || c.Pool == nil { - return &cmn.Error{Err: errors.New("postgres client not initialized"), Status: http.StatusInternalServerError} - } - - // Note: CREATE EXTENSION sometimes requires superuser privileges. - // If you can't create it inside a transaction in your environment, run it separately. - tx, err := c.Pool.Begin(ctx) - if err != nil { - return &cmn.Error{Err: fmt.Errorf("begin tx for schema: %w", err), Status: http.StatusInternalServerError} - } - defer tx.Rollback(ctx) - - stmts := []string{ - // extension for gen_random_uuid - `CREATE EXTENSION IF NOT EXISTS pgcrypto;`, - - `CREATE TABLE IF NOT EXISTS users ( - id uuid PRIMARY KEY DEFAULT gen_random_uuid(), - username text NOT NULL, - type text NOT NULL, - email text NOT NULL, - password text NOT NULL, - created_at timestamptz NOT NULL DEFAULT now(), - updated_at timestamptz NOT NULL DEFAULT now() - );`, - - `CREATE TABLE IF NOT EXISTS novels ( - id uuid PRIMARY KEY DEFAULT gen_random_uuid(), - title text NOT NULL, - author text NOT NULL, - description text NOT NULL, - chapter_count bigint NOT NULL DEFAULT 0, - created_at timestamptz NOT NULL DEFAULT now(), - updated_at timestamptz NOT NULL DEFAULT now() - );`, - - // IMPORTANT: Primary key must include partition key (novel_id) for partitioned tables. - `CREATE TABLE IF NOT EXISTS chapters ( - id uuid NOT NULL DEFAULT gen_random_uuid(), - novel_id uuid NOT NULL, - title text NOT NULL, - author text NOT NULL, - description text NOT NULL, - content text NOT NULL, - chapter_index bigint DEFAULT 0, - deleted boolean DEFAULT false, - created_at timestamptz NOT NULL DEFAULT now(), - updated_at timestamptz NOT NULL DEFAULT now(), - PRIMARY KEY (novel_id, id) - ) PARTITION BY HASH (novel_id);`, - - // Index for seek/pagination that includes the partition key (novel_id) - `CREATE INDEX IF NOT EXISTS idx_chapters_novel_index_id ON chapters (novel_id, chapter_index, id);`, - - `CREATE INDEX IF NOT EXISTS idx_novels_title_id ON novels (title, id);`, - `CREATE INDEX IF NOT EXISTS idx_users_email_id ON users (email, id);`, - } - - for _, s := range stmts { - if _, err := tx.Exec(ctx, s); err != nil { - return &cmn.Error{Err: fmt.Errorf("schema creation exec: %w", err), Status: http.StatusInternalServerError} - } - } - - // Create partitions (idempotent) - const partitionsCount = 16 - for i := 0; i < partitionsCount; i++ { - partitionName := fmt.Sprintf("chapters_p%d", i) - stmt := fmt.Sprintf( - `CREATE TABLE IF NOT EXISTS %s PARTITION OF chapters - FOR VALUES WITH (MODULUS %d, REMAINDER %d);`, - partitionName, partitionsCount, i, - ) - if _, err := tx.Exec(ctx, stmt); err != nil { - return &cmn.Error{Err: fmt.Errorf("creating partition %d: %w", i, err), Status: http.StatusInternalServerError} - } - - idxStmt := fmt.Sprintf( - `CREATE UNIQUE INDEX IF NOT EXISTS idx_%s_id ON %s (id);`, - partitionName, partitionName, - ) - if _, err := tx.Exec(ctx, idxStmt); err != nil { - return &cmn.Error{Err: fmt.Errorf("creating id index for partition %d: %w", i, err), Status: http.StatusInternalServerError} - } - } - - if err := tx.Commit(ctx); err != nil { - return &cmn.Error{Err: fmt.Errorf("commit schema creation: %w", err), Status: http.StatusInternalServerError} - } - - return nil -} diff --git a/api/internal/database/migration.go b/api/internal/database/migration.go new file mode 100644 index 0000000..e87b72d --- /dev/null +++ b/api/internal/database/migration.go @@ -0,0 +1,158 @@ +package db + +import ( + cmn "Codex-Backend/api/common" + "context" + "errors" + "fmt" + "io/fs" + "net/http" + "os" + "path/filepath" + "sort" + "strings" +) + +type MigrationRunner struct { + client *Client +} + +func NewMigrationRunner(client *Client) *MigrationRunner { + return &MigrationRunner{client: client} +} +func (mr *MigrationRunner) RunMigrations(ctx context.Context, migrationsDir string) error { + // Ensure migration tracking table exists + if err := mr.ensureMigrationTable(ctx); err != nil { + return fmt.Errorf("failed to create migration table: %w", err) + } + + // Get applied migrations + applied, err := mr.getAppliedMigrations(ctx) + if err != nil { + return fmt.Errorf("failed to get applied migrations: %w", err) + } + + // Get migration files + files, err := mr.getMigrationFiles(migrationsDir) + if err != nil { + return fmt.Errorf("failed to get migration files: %w", err) + } + + // Run pending migrations + for _, file := range files { + version := mr.extractVersion(file) + if applied[version] { + fmt.Printf("Migration %s already applied, skipping\n", version) + continue + } + + if err := mr.runMigration(ctx, filepath.Join(migrationsDir, file), version); err != nil { + return fmt.Errorf("failed to run migration %s: %w", file, err) + } + fmt.Printf("Applied migration: %s\n", file) + } + + return nil +} +func (mr *MigrationRunner) ensureMigrationTable(ctx context.Context) error { + query := ` + CREATE TABLE IF NOT EXISTS schema_migrations ( + version VARCHAR(255) PRIMARY KEY, + applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW() + );` + + _, err := mr.client.Pool.Exec(ctx, query) + return err +} +func (mr *MigrationRunner) getAppliedMigrations(ctx context.Context) (map[string]bool, error) { + query := "SELECT version FROM schema_migrations" + rows, err := mr.client.Pool.Query(ctx, query) + if err != nil { + return nil, err + } + defer rows.Close() + + applied := make(map[string]bool) + for rows.Next() { + var version string + if err := rows.Scan(&version); err != nil { + return nil, err + } + applied[version] = true + } + + return applied, rows.Err() +} +func (mr *MigrationRunner) getMigrationFiles(dir string) ([]string, error) { + var files []string + + err := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + + if !d.IsDir() && strings.HasSuffix(d.Name(), ".sql") { + files = append(files, d.Name()) + } + return nil + }) + + if err != nil { + return nil, err + } + + // Sort files to ensure correct order + sort.Strings(files) + return files, nil +} +func (mr *MigrationRunner) extractVersion(filename string) string { + // Extract version from filename like "001_initial_schema.sql" + parts := strings.SplitN(filename, "_", 2) + if len(parts) > 0 { + return strings.TrimSuffix(parts[0], ".sql") + } + return filename +} +func (mr *MigrationRunner) runMigration(ctx context.Context, filePath, version string) error { + // Read SQL file + content, err := os.ReadFile(filePath) + if err != nil { + return fmt.Errorf("failed to read migration file: %w", err) + } + + // Start transaction + tx, err := mr.client.Pool.Begin(ctx) + if err != nil { + return fmt.Errorf("failed to begin transaction: %w", err) + } + defer tx.Rollback(ctx) + + // Execute migration + if _, err := tx.Exec(ctx, string(content)); err != nil { + return fmt.Errorf("failed to execute migration SQL: %w", err) + } + + // Record migration as applied + if _, err := tx.Exec(ctx, "INSERT INTO schema_migrations (version) VALUES ($1)", version); err != nil { + return fmt.Errorf("failed to record migration: %w", err) + } + + // Commit transaction + if err := tx.Commit(ctx); err != nil { + return fmt.Errorf("failed to commit migration: %w", err) + } + + return nil +} +func (c *Client) EnsureSchema(ctx context.Context) error { + if c == nil || c.Pool == nil { + return &cmn.Error{Err: errors.New("postgres client not initialized"), Status: http.StatusInternalServerError} + } + + runner := NewMigrationRunner(c) + if err := runner.RunMigrations(ctx, "migrations"); err != nil { + return &cmn.Error{Err: fmt.Errorf("migration error: %w", err), Status: http.StatusInternalServerError} + } + + return nil +} diff --git a/api/internal/database/novels.go b/api/internal/database/novels.go index 0f3a878..882d919 100644 --- a/api/internal/database/novels.go +++ b/api/internal/database/novels.go @@ -13,7 +13,46 @@ import ( "github.com/jackc/pgx/v5/pgxpool" ) -func (c *Client) CreateNovel(novel domain.Novel, ctx context.Context) error { +func (c *Client) CreateNovelFromEpub(novel domain.Novel, chapters []domain.Chapter, ctx context.Context) error { + chunkSize := 500 + totalChapters := len(chapters) + + return c.WithTx(ctx, func(tx pgx.Tx) error { + var novelID string + + if err := tx.QueryRow(ctx, + `INSERT INTO novels (title, author, description, chapter_count) VALUES ($1, $2, $3, $4) RETURNING id`, + novel.Title, novel.Author, novel.Description, totalChapters, + ).Scan(&novelID); err != nil { + return &cmn.Error{Err: fmt.Errorf("insert novel: %w", err), Status: http.StatusInternalServerError} + } + + for i := 0; i < len(chapters); i += chunkSize { + end := min(i+chunkSize, len(chapters)) + chunk := chapters[i:end] + + // Batch insert this chunk + b := &pgx.Batch{} + insertSQL := `INSERT INTO chapters (novel_id, title, author, description, content, chapter_index, deleted) VALUES ($1,$2,$3,$4,$5,$6,$7)` + for _, ch := range chunk { + b.Queue(insertSQL, novelID, ch.Title, ch.Author, ch.Description, ch.Content, ch.Index, ch.Deleted) + } + + br := tx.SendBatch(ctx, b) + for range chunk { + if _, err := br.Exec(); err != nil { + br.Close() + return fmt.Errorf("batch exec chunk %d-%d: %w", i, end, err) + } + } + br.Close() + } + + return nil + }) +} + +func (c *Client) CreateNovel(novel domain.CreateNovel, ctx context.Context) error { return c.WithConn(ctx, func(conn *pgxpool.Conn) error { const insertSQL = `INSERT INTO novels (title, author, description) VALUES ($1,$2,$3)` if _, err := conn.Exec(ctx, insertSQL, novel.Title, novel.Author, novel.Description); err != nil { @@ -27,7 +66,7 @@ func (c *Client) GetNovelById(id string, ctx context.Context) (domain.Novel, err novel := domain.Novel{} if err := c.WithConn(ctx, func(conn *pgxpool.Conn) error { - if err := conn.QueryRow(ctx, "SELECT id, title, author, description FROM novels WHERE id = $1", id).Scan(&novel.ID, &novel.Title, &novel.Author, &novel.Description); err != nil { + if err := conn.QueryRow(ctx, "SELECT id, title, author, description, deleted, created_at, updated_at FROM novels WHERE id = $1", id).Scan(&novel.ID, &novel.Title, &novel.Author, &novel.Description, &novel.Deleted, &novel.CreatedAt, &novel.UpdatedAt); err != nil { if errors.Is(err, pgx.ErrNoRows) { return &cmn.Error{Err: fmt.Errorf("novel not found: %w", err), Status: http.StatusNotFound} } @@ -44,7 +83,7 @@ func (c *Client) GetAllNovels(ctx context.Context) ([]domain.Novel, error) { novels := []domain.Novel{} if err := c.WithConn(ctx, func(conn *pgxpool.Conn) error { - rows, err := conn.Query(ctx, "SELECT id, title, author, description FROM novels") + rows, err := conn.Query(ctx, "SELECT id, title, author, description, deleted, created_at, updated_at FROM novels") if err != nil { return &cmn.Error{Err: fmt.Errorf("get all novels: %w", err), Status: http.StatusInternalServerError} } @@ -52,7 +91,7 @@ func (c *Client) GetAllNovels(ctx context.Context) ([]domain.Novel, error) { for rows.Next() { novel := domain.Novel{} - if err := rows.Scan(&novel.ID, &novel.Title, &novel.Author, &novel.Description); err != nil { + if err := rows.Scan(&novel.ID, &novel.Title, &novel.Author, &novel.Description, &novel.Deleted, &novel.CreatedAt, &novel.UpdatedAt); err != nil { return &cmn.Error{Err: fmt.Errorf("scan novel row: %w", err), Status: http.StatusInternalServerError} } novels = append(novels, novel) diff --git a/api/internal/database/users.go b/api/internal/database/users.go index b2e2ea7..e392e82 100644 --- a/api/internal/database/users.go +++ b/api/internal/database/users.go @@ -13,8 +13,8 @@ import ( func (c *Client) CreateUser(user domain.User, ctx context.Context) error { return c.WithConn(ctx, func(conn *pgxpool.Conn) error { - const insertSQL = `INSERT INTO users (id, email, username, type, password) VALUES ($1,$2,$3,$4,$5)` - if _, err := conn.Exec(ctx, insertSQL, user.ID, user.Email, user.Username, user.Type, user.Password); err != nil { + const insertSQL = `INSERT INTO users (email, username, type, password) VALUES ($1,$2,$3,$4)` + if _, err := conn.Exec(ctx, insertSQL, user.Email, user.Username, user.Type, user.Password); err != nil { return &cmn.Error{Err: fmt.Errorf("insert user: %w", err), Status: http.StatusInternalServerError} } return nil From 2bdc21b74bce72fc093a4d6fcdddd0b1c922553d Mon Sep 17 00:00:00 2001 From: Aleksandre Date: Wed, 24 Sep 2025 23:34:29 +0400 Subject: [PATCH 38/49] Defines domain types for novel and chapter actions Introduces specific data structures for handling novel and chapter creation requests. This also includes dedicated types for uniquely identifying novels and chapters, streamlining API interactions and ensuring consistent data representation across the domain layer. --- api/internal/domain/chapter.go | 13 +++++++++++++ api/internal/domain/novel.go | 10 ++++++++++ 2 files changed, 23 insertions(+) diff --git a/api/internal/domain/chapter.go b/api/internal/domain/chapter.go index 0a17b34..26d304f 100644 --- a/api/internal/domain/chapter.go +++ b/api/internal/domain/chapter.go @@ -36,3 +36,16 @@ type FrontendChapter struct { UpdatedAt time.Time `json:"update_date"` Content string `json:"content"` } + +type CreateChapter struct { + NovelID string `json:"novel_id"` + Title string `json:"title"` + Author string `json:"author"` + Description string `json:"description"` + Content string `json:"content"` +} + +type IDs struct { + NovelId string `json:"novel_id"` + ChapterId string `json:"chapter_id"` +} diff --git a/api/internal/domain/novel.go b/api/internal/domain/novel.go index f277756..253b05d 100644 --- a/api/internal/domain/novel.go +++ b/api/internal/domain/novel.go @@ -22,3 +22,13 @@ type FrontendNovel struct { CreatedAt time.Time `json:"creation_date"` UpdatedAt time.Time `json:"update_date"` } + +type CreateNovel struct { + Title string `json:"title"` + Author string `json:"author"` + Description string `json:"description"` +} + +type ID struct { + ID string `json:"id"` +} From 95a27e73c5c7f834a4c34e3f2867be96ae3a875f Mon Sep 17 00:00:00 2001 From: Aleksandre Date: Wed, 24 Sep 2025 23:35:55 +0400 Subject: [PATCH 39/49] Refactors resource ID handling and API routes Moves resource IDs (novel and chapter) from URL path parameters to the request body for Create, Update, Delete, and specific chapter retrieval operations. Simplifies the FindNovel endpoint to exclusively use the novel ID from the path. Enhances validation for chapter pagination page size and ascending query parameter values. Temporarily disables authentication middleware. --- api/internal/server/handler/chapters.go | 71 ++++++++++++---- api/internal/server/handler/novels.go | 106 +++++++++++------------- api/internal/server/routes.go | 12 +-- 3 files changed, 109 insertions(+), 80 deletions(-) diff --git a/api/internal/server/handler/chapters.go b/api/internal/server/handler/chapters.go index 27695c0..5b0fdc1 100644 --- a/api/internal/server/handler/chapters.go +++ b/api/internal/server/handler/chapters.go @@ -31,7 +31,6 @@ func GetPaginatedChapters(c *gin.Context) { if cursor, exists := c.GetQuery("cursor"); exists { options.Cursor = cursor - } if limit, exists := c.GetQuery("limit"); exists { @@ -52,7 +51,7 @@ func GetPaginatedChapters(c *gin.Context) { } } - response, err := service.GetCursorPaginatedChapters(options, ctx) + response, err := service.GetPaginatedChapters(options, ctx) if e, ok := err.(*cmn.Error); ok { c.AbortWithStatusJSON(e.StatusCode(), gin.H{ "error": "Failed to retrieve chapters: " + e.Error(), @@ -75,17 +74,33 @@ func FindChapter(c *gin.Context) { ctx := c.Request.Context() defer ctx.Done() - novelId := c.Param("novel") - chapterId := c.Param("chapter") + IDs := domain.IDs{ + NovelId: "", + ChapterId: "", + } + + if err := c.ShouldBindJSON(&IDs); err != nil { + c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{ + "error": "Failed to get chapter IDs: " + err.Error(), + }) + return + } + + if IDs.NovelId == "" { + c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{ + "error": "Novel ID not found", + }) + return + } - if novelId == "" || chapterId == "" { + if IDs.ChapterId == "" { c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{ - "error": "IDs are not present in request", + "error": "Chapter ID not found", }) return } - chapter, err := service.GetChapter(novelId, chapterId, ctx) + chapter, err := service.GetChapter(IDs.NovelId, IDs.ChapterId, ctx) if e, ok := err.(*cmn.Error); ok { c.AbortWithStatusJSON(e.StatusCode(), gin.H{ "error": "Failed to retrieve chapter: " + e.Error(), @@ -127,15 +142,25 @@ func FindAllChapters(c *gin.Context) { }) return } + + if pageSize <= 0 && pageSize > 200 { + pageSize = 200 + } } ascending := false asc, exists := c.GetQuery("ascending") if exists { - if asc == "true" { + switch asc { + case "true": ascending = true - } else if asc == "false" { + case "false": ascending = false + default: + c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{ + "error": "Invalid ascending value", + }) + return } } @@ -161,9 +186,7 @@ func CreateChapter(c *gin.Context) { ctx := c.Request.Context() defer ctx.Done() - novelId := c.Param("novel") - - chapter := domain.Chapter{} + chapter := domain.CreateChapter{} if err := c.ShouldBindJSON(&chapter); err != nil { c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{ @@ -172,7 +195,7 @@ func CreateChapter(c *gin.Context) { return } - err := service.CreateChapter(novelId, chapter, ctx) + err := service.CreateChapter(chapter, ctx) if e, ok := err.(*cmn.Error); ok { c.AbortWithStatusJSON(e.StatusCode(), gin.H{ "error": "Failed to create chapter: " + e.Error(), @@ -227,10 +250,26 @@ func DeleteChapter(c *gin.Context) { ctx := c.Request.Context() defer ctx.Done() - novelId := c.Param("novel") - chapterId := c.Param("chapter") + IDs := domain.IDs{ + NovelId: "", + ChapterId: "", + } + + if err := c.ShouldBindJSON(&IDs); err != nil { + c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{ + "error": "Failed to get IDs: " + err.Error(), + }) + return + } + + if IDs.NovelId == "" || IDs.ChapterId == "" { + c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{ + "error": "No IDs provided", + }) + return + } - err := service.DeleteChapter(novelId, chapterId, ctx) + err := service.DeleteChapter(IDs.NovelId, IDs.ChapterId, ctx) if e, ok := err.(*cmn.Error); ok { c.AbortWithStatusJSON(e.StatusCode(), gin.H{ "error": "Failed to delete chapter: " + e.Error(), diff --git a/api/internal/server/handler/novels.go b/api/internal/server/handler/novels.go index f3b5699..5110954 100644 --- a/api/internal/server/handler/novels.go +++ b/api/internal/server/handler/novels.go @@ -8,7 +8,6 @@ import ( "Codex-Backend/api/internal/service/worker" "io" "net/http" - "strings" "github.com/gin-gonic/gin" ) @@ -69,56 +68,47 @@ func FindNovel(c *gin.Context) { defer ctx.Done() param := c.Param("novel") - - withId := false - withTitle := false - - if strings.HasPrefix(param, "novel_") { - withId = true - } else { - withTitle = true + if param == "" { + c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{ + "error": "Novel ID not found", + }) + return } - if withId { - novel, err := service.GetNovelById(param, ctx) - if e, ok := err.(*cmn.Error); ok { - c.AbortWithStatusJSON(e.StatusCode(), gin.H{ - "error": "Failed to retrieve novel: " + e.Error(), - }) - return - } else if err != nil { - c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{ - "error": "Failed to retrieve novel: " + err.Error(), - }) - return - } - - c.JSON(http.StatusOK, gin.H{ - "novel": novel, - }) - } else if withTitle { - novel, err := service.GetNovelByTitle(param, ctx) - if e, ok := err.(*cmn.Error); ok { - c.AbortWithStatusJSON(e.StatusCode(), gin.H{ - "error": "Failed to retrieve novel: " + e.Error(), - }) - return - } else if err != nil { - c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{ - "error": "Failed to retrieve novel: " + err.Error(), - }) - return - } - - c.JSON(http.StatusOK, gin.H{ - "novel": novel, + novel, err := service.GetNovelById(param, ctx) + if e, ok := err.(*cmn.Error); ok { + c.AbortWithStatusJSON(e.StatusCode(), gin.H{ + "error": "Failed to retrieve novel: " + e.Error(), }) - } else { - c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{ - "error": "Novel Title and ID not found", + return + } else if err != nil { + c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{ + "error": "Failed to retrieve novel: " + err.Error(), }) return } + + c.JSON(http.StatusOK, gin.H{ + "novel": novel, + }) + + // novel, err := service.GetNovelByTitle(param, ctx) + // if e, ok := err.(*cmn.Error); ok { + // c.AbortWithStatusJSON(e.StatusCode(), gin.H{ + // "error": "Failed to retrieve novel: " + e.Error(), + // }) + // return + // } else if err != nil { + // c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{ + // "error": "Failed to retrieve novel: " + err.Error(), + // }) + // return + // } + + // c.JSON(http.StatusOK, gin.H{ + // "novel": novel, + // }) + } func FindAllNovels(c *gin.Context) { @@ -147,7 +137,7 @@ func CreateNovel(c *gin.Context) { ctx := c.Request.Context() defer ctx.Done() - novel := domain.Novel{} + novel := domain.CreateNovel{} if err := c.ShouldBindJSON(&novel); err != nil { c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{ @@ -178,14 +168,6 @@ func UpdateNovel(c *gin.Context) { ctx := c.Request.Context() defer ctx.Done() - novelId := c.Param("novel") - if novelId == "" { - c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{ - "error": "Novel ID not found", - }) - return - } - novel := domain.Novel{} if err := c.ShouldBindJSON(&novel); err != nil { @@ -195,7 +177,7 @@ func UpdateNovel(c *gin.Context) { return } - err := service.UpdateNovel(novelId, novel, ctx) + err := service.UpdateNovel(novel, ctx) if e, ok := err.(*cmn.Error); ok { c.AbortWithStatusJSON(e.StatusCode(), gin.H{ "error": "Failed to update novel: " + e.Error(), @@ -217,15 +199,23 @@ func DeleteNovel(c *gin.Context) { ctx := c.Request.Context() defer ctx.Done() - novelId := c.Param("novel") - if novelId == "" { + novelId := domain.ID{} + + if err := c.ShouldBindJSON(&novelId); err != nil { + c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{ + "error": "Failed to get novel ID: " + err.Error(), + }) + return + } + + if novelId.ID == "" { c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{ "error": "Novel ID not found", }) return } - err := service.DeleteNovel(novelId, ctx) + err := service.DeleteNovel(novelId.ID, ctx) if e, ok := err.(*cmn.Error); ok { c.AbortWithStatusJSON(e.StatusCode(), gin.H{ "error": "Failed to delete novel: " + e.Error(), diff --git a/api/internal/server/routes.go b/api/internal/server/routes.go index ec45c5a..4f06f0f 100644 --- a/api/internal/server/routes.go +++ b/api/internal/server/routes.go @@ -56,7 +56,7 @@ func RegisteredRoutes(r *gin.Engine) { client := r.Group("/api/") { - client.Use(token.SetClaimsFromToken(), token.GlobalToken.UpdateAccessToken(), token.GlobalToken.LoadUser()) + // client.Use(token.SetClaimsFromToken(), token.GlobalToken.UpdateAccessToken(), token.GlobalToken.LoadUser()) // Potentially add user public profile view here as well. @@ -76,15 +76,15 @@ func RegisteredRoutes(r *gin.Engine) { // Create manage.POST("/create/novel", handler.CreateNovel) - manage.POST("/create/:novel/chapter", handler.CreateChapter) + manage.POST("/create/chapter", handler.CreateChapter) // Update - manage.PUT("/update/:novel", handler.UpdateNovel) - manage.PUT("/update/:novel/:chapter", handler.UpdateChapter) + manage.PUT("/update/novel", handler.UpdateNovel) + manage.PUT("/update/chapter", handler.UpdateChapter) // Delete - manage.DELETE("/delete/:novel", handler.DeleteNovel) - manage.DELETE("/delete/:novel/:chapter", handler.DeleteChapter) + manage.DELETE("/delete/novel", handler.DeleteNovel) + manage.DELETE("/delete/chapter", handler.DeleteChapter) } user := r.Group("/api/user") From 08e92f680aa99c71701d03e36a1705600a201a6f Mon Sep 17 00:00:00 2001 From: Aleksandre Date: Wed, 24 Sep 2025 23:36:46 +0400 Subject: [PATCH 40/49] Improves service API consistency and data handling Refactors chapter and novel services to use dedicated DTOs for creation and updates, enhancing API clarity. Streamlines the EPUB import process into a single database operation for novel and chapter creation. Introduces chapter indexing for correct ordering and enforces `novelId` for chapter deletion. Adjusts user registration by removing a redundant email existence check. --- api/internal/service/chapters.go | 14 +++++------- api/internal/service/novels.go | 38 ++++++++++---------------------- api/internal/service/user.go | 7 ------ 3 files changed, 17 insertions(+), 42 deletions(-) diff --git a/api/internal/service/chapters.go b/api/internal/service/chapters.go index 6bdf1f0..867a048 100644 --- a/api/internal/service/chapters.go +++ b/api/internal/service/chapters.go @@ -9,17 +9,13 @@ import ( "net/http" ) -func GetCursorPaginatedChapters(options domain.CursorOptions, ctx context.Context) (*domain.CursorResponse, error) { +func GetPaginatedChapters(options domain.CursorOptions, ctx context.Context) (*domain.CursorResponse, error) { client, err := db.GetClient(ctx) if err != nil { return nil, err } - if options.Limit > 100 || options.Limit <= 0 { - options.Limit = 100 - } - - chapters, nextCursor, err := client.ListChaptersSeek(options.NovelID, options.Limit, options.Cursor, options.Ascending, ctx) + chapters, nextCursor, err := client.ListChaptersSeek(options, ctx) if err != nil { return nil, err } @@ -32,13 +28,13 @@ func GetCursorPaginatedChapters(options domain.CursorOptions, ctx context.Contex return response, nil } -func CreateChapter(novelId string, chapter domain.Chapter, ctx context.Context) error { +func CreateChapter(chapter domain.CreateChapter, ctx context.Context) error { client, err := db.GetClient(ctx) if err != nil { return err } - if err = client.CreateChapter(novelId, chapter, ctx); err != nil { + if err = client.CreateChapter(chapter, ctx); err != nil { return err } @@ -96,7 +92,7 @@ func DeleteChapter(novelId, chapterId string, ctx context.Context) error { return err } - if err = client.DeleteChapter(chapterId, ctx); err != nil { + if err = client.DeleteChapter(novelId, chapterId, ctx); err != nil { return err } diff --git a/api/internal/service/novels.go b/api/internal/service/novels.go index 0cfd034..5701e87 100644 --- a/api/internal/service/novels.go +++ b/api/internal/service/novels.go @@ -8,7 +8,6 @@ import ( "errors" "net/http" "strings" - "time" htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown/v2" "github.com/PuerkitoBio/goquery" @@ -48,21 +47,16 @@ func CreateNovelFromEPUB(data []byte, ctx context.Context) error { return err } - createdAt, err := time.Parse(time.RFC3339, book.Date) - if err != nil { - return err - } + // TODO: Fix in future commits: Add book creation time + // createdAt, err := time.Parse(time.RFC3339, book.Date) + // if err != nil { + // return err + // } - novel := domain.Novel{ + newNovel := domain.Novel{ Title: book.Title, Author: book.Author, Description: description, - CreatedAt: createdAt, - } - - err = CreateNovel(novel, ctx) - if err != nil { - return err } // Create chapters @@ -87,7 +81,7 @@ func CreateNovelFromEPUB(data []byte, ctx context.Context) error { chapters := make([]domain.Chapter, len(orderedChapters)) for i, chapter := range orderedChapters { - chap, err := processChap(chapter, book.Author) + chap, err := processChap(chapter, i, book.Author) if err != nil { return err } @@ -100,16 +94,7 @@ func CreateNovelFromEPUB(data []byte, ctx context.Context) error { return err } - novel, err = GetNovelByTitle(novel.Title, ctx) - if err != nil { - return err - } - - if novel.ID == "" { - return &cmn.Error{Err: errors.New("Novel Service Error - Create Novel From EPUB - Novel Not Created/Found"), Status: http.StatusNotFound} - } - - err = client.BatchUploadChapters(novel.ID, chapters, ctx) + err = client.CreateNovelFromEpub(newNovel, chapters, ctx) if err != nil { return err } @@ -117,7 +102,7 @@ func CreateNovelFromEPUB(data []byte, ctx context.Context) error { return nil } -func processChap(chapter pamphlet.Chapter, author string) (*domain.Chapter, error) { +func processChap(chapter pamphlet.Chapter, index int, author string) (*domain.Chapter, error) { rawContent, err := chapter.GetContent() if err != nil { return nil, err @@ -141,10 +126,11 @@ func processChap(chapter pamphlet.Chapter, author string) (*domain.Chapter, erro Author: author, Description: "", Content: content, + Index: index, }, nil } -func CreateNovel(novel domain.Novel, ctx context.Context) error { +func CreateNovel(novel domain.CreateNovel, ctx context.Context) error { client, err := db.GetClient(ctx) if err != nil { return err @@ -203,7 +189,7 @@ func GetAllNovels(ctx context.Context) ([]domain.Novel, error) { return novels, nil } -func UpdateNovel(id string, novel domain.Novel, ctx context.Context) error { +func UpdateNovel(novel domain.Novel, ctx context.Context) error { client, err := db.GetClient(ctx) if err != nil { return err diff --git a/api/internal/service/user.go b/api/internal/service/user.go index 0482342..767568c 100644 --- a/api/internal/service/user.go +++ b/api/internal/service/user.go @@ -48,13 +48,6 @@ func RegisterUser(newUser domain.NewUser, ctx context.Context) error { return err } - _, err = client.GetUserByEmail(newUser.Email, ctx) - if e, ok := err.(*cmn.Error); ok { - if e.StatusCode() != http.StatusNotFound { - return &cmn.Error{Err: errors.New("Register Service Error - Getting User By Email: " + err.Error()), Status: http.StatusInternalServerError} - } - } - hashedPassword, err := cmn.HashPassword(newUser.Password) if err != nil { return err From 4e07d4c1ffe76f3f5708461c8ceb41ea2def36de Mon Sep 17 00:00:00 2001 From: Aleksandre Date: Wed, 24 Sep 2025 23:59:01 +0400 Subject: [PATCH 41/49] docs: Update README with latest project details and API documentation --- README.md | 64 ++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 40 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index d312c94..6d17d87 100644 --- a/README.md +++ b/README.md @@ -4,51 +4,67 @@ Backend for Codex - novel reading platform. ## Details -Codex-Backend is built in `GoLang`, using `Gin` for server and ~AWS-dynamoDB~ firestore (moving to Heroku Postgres) for database. +Codex-Backend is built in `GoLang`, using `Gin` for server and ~AWS-dynamoDB~ ~firestore (moving to Heroku Postgres)~ PostgreSQL for database. -It is deployed on `Heroku` (thats why the code is in api directory). +~It is deployed on `Heroku` (thats why the code is in api directory).~ -air config is outdated and not recommended. use [Run](Run guide instead) +Deployed on personal server in Docker. ## Run run server: ```bash -go run api/cmd/web/main.go +GIN_MODE=debug go run api/cmd/web/main.go ``` ```bash -go run api/cmd/worker/main.go +GIN_MODE=debug go run api/cmd/worker/main.go ``` Both are needed ## Endpoints -3 Groups of endpoints: Client, Manage and User. +5 Groups of endpoints: Client, Manage, User, Validate and Health. - Client is responsible for basic GET requests. -- Manage is responsible for Upload/Modification operations. +- Manage is responsible for Upload/Modification/Delete operations. - User is responsible for user authentication, authorization and Registration (Delete is not yet implemented). +- Validate is responsible for validating user tokens. +- Health is responsible for checking the health of the server. -### Client: base path followed by request path -- `/all` - Get all novels -- `/:novel` - Get a novel by id -- `/:novel/:chapter` - Get chapter from novel using both ids -- `/:novel/all` - Get all chapters from novel using id -- `/:novel/chapter` - Get cursor paginated chapters from novel using id +### Client: `/api` followed by request path +- GET `/all` - Get all novels +- GET `/:novel` - Get a novel by id +- GET `/:novel/:chapter` - Get chapter from novel using both ids +- GET `/:novel/all` - Get all chapters from novel using id +- GET `/:novel/chapter` - Get cursor paginated chapters from novel using id - Options: limit (max 100), cursor (chapter index (integer)) and sort ("asc" || "desc"). +Pagination querries: +- `?limit=100` - Limit the number of results returned, Max = 200, Min = 1 +- `?cursor=""` - Encoded offset, will be handled automatically +- `?sort="asc"` - Sort order, asc or desc - Defaults: limit=100, cursor=0, sort="desc" +### Manage: `/api/manage` followed by request path +- POST `/epub` Create Novel/Chapters from epub file. -### Manage: `/manage` followed by request path -- `/upload` - Upload novel -- `/:novel` - Update novel -- `/:novel/:chapter` - Update chapter +- POST `/create/novel` Create Novel +- POST `/create/chapter` Create Chapter -### User: `/user` followed by request path -- `/validate` - Validate user token -- `/login` - Login user -- `/register` - Register user -- `/logout` - Logout user +- PUT `/update/novel` - Update novel +- PUT `/update/chapter` - Update chapter + +- DELETE `/delete/novel` - Delete novel +- DELETE `/delete/chapter` - Delete chapter + +### User: `/api/user` followed by request path +- POST `/login` - Login user +- POST `/logout` - Logout user +- POST `/register` - Register user + +### Validate: `/api/validate` followed by request path +- GET `/` - Validate user token + +## Health: `/health` followed by request path +- GET `/` - Check health of server +For now this does nothing, but will be used to check the health of Docker image. From 29351ce2522df4d71f8ee0d3fe82834965e88df2 Mon Sep 17 00:00:00 2001 From: Aleksandre Date: Fri, 26 Sep 2025 00:56:56 +0400 Subject: [PATCH 42/49] Upgrades Go builder image to 1.25 Updates the base Go version for building both the web and worker Docker images to `golang:1.25-alpine`. This keeps the development environment current with the latest Go release. --- Dockerfile.web | 2 +- Dockerfile.worker | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile.web b/Dockerfile.web index 40387cf..702b8ef 100644 --- a/Dockerfile.web +++ b/Dockerfile.web @@ -1,4 +1,4 @@ -FROM golang:1.23-alpine AS builder +FROM golang:1.25-alpine AS builder WORKDIR /app COPY . . diff --git a/Dockerfile.worker b/Dockerfile.worker index feb369b..23235e6 100644 --- a/Dockerfile.worker +++ b/Dockerfile.worker @@ -1,4 +1,4 @@ -FROM golang:1.23-alpine AS builder +FROM golang:1.25-alpine AS builder WORKDIR /app COPY . . From 33b66e2f09e1f4383812577a2bc47e821d373673 Mon Sep 17 00:00:00 2001 From: Aleksandre Date: Fri, 26 Sep 2025 01:07:24 +0400 Subject: [PATCH 43/49] Sets proper ownership for Dockerfile assets Ensures critical application resources, specifically the .env configuration file and database migrations/ directory, are copied with appropriate ownership (appuser:appgroup). This prevents potential permission issues and aligns with security best practices for running applications as non-root users within Docker containers. --- Dockerfile.web | 4 ++-- Dockerfile.worker | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile.web b/Dockerfile.web index 702b8ef..318795d 100644 --- a/Dockerfile.web +++ b/Dockerfile.web @@ -19,8 +19,8 @@ WORKDIR /app # Copy binary with proper ownership and permissions COPY --from=builder --chown=appuser:appgroup /app/web ./web - -COPY .env .env +COPY --chown=appuser:appgroup .env .env +COPY --chown=appuser:appgroup migrations/ ./migrations/ RUN chmod +x ./web diff --git a/Dockerfile.worker b/Dockerfile.worker index 23235e6..25d6376 100644 --- a/Dockerfile.worker +++ b/Dockerfile.worker @@ -19,8 +19,8 @@ WORKDIR /app # Copy binary with proper ownership and permissions COPY --from=builder --chown=appuser:appgroup /app/worker ./worker - -COPY .env .env +COPY --chown=appuser:appgroup .env .env +COPY --chown=appuser:appgroup migrations/ ./migrations/ RUN chmod +x ./worker From 92549f692c68ae5b4079b16a9e8495f0d7fa723c Mon Sep 17 00:00:00 2001 From: Aleksandre Date: Mon, 29 Sep 2025 16:35:36 +0400 Subject: [PATCH 44/49] refactor: Remove commented out client middleware --- api/internal/server/routes.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/api/internal/server/routes.go b/api/internal/server/routes.go index 4f06f0f..5b1afbc 100644 --- a/api/internal/server/routes.go +++ b/api/internal/server/routes.go @@ -56,8 +56,6 @@ func RegisteredRoutes(r *gin.Engine) { client := r.Group("/api/") { - // client.Use(token.SetClaimsFromToken(), token.GlobalToken.UpdateAccessToken(), token.GlobalToken.LoadUser()) - // Potentially add user public profile view here as well. client.GET("/all", handler.FindAllNovels) From 94c403d3d3055b287019ac7e6881107760610e71 Mon Sep 17 00:00:00 2001 From: Aleksandre Date: Mon, 29 Sep 2025 16:35:36 +0400 Subject: [PATCH 45/49] feat: Standardize validate token route path --- api/internal/server/routes.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/internal/server/routes.go b/api/internal/server/routes.go index 5b1afbc..c014b3f 100644 --- a/api/internal/server/routes.go +++ b/api/internal/server/routes.go @@ -96,7 +96,7 @@ func RegisteredRoutes(r *gin.Engine) { { validate.Use(token.SetClaimsFromToken(), token.GlobalToken.UpdateAccessToken(), token.GlobalToken.LoadUser()) - validate.GET("/", handler.ValidateToken) + validate.GET("", handler.ValidateToken) } // For docker health check From 20a07ff2c013a46f389a4ad398a349ecb35484e6 Mon Sep 17 00:00:00 2001 From: Aleksandre Date: Mon, 29 Sep 2025 16:35:36 +0400 Subject: [PATCH 46/49] refactor: Rename 'role' to 'type' in validate token response --- api/internal/server/handler/token.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/internal/server/handler/token.go b/api/internal/server/handler/token.go index aba10dc..bcdba1f 100644 --- a/api/internal/server/handler/token.go +++ b/api/internal/server/handler/token.go @@ -29,6 +29,6 @@ func ValidateToken(c *gin.Context) { "id": claims.ID, "email": claims.Email, "username": claims.Username, - "role": claims.Type, + "type": claims.Type, }) } From 1f7946d97e4e735f6658f02c4e6ee64f62b71eb4 Mon Sep 17 00:00:00 2001 From: Aleksandre Date: Sat, 4 Oct 2025 01:45:40 +0400 Subject: [PATCH 47/49] feat(migrations): Add foreign key to chapters table --- migrations/v4_add_foreign_key.sql | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 migrations/v4_add_foreign_key.sql diff --git a/migrations/v4_add_foreign_key.sql b/migrations/v4_add_foreign_key.sql new file mode 100644 index 0000000..5aa0ff1 --- /dev/null +++ b/migrations/v4_add_foreign_key.sql @@ -0,0 +1,5 @@ +ALTER TABLE chapters +ADD CONSTRAINT fk_chapters_novel_id +FOREIGN KEY (novel_id) +REFERENCES novels(id) +ON DELETE CASCADE; From e8e683f327a0fa11a4845e03dee5ba606bb8d6a2 Mon Sep 17 00:00:00 2001 From: Aleksandre Date: Sat, 4 Oct 2025 01:45:40 +0400 Subject: [PATCH 48/49] feat(migrations): Add user type default and check constraint --- migrations/v5_add_default_user_type.sql | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 migrations/v5_add_default_user_type.sql diff --git a/migrations/v5_add_default_user_type.sql b/migrations/v5_add_default_user_type.sql new file mode 100644 index 0000000..9c8fd15 --- /dev/null +++ b/migrations/v5_add_default_user_type.sql @@ -0,0 +1,6 @@ +ALTER TABLE users +ALTER COLUMN type SET DEFAULT "User"; + +ALTER TABLE users +ADD CONSTRAINT chk_user_type +CHECK (type IN ('User', 'Admin')); From 16fc815174991b355ad7f0fd68b321584961fbf3 Mon Sep 17 00:00:00 2001 From: Aleksandre Date: Sat, 4 Oct 2025 01:45:40 +0400 Subject: [PATCH 49/49] feat(migrations): Create progress tracking table and indexes --- migrations/v6_create_progress_table.sql | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 migrations/v6_create_progress_table.sql diff --git a/migrations/v6_create_progress_table.sql b/migrations/v6_create_progress_table.sql new file mode 100644 index 0000000..7b8d82c --- /dev/null +++ b/migrations/v6_create_progress_table.sql @@ -0,0 +1,19 @@ +CREATE TABLE progress ( + id uuid NOT NULL DEFAULT gen_random_uui(), + user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE, + novel_id INTEGER NOT NULL REFERENCES novels(id) ON DELETE CASCADE, + chapter_id INTEGER NOT NULL REFERENCES chapters(id) ON DELETE CASCADE, + + scroll_position INTEGER DEFAULT 0, + progress_percentage DECIMAL(5,2) DEFAULT 0.00, + + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + completed boolean DEFAULT false, + + UNIQUE(user_id, chapter_id), + CHECK (progress_percentage >= 0 AND progress_percentage <= 100) +) + +CREATE INDEX idx_progress_user_id_novel_id ON progress (user_id, novel_id); +CREATE INDEX idx_progress_updated_at ON progress (updated_at);