From a19358767941b56c83791d752458440e46c9ea64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jordi=20G=C3=B3mez?= Date: Fri, 31 Jul 2026 12:53:35 +0200 Subject: [PATCH 01/20] feat: time-series download analytics backed by TimescaleDB --- docker-compose.yml | 4 +- server/build.gradle | 13 +- server/src/dev/resources/application.yml | 2 + .../analytics/DownloadAnalyticsAPI.java | 162 +++++++++ .../DownloadAnalyticsConfiguration.java | 49 +++ .../DownloadAnalyticsRepository.java | 33 ++ .../analytics/DownloadAnalyticsService.java | 148 ++++++++ .../openvsx/analytics/DownloadEvent.java | 76 ++++ .../DownloadSeriesGroupBy.java} | 12 +- .../analytics/DownloadSeriesInterval.java | 41 +++ .../openvsx/analytics/DownloadSeriesJson.java | 31 ++ .../analytics/DownloadSeriesPoint.java | 23 ++ .../analytics/DownloadSeriesRequest.java | 55 +++ .../openvsx/analytics/DownloadSeriesRow.java | 23 ++ .../analytics/ingestion/CountryCodes.java | 53 +++ .../ingestion/DownloadIngestionMetrics.java | 101 +++++ .../ingestion/DownloadIngestionProcessor.java | 344 ++++++++++++++++++ .../ingestion/DownloadIngestionRunner.java | 145 ++++++++ .../ingestion/DownloadRecordSource.java | 69 ++++ .../ingestion/RawDownloadRecord.java | 39 ++ .../ingestion/aws/AccessLogRecord.java | 62 ++++ .../aws/AwsDownloadRecordSource.java | 218 +++++++++++ .../aws/CloudFrontLogFileParser.java | 80 ++++ .../ingestion/aws/FastlyLogFileParser.java | 107 ++++++ .../ingestion/aws}/LogFileParser.java | 6 +- .../azure/AzureDownloadRecordSource.java} | 243 ++++++------- .../ingestion/jobs/IngestionJobRequest.java | 47 +++ .../ingestion/jobs/LogIngestionJob.java | 103 ++++++ .../TimescaleDownloadAnalyticsRepository.java | 129 +++++++ ...cessedItem.java => DownloadIngestion.java} | 5 +- .../aop/DownloadCountServiceAspect.java | 27 -- ....java => DownloadIngestionRepository.java} | 15 +- .../repositories/RepositoryService.java | 18 +- .../openvsx/storage/StorageUtilService.java | 16 +- .../storage/log/AwsDownloadCountHandler.java | 304 ---------------- .../storage/log/CloudFrontLogFileParser.java | 27 -- .../storage/log/DownloadCountProcessor.java | 147 -------- .../storage/log/DownloadCountService.java | 89 ----- .../storage/log/FastlyLogFileParser.java | 67 ---- .../org/eclipse/openvsx/jooq/Indexes.java | 3 + .../org/eclipse/openvsx/jooq/Public.java | 14 + .../org/eclipse/openvsx/jooq/Tables.java | 12 + .../openvsx/jooq/tables/DownloadEvent.java | 270 ++++++++++++++ .../jooq/tables/DownloadStatsDaily.java | 264 ++++++++++++++ .../tables/records/DownloadEventRecord.java | 205 +++++++++++ .../records/DownloadStatsDailyRecord.java | 145 ++++++++ .../migration/V1_72__Download_Analytics.sql | 51 +++ .../V1_72__Download_Analytics.sql.conf | 1 + .../AbstractPostgresContainerTest.java | 9 +- .../org/eclipse/openvsx/RegistryAPITest.java | 12 +- .../openvsx/adapter/VSCodeAPITest.java | 12 +- .../eclipse/openvsx/admin/AdminAPITest.java | 12 +- .../analytics/DownloadAnalyticsAPITest.java | 120 ++++++ .../DownloadAnalyticsDisabledTest.java | 54 +++ .../DownloadAnalyticsEndpointTest.java | 214 +++++++++++ .../DownloadAnalyticsServiceTest.java | 213 +++++++++++ .../openvsx/analytics/DownloadEventTest.java | 113 ++++++ .../analytics/ingestion/CountryCodesTest.java | 43 +++ .../DownloadIngestionMetricsTest.java | 65 ++++ .../DownloadIngestionProcessorTest.java | 270 ++++++++++++++ .../ingestion/RawDownloadRecordTest.java | 35 ++ .../ingestion/aws/AccessLogRecordTest.java | 71 ++++ .../aws/CloudFrontLogFileParserTest.java | 89 +++++ .../aws/FastlyLogFileParserTest.java | 91 +++++ .../jobs/AwsLogIngestionHandlerTest.java | 313 ++++++++++++++++ .../ingestion/jobs/IngestionJobsTest.java | 161 ++++++++ ...escaleDownloadAnalyticsRepositoryTest.java | 315 ++++++++++++++++ .../openvsx/eclipse/EclipseServiceTest.java | 12 +- .../RepositoryServiceSmokeTest.java | 5 +- .../storage/StorageUtilServiceTest.java | 110 +++++- .../StorageUtilServiceUploadFileTest.java | 11 +- .../log/CloudFrontLogFileParserTest.java | 45 --- .../storage/log/FastlyLogFileParserTest.java | 41 --- .../ingestion/aws}/cloudfront.log | 3 + .../analytics/ingestion/aws/fastly.log | 5 + .../eclipse/openvsx/storage/log/fastly.log | 1 - 76 files changed, 5612 insertions(+), 931 deletions(-) create mode 100644 server/src/main/java/org/eclipse/openvsx/analytics/DownloadAnalyticsAPI.java create mode 100644 server/src/main/java/org/eclipse/openvsx/analytics/DownloadAnalyticsConfiguration.java create mode 100644 server/src/main/java/org/eclipse/openvsx/analytics/DownloadAnalyticsRepository.java create mode 100644 server/src/main/java/org/eclipse/openvsx/analytics/DownloadAnalyticsService.java create mode 100644 server/src/main/java/org/eclipse/openvsx/analytics/DownloadEvent.java rename server/src/main/java/org/eclipse/openvsx/{storage/log/LogRecord.java => analytics/DownloadSeriesGroupBy.java} (67%) create mode 100644 server/src/main/java/org/eclipse/openvsx/analytics/DownloadSeriesInterval.java create mode 100644 server/src/main/java/org/eclipse/openvsx/analytics/DownloadSeriesJson.java create mode 100644 server/src/main/java/org/eclipse/openvsx/analytics/DownloadSeriesPoint.java create mode 100644 server/src/main/java/org/eclipse/openvsx/analytics/DownloadSeriesRequest.java create mode 100644 server/src/main/java/org/eclipse/openvsx/analytics/DownloadSeriesRow.java create mode 100644 server/src/main/java/org/eclipse/openvsx/analytics/ingestion/CountryCodes.java create mode 100644 server/src/main/java/org/eclipse/openvsx/analytics/ingestion/DownloadIngestionMetrics.java create mode 100644 server/src/main/java/org/eclipse/openvsx/analytics/ingestion/DownloadIngestionProcessor.java create mode 100644 server/src/main/java/org/eclipse/openvsx/analytics/ingestion/DownloadIngestionRunner.java create mode 100644 server/src/main/java/org/eclipse/openvsx/analytics/ingestion/DownloadRecordSource.java create mode 100644 server/src/main/java/org/eclipse/openvsx/analytics/ingestion/RawDownloadRecord.java create mode 100644 server/src/main/java/org/eclipse/openvsx/analytics/ingestion/aws/AccessLogRecord.java create mode 100644 server/src/main/java/org/eclipse/openvsx/analytics/ingestion/aws/AwsDownloadRecordSource.java create mode 100644 server/src/main/java/org/eclipse/openvsx/analytics/ingestion/aws/CloudFrontLogFileParser.java create mode 100644 server/src/main/java/org/eclipse/openvsx/analytics/ingestion/aws/FastlyLogFileParser.java rename server/src/main/java/org/eclipse/openvsx/{storage/log => analytics/ingestion/aws}/LogFileParser.java (83%) rename server/src/main/java/org/eclipse/openvsx/{storage/log/AzureDownloadCountHandler.java => analytics/ingestion/azure/AzureDownloadRecordSource.java} (58%) create mode 100644 server/src/main/java/org/eclipse/openvsx/analytics/ingestion/jobs/IngestionJobRequest.java create mode 100644 server/src/main/java/org/eclipse/openvsx/analytics/ingestion/jobs/LogIngestionJob.java create mode 100644 server/src/main/java/org/eclipse/openvsx/analytics/timescale/TimescaleDownloadAnalyticsRepository.java rename server/src/main/java/org/eclipse/openvsx/entities/{DownloadCountProcessedItem.java => DownloadIngestion.java} (91%) delete mode 100644 server/src/main/java/org/eclipse/openvsx/mirror/aop/DownloadCountServiceAspect.java rename server/src/main/java/org/eclipse/openvsx/repositories/{DownloadCountProcessedItemRepository.java => DownloadIngestionRepository.java} (51%) delete mode 100644 server/src/main/java/org/eclipse/openvsx/storage/log/AwsDownloadCountHandler.java delete mode 100644 server/src/main/java/org/eclipse/openvsx/storage/log/CloudFrontLogFileParser.java delete mode 100644 server/src/main/java/org/eclipse/openvsx/storage/log/DownloadCountProcessor.java delete mode 100644 server/src/main/java/org/eclipse/openvsx/storage/log/DownloadCountService.java delete mode 100644 server/src/main/java/org/eclipse/openvsx/storage/log/FastlyLogFileParser.java create mode 100644 server/src/main/jooq-gen/org/eclipse/openvsx/jooq/tables/DownloadEvent.java create mode 100644 server/src/main/jooq-gen/org/eclipse/openvsx/jooq/tables/DownloadStatsDaily.java create mode 100644 server/src/main/jooq-gen/org/eclipse/openvsx/jooq/tables/records/DownloadEventRecord.java create mode 100644 server/src/main/jooq-gen/org/eclipse/openvsx/jooq/tables/records/DownloadStatsDailyRecord.java create mode 100644 server/src/main/resources/db/migration/V1_72__Download_Analytics.sql create mode 100644 server/src/main/resources/db/migration/V1_72__Download_Analytics.sql.conf create mode 100644 server/src/test/java/org/eclipse/openvsx/analytics/DownloadAnalyticsAPITest.java create mode 100644 server/src/test/java/org/eclipse/openvsx/analytics/DownloadAnalyticsDisabledTest.java create mode 100644 server/src/test/java/org/eclipse/openvsx/analytics/DownloadAnalyticsEndpointTest.java create mode 100644 server/src/test/java/org/eclipse/openvsx/analytics/DownloadAnalyticsServiceTest.java create mode 100644 server/src/test/java/org/eclipse/openvsx/analytics/DownloadEventTest.java create mode 100644 server/src/test/java/org/eclipse/openvsx/analytics/ingestion/CountryCodesTest.java create mode 100644 server/src/test/java/org/eclipse/openvsx/analytics/ingestion/DownloadIngestionMetricsTest.java create mode 100644 server/src/test/java/org/eclipse/openvsx/analytics/ingestion/DownloadIngestionProcessorTest.java create mode 100644 server/src/test/java/org/eclipse/openvsx/analytics/ingestion/RawDownloadRecordTest.java create mode 100644 server/src/test/java/org/eclipse/openvsx/analytics/ingestion/aws/AccessLogRecordTest.java create mode 100644 server/src/test/java/org/eclipse/openvsx/analytics/ingestion/aws/CloudFrontLogFileParserTest.java create mode 100644 server/src/test/java/org/eclipse/openvsx/analytics/ingestion/aws/FastlyLogFileParserTest.java create mode 100644 server/src/test/java/org/eclipse/openvsx/analytics/ingestion/jobs/AwsLogIngestionHandlerTest.java create mode 100644 server/src/test/java/org/eclipse/openvsx/analytics/ingestion/jobs/IngestionJobsTest.java create mode 100644 server/src/test/java/org/eclipse/openvsx/analytics/timescale/TimescaleDownloadAnalyticsRepositoryTest.java delete mode 100644 server/src/test/java/org/eclipse/openvsx/storage/log/CloudFrontLogFileParserTest.java delete mode 100644 server/src/test/java/org/eclipse/openvsx/storage/log/FastlyLogFileParserTest.java rename server/src/test/resources/org/eclipse/openvsx/{storage/log => analytics/ingestion/aws}/cloudfront.log (55%) create mode 100644 server/src/test/resources/org/eclipse/openvsx/analytics/ingestion/aws/fastly.log delete mode 100644 server/src/test/resources/org/eclipse/openvsx/storage/log/fastly.log diff --git a/docker-compose.yml b/docker-compose.yml index b985fd304..d7506eca2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,9 @@ services: postgres: - image: postgres:16.2 + # PostgreSQL with the timescaledb extension: the main migration chain contains the + # download analytics schema, which requires the extension + image: timescale/timescaledb:2.17.2-pg16 environment: - POSTGRES_USER=openvsx - POSTGRES_PASSWORD=openvsx diff --git a/server/build.gradle b/server/build.gradle index bee01674c..10ca436c7 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -200,7 +200,12 @@ jooq { name = 'org.jooq.meta.postgres.PostgresDatabase' inputSchema = 'public' includes = '.*' - excludes = 'jobrunr.*' + // jobrunr manages its own tables; the timescaledb extension contributes + // public-schema (table-valued) functions that we never call through jOOQ + excludes = 'jobrunr.*' + + '|add_dimension|alter_job|create_hypertable|disable_chunk_skipping|drop_chunks' + + '|enable_chunk_skipping|show_chunks|show_tablespaces' + + '|chunk_compression_stats|chunks_detailed_size|hypertable_.*' includeRoutines = false } target { @@ -281,6 +286,12 @@ test { // observed as an OutOfMemoryError during unrelated context bootstrapping on CI. jvmArgs = ['--enable-native-access=ALL-UNNAMED', '-Xmx6144m', '-Xshare:off'] // due to https://github.com/netty/netty/issues/15161 useJUnitPlatform() + + // the test database image is timescale/timescaledb by default (the main migration chain + // requires the extension); override with -Dovsx.test.postgres.image=... if needed + if (System.getProperty('ovsx.test.postgres.image') != null) { + systemProperty 'ovsx.test.postgres.image', System.getProperty('ovsx.test.postgres.image') + } } tasks.register('unitTests', Test) { diff --git a/server/src/dev/resources/application.yml b/server/src/dev/resources/application.yml index 7993ed138..11b55392e 100644 --- a/server/src/dev/resources/application.yml +++ b/server/src/dev/resources/application.yml @@ -149,6 +149,8 @@ ovsx: # path-style-access: true local: directory: /tmp/ovsx + analytics: + enabled: true access-token: prefix: dev_ovsxat_ # use a token prefix that clearly indicates that it's for development expiration: 0 # do not expire tokens in a dev environment diff --git a/server/src/main/java/org/eclipse/openvsx/analytics/DownloadAnalyticsAPI.java b/server/src/main/java/org/eclipse/openvsx/analytics/DownloadAnalyticsAPI.java new file mode 100644 index 000000000..822bf9f0c --- /dev/null +++ b/server/src/main/java/org/eclipse/openvsx/analytics/DownloadAnalyticsAPI.java @@ -0,0 +1,162 @@ +/****************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation. + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * https://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + *****************************************************************************/ +package org.eclipse.openvsx.analytics; + +import java.time.Clock; +import java.time.LocalDate; +import java.time.ZoneOffset; +import java.time.format.DateTimeParseException; + +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.media.Content; +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.CrossOrigin; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.server.ResponseStatusException; + +import org.eclipse.openvsx.repositories.RepositoryService; +import org.eclipse.openvsx.util.NotFoundException; + +/** + * Minimal REST surface over {@link DownloadAnalyticsService}. The bean only exists when download + * analytics is enabled, so the path stays unmapped (404) otherwise. + */ +@RestController +@ConditionalOnProperty(name = "ovsx.analytics.enabled", havingValue = "true") +public class DownloadAnalyticsAPI { + + private static final int MAX_RANGE_YEARS = 5; + + private final DownloadAnalyticsService service; + private final RepositoryService repositories; + private final Clock clock; + + @Autowired + public DownloadAnalyticsAPI(DownloadAnalyticsService service, RepositoryService repositories) { + this(service, repositories, Clock.systemUTC()); + } + + DownloadAnalyticsAPI( + DownloadAnalyticsService service, + RepositoryService repositories, + Clock clock + ) { + this.service = service; + this.repositories = repositories; + this.clock = clock; + } + + @GetMapping(path = "/api/{namespace}/{extension}/analytics/downloads", produces = MediaType.APPLICATION_JSON_VALUE) + @CrossOrigin + @Operation(summary = "Provides the download counts of an extension over time") + @ApiResponse( + responseCode = "200", + description = "The dense, zero-filled download series is returned in JSON format; the last point may still be partial" + ) + @ApiResponse( + responseCode = "400", + description = "A query parameter is invalid", + content = @Content() + ) + @ApiResponse( + responseCode = "404", + description = "The specified extension could not be found, or download analytics is disabled", + content = @Content() + ) + public ResponseEntity getDownloads( + @PathVariable + @Parameter(description = "Extension namespace", example = "redhat") String namespace, + @PathVariable + @Parameter(description = "Extension name", example = "java") String extension, + @RequestParam(required = false) + @Parameter( + description = "UTC start date (inclusive), defaults to 30 buckets before 'to'", + example = "2026-06-16" + ) String from, + @RequestParam(required = false) + @Parameter( + description = "UTC end date (exclusive), defaults to tomorrow", + example = "2026-07-16" + ) String to, + @RequestParam(defaultValue = "day") + @Parameter( + description = "Bucket interval", + schema = @Schema(type = "string", allowableValues = { "day", "week", "month" }, defaultValue = "day") + ) String interval + ) { + var extensionEntity = repositories.findActiveExtension(extension, namespace); + if (extensionEntity == null) { + throw new NotFoundException(); + } + + var request = buildRequest(extensionEntity.getId(), from, to, interval); + var points = service.getSeries(request).stream() + .map( + point -> new DownloadSeriesJson.DownloadSeriesPointJson( + LocalDate.ofInstant(point.bucketStart(), ZoneOffset.UTC).toString(), + point.count())) + .toList(); + return ResponseEntity.ok(new DownloadSeriesJson(points)); + } + + private DownloadSeriesRequest buildRequest(long extensionId, String from, String to, String interval) { + DownloadSeriesInterval seriesInterval; + try { + seriesInterval = DownloadSeriesInterval.fromValue(interval); + } catch (IllegalArgumentException e) { + throw new ResponseStatusException(HttpStatus.BAD_REQUEST, e.getMessage()); + } + + var today = LocalDate.ofInstant(clock.instant(), ZoneOffset.UTC); + var toDate = parseDate(to, "to", today.plusDays(1)); + var fromDate = parseDate(from, "from", toDate.minusDays(30)); + if (!fromDate.isBefore(toDate)) { + throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "'from' must be before 'to'"); + } + if (fromDate.plusYears(MAX_RANGE_YEARS).isBefore(toDate)) { + throw new ResponseStatusException( + HttpStatus.BAD_REQUEST, + "the requested range must not exceed " + MAX_RANGE_YEARS + " years"); + } + + return DownloadSeriesRequest.of( + extensionId, + fromDate.atStartOfDay(ZoneOffset.UTC).toInstant(), + toDate.atStartOfDay(ZoneOffset.UTC).toInstant(), + seriesInterval); + } + + private LocalDate parseDate(String value, String name, LocalDate defaultValue) { + if (value == null) { + return defaultValue; + } + + try { + return LocalDate.parse(value); + } catch (DateTimeParseException e) { + throw new ResponseStatusException( + HttpStatus.BAD_REQUEST, + "parameter '" + name + "' must be a date in the format yyyy-mm-dd"); + } + } +} diff --git a/server/src/main/java/org/eclipse/openvsx/analytics/DownloadAnalyticsConfiguration.java b/server/src/main/java/org/eclipse/openvsx/analytics/DownloadAnalyticsConfiguration.java new file mode 100644 index 000000000..a76383416 --- /dev/null +++ b/server/src/main/java/org/eclipse/openvsx/analytics/DownloadAnalyticsConfiguration.java @@ -0,0 +1,49 @@ +/****************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation. + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * https://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + *****************************************************************************/ +package org.eclipse.openvsx.analytics; + +import java.time.Clock; +import java.time.Duration; + +import org.jooq.DSLContext; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.core.env.Environment; + +import org.eclipse.openvsx.analytics.timescale.TimescaleDownloadAnalyticsRepository; + +/** + * Wires download analytics when {@code ovsx.analytics.enabled=true}. The download_event schema + * is part of the main migration chain, so the database image must provide the timescaledb + * extension. + */ +@Configuration +@ConditionalOnProperty(name = "ovsx.analytics.enabled", havingValue = "true") +class DownloadAnalyticsConfiguration { + + @Bean + DownloadAnalyticsRepository downloadAnalyticsRepository(DSLContext dsl) { + return new TimescaleDownloadAnalyticsRepository(dsl); + } + + @Bean + DownloadAnalyticsService downloadAnalyticsService( + DownloadAnalyticsRepository repository, + Environment environment + ) { + var settlingMargin = environment + .getProperty("ovsx.analytics.settling-margin", Duration.class, Duration.ofHours(2)); + return new DownloadAnalyticsService(repository, settlingMargin, Clock.systemUTC()); + } +} diff --git a/server/src/main/java/org/eclipse/openvsx/analytics/DownloadAnalyticsRepository.java b/server/src/main/java/org/eclipse/openvsx/analytics/DownloadAnalyticsRepository.java new file mode 100644 index 000000000..c54dc08eb --- /dev/null +++ b/server/src/main/java/org/eclipse/openvsx/analytics/DownloadAnalyticsRepository.java @@ -0,0 +1,33 @@ +/****************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation. + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * https://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + *****************************************************************************/ +package org.eclipse.openvsx.analytics; + +import java.util.List; + +/** + * Storage for download analytics: one interface for writing events and reading series. + */ +public interface DownloadAnalyticsRepository { + + /** + * Persists the given events. Implementations must participate in the caller's transaction, + * so that events, the extension download counter and the download ingestion entry commit atomically. + */ + void save(List events); + + /** + * Returns the (sparse) aggregated download series for the given request. Buckets without + * downloads are absent; zero-filling is the {@link DownloadAnalyticsService}'s concern. + */ + List findSeries(DownloadSeriesRequest request); +} diff --git a/server/src/main/java/org/eclipse/openvsx/analytics/DownloadAnalyticsService.java b/server/src/main/java/org/eclipse/openvsx/analytics/DownloadAnalyticsService.java new file mode 100644 index 000000000..ceda31e66 --- /dev/null +++ b/server/src/main/java/org/eclipse/openvsx/analytics/DownloadAnalyticsService.java @@ -0,0 +1,148 @@ +/****************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation. + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * https://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + *****************************************************************************/ +package org.eclipse.openvsx.analytics; + +import java.time.Clock; +import java.time.DayOfWeek; +import java.time.Duration; +import java.time.Instant; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.time.temporal.ChronoUnit; +import java.time.temporal.TemporalAdjusters; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; +import org.jspecify.annotations.Nullable; + +/** + * Query facade over a {@link DownloadAnalyticsRepository}: aligns ranges to UTC buckets, returns + * dense zero-filled series, marks trailing buckets that may still change as partial, and caches + * settled sub-ranges (data older than the settling margin never changes). + */ +public class DownloadAnalyticsService { + + private final DownloadAnalyticsRepository repository; + private final Duration settlingMargin; + private final Clock clock; + + private final Cache> settledCache = Caffeine.newBuilder() + .maximumSize(10_000) + .expireAfterWrite(Duration.ofHours(1)) + .build(); + + public DownloadAnalyticsService(DownloadAnalyticsRepository repository, Duration settlingMargin, Clock clock) { + this.repository = repository; + this.settlingMargin = settlingMargin; + this.clock = clock; + } + + /** + * Returns the dense, zero-filled download series for the given request, ordered by bucket + * start and group. The range is aligned outwards to full UTC buckets. + */ + public List getSeries(DownloadSeriesRequest request) { + var now = clock.instant(); + var interval = request.interval(); + var from = truncate(request.from(), interval).toInstant(); + var to = alignUp(request.to(), interval); + var aligned = new DownloadSeriesRequest(request.extensionIds(), from, to, interval, request.groupBy()); + + var settledEnd = truncate(now.minus(settlingMargin), interval).toInstant(); + List rows; + if (!to.isAfter(settledEnd)) { + rows = settledCache.get(aligned, repository::findSeries); + } else if (from.isBefore(settledEnd)) { + var settled = new DownloadSeriesRequest( + request.extensionIds(), + from, + settledEnd, + interval, + request.groupBy()); + var live = new DownloadSeriesRequest(request.extensionIds(), settledEnd, to, interval, request.groupBy()); + rows = Stream + .concat( + settledCache.get(settled, repository::findSeries).stream(), + repository.findSeries(live).stream()) + .toList(); + } else { + rows = repository.findSeries(aligned); + } + + return zeroFill(aligned, rows, now); + } + + private List zeroFill( + DownloadSeriesRequest request, + List rows, + Instant now + ) { + var interval = request.interval(); + var groups = rows.stream() + .map(DownloadSeriesRow::group) + .distinct() + .sorted(Comparator.nullsFirst(Comparator.naturalOrder())) + .toList(); + if (groups.isEmpty()) { + groups = Collections.singletonList(null); + } + + var counts = rows.stream().collect( + Collectors.toMap(row -> new BucketKey(row.bucketStart(), row.group()), DownloadSeriesRow::count)); + + var points = new ArrayList(); + for (var bucket = truncate(request.from(), interval); bucket.toInstant() + .isBefore(request.to()); bucket = next(bucket, interval)) { + var bucketEnd = next(bucket, interval).toInstant(); + var partial = bucketEnd.plus(settlingMargin).isAfter(now); + for (var group : groups) { + var count = counts.getOrDefault(new BucketKey(bucket.toInstant(), group), 0L); + points.add(new DownloadSeriesPoint(bucket.toInstant(), group, count, partial)); + } + } + + return points; + } + + private record BucketKey(Instant bucketStart, @Nullable String group) {} + + private ZonedDateTime truncate(Instant instant, DownloadSeriesInterval interval) { + var day = instant.atZone(ZoneOffset.UTC).truncatedTo(ChronoUnit.DAYS); + return switch (interval) { + case DAY -> day; + case WEEK -> day.with(TemporalAdjusters.previousOrSame(DayOfWeek.MONDAY)); + case MONTH -> day.with(TemporalAdjusters.firstDayOfMonth()); + }; + } + + private Instant alignUp(Instant instant, DownloadSeriesInterval interval) { + var truncated = truncate(instant, interval); + return truncated.toInstant().equals(instant) + ? instant + : next(truncated, interval).toInstant(); + } + + private ZonedDateTime next(ZonedDateTime bucket, DownloadSeriesInterval interval) { + return switch (interval) { + case DAY -> bucket.plusDays(1); + case WEEK -> bucket.plusWeeks(1); + case MONTH -> bucket.plusMonths(1); + }; + } +} diff --git a/server/src/main/java/org/eclipse/openvsx/analytics/DownloadEvent.java b/server/src/main/java/org/eclipse/openvsx/analytics/DownloadEvent.java new file mode 100644 index 000000000..68fc3e17f --- /dev/null +++ b/server/src/main/java/org/eclipse/openvsx/analytics/DownloadEvent.java @@ -0,0 +1,76 @@ +/****************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation. + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * https://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + *****************************************************************************/ +package org.eclipse.openvsx.analytics; + +import java.time.Instant; +import java.util.Objects; + +import org.apache.commons.lang3.StringUtils; +import org.jspecify.annotations.Nullable; + +/** + * A time-bucketed, strictly additive download fact: {@code SUM(count)} over any grouping is a + * valid total. Every event correlates with the concrete extension version it was downloaded + * from: {@code extensionId} and {@code extensionVersionId} are the keys (no foreign keys into + * the registry tables), while namespace, extension name, version and target platform are + * write-time snapshots. + *

+ * The client IP and raw user agent are persisted as found in the source (first iteration); + * deriving client classifications from the user agent is left to future consumers. + */ +public record DownloadEvent( + Instant time, + long extensionId, + long extensionVersionId, + String namespace, + String extensionName, + String version, + String targetPlatform, + @Nullable String country, + @Nullable String ip, + @Nullable String userAgent, + int count +) { + public DownloadEvent { + Objects.requireNonNull(time, "time must not be null"); + requireNonBlank(namespace, "namespace"); + requireNonBlank(extensionName, "extensionName"); + requireNonBlank(version, "version"); + requireNonBlank(targetPlatform, "targetPlatform"); + if (count < 1) { + throw new IllegalArgumentException("count must be a positive increment, got " + count); + } + + country = normalizeCountry(country); + ip = StringUtils.trimToNull(ip); + userAgent = StringUtils.trimToNull(userAgent); + } + + private static void requireNonBlank(String value, String name) { + Objects.requireNonNull(value, name + " must not be null"); + if (value.isBlank()) { + throw new IllegalArgumentException(name + " must not be blank"); + } + } + + private static @Nullable String normalizeCountry(@Nullable String country) { + if (country == null) { + return null; + } + if (country.length() != 2 || !country.chars().allMatch(Character::isLetter)) { + throw new IllegalArgumentException("country must be a two-letter ISO code, got '" + country + "'"); + } + + return country.toUpperCase(); + } +} diff --git a/server/src/main/java/org/eclipse/openvsx/storage/log/LogRecord.java b/server/src/main/java/org/eclipse/openvsx/analytics/DownloadSeriesGroupBy.java similarity index 67% rename from server/src/main/java/org/eclipse/openvsx/storage/log/LogRecord.java rename to server/src/main/java/org/eclipse/openvsx/analytics/DownloadSeriesGroupBy.java index 39cbe7fc1..343378a44 100644 --- a/server/src/main/java/org/eclipse/openvsx/storage/log/LogRecord.java +++ b/server/src/main/java/org/eclipse/openvsx/analytics/DownloadSeriesGroupBy.java @@ -10,8 +10,12 @@ * * SPDX-License-Identifier: EPL-2.0 *****************************************************************************/ -package org.eclipse.openvsx.storage.log; +package org.eclipse.openvsx.analytics; -import org.jspecify.annotations.NonNull; - -public record LogRecord(@NonNull String method, int status, @NonNull String url) {} +/** + * Optional grouping dimension of a download series. Because event counts are strictly + * additive, any grouping sums to the same total. + */ +public enum DownloadSeriesGroupBy { + NONE, VERSION, TARGET_PLATFORM, COUNTRY +} diff --git a/server/src/main/java/org/eclipse/openvsx/analytics/DownloadSeriesInterval.java b/server/src/main/java/org/eclipse/openvsx/analytics/DownloadSeriesInterval.java new file mode 100644 index 000000000..7bff07b82 --- /dev/null +++ b/server/src/main/java/org/eclipse/openvsx/analytics/DownloadSeriesInterval.java @@ -0,0 +1,41 @@ +/****************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation. + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * https://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + *****************************************************************************/ +package org.eclipse.openvsx.analytics; + +/** + * Bucket size of a download series. Buckets start at UTC midnight (day), UTC Monday (week) + * or the first of the month (month). + */ +public enum DownloadSeriesInterval { + DAY("day"), WEEK("week"), MONTH("month"); + + private final String value; + + DownloadSeriesInterval(String value) { + this.value = value; + } + + public String value() { + return value; + } + + public static DownloadSeriesInterval fromValue(String value) { + for (var interval : values()) { + if (interval.value.equals(value)) { + return interval; + } + } + + throw new IllegalArgumentException("unknown interval '" + value + "', expected day, week or month"); + } +} diff --git a/server/src/main/java/org/eclipse/openvsx/analytics/DownloadSeriesJson.java b/server/src/main/java/org/eclipse/openvsx/analytics/DownloadSeriesJson.java new file mode 100644 index 000000000..d268dbb87 --- /dev/null +++ b/server/src/main/java/org/eclipse/openvsx/analytics/DownloadSeriesJson.java @@ -0,0 +1,31 @@ +/****************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation. + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * https://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + *****************************************************************************/ +package org.eclipse.openvsx.analytics; + +import java.util.List; + +import io.swagger.v3.oas.annotations.media.Schema; + +/** + * REST payload of the download series endpoint. Points are dense and zero-filled; the last point + * may still be partial (its bucket has not ended, or logs are still being ingested). + */ +@Schema(name = "DownloadSeries", description = "Time series of download counts") +public record DownloadSeriesJson(List points) { + + @Schema(name = "DownloadSeriesPoint") + public record DownloadSeriesPointJson( + @Schema(description = "UTC start date of the bucket", example = "2026-07-01") String t, + @Schema(description = "Number of downloads in the bucket", example = "4321") long count + ) {} +} diff --git a/server/src/main/java/org/eclipse/openvsx/analytics/DownloadSeriesPoint.java b/server/src/main/java/org/eclipse/openvsx/analytics/DownloadSeriesPoint.java new file mode 100644 index 000000000..c280c00cc --- /dev/null +++ b/server/src/main/java/org/eclipse/openvsx/analytics/DownloadSeriesPoint.java @@ -0,0 +1,23 @@ +/****************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation. + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * https://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + *****************************************************************************/ +package org.eclipse.openvsx.analytics; + +import java.time.Instant; + +import org.jspecify.annotations.Nullable; + +/** + * One bucket of a dense, zero-filled download series. {@code partial} marks buckets whose data + * may still change: the bucket has not yet ended, or the ingestion settling margin has not passed. + */ +public record DownloadSeriesPoint(Instant bucketStart, @Nullable String group, long count, boolean partial) {} diff --git a/server/src/main/java/org/eclipse/openvsx/analytics/DownloadSeriesRequest.java b/server/src/main/java/org/eclipse/openvsx/analytics/DownloadSeriesRequest.java new file mode 100644 index 000000000..aa38c9f23 --- /dev/null +++ b/server/src/main/java/org/eclipse/openvsx/analytics/DownloadSeriesRequest.java @@ -0,0 +1,55 @@ +/****************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation. + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * https://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + *****************************************************************************/ +package org.eclipse.openvsx.analytics; + +import java.time.Instant; +import java.util.List; +import java.util.Objects; + +/** + * A download series query: one or more extensions, a UTC time range ({@code from} inclusive, + * {@code to} exclusive), a bucket interval and an optional grouping dimension. Deliberately + * richer than what the REST endpoint exposes, so downstream deployments can compose on it. + */ +public record DownloadSeriesRequest( + List extensionIds, + Instant from, + Instant to, + DownloadSeriesInterval interval, + DownloadSeriesGroupBy groupBy +) { + public DownloadSeriesRequest { + Objects.requireNonNull(extensionIds, "extensionIds must not be null"); + Objects.requireNonNull(from, "from must not be null"); + Objects.requireNonNull(to, "to must not be null"); + Objects.requireNonNull(interval, "interval must not be null"); + Objects.requireNonNull(groupBy, "groupBy must not be null"); + if (extensionIds.isEmpty()) { + throw new IllegalArgumentException("extensionIds must not be empty"); + } + if (!from.isBefore(to)) { + throw new IllegalArgumentException("from must be before to"); + } + + extensionIds = List.copyOf(extensionIds); + } + + public static DownloadSeriesRequest of( + long extensionId, + Instant from, + Instant to, + DownloadSeriesInterval interval + ) { + return new DownloadSeriesRequest(List.of(extensionId), from, to, interval, DownloadSeriesGroupBy.NONE); + } +} diff --git a/server/src/main/java/org/eclipse/openvsx/analytics/DownloadSeriesRow.java b/server/src/main/java/org/eclipse/openvsx/analytics/DownloadSeriesRow.java new file mode 100644 index 000000000..d70eb870e --- /dev/null +++ b/server/src/main/java/org/eclipse/openvsx/analytics/DownloadSeriesRow.java @@ -0,0 +1,23 @@ +/****************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation. + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * https://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + *****************************************************************************/ +package org.eclipse.openvsx.analytics; + +import java.time.Instant; + +import org.jspecify.annotations.Nullable; + +/** + * One bucket of an aggregated download series. {@code group} is the value of the requested + * grouping dimension, or null when grouping by {@link DownloadSeriesGroupBy#NONE}. + */ +public record DownloadSeriesRow(Instant bucketStart, @Nullable String group, long count) {} diff --git a/server/src/main/java/org/eclipse/openvsx/analytics/ingestion/CountryCodes.java b/server/src/main/java/org/eclipse/openvsx/analytics/ingestion/CountryCodes.java new file mode 100644 index 000000000..ba72f733a --- /dev/null +++ b/server/src/main/java/org/eclipse/openvsx/analytics/ingestion/CountryCodes.java @@ -0,0 +1,53 @@ +/****************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation. + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * https://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + *****************************************************************************/ +package org.eclipse.openvsx.analytics.ingestion; + +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import org.apache.commons.lang3.StringUtils; +import org.jspecify.annotations.Nullable; + +/** + * Normalizes log-provided country values (ISO codes or English country names, as emitted by + * Fastly's {@code geo_country}) to two-letter ISO 3166-1 codes. Unknown values map to null. + */ +final class CountryCodes { + + private static final Set ISO_CODES = Set.of(Locale.getISOCountries()); + + private static final Map NAME_TO_CODE = ISO_CODES.stream().collect( + Collectors.toMap( + code -> Locale.of("", code).getDisplayCountry(Locale.ENGLISH).toLowerCase(Locale.ENGLISH), + code -> code, + (first, second) -> first)); + + private CountryCodes() { + } + + public static @Nullable String toIsoCode(@Nullable String country) { + if (StringUtils.isBlank(country)) { + return null; + } + + var trimmed = country.trim(); + if (trimmed.length() == 2) { + var code = trimmed.toUpperCase(Locale.ENGLISH); + return ISO_CODES.contains(code) ? code : null; + } + + return NAME_TO_CODE.get(trimmed.toLowerCase(Locale.ENGLISH)); + } +} diff --git a/server/src/main/java/org/eclipse/openvsx/analytics/ingestion/DownloadIngestionMetrics.java b/server/src/main/java/org/eclipse/openvsx/analytics/ingestion/DownloadIngestionMetrics.java new file mode 100644 index 000000000..7255392d9 --- /dev/null +++ b/server/src/main/java/org/eclipse/openvsx/analytics/ingestion/DownloadIngestionMetrics.java @@ -0,0 +1,101 @@ +/****************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation. + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * https://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + *****************************************************************************/ +package org.eclipse.openvsx.analytics.ingestion; + +import java.time.Duration; +import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; + +import com.google.common.base.Suppliers; +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.Gauge; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Timer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import org.eclipse.openvsx.repositories.RepositoryService; + +/** + * Operational metrics of the download ingestion pipeline: parse skip rate, load volume, + * extract lag and dead-letter depth. + */ +@Component +public class DownloadIngestionMetrics { + + public static final String LINES_METRIC = "openvsx_analytics_log_lines_total"; + public static final String SKIPPED_LINES_METRIC = "openvsx_analytics_log_lines_skipped_total"; + public static final String EVENTS_METRIC = "openvsx_analytics_events_loaded_total"; + public static final String DOWNLOADS_METRIC = "openvsx_analytics_downloads_loaded_total"; + public static final String EXTRACT_LAG_METRIC = "openvsx_analytics_extract_lag"; + public static final String DEAD_LETTER_METRIC = "openvsx_analytics_dead_letter_depth"; + + private static final Logger logger = LoggerFactory.getLogger(DownloadIngestionMetrics.class); + + private final Counter lines; + private final Counter skippedLines; + private final Counter events; + private final Counter downloads; + private final Timer extractLag; + + public DownloadIngestionMetrics(MeterRegistry registry, RepositoryService repositories) { + this.lines = Counter.builder(LINES_METRIC) + .description("Access log lines read by the download ingestion pipeline") + .register(registry); + this.skippedLines = Counter.builder(SKIPPED_LINES_METRIC) + .description("Access log lines skipped as malformed") + .register(registry); + this.events = Counter.builder(EVENTS_METRIC) + .description("Aggregated download events loaded into the analytics store") + .register(registry); + this.downloads = Counter.builder(DOWNLOADS_METRIC) + .description("Downloads counted by the ingestion pipeline") + .register(registry); + this.extractLag = Timer.builder(EXTRACT_LAG_METRIC) + .description("Delay between a download and its ingestion from access logs") + .register(registry); + + // scraped frequently, so the underlying count query is memoized for a minute + Supplier deadLetterDepth = Suppliers + .memoizeWithExpiration(() -> countFailedItems(repositories), 1, TimeUnit.MINUTES)::get; + Gauge.builder(DEAD_LETTER_METRIC, deadLetterDepth::get) + .description("Number of log files that failed processing and await manual attention") + .register(registry); + } + + private long countFailedItems(RepositoryService repositories) { + try { + return repositories.countFailedDownloadIngestions(); + } catch (Exception e) { + logger.warn("could not determine dead-letter depth", e); + return -1; + } + } + + public void recordParsedLines(int total, int skipped) { + lines.increment(total); + skippedLines.increment(skipped); + } + + public void recordLoaded(int eventCount, int downloadCount) { + events.increment(eventCount); + downloads.increment(downloadCount); + } + + public void recordExtractLag(Duration lag) { + if (!lag.isNegative()) { + extractLag.record(lag); + } + } +} diff --git a/server/src/main/java/org/eclipse/openvsx/analytics/ingestion/DownloadIngestionProcessor.java b/server/src/main/java/org/eclipse/openvsx/analytics/ingestion/DownloadIngestionProcessor.java new file mode 100644 index 000000000..4286d36f1 --- /dev/null +++ b/server/src/main/java/org/eclipse/openvsx/analytics/ingestion/DownloadIngestionProcessor.java @@ -0,0 +1,344 @@ +/******************************************************************************** + * Copyright (c) 2022 Precies. Software Ltd and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + ********************************************************************************/ +package org.eclipse.openvsx.analytics.ingestion; + +import java.time.Duration; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; +import com.google.common.collect.Lists; +import io.micrometer.observation.Observation; +import io.micrometer.observation.ObservationRegistry; +import jakarta.persistence.EntityManager; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.transaction.Transactional; +import org.apache.commons.lang3.StringUtils; +import org.jspecify.annotations.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.stereotype.Component; +import org.springframework.web.context.request.RequestContextHolder; +import org.springframework.web.context.request.ServletRequestAttributes; + +import org.eclipse.openvsx.analytics.DownloadAnalyticsRepository; +import org.eclipse.openvsx.analytics.DownloadEvent; +import org.eclipse.openvsx.cache.CacheService; +import org.eclipse.openvsx.entities.DownloadIngestion; +import org.eclipse.openvsx.entities.Extension; +import org.eclipse.openvsx.entities.FileResource; +import org.eclipse.openvsx.repositories.RepositoryService; +import org.eclipse.openvsx.search.SearchUtilService; + +@Component +public class DownloadIngestionProcessor { + + protected final Logger logger = LoggerFactory.getLogger(DownloadIngestionProcessor.class); + + private final EntityManager entityManager; + private final RepositoryService repositories; + private final CacheService cache; + private final SearchUtilService search; + private final ObservationRegistry observations; + private final ObjectProvider analyticsRepository; + private final DownloadIngestionMetrics metrics; + + private final Cache resolutionCache = Caffeine.newBuilder() + .maximumSize(65_536) + .expireAfterWrite(Duration.ofHours(24)) + .build(); + + public DownloadIngestionProcessor( + EntityManager entityManager, + RepositoryService repositories, + CacheService cache, + SearchUtilService search, + ObservationRegistry observations, + ObjectProvider analyticsRepository, + DownloadIngestionMetrics metrics + ) { + this.entityManager = entityManager; + this.repositories = repositories; + this.cache = cache; + this.search = search; + this.observations = observations; + this.analyticsRepository = analyticsRepository; + this.metrics = metrics; + } + + /** + * A download-relevant snapshot of the extension version a vsix file belongs to. + */ + public record ResolvedExtension( + long extensionId, + long extensionVersionId, + String namespace, + String extensionName, + String version, + String targetPlatform + ) {} + + /** + * Processes one log file's download records: resolves vsix filenames to extension versions, + * aggregates them into hourly {@link DownloadEvent}s and, in a single transaction, saves the + * events, increments the extension download counters and writes the download ingestion entry. + * Returns the extensions whose counters changed, for cache eviction and search updates. + */ + @Transactional + public List process( + String storageType, + String fileName, + LocalDateTime processedOn, + int executionTime, + List records + ) { + return Observation.createNotStarted("DownloadIngestionProcessor#process", observations).observe(() -> { + var resolved = resolveExtensions(storageType, records); + var events = aggregate(records, resolved); + if (!events.isEmpty()) { + analyticsRepository.ifAvailable(repository -> repository.save(events)); + } + + var extensionDownloads = events.stream().collect( + Collectors.groupingBy(DownloadEvent::extensionId, Collectors.summingInt(DownloadEvent::count))); + var extensions = extensionDownloads.isEmpty() + ? List.of() + : increaseDownloadCounts(extensionDownloads); + persistIngestion(fileName, storageType, processedOn, executionTime, true); + + metrics.recordLoaded(events.size(), events.stream().mapToInt(DownloadEvent::count).sum()); + records.stream().map(RawDownloadRecord::time).max(Instant::compareTo).ifPresent( + latest -> metrics.recordExtractLag(Duration.between(latest, Instant.now()))); + return extensions; + }); + } + + /** + * Records a single request-path download of a file that no {@link DownloadRecordSource} + * covers. Client IP and user agent are taken from the current HTTP request, if any. The + * event save participates in the caller's transaction, so it commits atomically with the + * download counter. + */ + public void captureDownload(FileResource resource) { + analyticsRepository.ifAvailable(repository -> { + var extVersion = resource.getExtension(); + if (extVersion == null) { + logger.warn("no extension version found for download {}, skipping", resource.getName()); + return; + } + + var extension = extVersion.getExtension(); + var request = currentRequest(); + var userAgent = request != null ? StringUtils.trimToNull(request.getHeader("User-Agent")) : null; + var event = new DownloadEvent( + Instant.now(), + extension.getId(), + extVersion.getId(), + extension.getNamespace().getName(), + extension.getName(), + extVersion.getVersion(), + extVersion.getTargetPlatform(), + // no country information is available on the request path + null, + clientIp(request), + userAgent, + 1); + repository.save(List.of(event)); + metrics.recordLoaded(1, 1); + }); + } + + private @Nullable HttpServletRequest currentRequest() { + return RequestContextHolder.getRequestAttributes() instanceof ServletRequestAttributes attributes + ? attributes.getRequest() + : null; + } + + private @Nullable String clientIp(@Nullable HttpServletRequest request) { + if (request == null) { + return null; + } + + var forwardedFor = request.getHeader("X-Forwarded-For"); + if (StringUtils.isNotBlank(forwardedFor)) { + return forwardedFor.split(",")[0].trim(); + } + + return request.getRemoteAddr(); + } + + private Map resolveExtensions(String storageType, List records) { + var filenames = records.stream().map(RawDownloadRecord::vsixFilename).distinct().sorted().toList(); + var resolved = new HashMap(); + var misses = new ArrayList(); + for (var filename : filenames) { + var cached = resolutionCache.getIfPresent(cacheKey(storageType, filename)); + if (cached != null) { + resolved.put(filename, cached); + } else { + misses.add(filename); + } + } + + if (!misses.isEmpty()) { + for (var resource : repositories.findDownloadsByStorageTypeAndName(storageType, misses)) { + var extVersion = resource.getExtension(); + if (extVersion == null) { + logger.warn("no extension version found for download {}, skipping", resource.getName()); + continue; + } + + var extension = extVersion.getExtension(); + var entry = new ResolvedExtension( + extension.getId(), + extVersion.getId(), + extension.getNamespace().getName(), + extension.getName(), + extVersion.getVersion(), + extVersion.getTargetPlatform()); + var filename = resource.getName().toUpperCase(); + resolved.put(filename, entry); + resolutionCache.put(cacheKey(storageType, filename), entry); + } + } + + return resolved; + } + + private String cacheKey(String storageType, String filename) { + return storageType + '|' + filename; + } + + private List aggregate(List records, Map resolved) { + var counts = new LinkedHashMap(); + var skipped = 0; + for (var record : records) { + var extension = resolved.get(record.vsixFilename()); + if (extension == null) { + skipped++; + continue; + } + + var key = new EventKey( + record.time().truncatedTo(ChronoUnit.HOURS), + extension, + CountryCodes.toIsoCode(record.country()), + record.ip(), + record.rawUserAgent()); + counts.merge(key, 1, Integer::sum); + } + if (skipped > 0) { + logger.warn("skipped {} download records referring to unknown vsix files", skipped); + } + + return counts.entrySet().stream() + .map( + entry -> new DownloadEvent( + entry.getKey().time(), + entry.getKey().extension().extensionId(), + entry.getKey().extension().extensionVersionId(), + entry.getKey().extension().namespace(), + entry.getKey().extension().extensionName(), + entry.getKey().extension().version(), + entry.getKey().extension().targetPlatform(), + entry.getKey().country(), + entry.getKey().ip(), + entry.getKey().userAgent(), + entry.getValue())) + .toList(); + } + + private record EventKey( + Instant time, + ResolvedExtension extension, + @Nullable String country, + @Nullable String ip, + @Nullable String userAgent + ) {} + + @Transactional + public void persistIngestion( + String name, + String storageType, + LocalDateTime processedOn, + int executionTime, + boolean success + ) { + Observation.createNotStarted("DownloadIngestionProcessor#persistIngestion", observations).observe(() -> { + var processedItem = new DownloadIngestion(); + processedItem.setName(name); + processedItem.setStorageType(storageType); + processedItem.setProcessedOn(processedOn); + processedItem.setExecutionTime(executionTime); + processedItem.setSuccess(success); + entityManager.persist(processedItem); + }); + } + + @Transactional + public List increaseDownloadCounts(Map extensionDownloads) { + return Observation.createNotStarted("DownloadIngestionProcessor#increaseDownloadCounts", observations) + .observe(() -> { + var extensions = repositories.findExtensions(extensionDownloads.keySet()).toList(); + extensions.forEach(extension -> { + var downloads = extensionDownloads.get(extension.getId()); + extension.setDownloadCount(extension.getDownloadCount() + downloads); + }); + + return extensions; + }); + } + + @Transactional // needs transaction for lazy-loading versions + public void evictCaches(Extension extension) { + Observation.createNotStarted("DownloadIngestionProcessor#evictCaches", observations).observe(() -> { + var mergedExtension = entityManager.merge(extension); + cache.evictExtensionJsons(mergedExtension); + cache.evictLatestExtensionVersion(mergedExtension); + }); + } + + public void updateSearchEntries(List extensions) { + Observation.createNotStarted("DownloadIngestionProcessor#updateSearchEntries", observations).observe(() -> { + logger.info("[DownloadIngestionProcessor] >> updateSearchEntries"); + var activeExtensions = extensions.stream() + .filter(Extension::isActive) + .collect(Collectors.toList()); + + logger.info("[DownloadIngestionProcessor] total active extensions: {}", activeExtensions.size()); + var parts = Lists.partition(activeExtensions, 100); + logger.info("[DownloadIngestionProcessor] partitions: {} | partition size: 100", parts.size()); + + parts.forEach(search::updateSearchEntriesAsync); + logger.info("[DownloadIngestionProcessor] << updateSearchEntries"); + }); + } + + public List succeededIngestions(String storageType, List blobNames) { + return Observation.createNotStarted("DownloadIngestionProcessor#succeededIngestions", observations).observe( + () -> repositories + .findAllSucceededDownloadIngestionsByStorageTypeAndNameIn(storageType, blobNames)); + } + + public List failedIngestions(String storageType, List blobNames) { + return Observation.createNotStarted("DownloadIngestionProcessor#failedIngestions", observations).observe( + () -> repositories + .findAllFailedDownloadIngestionsByStorageTypeAndNameIn(storageType, blobNames)); + } +} diff --git a/server/src/main/java/org/eclipse/openvsx/analytics/ingestion/DownloadIngestionRunner.java b/server/src/main/java/org/eclipse/openvsx/analytics/ingestion/DownloadIngestionRunner.java new file mode 100644 index 000000000..026268b8c --- /dev/null +++ b/server/src/main/java/org/eclipse/openvsx/analytics/ingestion/DownloadIngestionRunner.java @@ -0,0 +1,145 @@ +/****************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation. + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * https://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + *****************************************************************************/ +package org.eclipse.openvsx.analytics.ingestion; + +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; +import org.springframework.util.StopWatch; + +import org.eclipse.openvsx.entities.Extension; +import org.eclipse.openvsx.settings.SettingsService; + +/** + * Drives the ingestion of one {@link DownloadRecordSource}: skips already-processed and + * previously failed items (idempotency and dead-lettering via the ingestion entries), reads and + * processes the remaining items within a time budget, cleans up successful ones and records + * failures. + */ +@Component +public class DownloadIngestionRunner { + + private static final int TIME_BUDGET_MINUTES = 50; + + protected final Logger logger = LoggerFactory.getLogger(DownloadIngestionRunner.class); + + private final SettingsService settings; + private final DownloadIngestionProcessor processor; + + public DownloadIngestionRunner(SettingsService settings, DownloadIngestionProcessor processor) { + this.settings = settings; + this.processor = processor; + } + + public void run(DownloadRecordSource source) { + var storageType = source.getStorageType(); + if (settings.isReadOnly()) { + logger.info("registry is in read-only mode, skipping {} ingestion", storageType); + return; + } + + logger.info(">> ingesting downloads from {}", storageType); + var maxExecutionTime = LocalDateTime.now().plusMinutes(TIME_BUDGET_MINUTES); + var stopWatch = new StopWatch(); + var allUpdatedExtensions = new HashMap(); + + try { + var batches = source.listBatches(); + batches : while (batches.hasNext()) { + for (var name : itemsToProcess(source, batches.next())) { + var processedOn = LocalDateTime.now(); + + if (processedOn.isAfter(maxExecutionTime)) { + logger.info( + "could not ingest all {} items within the time budget, the rest is picked up by the next run", + storageType); + break batches; + } + + if (settings.isReadOnly()) { + logger.info("registry is in read-only mode, stopping {} ingestion", storageType); + break batches; + } + + var success = false; + stopWatch.start(); + List records = null; + try { + records = source.read(name); + } catch (Exception e) { + logger.error("failed to read item: {}", name, e); + } finally { + stopWatch.stop(); + } + + var executionTime = (int) stopWatch.lastTaskInfo().getTimeMillis(); + if (records != null) { + try { + // saves analytics events, increments download counters and writes the + // download ingestion entry in one transaction + var updatedExtensions = processor + .process(storageType, name, processedOn, executionTime, records); + updatedExtensions + .forEach(extension -> allUpdatedExtensions.put(extension.getId(), extension)); + success = true; + } catch (Exception e) { + logger.error("failed to process item: {}", name, e); + } + } + + if (success) { + source.finish(name); + } else { + processor.persistIngestion(name, storageType, processedOn, executionTime, false); + } + } + } + } finally { + // evict caches and update search entries for all updated extensions + allUpdatedExtensions.values().forEach(processor::evictCaches); + processor.updateSearchEntries(allUpdatedExtensions.values().stream().toList()); + } + + logger.info("<< ingesting downloads from {}", storageType); + } + + /** + * Removes already-processed items (cleaning them up along the way) and previously failed + * ones (kept for analysis) from a batch. + */ + private List itemsToProcess(DownloadRecordSource source, List batch) { + var names = new ArrayList<>(batch); + + var succeeded = processor.succeededIngestions(source.getStorageType(), names); + succeeded.forEach(source::finish); + if (!succeeded.isEmpty()) { + logger.info("cleaning up already ingested items:"); + succeeded.forEach(item -> logger.info(" - {}", item)); + } + names.removeAll(succeeded); + + var failed = processor.failedIngestions(source.getStorageType(), names); + if (!failed.isEmpty()) { + logger.info("skipping previously failed items:"); + failed.forEach(item -> logger.info(" - {}", item)); + } + names.removeAll(failed); + + return names; + } +} diff --git a/server/src/main/java/org/eclipse/openvsx/analytics/ingestion/DownloadRecordSource.java b/server/src/main/java/org/eclipse/openvsx/analytics/ingestion/DownloadRecordSource.java new file mode 100644 index 000000000..564b8e3b6 --- /dev/null +++ b/server/src/main/java/org/eclipse/openvsx/analytics/ingestion/DownloadRecordSource.java @@ -0,0 +1,69 @@ +/****************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation. + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * https://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + *****************************************************************************/ +package org.eclipse.openvsx.analytics.ingestion; + +import java.io.IOException; +import java.util.Iterator; +import java.util.List; + +import org.eclipse.openvsx.entities.FileResource; + +/** + * Produces raw download records from some backing store, typically CDN or storage access logs. + * A source knows how to list, read and clean up its named items (e.g. log files) and declares its + * storage type, coverage, enablement and recurring schedule; the {@link DownloadIngestionRunner} + * drives the ingestion and owns idempotency, failure handling and the time budget. + *

+ * Sources are conditional beans that only exist when their configuration is present. + */ +public interface DownloadRecordSource { + + /** + * The storage type this source ingests downloads for, see {@code FileResource.STORAGE_*}. + */ + String getStorageType(); + + /** + * Whether this source is fully configured and ready to ingest. A source bean may exist (its + * primary property is set) while still being disabled because a dependent service is not. + */ + boolean isEnabled(); + + /** + * The cron expression on which this source's ingestion job should recur, in UTC. + */ + String getCronSchedule(); + + /** + * Returns whether downloads of the given file are counted by this source. Download requests + * for covered files are not counted on the request path, otherwise they would be counted + * twice. + */ + boolean covers(FileResource resource); + + /** + * Lists the names of the items that are candidates for ingestion, in pages. + */ + Iterator> listBatches(); + + /** + * Reads one item and returns the downloads it contains. + */ + List read(String name) throws IOException; + + /** + * Cleans up one successfully processed (or previously processed) item, e.g. by deleting or + * archiving it. + */ + void finish(String name); +} diff --git a/server/src/main/java/org/eclipse/openvsx/analytics/ingestion/RawDownloadRecord.java b/server/src/main/java/org/eclipse/openvsx/analytics/ingestion/RawDownloadRecord.java new file mode 100644 index 000000000..35c02c450 --- /dev/null +++ b/server/src/main/java/org/eclipse/openvsx/analytics/ingestion/RawDownloadRecord.java @@ -0,0 +1,39 @@ +/****************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation. + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * https://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + *****************************************************************************/ +package org.eclipse.openvsx.analytics.ingestion; + +import java.time.Instant; +import java.util.Objects; + +import org.jspecify.annotations.Nullable; + +/** + * A single extension package download as produced by a {@link DownloadRecordSource}, carrying + * the client IP and raw user agent as found in the source. The vsix filename is upper-cased; + * country values are normalized during ingestion. + */ +public record RawDownloadRecord( + Instant time, + String vsixFilename, + @Nullable String country, + @Nullable String ip, + @Nullable String rawUserAgent +) { + public RawDownloadRecord { + Objects.requireNonNull(time, "time must not be null"); + Objects.requireNonNull(vsixFilename, "vsixFilename must not be null"); + if (vsixFilename.isBlank()) { + throw new IllegalArgumentException("vsixFilename must not be blank"); + } + } +} diff --git a/server/src/main/java/org/eclipse/openvsx/analytics/ingestion/aws/AccessLogRecord.java b/server/src/main/java/org/eclipse/openvsx/analytics/ingestion/aws/AccessLogRecord.java new file mode 100644 index 000000000..1935c2cfd --- /dev/null +++ b/server/src/main/java/org/eclipse/openvsx/analytics/ingestion/aws/AccessLogRecord.java @@ -0,0 +1,62 @@ +/****************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation. + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * https://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + *****************************************************************************/ +package org.eclipse.openvsx.analytics.ingestion.aws; + +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.util.Objects; + +import org.jspecify.annotations.NonNull; +import org.jspecify.annotations.Nullable; +import org.springframework.web.util.UriUtils; + +import org.eclipse.openvsx.analytics.ingestion.RawDownloadRecord; + +/** + * A parsed access log line. Timestamp, country, client IP and user agent are optional: not + * every log format carries them. Values are kept as found in the log; normalization happens + * during ingestion. + */ +record AccessLogRecord( + @NonNull String method, + int status, + @NonNull String url, + @Nullable Instant timestamp, + @Nullable String country, + @Nullable String ip, + @Nullable String userAgent +) { + /** + * Turns this log line into a download record, or returns {@code null} if it is not a + * successful extension package download. Lines without a timestamp fall back to + * {@code fallbackTime} (typically the log file's date). + */ + @Nullable + RawDownloadRecord toDownloadRecord(Instant fallbackTime) { + if (!isVsixDownload()) { + return null; + } + + var uriComponents = url.split("/"); + var vsixFilename = UriUtils.decode(uriComponents[uriComponents.length - 1], StandardCharsets.UTF_8) + .toUpperCase(); + var time = timestamp != null + ? timestamp + : Objects.requireNonNull(fallbackTime, "fallbackTime must not be null"); + return new RawDownloadRecord(time, vsixFilename, country, ip, userAgent); + } + + private boolean isVsixDownload() { + return method.equalsIgnoreCase("GET") && status == 200 && url.endsWith(".vsix"); + } +} diff --git a/server/src/main/java/org/eclipse/openvsx/analytics/ingestion/aws/AwsDownloadRecordSource.java b/server/src/main/java/org/eclipse/openvsx/analytics/ingestion/aws/AwsDownloadRecordSource.java new file mode 100644 index 000000000..d29270e21 --- /dev/null +++ b/server/src/main/java/org/eclipse/openvsx/analytics/ingestion/aws/AwsDownloadRecordSource.java @@ -0,0 +1,218 @@ +/******************************************************************************** + * Copyright (c) 2025 Eclipse Foundation and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + ********************************************************************************/ +package org.eclipse.openvsx.analytics.ingestion.aws; + +import java.io.*; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.StandardCopyOption; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.zip.GZIPInputStream; + +import jakarta.annotation.PostConstruct; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Component; +import software.amazon.awssdk.core.sync.ResponseTransformer; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.*; + +import org.eclipse.openvsx.analytics.ingestion.DownloadIngestionMetrics; +import org.eclipse.openvsx.analytics.ingestion.DownloadRecordSource; +import org.eclipse.openvsx.analytics.ingestion.RawDownloadRecord; +import org.eclipse.openvsx.entities.FileResource; +import org.eclipse.openvsx.storage.AwsStorageService; +import org.eclipse.openvsx.util.TempFile; + +/** + * Reads downloads from access logs in an Amazon S3 bucket. + *

+ * The following log file formats are supported: + *

    + *
  • cloudfront
    • + *
  • fastly
    • + *
+ *

+ * See + *

+ */ +@Component +@ConditionalOnProperty(name = "ovsx.logs.aws.bucket") +public class AwsDownloadRecordSource implements DownloadRecordSource { + + private static final String LOG_LOCATION_PREFIX = "AWSLogs/"; + + private final AwsStorageService awsStorageService; + private final DownloadIngestionMetrics metrics; + + @Value("${ovsx.logs.aws.bucket:}") + String bucket; + + @Value("${ovsx.logs.aws.log-location-prefix:" + LOG_LOCATION_PREFIX + "}") + String logLocationPrefix; + + @Value("${ovsx.logs.aws.format:cloudfront}") + String logFormat; + + @Value("${ovsx.logs.aws.max-keys:100}") + int maxKeys; + + @Value("${ovsx.logs.aws.archive-prefix:}") + String archivePrefix; + + @Value("${ovsx.logs.aws.cron:0 10 * * * *}") + String cronSchedule; + + LogFileParser logFileParser; + + public AwsDownloadRecordSource(AwsStorageService awsStorageService, DownloadIngestionMetrics metrics) { + this.awsStorageService = awsStorageService; + this.metrics = metrics; + } + + @PostConstruct + public void initialize() { + logFileParser = switch (logFormat.toLowerCase()) { + case "cloudfront" -> new CloudFrontLogFileParser(); + case "fastly" -> new FastlyLogFileParser(); + default -> throw new IllegalArgumentException("unsupported log file format '" + logFormat + "'"); + }; + } + + /** + * Indicates whether this source is enabled by application config. + */ + @Override + public boolean isEnabled() { + return !StringUtils.isEmpty(bucket) && awsStorageService.isEnabled(); + } + + @Override + public String getStorageType() { + return FileResource.STORAGE_AWS; + } + + @Override + public String getCronSchedule() { + return cronSchedule; + } + + @Override + public boolean covers(FileResource resource) { + return FileResource.STORAGE_AWS.equals(resource.getStorageType()) && isEnabled(); + } + + @Override + public Iterator> listBatches() { + return new Iterator<>() { + private String continuationToken; + private boolean done; + + @Override + public boolean hasNext() { + return !done; + } + + @Override + public List next() { + var response = listObjects(continuationToken); + continuationToken = response.isTruncated() ? response.nextContinuationToken() : null; + done = continuationToken == null; + return response.contents().stream() + .map(S3Object::key) + .filter(key -> key.endsWith(".gz")) + .toList(); + } + }; + } + + @Override + public List read(String name) throws IOException { + var inputStream = getS3Client().getObject( + GetObjectRequest.builder() + .bucket(bucket) + .key(name) + .build(), + ResponseTransformer.toInputStream()); + + // records without their own timestamp fall back to the log file's date + var lastModified = inputStream.response().lastModified(); + var fallbackTime = lastModified != null ? lastModified : Instant.now(); + + try (var downloadsTempFile = new TempFile("aws-downloads-", ".gz")) { + Files.copy(inputStream, downloadsTempFile.getPath(), StandardCopyOption.REPLACE_EXISTING); + try ( + var fileStream = new FileInputStream(downloadsTempFile.getPath().toFile()); + var gzipStream = new GZIPInputStream(fileStream); + var reader = new BufferedReader(new InputStreamReader(gzipStream, StandardCharsets.UTF_8)); + ) { + var records = new ArrayList(); + var totalLines = 0; + var skippedLines = 0; + var lines = reader.lines().iterator(); + while (lines.hasNext()) { + totalLines++; + var record = logFileParser.parse(lines.next()); + if (record == null) { + skippedLines++; + continue; + } + + var download = record.toDownloadRecord(fallbackTime); + if (download != null) { + records.add(download); + } + } + metrics.recordParsedLines(totalLines, skippedLines); + return records; + } + } + } + + /** + * Deletes a processed log file, or moves it below the configured + * {@code ovsx.logs.aws.archive-prefix} instead. + */ + @Override + public void finish(String name) { + if (!StringUtils.isEmpty(archivePrefix)) { + getS3Client().copyObject( + CopyObjectRequest.builder() + .sourceBucket(bucket) + .sourceKey(name) + .destinationBucket(bucket) + .destinationKey(archivePrefix + name) + .build()); + } + getS3Client().deleteObject(DeleteObjectRequest.builder().bucket(bucket).key(name).build()); + } + + private S3Client getS3Client() { + return awsStorageService.getS3Client(); + } + + private ListObjectsV2Response listObjects(String continuationToken) { + var builder = ListObjectsV2Request.builder().bucket(bucket).maxKeys(maxKeys).prefix(logLocationPrefix); + + if (continuationToken != null) { + builder.continuationToken(continuationToken); + } + + return getS3Client().listObjectsV2(builder.build()); + } +} diff --git a/server/src/main/java/org/eclipse/openvsx/analytics/ingestion/aws/CloudFrontLogFileParser.java b/server/src/main/java/org/eclipse/openvsx/analytics/ingestion/aws/CloudFrontLogFileParser.java new file mode 100644 index 000000000..602a474ab --- /dev/null +++ b/server/src/main/java/org/eclipse/openvsx/analytics/ingestion/aws/CloudFrontLogFileParser.java @@ -0,0 +1,80 @@ +/****************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation. + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * https://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + *****************************************************************************/ +package org.eclipse.openvsx.analytics.ingestion.aws; + +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalTime; +import java.time.ZoneOffset; + +import org.jspecify.annotations.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.web.util.UriUtils; + +class CloudFrontLogFileParser implements LogFileParser { + private final Logger logger = LoggerFactory.getLogger(CloudFrontLogFileParser.class); + + private static final String EMPTY_FIELD = "-"; + + @Override + public @Nullable AccessLogRecord parse(String line) { + if (line.startsWith("#")) { + return null; + } + + // Format: + // date time x-edge-location sc-bytes c-ip cs-method cs(Host) cs-uri-stem sc-status cs(Referer) cs(User-Agent) cs-uri-query cs(Cookie) x-edge-result-type x-edge-request-id x-host-header cs-protocol cs-bytes time-taken x-forwarded-for ssl-protocol ssl-cipher x-edge-response-result-type cs-protocol-version fle-status fle-encrypted-fields c-port time-to-first-byte x-edge-detailed-result-type sc-content-type sc-content-len sc-range-start sc-range-end + var components = line.split("[ \t]+"); + if (components.length < 11) { + logger.warn("skipping malformed log line '{}'", line); + return null; + } + + try { + // CloudFront standard logs carry no country information + return new AccessLogRecord( + components[5], + Integer.parseInt(components[8]), + components[7], + parseTimestamp(components[0], components[1]), + null, + parseField(components[4]), + parseUserAgent(components[10])); + } catch (RuntimeException e) { + logger.warn("skipping malformed log line '{}'", line, e); + return null; + } + } + + private @Nullable String parseField(String value) { + return EMPTY_FIELD.equals(value) ? null : value; + } + + private @Nullable Instant parseTimestamp(String date, String time) { + if (EMPTY_FIELD.equals(date) || EMPTY_FIELD.equals(time)) { + return null; + } + + return LocalDate.parse(date).atTime(LocalTime.parse(time)).toInstant(ZoneOffset.UTC); + } + + private @Nullable String parseUserAgent(String userAgent) { + if (EMPTY_FIELD.equals(userAgent)) { + return null; + } + + return UriUtils.decode(userAgent, StandardCharsets.UTF_8); + } +} diff --git a/server/src/main/java/org/eclipse/openvsx/analytics/ingestion/aws/FastlyLogFileParser.java b/server/src/main/java/org/eclipse/openvsx/analytics/ingestion/aws/FastlyLogFileParser.java new file mode 100644 index 000000000..c953004d1 --- /dev/null +++ b/server/src/main/java/org/eclipse/openvsx/analytics/ingestion/aws/FastlyLogFileParser.java @@ -0,0 +1,107 @@ +/****************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation. + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * https://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + *****************************************************************************/ +package org.eclipse.openvsx.analytics.ingestion.aws; + +import java.time.Instant; +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.util.Locale; + +import org.apache.commons.lang3.StringUtils; +import org.jspecify.annotations.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import tools.jackson.core.JacksonException; +import tools.jackson.core.JsonParser; +import tools.jackson.databind.DeserializationContext; +import tools.jackson.databind.JsonNode; +import tools.jackson.databind.deser.std.StdDeserializer; +import tools.jackson.databind.json.JsonMapper; +import tools.jackson.databind.module.SimpleModule; + +class FastlyLogFileParser implements LogFileParser { + private final Logger logger = LoggerFactory.getLogger(FastlyLogFileParser.class); + + private final JsonMapper mapper; + + public FastlyLogFileParser() { + var module = new SimpleModule(); + module.addDeserializer(AccessLogRecord.class, new AccessLogRecordDeserializer()); + this.mapper = JsonMapper.builder().addModule(module).build(); + } + + @Override + public @Nullable AccessLogRecord parse(String line) { + try { + var jsonStartIndex = line.indexOf("{"); + if (jsonStartIndex != -1) { + return mapper.readValue(line.substring(jsonStartIndex), AccessLogRecord.class); + } else { + return null; + } + } catch (JacksonException | IllegalArgumentException | NullPointerException ex) { + logger.error("could not parse log line '{}'", line, ex); + return null; + } + } +} + +class AccessLogRecordDeserializer extends StdDeserializer { + + // Fastly emits RFC 822 zone offsets, e.g. 2026-02-09T04:20:50+0000 + private static final DateTimeFormatter FASTLY_TIMESTAMP = DateTimeFormatter + .ofPattern("yyyy-MM-dd'T'HH:mm:ssZ", Locale.ROOT); + + public AccessLogRecordDeserializer() { + super(AccessLogRecord.class); + } + + @Override + public AccessLogRecord deserialize(JsonParser jp, DeserializationContext ctxt) throws JacksonException { + JsonNode node = ctxt.readTree(jp); + String operation = node.get("request_method").asString(); + int status = (Integer) node.get("response_status").numberValue(); + String url = node.get("url").asString(); + return new AccessLogRecord( + operation, + status, + url, + parseTimestamp(optionalString(node, "timestamp")), + optionalString(node, "geo_country"), + optionalString(node, "client_ip"), + optionalString(node, "request_user_agent")); + } + + private @Nullable String optionalString(JsonNode node, String field) { + var value = node.path(field); + return value.isString() ? StringUtils.trimToNull(value.asString()) : null; + } + + private @Nullable Instant parseTimestamp(@Nullable String timestamp) { + if (timestamp == null) { + return null; + } + + try { + return OffsetDateTime.parse(timestamp, FASTLY_TIMESTAMP).toInstant(); + } catch (DateTimeParseException e) { + // fall through to the ISO format, e.g. 2026-02-09T04:20:50+00:00 + } + try { + return OffsetDateTime.parse(timestamp).toInstant(); + } catch (DateTimeParseException e) { + return null; + } + } +} diff --git a/server/src/main/java/org/eclipse/openvsx/storage/log/LogFileParser.java b/server/src/main/java/org/eclipse/openvsx/analytics/ingestion/aws/LogFileParser.java similarity index 83% rename from server/src/main/java/org/eclipse/openvsx/storage/log/LogFileParser.java rename to server/src/main/java/org/eclipse/openvsx/analytics/ingestion/aws/LogFileParser.java index f426b0041..bfe6c7c37 100644 --- a/server/src/main/java/org/eclipse/openvsx/storage/log/LogFileParser.java +++ b/server/src/main/java/org/eclipse/openvsx/analytics/ingestion/aws/LogFileParser.java @@ -10,11 +10,11 @@ * * SPDX-License-Identifier: EPL-2.0 *****************************************************************************/ -package org.eclipse.openvsx.storage.log; +package org.eclipse.openvsx.analytics.ingestion.aws; import org.jspecify.annotations.Nullable; -public interface LogFileParser { +interface LogFileParser { @Nullable - LogRecord parse(String line); + AccessLogRecord parse(String line); } diff --git a/server/src/main/java/org/eclipse/openvsx/storage/log/AzureDownloadCountHandler.java b/server/src/main/java/org/eclipse/openvsx/analytics/ingestion/azure/AzureDownloadRecordSource.java similarity index 58% rename from server/src/main/java/org/eclipse/openvsx/storage/log/AzureDownloadCountHandler.java rename to server/src/main/java/org/eclipse/openvsx/analytics/ingestion/azure/AzureDownloadRecordSource.java index bc6e989d6..c788e3ca4 100644 --- a/server/src/main/java/org/eclipse/openvsx/storage/log/AzureDownloadCountHandler.java +++ b/server/src/main/java/org/eclipse/openvsx/analytics/ingestion/azure/AzureDownloadRecordSource.java @@ -7,22 +7,20 @@ * * SPDX-License-Identifier: EPL-2.0 ********************************************************************************/ -package org.eclipse.openvsx.storage.log; +package org.eclipse.openvsx.analytics.ingestion.azure; import java.io.IOException; import java.net.URI; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.time.Duration; -import java.time.LocalDateTime; +import java.time.Instant; +import java.time.format.DateTimeParseException; import java.util.ArrayList; -import java.util.HashMap; +import java.util.Iterator; import java.util.List; -import java.util.Map; import java.util.regex.Pattern; -import com.azure.core.http.rest.PagedIterable; -import com.azure.core.http.rest.PagedResponse; import com.azure.storage.blob.BlobContainerClient; import com.azure.storage.blob.BlobContainerClientBuilder; import com.azure.storage.blob.models.BlobItem; @@ -30,36 +28,35 @@ import com.azure.storage.blob.models.BlobStorageException; import com.azure.storage.blob.models.ListBlobsOptions; import org.apache.commons.lang3.StringUtils; -import org.jobrunr.jobs.annotations.Job; -import org.jobrunr.jobs.lambdas.JobRequestHandler; +import org.jspecify.annotations.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.http.HttpStatus; import org.springframework.stereotype.Component; -import org.springframework.util.StopWatch; import org.springframework.web.util.UriUtils; import tools.jackson.databind.JsonNode; import tools.jackson.databind.json.JsonMapper; +import org.eclipse.openvsx.analytics.ingestion.DownloadIngestionMetrics; +import org.eclipse.openvsx.analytics.ingestion.DownloadRecordSource; +import org.eclipse.openvsx.analytics.ingestion.RawDownloadRecord; import org.eclipse.openvsx.entities.FileResource; -import org.eclipse.openvsx.migration.HandlerJobRequest; -import org.eclipse.openvsx.settings.SettingsService; import org.eclipse.openvsx.util.TempFile; import static org.eclipse.openvsx.storage.AzureBlobStorageService.AZURE_USER_AGENT; /** - * Pulls logs from Azure Blob Storage, extracts downloads from the logs - * and updates download counts in the database. + * Reads downloads from Azure Blob Storage access logs. */ @Component -public class AzureDownloadCountHandler implements JobRequestHandler> { +@ConditionalOnProperty(name = "ovsx.logs.azure.service-endpoint") +public class AzureDownloadRecordSource implements DownloadRecordSource { - protected final Logger logger = LoggerFactory.getLogger(AzureDownloadCountHandler.class); + protected final Logger logger = LoggerFactory.getLogger(AzureDownloadRecordSource.class); - private final SettingsService settings; - private final DownloadCountProcessor processor; + private final DownloadIngestionMetrics metrics; private final JsonMapper jsonMapper; private BlobContainerClient containerClient; private Pattern blobItemNamePattern; @@ -82,143 +79,70 @@ public class AzureDownloadCountHandler implements JobRequestHandler jobRequest) throws Exception { - if (!isEnabled()) { - return; - } - - if (settings.isReadOnly()) { - logger.info("[AzureDownloadCountService] registry is in read-only mode, skipping job"); - return; - } - - logger.info("[AzureDownloadCountService] >> updateDownloadCounts"); - - var maxExecutionTime = LocalDateTime.now().plusMinutes(50); - var blobs = listBlobs(); - var iterableByPage = blobs.iterableByPage(); - - var stopWatch = new StopWatch(); - while (iterableByPage != null) { - PagedResponse response = null; - var iterator = iterableByPage.iterator(); - if (iterator.hasNext()) { - response = iterator.next(); - if (!processResponse(response, stopWatch, maxExecutionTime)) { - break; - } - } - - var continuationToken = response != null ? response.getContinuationToken() : ""; - iterableByPage = !StringUtils.isEmpty(continuationToken) ? blobs.iterableByPage(continuationToken) : null; - } - - logger.info("[AzureDownloadCountService] << updateDownloadCounts"); + @Override + public String getStorageType() { + return FileResource.STORAGE_AZURE; } - private boolean processResponse( - PagedResponse response, - StopWatch stopWatch, - LocalDateTime maxExecutionTime - ) { - var blobNames = getBlobNames(response.getValue()); - var processedItems = processor.processedItems(FileResource.STORAGE_AZURE, blobNames); - processedItems.forEach(this::deleteBlob); - blobNames.removeAll(processedItems); - for (var name : blobNames) { - if (LocalDateTime.now().isAfter(maxExecutionTime)) { - var nextJobRunTime = LocalDateTime.now().plusHours(1).withMinute(5); - logger.info( - "Failed to process all download counts within timeslot, next job run is at {}", - nextJobRunTime); - return false; - } - - if (settings.isReadOnly()) { - logger.info("skip processing log files as registry is in read-only mode"); - return false; - } + @Override + public String getCronSchedule() { + return cronSchedule; + } - var processedOn = LocalDateTime.now(); - var success = false; - stopWatch.start(); - try { - var files = processBlobItem(name); - if (!files.isEmpty()) { - var extensionDownloads = processor.processDownloadCounts(FileResource.STORAGE_AZURE, files); - var updatedExtensions = processor.increaseDownloadCounts(extensionDownloads); - updatedExtensions.forEach(processor::evictCaches); - processor.updateSearchEntries(updatedExtensions); - } + @Override + public boolean covers(FileResource resource) { + return FileResource.STORAGE_AZURE.equals(resource.getStorageType()) && isEnabled(); + } - success = true; - } catch (Exception e) { - logger.error("Failed to process BlobItem: {}", name, e); + @Override + public Iterator> listBatches() { + var pages = listBlobs().iterator(); + return new Iterator<>() { + @Override + public boolean hasNext() { + return pages.hasNext(); } - stopWatch.stop(); - var executionTime = (int) stopWatch.lastTaskInfo().getTimeMillis(); - processor.persistProcessedItem(name, FileResource.STORAGE_AZURE, processedOn, executionTime, success); - if (success) { - deleteBlob(name); + @Override + public List next() { + return getBlobNames(pages.next().getValue()); } - } - - return true; + }; } - private void deleteBlob(String blobName) { - try { - getContainerClient().getBlobClient(blobName).delete(); - } catch (BlobStorageException e) { - if (e.getStatusCode() != HttpStatus.NOT_FOUND.value()) { - // 404 indicates that the file is already deleted - // so only throw an exception for other status codes - throw e; - } - } - } - - private Map processBlobItem(String blobName) throws IOException { + @Override + public List read(String name) throws IOException { try ( - var downloadsTempFile = downloadBlobItem(blobName); + var downloadsTempFile = downloadBlobItem(name); var reader = Files.newBufferedReader(downloadsTempFile.getPath()) ) { - var fileCounts = new HashMap(); + // records without their own timestamp fall back to the processing time + var fallbackTime = Instant.now(); + var records = new ArrayList(); + var totalLines = 0; var lines = reader.lines().iterator(); while (lines.hasNext()) { + totalLines++; var line = lines.next(); var node = jsonMapper.readTree(line); String[] pathParams = null; @@ -230,13 +154,74 @@ && isNotOpenVSXUserAgent(node)) { if (pathParams != null && storageBlobContainer.equals(pathParams[1])) { var fileName = UriUtils.decode(pathParams[pathParams.length - 1], StandardCharsets.UTF_8) .toUpperCase(); - fileCounts.merge(fileName, 1, Integer::sum); + // Azure Storage logs carry no country information + records.add( + new RawDownloadRecord( + parseTime(node, fallbackTime), + fileName, + null, + callerIp(node), + userAgent(node))); } } - return fileCounts; + metrics.recordParsedLines(totalLines, 0); + return records; } } + @Override + public void finish(String name) { + try { + getContainerClient().getBlobClient(name).delete(); + } catch (BlobStorageException e) { + if (e.getStatusCode() != HttpStatus.NOT_FOUND.value()) { + // 404 indicates that the file is already deleted + // so only throw an exception for other status codes + throw e; + } + } + } + + private Instant parseTime(JsonNode node, Instant fallbackTime) { + var time = node.path("time"); + if (time.isString()) { + try { + return Instant.parse(time.asString()); + } catch (DateTimeParseException e) { + // fall back to the processing time below + } + } + + return fallbackTime; + } + + private @Nullable String userAgent(JsonNode node) { + var userAgent = node.path("properties").path("userAgentHeader"); + return userAgent.isString() ? StringUtils.trimToNull(userAgent.asString()) : null; + } + + /** + * Azure logs report the caller as {@code ip:port} (or {@code [ipv6]:port}); only the address + * part is kept. + */ + private @Nullable String callerIp(JsonNode node) { + var value = node.path("callerIpAddress"); + if (!value.isString() || StringUtils.isBlank(value.asString())) { + return null; + } + + var address = value.asString().trim(); + if (address.startsWith("[") && address.contains("]")) { + return address.substring(1, address.indexOf(']')); + } + var colon = address.lastIndexOf(':'); + if (colon > -1 && address.indexOf(':') == colon) { + return address.substring(0, colon); + } + + return address; + } + private boolean isGetBlobOperation(JsonNode node) { return node.get("operationName").asString().equals("GetBlob"); } @@ -273,7 +258,7 @@ private List getBlobNames(List items) { return blobNames; } - private PagedIterable listBlobs() { + private Iterable> listBlobs() { var details = new BlobListDetails() .setRetrieveCopy(false) .setRetrieveMetadata(false) @@ -284,7 +269,7 @@ private PagedIterable listBlobs() { .setRetrieveVersions(false); var options = new ListBlobsOptions().setMaxResultsPerPage(100).setDetails(details); - return getContainerClient().listBlobs(options, Duration.ofMinutes(5)); + return getContainerClient().listBlobs(options, Duration.ofMinutes(5)).iterableByPage(); } private BlobContainerClient getContainerClient() { diff --git a/server/src/main/java/org/eclipse/openvsx/analytics/ingestion/jobs/IngestionJobRequest.java b/server/src/main/java/org/eclipse/openvsx/analytics/ingestion/jobs/IngestionJobRequest.java new file mode 100644 index 000000000..8683a95a5 --- /dev/null +++ b/server/src/main/java/org/eclipse/openvsx/analytics/ingestion/jobs/IngestionJobRequest.java @@ -0,0 +1,47 @@ +/****************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation. + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * https://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + *****************************************************************************/ +package org.eclipse.openvsx.analytics.ingestion.jobs; + +import org.jobrunr.jobs.lambdas.JobRequest; +import org.jobrunr.jobs.lambdas.JobRequestHandler; + +/** + * JobRunr request for one storage type's download log ingestion. The {@code storageType} selects + * which {@link org.eclipse.openvsx.analytics.ingestion.DownloadRecordSource} the handler drives, so + * a single generic handler can serve every source. The type bound mirrors {@code HandlerJobRequest} + * (rather than the self-referential {@code MigrationJobRequest}) so JobRunr can serialize the + * recurring job without JSON type resolution recursing. + */ +public class IngestionJobRequest> implements JobRequest { + + private Class handler; + private String storageType; + + // needed for serialization by jobrunr + public IngestionJobRequest() { + } + + public IngestionJobRequest(Class handler, String storageType) { + this.handler = handler; + this.storageType = storageType; + } + + @Override + public Class> getJobRequestHandler() { + return handler; + } + + public String getStorageType() { + return storageType; + } +} diff --git a/server/src/main/java/org/eclipse/openvsx/analytics/ingestion/jobs/LogIngestionJob.java b/server/src/main/java/org/eclipse/openvsx/analytics/ingestion/jobs/LogIngestionJob.java new file mode 100644 index 000000000..7420a7a77 --- /dev/null +++ b/server/src/main/java/org/eclipse/openvsx/analytics/ingestion/jobs/LogIngestionJob.java @@ -0,0 +1,103 @@ +/****************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation. + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * https://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + *****************************************************************************/ +package org.eclipse.openvsx.analytics.ingestion.jobs; + +import java.time.ZoneId; +import java.util.List; + +import org.jobrunr.jobs.annotations.Job; +import org.jobrunr.jobs.lambdas.JobRequestHandler; +import org.jobrunr.scheduling.JobRequestScheduler; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.boot.context.event.ApplicationStartedEvent; +import org.springframework.context.event.EventListener; +import org.springframework.stereotype.Component; + +import org.eclipse.openvsx.analytics.ingestion.DownloadIngestionRunner; +import org.eclipse.openvsx.analytics.ingestion.DownloadRecordSource; +import org.eclipse.openvsx.entities.FileResource; + +/** + * The recurring download log ingestion job. On startup it registers one recurring job per configured + * source, and deletes the job of any storage type whose source is absent or disabled, so JobRunr + * never fires a job it cannot serve. JobRunr then invokes {@link #run} on a single node per tick with + * the storage type to ingest, which is handed to the {@link DownloadIngestionRunner}. Storage-agnostic, + * so one job serves every source. + */ +@Component +public class LogIngestionJob implements JobRequestHandler> { + + private static final List KNOWN_STORAGE_TYPES = List + .of(FileResource.STORAGE_AWS, FileResource.STORAGE_AZURE); + + protected final Logger logger = LoggerFactory.getLogger(LogIngestionJob.class); + + private final ObjectProvider sources; + private final DownloadIngestionRunner runner; + private final JobRequestScheduler scheduler; + + public LogIngestionJob( + ObjectProvider sources, + DownloadIngestionRunner runner, + JobRequestScheduler scheduler + ) { + this.sources = sources; + this.runner = runner; + this.scheduler = scheduler; + } + + @EventListener + public void scheduleJobs(ApplicationStartedEvent event) { + for (var storageType : KNOWN_STORAGE_TYPES) { + var jobId = recurringJobId(storageType); + var source = findEnabledSource(storageType); + if (source == null) { + scheduler.deleteRecurringJob(jobId); + } else { + logger.info("Scheduling {} log ingestion with cron '{}'", storageType, source.getCronSchedule()); + scheduler.scheduleRecurrently( + jobId, + source.getCronSchedule(), + ZoneId.of("UTC"), + new IngestionJobRequest<>(LogIngestionJob.class, storageType)); + } + } + } + + @Override + @Job(name = "Ingest download logs", retries = 0) + public void run(IngestionJobRequest jobRequest) { + var source = findEnabledSource(jobRequest.getStorageType()); + if (source == null) { + logger.warn( + "no enabled download record source for storage type {}, skipping", + jobRequest.getStorageType()); + return; + } + + runner.run(source); + } + + private DownloadRecordSource findEnabledSource(String storageType) { + return sources.stream() + .filter(source -> source.getStorageType().equals(storageType) && source.isEnabled()) + .findFirst() + .orElse(null); + } + + private String recurringJobId(String storageType) { + return "update-" + storageType + "-download-counts"; + } +} diff --git a/server/src/main/java/org/eclipse/openvsx/analytics/timescale/TimescaleDownloadAnalyticsRepository.java b/server/src/main/java/org/eclipse/openvsx/analytics/timescale/TimescaleDownloadAnalyticsRepository.java new file mode 100644 index 000000000..15de4528b --- /dev/null +++ b/server/src/main/java/org/eclipse/openvsx/analytics/timescale/TimescaleDownloadAnalyticsRepository.java @@ -0,0 +1,129 @@ +/****************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation. + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * https://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + *****************************************************************************/ +package org.eclipse.openvsx.analytics.timescale; + +import java.time.OffsetDateTime; +import java.time.ZoneOffset; +import java.util.List; + +import com.google.common.collect.Lists; +import org.jooq.DSLContext; +import org.jooq.Field; +import org.jooq.impl.DSL; + +import org.eclipse.openvsx.analytics.DownloadAnalyticsRepository; +import org.eclipse.openvsx.analytics.DownloadEvent; +import org.eclipse.openvsx.analytics.DownloadSeriesGroupBy; +import org.eclipse.openvsx.analytics.DownloadSeriesInterval; +import org.eclipse.openvsx.analytics.DownloadSeriesRequest; +import org.eclipse.openvsx.analytics.DownloadSeriesRow; + +import static org.eclipse.openvsx.jooq.Tables.DOWNLOAD_EVENT; +import static org.eclipse.openvsx.jooq.Tables.DOWNLOAD_STATS_DAILY; + +/** + * {@link DownloadAnalyticsRepository} backed by TimescaleDB: writes to the download_event + * hypertable and reads from the download_stats_daily continuous aggregate. Queries run through + * the application's transaction-aware {@link DSLContext}, so writes commit atomically with the + * download counter and the ingestion entry. + */ +public class TimescaleDownloadAnalyticsRepository implements DownloadAnalyticsRepository { + + private static final int BATCH_SIZE = 500; + + private final DSLContext dsl; + + public TimescaleDownloadAnalyticsRepository(DSLContext dsl) { + this.dsl = dsl; + } + + @Override + public void save(List events) { + for (var batch : Lists.partition(events, BATCH_SIZE)) { + var insert = dsl.insertInto( + DOWNLOAD_EVENT, + DOWNLOAD_EVENT.TIME, + DOWNLOAD_EVENT.EXTENSION_ID, + DOWNLOAD_EVENT.EXTENSION_VERSION_ID, + DOWNLOAD_EVENT.NAMESPACE, + DOWNLOAD_EVENT.EXTENSION_NAME, + DOWNLOAD_EVENT.VERSION, + DOWNLOAD_EVENT.TARGET_PLATFORM, + DOWNLOAD_EVENT.COUNTRY, + DOWNLOAD_EVENT.IP, + DOWNLOAD_EVENT.USER_AGENT, + DOWNLOAD_EVENT.COUNT); + for (var event : batch) { + insert = insert.values( + OffsetDateTime.ofInstant(event.time(), ZoneOffset.UTC), + event.extensionId(), + event.extensionVersionId(), + event.namespace(), + event.extensionName(), + event.version(), + event.targetPlatform(), + event.country(), + event.ip(), + event.userAgent(), + event.count()); + } + insert.execute(); + } + } + + @Override + public List findSeries(DownloadSeriesRequest request) { + var bucket = bucketField(request.interval()); + var group = groupField(request.groupBy()); + var total = DSL.sum(DOWNLOAD_STATS_DAILY.DOWNLOADS).cast(Long.class); + + List> groupByFields = request.groupBy() == DownloadSeriesGroupBy.NONE + ? List.>of(bucket) + : List.>of(bucket, group); + return dsl.select(bucket, group, total) + .from(DOWNLOAD_STATS_DAILY) + .where( + DOWNLOAD_STATS_DAILY.EXTENSION_ID.in(request.extensionIds()), + DOWNLOAD_STATS_DAILY.DAY + .greaterOrEqual(OffsetDateTime.ofInstant(request.from(), ZoneOffset.UTC)), + DOWNLOAD_STATS_DAILY.DAY.lessThan(OffsetDateTime.ofInstant(request.to(), ZoneOffset.UTC))) + .groupBy(groupByFields) + .orderBy(groupByFields) + .fetch(record -> new DownloadSeriesRow(record.value1().toInstant(), record.value2(), record.value3())); + } + + private Field bucketField(DownloadSeriesInterval interval) { + // `day` holds UTC-aligned buckets; date_trunc must not depend on the session time zone, + // hence the AT TIME ZONE round-trip + return switch (interval) { + case DAY -> DOWNLOAD_STATS_DAILY.DAY; + case WEEK -> DSL.field( + "(date_trunc('week', {0} AT TIME ZONE 'UTC') AT TIME ZONE 'UTC')", + OffsetDateTime.class, + DOWNLOAD_STATS_DAILY.DAY); + case MONTH -> DSL.field( + "(date_trunc('month', {0} AT TIME ZONE 'UTC') AT TIME ZONE 'UTC')", + OffsetDateTime.class, + DOWNLOAD_STATS_DAILY.DAY); + }; + } + + private Field groupField(DownloadSeriesGroupBy groupBy) { + return switch (groupBy) { + case NONE -> DSL.inline(null, String.class); + case VERSION -> DOWNLOAD_STATS_DAILY.VERSION; + case TARGET_PLATFORM -> DOWNLOAD_STATS_DAILY.TARGET_PLATFORM; + case COUNTRY -> DOWNLOAD_STATS_DAILY.COUNTRY; + }; + } +} diff --git a/server/src/main/java/org/eclipse/openvsx/entities/DownloadCountProcessedItem.java b/server/src/main/java/org/eclipse/openvsx/entities/DownloadIngestion.java similarity index 91% rename from server/src/main/java/org/eclipse/openvsx/entities/DownloadCountProcessedItem.java rename to server/src/main/java/org/eclipse/openvsx/entities/DownloadIngestion.java index 025a71000..21d296390 100644 --- a/server/src/main/java/org/eclipse/openvsx/entities/DownloadCountProcessedItem.java +++ b/server/src/main/java/org/eclipse/openvsx/entities/DownloadIngestion.java @@ -15,9 +15,12 @@ import jakarta.persistence.GeneratedValue; import jakarta.persistence.Id; import jakarta.persistence.SequenceGenerator; +import jakarta.persistence.Table; +// The table keeps its historical name; only the Java class was renamed. @Entity -public class DownloadCountProcessedItem { +@Table(name = "download_count_processed_item") +public class DownloadIngestion { @Id @GeneratedValue(generator = "downloadCountProcessedItemSeq") diff --git a/server/src/main/java/org/eclipse/openvsx/mirror/aop/DownloadCountServiceAspect.java b/server/src/main/java/org/eclipse/openvsx/mirror/aop/DownloadCountServiceAspect.java deleted file mode 100644 index 781898c01..000000000 --- a/server/src/main/java/org/eclipse/openvsx/mirror/aop/DownloadCountServiceAspect.java +++ /dev/null @@ -1,27 +0,0 @@ -/** ****************************************************************************** - * Copyright (c) 2022 Precies. Software Ltd and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Public License v. 2.0 which is available at - * http://www.eclipse.org/legal/epl-2.0. - * - * SPDX-License-Identifier: EPL-2.0 - * ****************************************************************************** */ -package org.eclipse.openvsx.mirror.aop; - -import org.aspectj.lang.ProceedingJoinPoint; -import org.aspectj.lang.annotation.Around; -import org.aspectj.lang.annotation.Aspect; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.stereotype.Component; - -@Aspect -@Component -@ConditionalOnProperty(value = "ovsx.data.mirror.enabled", havingValue = "true") -public class DownloadCountServiceAspect { - - @Around("execution(* org.eclipse.openvsx.storage.log.*DownloadCountService.isEnabled(..))") - public Object isEnabled(ProceedingJoinPoint ignoredPjp) throws Throwable { - return false; - } -} diff --git a/server/src/main/java/org/eclipse/openvsx/repositories/DownloadCountProcessedItemRepository.java b/server/src/main/java/org/eclipse/openvsx/repositories/DownloadIngestionRepository.java similarity index 51% rename from server/src/main/java/org/eclipse/openvsx/repositories/DownloadCountProcessedItemRepository.java rename to server/src/main/java/org/eclipse/openvsx/repositories/DownloadIngestionRepository.java index a697bb879..244bf1cda 100644 --- a/server/src/main/java/org/eclipse/openvsx/repositories/DownloadCountProcessedItemRepository.java +++ b/server/src/main/java/org/eclipse/openvsx/repositories/DownloadIngestionRepository.java @@ -14,20 +14,23 @@ import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.Repository; -import org.eclipse.openvsx.entities.DownloadCountProcessedItem; +import org.eclipse.openvsx.entities.DownloadIngestion; -public interface DownloadCountProcessedItemRepository extends Repository { +public interface DownloadIngestionRepository extends Repository { @Query( - "select dc.name from DownloadCountProcessedItem dc where dc.success = true and dc.storageType = ?1 and dc.name in(?2)" + "select dc.name from DownloadIngestion dc where dc.success = true and dc.storageType = ?1 and dc.name in(?2)" ) - List findAllSucceededDownloadCountProcessedItemsByStorageTypeAndNameIn( + List findAllSucceededDownloadIngestionsByStorageTypeAndNameIn( String storageType, List names ); @Query( - "select dc.name from DownloadCountProcessedItem dc where dc.success = false and dc.storageType = ?1 and dc.name in(?2)" + "select dc.name from DownloadIngestion dc where dc.success = false and dc.storageType = ?1 and dc.name in(?2)" ) - List findAllFailedDownloadCountProcessedItemsByStorageTypeAndNameIn(String storageType, List names); + List findAllFailedDownloadIngestionsByStorageTypeAndNameIn(String storageType, List names); + + @Query("select count(dc) from DownloadIngestion dc where dc.success = false") + long countFailedDownloadIngestions(); } diff --git a/server/src/main/java/org/eclipse/openvsx/repositories/RepositoryService.java b/server/src/main/java/org/eclipse/openvsx/repositories/RepositoryService.java index c32cf71c1..5418b2706 100644 --- a/server/src/main/java/org/eclipse/openvsx/repositories/RepositoryService.java +++ b/server/src/main/java/org/eclipse/openvsx/repositories/RepositoryService.java @@ -88,7 +88,7 @@ public class RepositoryService { private final NamespaceMembershipRepository membershipRepo; private final PersonalAccessTokenRepository personalAccessTokenRepo; private final PersistedLogRepository persistedLogRepo; - private final DownloadCountProcessedItemRepository downloadCountRepo; + private final DownloadIngestionRepository downloadIngestionRepo; private final ExtensionJooqRepository extensionJooqRepo; private final ExtensionVersionJooqRepository extensionVersionJooqRepo; private final FileResourceJooqRepository fileResourceJooqRepo; @@ -127,7 +127,7 @@ public RepositoryService( NamespaceMembershipRepository membershipRepo, PersonalAccessTokenRepository personalAccessTokenRepo, PersistedLogRepository persistedLogRepo, - DownloadCountProcessedItemRepository downloadCountRepo, + DownloadIngestionRepository downloadIngestionRepo, ExtensionJooqRepository extensionJooqRepo, ExtensionVersionJooqRepository extensionVersionJooqRepo, FileResourceJooqRepository fileResourceJooqRepo, @@ -165,7 +165,7 @@ public RepositoryService( this.membershipRepo = membershipRepo; this.personalAccessTokenRepo = personalAccessTokenRepo; this.persistedLogRepo = persistedLogRepo; - this.downloadCountRepo = downloadCountRepo; + this.downloadIngestionRepo = downloadIngestionRepo; this.extensionJooqRepo = extensionJooqRepo; this.extensionVersionJooqRepo = extensionVersionJooqRepo; this.fileResourceJooqRepo = fileResourceJooqRepo; @@ -619,18 +619,22 @@ public long countPersistedLogs(UserData user) { return persistedLogRepo.countByUser(user); } - public List findAllSucceededDownloadCountProcessedItemsByStorageTypeAndNameIn( + public List findAllSucceededDownloadIngestionsByStorageTypeAndNameIn( String storageType, List names ) { - return downloadCountRepo.findAllSucceededDownloadCountProcessedItemsByStorageTypeAndNameIn(storageType, names); + return downloadIngestionRepo.findAllSucceededDownloadIngestionsByStorageTypeAndNameIn(storageType, names); } - public List findAllFailedDownloadCountProcessedItemsByStorageTypeAndNameIn( + public List findAllFailedDownloadIngestionsByStorageTypeAndNameIn( String storageType, List names ) { - return downloadCountRepo.findAllFailedDownloadCountProcessedItemsByStorageTypeAndNameIn(storageType, names); + return downloadIngestionRepo.findAllFailedDownloadIngestionsByStorageTypeAndNameIn(storageType, names); + } + + public long countFailedDownloadIngestions() { + return downloadIngestionRepo.countFailedDownloadIngestions(); } public List findActiveExtensionsByPublicId(Collection publicIds, String... namespacesToExclude) { diff --git a/server/src/main/java/org/eclipse/openvsx/storage/StorageUtilService.java b/server/src/main/java/org/eclipse/openvsx/storage/StorageUtilService.java index 6629ddc51..b7cf4cebd 100644 --- a/server/src/main/java/org/eclipse/openvsx/storage/StorageUtilService.java +++ b/server/src/main/java/org/eclipse/openvsx/storage/StorageUtilService.java @@ -20,6 +20,7 @@ import jakarta.persistence.EntityManager; import jakarta.transaction.Transactional; import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.ObjectProvider; import org.springframework.beans.factory.annotation.Value; import org.springframework.data.util.Pair; import org.springframework.http.*; @@ -29,6 +30,8 @@ import tools.jackson.databind.json.JsonMapper; import tools.jackson.databind.node.ArrayNode; +import org.eclipse.openvsx.analytics.ingestion.DownloadIngestionProcessor; +import org.eclipse.openvsx.analytics.ingestion.DownloadRecordSource; import org.eclipse.openvsx.cache.CacheService; import org.eclipse.openvsx.entities.ExtensionVersion; import org.eclipse.openvsx.entities.FileResource; @@ -36,7 +39,6 @@ import org.eclipse.openvsx.metrics.ExtensionDownloadMetrics; import org.eclipse.openvsx.repositories.RepositoryService; import org.eclipse.openvsx.search.SearchUtilService; -import org.eclipse.openvsx.storage.log.DownloadCountService; import org.eclipse.openvsx.util.HttpHeadersUtil; import org.eclipse.openvsx.util.TempFile; import org.eclipse.openvsx.util.UrlUtil; @@ -56,7 +58,8 @@ public class StorageUtilService implements IStorageService { private final AzureBlobStorageService azureStorage; private final LocalStorageService localStorage; private final AwsStorageService awsStorage; - private final DownloadCountService downloadCountService; + private final ObjectProvider ingestionSources; + private final DownloadIngestionProcessor ingestionProcessor; private final ExtensionDownloadMetrics downloadMetrics; private final SearchUtilService search; private final CacheService cache; @@ -79,7 +82,8 @@ public StorageUtilService( AzureBlobStorageService azureStorage, LocalStorageService localStorage, AwsStorageService awsStorage, - DownloadCountService downloadCountService, + ObjectProvider ingestionSources, + DownloadIngestionProcessor ingestionProcessor, ExtensionDownloadMetrics downloadMetrics, SearchUtilService search, CacheService cache, @@ -92,7 +96,8 @@ public StorageUtilService( this.azureStorage = azureStorage; this.localStorage = localStorage; this.awsStorage = awsStorage; - this.downloadCountService = downloadCountService; + this.ingestionSources = ingestionSources; + this.ingestionProcessor = ingestionProcessor; this.downloadMetrics = downloadMetrics; this.search = search; this.cache = cache; @@ -326,7 +331,7 @@ public Map> getFileUrls( public void increaseDownloadCount(FileResource resource) { downloadMetrics.recordDownload(resource); - if (downloadCountService.isEnabled(resource)) { + if (ingestionSources.stream().anyMatch(source -> source.covers(resource))) { // don't count downloads twice return; } @@ -334,6 +339,7 @@ public void increaseDownloadCount(FileResource resource) { var managedResource = entityManager.find(FileResource.class, resource.getId()); var extension = managedResource.getExtension().getExtension(); extension.setDownloadCount(extension.getDownloadCount() + 1); + ingestionProcessor.captureDownload(managedResource); cache.evictNamespaceDetails(extension); cache.evictExtensionJsons(extension); diff --git a/server/src/main/java/org/eclipse/openvsx/storage/log/AwsDownloadCountHandler.java b/server/src/main/java/org/eclipse/openvsx/storage/log/AwsDownloadCountHandler.java deleted file mode 100644 index 2e70996b5..000000000 --- a/server/src/main/java/org/eclipse/openvsx/storage/log/AwsDownloadCountHandler.java +++ /dev/null @@ -1,304 +0,0 @@ -/******************************************************************************** - * Copyright (c) 2025 Eclipse Foundation and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Public License v. 2.0 which is available at - * http://www.eclipse.org/legal/epl-2.0. - * - * SPDX-License-Identifier: EPL-2.0 - ********************************************************************************/ -package org.eclipse.openvsx.storage.log; - -import java.io.*; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.StandardCopyOption; -import java.time.LocalDateTime; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import java.util.zip.GZIPInputStream; - -import jakarta.annotation.PostConstruct; -import org.apache.commons.lang3.StringUtils; -import org.jobrunr.jobs.annotations.Job; -import org.jobrunr.jobs.lambdas.JobRequestHandler; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; -import org.springframework.util.StopWatch; -import org.springframework.web.util.UriUtils; -import software.amazon.awssdk.core.sync.ResponseTransformer; -import software.amazon.awssdk.services.s3.S3Client; -import software.amazon.awssdk.services.s3.model.*; - -import org.eclipse.openvsx.entities.Extension; -import org.eclipse.openvsx.entities.FileResource; -import org.eclipse.openvsx.migration.HandlerJobRequest; -import org.eclipse.openvsx.settings.SettingsService; -import org.eclipse.openvsx.storage.AwsStorageService; -import org.eclipse.openvsx.util.TempFile; - -/** - * Pulls logs from an Amazon S3 bucket, extracts downloads from the logs and updates download counts in the database. - *

- * The following log file formats are supported: - *

    - *
  • cloudfront
    • - *
  • fastly
    • - *
- *

- * See - *

- */ -@Component -public class AwsDownloadCountHandler implements JobRequestHandler> { - private final Logger logger = LoggerFactory.getLogger(AwsDownloadCountHandler.class); - - private static final String LOG_LOCATION_PREFIX = "AWSLogs/"; - - private final SettingsService settings; - private final AwsStorageService awsStorageService; - private final DownloadCountProcessor processor; - - @Value("${ovsx.logs.aws.bucket:}") - String bucket; - - @Value("${ovsx.logs.aws.log-location-prefix:" + LOG_LOCATION_PREFIX + "}") - String logLocationPrefix; - - @Value("${ovsx.logs.aws.format:cloudfront}") - String logFormat; - - @Value("${ovsx.logs.aws.cron:0 10 * * * *}") - String cronSchedule; - - @Value("${ovsx.logs.aws.max-keys:100}") - int maxKeys; - - LogFileParser logFileParser; - - public AwsDownloadCountHandler( - SettingsService settings, - AwsStorageService awsStorageService, - DownloadCountProcessor processor - ) { - this.settings = settings; - this.awsStorageService = awsStorageService; - this.processor = processor; - } - - @PostConstruct - public void initialize() { - logFileParser = switch (logFormat.toLowerCase()) { - case "cloudfront" -> new CloudFrontLogFileParser(); - case "fastly" -> new FastlyLogFileParser(); - default -> throw new IllegalArgumentException("unsupported log file format '" + logFormat + "'"); - }; - } - - public String getRecurringJobId() { - return "update-aws-download-counts"; - } - - public String getCronSchedule() { - return cronSchedule; - } - - /** - * Indicates whether the download service is enabled by application config. - */ - public boolean isEnabled() { - return !StringUtils.isEmpty(bucket) && awsStorageService.isEnabled(); - } - - private S3Client getS3Client() { - return awsStorageService.getS3Client(); - } - - /** - * Scheduled task to pull logs from AWS S3 Storage and update extension download counts. - */ - @Override - @Job(name = "Update AWS Download Counts", retries = 0) - public void run(HandlerJobRequest jobRequest) throws Exception { - if (!isEnabled()) { - return; - } - - if (settings.isReadOnly()) { - logger.info("[AwsDownloadCountService] registry is in read-only mode, skipping job"); - return; - } - - logger.info("[AwsDownloadCountService] >> updateDownloadCounts"); - - // Note: need to align the next jobRunTime with the cron schedule when changing it. - var nextJobRunTime = LocalDateTime.now().plusHours(1).withMinute(10); - var maxExecutionTime = LocalDateTime.now().plusMinutes(50); - - var stopWatch = new StopWatch(); - - String continuationToken = null; - - do { - var objects = listObjects(continuationToken); - - var files = objects.contents().stream().map(S3Object::key).toList(); - if (!processResponse(files, stopWatch, maxExecutionTime, nextJobRunTime)) { - break; - } - - continuationToken = objects.isTruncated() ? objects.nextContinuationToken() : null; - } while (continuationToken != null); - - logger.info("[AwsDownloadCountService] << updateDownloadCounts"); - } - - private boolean processResponse( - List files, - StopWatch stopWatch, - LocalDateTime maxExecutionTime, - LocalDateTime nextJobRunTime - ) { - var logFiles = files.stream().filter(logFile -> logFile.endsWith(".gz")).collect(Collectors.toList()); - - // determine log files that have already been processed -> delete them and do not re-process them - var processedItems = processor.processedItems(FileResource.STORAGE_AWS, logFiles); - processedItems.forEach(this::deleteFile); - if (!processedItems.isEmpty()) { - logger.info("[AwsDownloadCountService] deleting already analysed log files:"); - processedItems.forEach(item -> logger.info(" - {}", item)); - } - logFiles.removeAll(processedItems); - - // determine log files that could not be processed before -> keep them for analysis and skip processing - var failedItems = processor.failedItems(FileResource.STORAGE_AWS, logFiles); - if (!failedItems.isEmpty()) { - logger.info("[AwsDownloadCountService] skipping previously failed log files:"); - failedItems.forEach(item -> logger.info(" - {}", item)); - } - logFiles.removeAll(failedItems); - - var allUpdatedExtensions = new HashMap(); - - try { - for (var name : logFiles) { - var processedOn = LocalDateTime.now(); - - if (processedOn.isAfter(maxExecutionTime)) { - logger.info( - "Failed to process all download counts within timeslot, next job run is at {}", - nextJobRunTime); - return false; - } - - if (settings.isReadOnly()) { - logger.info("skip processing log files as registry is in read-only mode"); - return false; - } - - var success = false; - stopWatch.start(); - try { - var counts = processLogFile(name); - if (!counts.isEmpty()) { - var extensionDownloads = processor.processDownloadCounts(FileResource.STORAGE_AWS, counts); - var updatedExtensions = processor.increaseDownloadCounts(extensionDownloads); - updatedExtensions.forEach(extension -> allUpdatedExtensions.put(extension.getId(), extension)); - } - - success = true; - } catch (Exception e) { - logger.error("failed to process log file: {}", name, e); - } - - stopWatch.stop(); - var executionTime = (int) stopWatch.lastTaskInfo().getTimeMillis(); - processor.persistProcessedItem(name, FileResource.STORAGE_AWS, processedOn, executionTime, success); - if (success) { - deleteFile(name); - } - } - - return true; - } finally { - // evict caches and update search entries for all updated extensions - allUpdatedExtensions.values().forEach(processor::evictCaches); - processor.updateSearchEntries(allUpdatedExtensions.values().stream().toList()); - } - } - - private Map processLogFile(String fileName) throws IOException { - try ( - var downloadsTempFile = downloadFile(fileName); - var fileStream = new FileInputStream(downloadsTempFile.getPath().toFile()); - var gzipStream = new GZIPInputStream(fileStream); - var reader = new BufferedReader(new InputStreamReader(gzipStream, StandardCharsets.UTF_8)); - ) { - var fileCounts = new HashMap(); - var lines = reader.lines().iterator(); - while (lines.hasNext()) { - var line = lines.next(); - - var record = logFileParser.parse(line); - if (record == null) { - continue; - } - - if (isGetOperation(record) && isStatusOk(record) && isExtensionPackageUri(record)) { - var uri = record.url(); - var uriComponents = uri.split("/"); - var vsixFile = UriUtils.decode(uriComponents[uriComponents.length - 1], StandardCharsets.UTF_8) - .toUpperCase(); - fileCounts.merge(vsixFile, 1, Integer::sum); - } - } - return fileCounts; - } - } - - private boolean isGetOperation(LogRecord record) { - return record.method().equalsIgnoreCase("GET"); - } - - private boolean isStatusOk(LogRecord record) { - return record.status() == 200; - } - - private boolean isExtensionPackageUri(LogRecord record) { - return record.url().endsWith(".vsix"); - } - - private TempFile downloadFile(String objectKey) throws IOException { - var downloadsTempFile = new TempFile("aws-downloads-", ".gz"); - var inputStream = getS3Client().getObject( - GetObjectRequest.builder() - .bucket(bucket) - .key(objectKey) - .build(), - ResponseTransformer.toInputStream()); - Files.copy(inputStream, downloadsTempFile.getPath(), StandardCopyOption.REPLACE_EXISTING); - return downloadsTempFile; - } - - private void deleteFile(String objectKey) { - getS3Client().deleteObject(DeleteObjectRequest.builder().bucket(bucket).key(objectKey).build()); - } - - private ListObjectsV2Response listObjects(String continuationToken) { - var builder = ListObjectsV2Request.builder().bucket(bucket).maxKeys(maxKeys).prefix(logLocationPrefix); - - if (continuationToken != null) { - builder.continuationToken(continuationToken); - } - - return getS3Client().listObjectsV2(builder.build()); - } -} diff --git a/server/src/main/java/org/eclipse/openvsx/storage/log/CloudFrontLogFileParser.java b/server/src/main/java/org/eclipse/openvsx/storage/log/CloudFrontLogFileParser.java deleted file mode 100644 index 73ea9e8a7..000000000 --- a/server/src/main/java/org/eclipse/openvsx/storage/log/CloudFrontLogFileParser.java +++ /dev/null @@ -1,27 +0,0 @@ -/****************************************************************************** - * Copyright (c) 2026 Contributors to the Eclipse Foundation. - * - * See the NOTICE file(s) distributed with this work for additional - * information regarding copyright ownership. - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Public License 2.0 which is available at - * https://www.eclipse.org/legal/epl-2.0. - * - * SPDX-License-Identifier: EPL-2.0 - *****************************************************************************/ -package org.eclipse.openvsx.storage.log; - -class CloudFrontLogFileParser implements LogFileParser { - @Override - public LogRecord parse(String line) { - if (line.startsWith("#")) { - return null; - } - - // Format: - // date time x-edge-location sc-bytes c-ip cs-method cs(Host) cs-uri-stem sc-status cs(Referer) cs(User-Agent) cs-uri-query cs(Cookie) x-edge-result-type x-edge-request-id x-host-header cs-protocol cs-bytes time-taken x-forwarded-for ssl-protocol ssl-cipher x-edge-response-result-type cs-protocol-version fle-status fle-encrypted-fields c-port time-to-first-byte x-edge-detailed-result-type sc-content-type sc-content-len sc-range-start sc-range-end - var components = line.split("[ \t]+"); - return new LogRecord(components[5], Integer.parseInt(components[8]), components[7]); - } -} diff --git a/server/src/main/java/org/eclipse/openvsx/storage/log/DownloadCountProcessor.java b/server/src/main/java/org/eclipse/openvsx/storage/log/DownloadCountProcessor.java deleted file mode 100644 index 0742de673..000000000 --- a/server/src/main/java/org/eclipse/openvsx/storage/log/DownloadCountProcessor.java +++ /dev/null @@ -1,147 +0,0 @@ -/******************************************************************************** - * Copyright (c) 2022 Precies. Software Ltd and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Public License v. 2.0 which is available at - * http://www.eclipse.org/legal/epl-2.0. - * - * SPDX-License-Identifier: EPL-2.0 - ********************************************************************************/ -package org.eclipse.openvsx.storage.log; - -import java.time.LocalDateTime; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -import com.google.common.collect.Lists; -import io.micrometer.observation.Observation; -import io.micrometer.observation.ObservationRegistry; -import jakarta.persistence.EntityManager; -import jakarta.transaction.Transactional; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.stereotype.Component; - -import org.eclipse.openvsx.cache.CacheService; -import org.eclipse.openvsx.entities.DownloadCountProcessedItem; -import org.eclipse.openvsx.entities.Extension; -import org.eclipse.openvsx.repositories.RepositoryService; -import org.eclipse.openvsx.search.SearchUtilService; - -@Component -public class DownloadCountProcessor { - - protected final Logger logger = LoggerFactory.getLogger(DownloadCountProcessor.class); - - private final EntityManager entityManager; - private final RepositoryService repositories; - private final CacheService cache; - private final SearchUtilService search; - private final ObservationRegistry observations; - - public DownloadCountProcessor( - EntityManager entityManager, - RepositoryService repositories, - CacheService cache, - SearchUtilService search, - ObservationRegistry observations - ) { - this.entityManager = entityManager; - this.repositories = repositories; - this.cache = cache; - this.search = search; - this.observations = observations; - } - - @Transactional - public void persistProcessedItem( - String name, - String storageType, - LocalDateTime processedOn, - int executionTime, - boolean success - ) { - Observation.createNotStarted("DownloadCountProcessor#persistProcessedItem", observations).observe(() -> { - var processedItem = new DownloadCountProcessedItem(); - processedItem.setName(name); - processedItem.setStorageType(storageType); - processedItem.setProcessedOn(processedOn); - processedItem.setExecutionTime(executionTime); - processedItem.setSuccess(success); - entityManager.persist(processedItem); - }); - } - - public Map processDownloadCounts(String storageType, Map files) { - return Observation.createNotStarted("DownloadCountProcessor#processDownloadCounts", observations).observe( - () -> repositories.findDownloadsByStorageTypeAndName(storageType, files.keySet()).stream() - .map(fileResource -> Map.entry(fileResource, files.get(fileResource.getName().toUpperCase()))) - .filter(fileResource -> { - var ev = fileResource.getKey().getExtension(); - if (ev == null) { - logger.warn( - "no extension version found for download {}, skipping", - fileResource.getKey().getName()); - return false; - } else { - return true; - } - }) - .collect( - Collectors.groupingBy( - e -> e.getKey().getExtension().getExtension().getId(), - Collectors.summingInt(Map.Entry::getValue)))); - } - - @Transactional - public List increaseDownloadCounts(Map extensionDownloads) { - return Observation.createNotStarted("DownloadCountProcessor#increaseDownloadCounts", observations) - .observe(() -> { - var extensions = repositories.findExtensions(extensionDownloads.keySet()).toList(); - extensions.forEach(extension -> { - var downloads = extensionDownloads.get(extension.getId()); - extension.setDownloadCount(extension.getDownloadCount() + downloads); - }); - - return extensions; - }); - } - - @Transactional // needs transaction for lazy-loading versions - public void evictCaches(Extension extension) { - Observation.createNotStarted("DownloadCountProcessor#evictCaches", observations).observe(() -> { - var mergedExtension = entityManager.merge(extension); - cache.evictExtensionJsons(mergedExtension); - cache.evictLatestExtensionVersion(mergedExtension); - }); - } - - public void updateSearchEntries(List extensions) { - Observation.createNotStarted("DownloadCountProcessor#updateSearchEntries", observations).observe(() -> { - logger.info("[DownloadCountProcessor] >> updateSearchEntries"); - var activeExtensions = extensions.stream() - .filter(Extension::isActive) - .collect(Collectors.toList()); - - logger.info("[DownloadCountProcessor] total active extensions: {}", activeExtensions.size()); - var parts = Lists.partition(activeExtensions, 100); - logger.info("[DownloadCountProcessor] partitions: {} | partition size: 100", parts.size()); - - parts.forEach(search::updateSearchEntriesAsync); - logger.info("[DownloadCountProcessor] << updateSearchEntries"); - }); - } - - public List processedItems(String storageType, List blobNames) { - return Observation.createNotStarted("DownloadCountProcessor#processedItems", observations).observe( - () -> repositories - .findAllSucceededDownloadCountProcessedItemsByStorageTypeAndNameIn(storageType, blobNames)); - } - - public List failedItems(String storageType, List blobNames) { - return Observation.createNotStarted("DownloadCountProcessor#failedItems", observations).observe( - () -> repositories - .findAllFailedDownloadCountProcessedItemsByStorageTypeAndNameIn(storageType, blobNames)); - } -} diff --git a/server/src/main/java/org/eclipse/openvsx/storage/log/DownloadCountService.java b/server/src/main/java/org/eclipse/openvsx/storage/log/DownloadCountService.java deleted file mode 100644 index b2193f071..000000000 --- a/server/src/main/java/org/eclipse/openvsx/storage/log/DownloadCountService.java +++ /dev/null @@ -1,89 +0,0 @@ -/******************************************************************************** - * Copyright (c) 2025 Eclipse Foundation and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Public License v. 2.0 which is available at - * http://www.eclipse.org/legal/epl-2.0. - * - * SPDX-License-Identifier: EPL-2.0 - ********************************************************************************/ -package org.eclipse.openvsx.storage.log; - -import java.time.ZoneId; - -import org.jobrunr.scheduling.JobRequestScheduler; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.boot.context.event.ApplicationStartedEvent; -import org.springframework.context.event.EventListener; -import org.springframework.stereotype.Service; - -import org.eclipse.openvsx.entities.FileResource; -import org.eclipse.openvsx.migration.HandlerJobRequest; - -/** - * A utility service to determine whether an optimized download count service - * is available for a specific {@link FileResource}. - */ -@Service -public class DownloadCountService { - - protected final Logger logger = LoggerFactory.getLogger(DownloadCountService.class); - - private final JobRequestScheduler scheduler; - private final AwsDownloadCountHandler awsDownloadCountHandler; - private final AzureDownloadCountHandler azureDownloadCountHandler; - - public DownloadCountService( - JobRequestScheduler scheduler, - AwsDownloadCountHandler awsDownloadCountHandler, - AzureDownloadCountHandler azureDownloadCountHandler - ) { - this.scheduler = scheduler; - this.awsDownloadCountHandler = awsDownloadCountHandler; - this.azureDownloadCountHandler = azureDownloadCountHandler; - } - - /** - * Returns whether an optimized download count service is enabled for the given {@link FileResource}. - * - * @param resource the {@link FileResource} to check - * @return {@code true} if an optimized download count service is available, {@code false} otherwise - */ - public boolean isEnabled(FileResource resource) { - return switch (resource.getStorageType()) { - case FileResource.STORAGE_AWS -> awsDownloadCountHandler.isEnabled(); - case FileResource.STORAGE_AZURE -> azureDownloadCountHandler.isEnabled(); - default -> false; - }; - } - - @EventListener - public void applicationStarted(ApplicationStartedEvent event) { - if (awsDownloadCountHandler.isEnabled()) { - logger.info( - "Scheduling AWS download count handler with cron '{}'", - awsDownloadCountHandler.getCronSchedule()); - scheduler.scheduleRecurrently( - awsDownloadCountHandler.getRecurringJobId(), - awsDownloadCountHandler.getCronSchedule(), - ZoneId.of("UTC"), - new HandlerJobRequest<>(AwsDownloadCountHandler.class)); - } else { - scheduler.deleteRecurringJob(awsDownloadCountHandler.getRecurringJobId()); - } - - if (azureDownloadCountHandler.isEnabled()) { - logger.info( - "Scheduling Azure download count handler with cron '{}'", - azureDownloadCountHandler.getCronSchedule()); - scheduler.scheduleRecurrently( - azureDownloadCountHandler.getRecurringJobId(), - azureDownloadCountHandler.getCronSchedule(), - ZoneId.of("UTC"), - new HandlerJobRequest<>(AzureDownloadCountHandler.class)); - } else { - scheduler.deleteRecurringJob(azureDownloadCountHandler.getRecurringJobId()); - } - } -} diff --git a/server/src/main/java/org/eclipse/openvsx/storage/log/FastlyLogFileParser.java b/server/src/main/java/org/eclipse/openvsx/storage/log/FastlyLogFileParser.java deleted file mode 100644 index 1374e812e..000000000 --- a/server/src/main/java/org/eclipse/openvsx/storage/log/FastlyLogFileParser.java +++ /dev/null @@ -1,67 +0,0 @@ -/****************************************************************************** - * Copyright (c) 2026 Contributors to the Eclipse Foundation. - * - * See the NOTICE file(s) distributed with this work for additional - * information regarding copyright ownership. - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Public License 2.0 which is available at - * https://www.eclipse.org/legal/epl-2.0. - * - * SPDX-License-Identifier: EPL-2.0 - *****************************************************************************/ -package org.eclipse.openvsx.storage.log; - -import org.jspecify.annotations.Nullable; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import tools.jackson.core.JacksonException; -import tools.jackson.core.JsonParser; -import tools.jackson.databind.DeserializationContext; -import tools.jackson.databind.JsonNode; -import tools.jackson.databind.deser.std.StdDeserializer; -import tools.jackson.databind.json.JsonMapper; -import tools.jackson.databind.module.SimpleModule; - -class FastlyLogFileParser implements LogFileParser { - private final Logger logger = LoggerFactory.getLogger(FastlyLogFileParser.class); - - private final JsonMapper mapper; - - public FastlyLogFileParser() { - var module = new SimpleModule(); - module.addDeserializer(LogRecord.class, new LogRecordDeserializer()); - this.mapper = JsonMapper.builder().addModule(module).build(); - } - - @Override - public @Nullable LogRecord parse(String line) { - try { - var jsonStartIndex = line.indexOf("{"); - if (jsonStartIndex != -1) { - return mapper.readValue(line.substring(jsonStartIndex), LogRecord.class); - } else { - return null; - } - } catch (JacksonException ex) { - logger.error("could not parse log line '{}'", line, ex); - return null; - } - } -} - -class LogRecordDeserializer extends StdDeserializer { - - public LogRecordDeserializer() { - super(LogRecord.class); - } - - @Override - public LogRecord deserialize(JsonParser jp, DeserializationContext ctxt) throws JacksonException { - JsonNode node = ctxt.readTree(jp); - String operation = node.get("request_method").asString(); - int status = (Integer) node.get("response_status").numberValue(); - String url = node.get("url").asString(); - return new LogRecord(operation, status, url); - } -} diff --git a/server/src/main/jooq-gen/org/eclipse/openvsx/jooq/Indexes.java b/server/src/main/jooq-gen/org/eclipse/openvsx/jooq/Indexes.java index 7bab676fc..b60338a95 100644 --- a/server/src/main/jooq-gen/org/eclipse/openvsx/jooq/Indexes.java +++ b/server/src/main/jooq-gen/org/eclipse/openvsx/jooq/Indexes.java @@ -8,6 +8,7 @@ import org.eclipse.openvsx.jooq.tables.AdminStatistics; import org.eclipse.openvsx.jooq.tables.CustomerMembership; import org.eclipse.openvsx.jooq.tables.DownloadCountProcessedItem; +import org.eclipse.openvsx.jooq.tables.DownloadEvent; import org.eclipse.openvsx.jooq.tables.Extension; import org.eclipse.openvsx.jooq.tables.ExtensionReview; import org.eclipse.openvsx.jooq.tables.ExtensionScan; @@ -45,8 +46,10 @@ public class Indexes { public static final Index CUSTOMER_MEMBERSHIP_NAMESPACE_IDX = Internal.createIndex(DSL.name("customer_membership_namespace_idx"), CustomerMembership.CUSTOMER_MEMBERSHIP, new OrderField[] { CustomerMembership.CUSTOMER_MEMBERSHIP.CUSTOMER }, false); public static final Index CUSTOMER_MEMBERSHIP_USER_DATA_IDX = Internal.createIndex(DSL.name("customer_membership_user_data_idx"), CustomerMembership.CUSTOMER_MEMBERSHIP, new OrderField[] { CustomerMembership.CUSTOMER_MEMBERSHIP.USER_DATA }, false); + public static final Index DE_EXT_TIME = Internal.createIndex(DSL.name("de_ext_time"), DownloadEvent.DOWNLOAD_EVENT, new OrderField[] { DownloadEvent.DOWNLOAD_EVENT.EXTENSION_ID, DownloadEvent.DOWNLOAD_EVENT.TIME.desc() }, false); public static final Index DOWNLOAD_COUNT_PROCESSED_ITEM_NAME = Internal.createIndex(DSL.name("download_count_processed_item_name"), DownloadCountProcessedItem.DOWNLOAD_COUNT_PROCESSED_ITEM, new OrderField[] { DownloadCountProcessedItem.DOWNLOAD_COUNT_PROCESSED_ITEM.NAME }, false); public static final Index DOWNLOAD_COUNT_PROCESSED_ITEM_STORAGE_TYPE = Internal.createIndex(DSL.name("download_count_processed_item_storage_type"), DownloadCountProcessedItem.DOWNLOAD_COUNT_PROCESSED_ITEM, new OrderField[] { DownloadCountProcessedItem.DOWNLOAD_COUNT_PROCESSED_ITEM.STORAGE_TYPE }, false); + public static final Index DOWNLOAD_EVENT_TIME_IDX = Internal.createIndex(DSL.name("download_event_time_idx"), DownloadEvent.DOWNLOAD_EVENT, new OrderField[] { DownloadEvent.DOWNLOAD_EVENT.TIME.desc() }, false); public static final Index EXTENSION__NAMESPACE_ID__IDX = Internal.createIndex(DSL.name("extension__namespace_id__idx"), Extension.EXTENSION, new OrderField[] { Extension.EXTENSION.NAMESPACE_ID }, false); public static final Index EXTENSION_REVIEW__EXTENSION_ID__IDX = Internal.createIndex(DSL.name("extension_review__extension_id__idx"), ExtensionReview.EXTENSION_REVIEW, new OrderField[] { ExtensionReview.EXTENSION_REVIEW.EXTENSION_ID }, false); public static final Index EXTENSION_REVIEW__USER_ID__IDX = Internal.createIndex(DSL.name("extension_review__user_id__idx"), ExtensionReview.EXTENSION_REVIEW, new OrderField[] { ExtensionReview.EXTENSION_REVIEW.USER_ID }, false); diff --git a/server/src/main/jooq-gen/org/eclipse/openvsx/jooq/Public.java b/server/src/main/jooq-gen/org/eclipse/openvsx/jooq/Public.java index 649d54b9a..b7188100a 100644 --- a/server/src/main/jooq-gen/org/eclipse/openvsx/jooq/Public.java +++ b/server/src/main/jooq-gen/org/eclipse/openvsx/jooq/Public.java @@ -19,6 +19,8 @@ import org.eclipse.openvsx.jooq.tables.CustomerMembership; import org.eclipse.openvsx.jooq.tables.DailyUsageStats; import org.eclipse.openvsx.jooq.tables.DownloadCountProcessedItem; +import org.eclipse.openvsx.jooq.tables.DownloadEvent; +import org.eclipse.openvsx.jooq.tables.DownloadStatsDaily; import org.eclipse.openvsx.jooq.tables.Extension; import org.eclipse.openvsx.jooq.tables.ExtensionReview; import org.eclipse.openvsx.jooq.tables.ExtensionScan; @@ -129,6 +131,16 @@ public class Public extends SchemaImpl { */ public final DownloadCountProcessedItem DOWNLOAD_COUNT_PROCESSED_ITEM = DownloadCountProcessedItem.DOWNLOAD_COUNT_PROCESSED_ITEM; + /** + * The table public.download_event. + */ + public final DownloadEvent DOWNLOAD_EVENT = DownloadEvent.DOWNLOAD_EVENT; + + /** + * The table public.download_stats_daily. + */ + public final DownloadStatsDaily DOWNLOAD_STATS_DAILY = DownloadStatsDaily.DOWNLOAD_STATS_DAILY; + /** * The table public.extension. */ @@ -328,6 +340,8 @@ public final List> getTables() { CustomerMembership.CUSTOMER_MEMBERSHIP, DailyUsageStats.DAILY_USAGE_STATS, DownloadCountProcessedItem.DOWNLOAD_COUNT_PROCESSED_ITEM, + DownloadEvent.DOWNLOAD_EVENT, + DownloadStatsDaily.DOWNLOAD_STATS_DAILY, Extension.EXTENSION, ExtensionReview.EXTENSION_REVIEW, ExtensionScan.EXTENSION_SCAN, diff --git a/server/src/main/jooq-gen/org/eclipse/openvsx/jooq/Tables.java b/server/src/main/jooq-gen/org/eclipse/openvsx/jooq/Tables.java index d965794e6..f3acab068 100644 --- a/server/src/main/jooq-gen/org/eclipse/openvsx/jooq/Tables.java +++ b/server/src/main/jooq-gen/org/eclipse/openvsx/jooq/Tables.java @@ -16,6 +16,8 @@ import org.eclipse.openvsx.jooq.tables.CustomerMembership; import org.eclipse.openvsx.jooq.tables.DailyUsageStats; import org.eclipse.openvsx.jooq.tables.DownloadCountProcessedItem; +import org.eclipse.openvsx.jooq.tables.DownloadEvent; +import org.eclipse.openvsx.jooq.tables.DownloadStatsDaily; import org.eclipse.openvsx.jooq.tables.Extension; import org.eclipse.openvsx.jooq.tables.ExtensionReview; import org.eclipse.openvsx.jooq.tables.ExtensionScan; @@ -115,6 +117,16 @@ public class Tables { */ public static final DownloadCountProcessedItem DOWNLOAD_COUNT_PROCESSED_ITEM = DownloadCountProcessedItem.DOWNLOAD_COUNT_PROCESSED_ITEM; + /** + * The table public.download_event. + */ + public static final DownloadEvent DOWNLOAD_EVENT = DownloadEvent.DOWNLOAD_EVENT; + + /** + * The table public.download_stats_daily. + */ + public static final DownloadStatsDaily DOWNLOAD_STATS_DAILY = DownloadStatsDaily.DOWNLOAD_STATS_DAILY; + /** * The table public.extension. */ diff --git a/server/src/main/jooq-gen/org/eclipse/openvsx/jooq/tables/DownloadEvent.java b/server/src/main/jooq-gen/org/eclipse/openvsx/jooq/tables/DownloadEvent.java new file mode 100644 index 000000000..3aff1eb51 --- /dev/null +++ b/server/src/main/jooq-gen/org/eclipse/openvsx/jooq/tables/DownloadEvent.java @@ -0,0 +1,270 @@ +/* + * This file is generated by jOOQ. + */ +package org.eclipse.openvsx.jooq.tables; + + +import java.time.OffsetDateTime; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; + +import org.eclipse.openvsx.jooq.Indexes; +import org.eclipse.openvsx.jooq.Public; +import org.eclipse.openvsx.jooq.tables.records.DownloadEventRecord; +import org.jooq.Condition; +import org.jooq.Field; +import org.jooq.Index; +import org.jooq.Name; +import org.jooq.PlainSQL; +import org.jooq.QueryPart; +import org.jooq.SQL; +import org.jooq.Schema; +import org.jooq.Select; +import org.jooq.Stringly; +import org.jooq.Table; +import org.jooq.TableField; +import org.jooq.TableOptions; +import org.jooq.impl.DSL; +import org.jooq.impl.SQLDataType; +import org.jooq.impl.TableImpl; + + +/** + * This class is generated by jOOQ. + */ +@SuppressWarnings({ "all", "unchecked", "rawtypes", "this-escape" }) +public class DownloadEvent extends TableImpl { + + private static final long serialVersionUID = 1L; + + /** + * The reference instance of public.download_event + */ + public static final DownloadEvent DOWNLOAD_EVENT = new DownloadEvent(); + + /** + * The class holding records for this type + */ + @Override + public Class getRecordType() { + return DownloadEventRecord.class; + } + + /** + * The column public.download_event.time. + */ + public final TableField TIME = createField(DSL.name("time"), SQLDataType.TIMESTAMPWITHTIMEZONE(6).nullable(false), this, ""); + + /** + * The column public.download_event.extension_id. + */ + public final TableField EXTENSION_ID = createField(DSL.name("extension_id"), SQLDataType.BIGINT.nullable(false), this, ""); + + /** + * The column public.download_event.extension_version_id. + */ + public final TableField EXTENSION_VERSION_ID = createField(DSL.name("extension_version_id"), SQLDataType.BIGINT.nullable(false), this, ""); + + /** + * The column public.download_event.namespace. + */ + public final TableField NAMESPACE = createField(DSL.name("namespace"), SQLDataType.VARCHAR(255).nullable(false), this, ""); + + /** + * The column public.download_event.extension_name. + */ + public final TableField EXTENSION_NAME = createField(DSL.name("extension_name"), SQLDataType.VARCHAR(255).nullable(false), this, ""); + + /** + * The column public.download_event.version. + */ + public final TableField VERSION = createField(DSL.name("version"), SQLDataType.VARCHAR(255).nullable(false), this, ""); + + /** + * The column public.download_event.target_platform. + */ + public final TableField TARGET_PLATFORM = createField(DSL.name("target_platform"), SQLDataType.VARCHAR(255).nullable(false), this, ""); + + /** + * The column public.download_event.country. + */ + public final TableField COUNTRY = createField(DSL.name("country"), SQLDataType.CHAR(2), this, ""); + + /** + * The column public.download_event.ip. + */ + public final TableField IP = createField(DSL.name("ip"), SQLDataType.VARCHAR(45), this, ""); + + /** + * The column public.download_event.user_agent. + */ + public final TableField USER_AGENT = createField(DSL.name("user_agent"), SQLDataType.CLOB, this, ""); + + /** + * The column public.download_event.count. + */ + public final TableField COUNT = createField(DSL.name("count"), SQLDataType.INTEGER.nullable(false).defaultValue(DSL.field(DSL.raw("1"), SQLDataType.INTEGER)), this, ""); + + private DownloadEvent(Name alias, Table aliased) { + this(alias, aliased, (Field[]) null, null); + } + + private DownloadEvent(Name alias, Table aliased, Field[] parameters, Condition where) { + super(alias, null, aliased, parameters, DSL.comment(""), TableOptions.table(), where); + } + + /** + * Create an aliased public.download_event table reference + */ + public DownloadEvent(String alias) { + this(DSL.name(alias), DOWNLOAD_EVENT); + } + + /** + * Create an aliased public.download_event table reference + */ + public DownloadEvent(Name alias) { + this(alias, DOWNLOAD_EVENT); + } + + /** + * Create a public.download_event table reference + */ + public DownloadEvent() { + this(DSL.name("download_event"), null); + } + + @Override + public Schema getSchema() { + return aliased() ? null : Public.PUBLIC; + } + + @Override + public List getIndexes() { + return Arrays.asList(Indexes.DE_EXT_TIME, Indexes.DOWNLOAD_EVENT_TIME_IDX); + } + + @Override + public DownloadEvent as(String alias) { + return new DownloadEvent(DSL.name(alias), this); + } + + @Override + public DownloadEvent as(Name alias) { + return new DownloadEvent(alias, this); + } + + @Override + public DownloadEvent as(Table alias) { + return new DownloadEvent(alias.getQualifiedName(), this); + } + + /** + * Rename this table + */ + @Override + public DownloadEvent rename(String name) { + return new DownloadEvent(DSL.name(name), null); + } + + /** + * Rename this table + */ + @Override + public DownloadEvent rename(Name name) { + return new DownloadEvent(name, null); + } + + /** + * Rename this table + */ + @Override + public DownloadEvent rename(Table name) { + return new DownloadEvent(name.getQualifiedName(), null); + } + + /** + * Create an inline derived table from this table + */ + @Override + public DownloadEvent where(Condition condition) { + return new DownloadEvent(getQualifiedName(), aliased() ? this : null, null, condition); + } + + /** + * Create an inline derived table from this table + */ + @Override + public DownloadEvent where(Collection conditions) { + return where(DSL.and(conditions)); + } + + /** + * Create an inline derived table from this table + */ + @Override + public DownloadEvent where(Condition... conditions) { + return where(DSL.and(conditions)); + } + + /** + * Create an inline derived table from this table + */ + @Override + public DownloadEvent where(Field condition) { + return where(DSL.condition(condition)); + } + + /** + * Create an inline derived table from this table + */ + @Override + @PlainSQL + public DownloadEvent where(SQL condition) { + return where(DSL.condition(condition)); + } + + /** + * Create an inline derived table from this table + */ + @Override + @PlainSQL + public DownloadEvent where(@Stringly.SQL String condition) { + return where(DSL.condition(condition)); + } + + /** + * Create an inline derived table from this table + */ + @Override + @PlainSQL + public DownloadEvent where(@Stringly.SQL String condition, Object... binds) { + return where(DSL.condition(condition, binds)); + } + + /** + * Create an inline derived table from this table + */ + @Override + @PlainSQL + public DownloadEvent where(@Stringly.SQL String condition, QueryPart... parts) { + return where(DSL.condition(condition, parts)); + } + + /** + * Create an inline derived table from this table + */ + @Override + public DownloadEvent whereExists(Select select) { + return where(DSL.exists(select)); + } + + /** + * Create an inline derived table from this table + */ + @Override + public DownloadEvent whereNotExists(Select select) { + return where(DSL.notExists(select)); + } +} diff --git a/server/src/main/jooq-gen/org/eclipse/openvsx/jooq/tables/DownloadStatsDaily.java b/server/src/main/jooq-gen/org/eclipse/openvsx/jooq/tables/DownloadStatsDaily.java new file mode 100644 index 000000000..c9900bf28 --- /dev/null +++ b/server/src/main/jooq-gen/org/eclipse/openvsx/jooq/tables/DownloadStatsDaily.java @@ -0,0 +1,264 @@ +/* + * This file is generated by jOOQ. + */ +package org.eclipse.openvsx.jooq.tables; + + +import java.time.OffsetDateTime; +import java.util.Collection; + +import org.eclipse.openvsx.jooq.Public; +import org.eclipse.openvsx.jooq.tables.records.DownloadStatsDailyRecord; +import org.jooq.Condition; +import org.jooq.Field; +import org.jooq.Name; +import org.jooq.PlainSQL; +import org.jooq.QueryPart; +import org.jooq.SQL; +import org.jooq.Schema; +import org.jooq.Select; +import org.jooq.Stringly; +import org.jooq.Table; +import org.jooq.TableField; +import org.jooq.TableOptions; +import org.jooq.impl.DSL; +import org.jooq.impl.SQLDataType; +import org.jooq.impl.TableImpl; + + +/** + * This class is generated by jOOQ. + */ +@SuppressWarnings({ "all", "unchecked", "rawtypes", "this-escape" }) +public class DownloadStatsDaily extends TableImpl { + + private static final long serialVersionUID = 1L; + + /** + * The reference instance of public.download_stats_daily + */ + public static final DownloadStatsDaily DOWNLOAD_STATS_DAILY = new DownloadStatsDaily(); + + /** + * The class holding records for this type + */ + @Override + public Class getRecordType() { + return DownloadStatsDailyRecord.class; + } + + /** + * The column public.download_stats_daily.day. + */ + public final TableField DAY = createField(DSL.name("day"), SQLDataType.TIMESTAMPWITHTIMEZONE(6), this, ""); + + /** + * The column public.download_stats_daily.extension_id. + */ + public final TableField EXTENSION_ID = createField(DSL.name("extension_id"), SQLDataType.BIGINT, this, ""); + + /** + * The column public.download_stats_daily.extension_version_id. + */ + public final TableField EXTENSION_VERSION_ID = createField(DSL.name("extension_version_id"), SQLDataType.BIGINT, this, ""); + + /** + * The column public.download_stats_daily.version. + */ + public final TableField VERSION = createField(DSL.name("version"), SQLDataType.VARCHAR(255), this, ""); + + /** + * The column public.download_stats_daily.target_platform. + */ + public final TableField TARGET_PLATFORM = createField(DSL.name("target_platform"), SQLDataType.VARCHAR(255), this, ""); + + /** + * The column public.download_stats_daily.country. + */ + public final TableField COUNTRY = createField(DSL.name("country"), SQLDataType.CHAR(2), this, ""); + + /** + * The column public.download_stats_daily.downloads. + */ + public final TableField DOWNLOADS = createField(DSL.name("downloads"), SQLDataType.BIGINT, this, ""); + + private DownloadStatsDaily(Name alias, Table aliased) { + this(alias, aliased, (Field[]) null, null); + } + + private DownloadStatsDaily(Name alias, Table aliased, Field[] parameters, Condition where) { + super(alias, null, aliased, parameters, DSL.comment(""), TableOptions.view(""" + create view "download_stats_daily" as SELECT _materialized_hypertable_2.day, + _materialized_hypertable_2.extension_id, + _materialized_hypertable_2.extension_version_id, + _materialized_hypertable_2.version, + _materialized_hypertable_2.target_platform, + _materialized_hypertable_2.country, + _materialized_hypertable_2.downloads + FROM _timescaledb_internal._materialized_hypertable_2 + WHERE (_materialized_hypertable_2.day < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(2)), '-infinity'::timestamp with time zone)) + UNION ALL + SELECT time_bucket('1 day'::interval, download_event."time") AS day, + download_event.extension_id, + download_event.extension_version_id, + download_event.version, + download_event.target_platform, + download_event.country, + sum(download_event.count) AS downloads + FROM download_event + WHERE (download_event."time" >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(2)), '-infinity'::timestamp with time zone)) + GROUP BY (time_bucket('1 day'::interval, download_event."time")), download_event.extension_id, download_event.extension_version_id, download_event.version, download_event.target_platform, download_event.country; + """), where); + } + + /** + * Create an aliased public.download_stats_daily table + * reference + */ + public DownloadStatsDaily(String alias) { + this(DSL.name(alias), DOWNLOAD_STATS_DAILY); + } + + /** + * Create an aliased public.download_stats_daily table + * reference + */ + public DownloadStatsDaily(Name alias) { + this(alias, DOWNLOAD_STATS_DAILY); + } + + /** + * Create a public.download_stats_daily table reference + */ + public DownloadStatsDaily() { + this(DSL.name("download_stats_daily"), null); + } + + @Override + public Schema getSchema() { + return aliased() ? null : Public.PUBLIC; + } + + @Override + public DownloadStatsDaily as(String alias) { + return new DownloadStatsDaily(DSL.name(alias), this); + } + + @Override + public DownloadStatsDaily as(Name alias) { + return new DownloadStatsDaily(alias, this); + } + + @Override + public DownloadStatsDaily as(Table alias) { + return new DownloadStatsDaily(alias.getQualifiedName(), this); + } + + /** + * Rename this table + */ + @Override + public DownloadStatsDaily rename(String name) { + return new DownloadStatsDaily(DSL.name(name), null); + } + + /** + * Rename this table + */ + @Override + public DownloadStatsDaily rename(Name name) { + return new DownloadStatsDaily(name, null); + } + + /** + * Rename this table + */ + @Override + public DownloadStatsDaily rename(Table name) { + return new DownloadStatsDaily(name.getQualifiedName(), null); + } + + /** + * Create an inline derived table from this table + */ + @Override + public DownloadStatsDaily where(Condition condition) { + return new DownloadStatsDaily(getQualifiedName(), aliased() ? this : null, null, condition); + } + + /** + * Create an inline derived table from this table + */ + @Override + public DownloadStatsDaily where(Collection conditions) { + return where(DSL.and(conditions)); + } + + /** + * Create an inline derived table from this table + */ + @Override + public DownloadStatsDaily where(Condition... conditions) { + return where(DSL.and(conditions)); + } + + /** + * Create an inline derived table from this table + */ + @Override + public DownloadStatsDaily where(Field condition) { + return where(DSL.condition(condition)); + } + + /** + * Create an inline derived table from this table + */ + @Override + @PlainSQL + public DownloadStatsDaily where(SQL condition) { + return where(DSL.condition(condition)); + } + + /** + * Create an inline derived table from this table + */ + @Override + @PlainSQL + public DownloadStatsDaily where(@Stringly.SQL String condition) { + return where(DSL.condition(condition)); + } + + /** + * Create an inline derived table from this table + */ + @Override + @PlainSQL + public DownloadStatsDaily where(@Stringly.SQL String condition, Object... binds) { + return where(DSL.condition(condition, binds)); + } + + /** + * Create an inline derived table from this table + */ + @Override + @PlainSQL + public DownloadStatsDaily where(@Stringly.SQL String condition, QueryPart... parts) { + return where(DSL.condition(condition, parts)); + } + + /** + * Create an inline derived table from this table + */ + @Override + public DownloadStatsDaily whereExists(Select select) { + return where(DSL.exists(select)); + } + + /** + * Create an inline derived table from this table + */ + @Override + public DownloadStatsDaily whereNotExists(Select select) { + return where(DSL.notExists(select)); + } +} diff --git a/server/src/main/jooq-gen/org/eclipse/openvsx/jooq/tables/records/DownloadEventRecord.java b/server/src/main/jooq-gen/org/eclipse/openvsx/jooq/tables/records/DownloadEventRecord.java new file mode 100644 index 000000000..26d7ee63f --- /dev/null +++ b/server/src/main/jooq-gen/org/eclipse/openvsx/jooq/tables/records/DownloadEventRecord.java @@ -0,0 +1,205 @@ +/* + * This file is generated by jOOQ. + */ +package org.eclipse.openvsx.jooq.tables.records; + + +import java.time.OffsetDateTime; + +import org.eclipse.openvsx.jooq.tables.DownloadEvent; +import org.jooq.impl.TableRecordImpl; + + +/** + * This class is generated by jOOQ. + */ +@SuppressWarnings({ "all", "unchecked", "rawtypes", "this-escape" }) +public class DownloadEventRecord extends TableRecordImpl { + + private static final long serialVersionUID = 1L; + + /** + * Setter for public.download_event.time. + */ + public void setTime(OffsetDateTime value) { + set(0, value); + } + + /** + * Getter for public.download_event.time. + */ + public OffsetDateTime getTime() { + return (OffsetDateTime) get(0); + } + + /** + * Setter for public.download_event.extension_id. + */ + public void setExtensionId(Long value) { + set(1, value); + } + + /** + * Getter for public.download_event.extension_id. + */ + public Long getExtensionId() { + return (Long) get(1); + } + + /** + * Setter for public.download_event.extension_version_id. + */ + public void setExtensionVersionId(Long value) { + set(2, value); + } + + /** + * Getter for public.download_event.extension_version_id. + */ + public Long getExtensionVersionId() { + return (Long) get(2); + } + + /** + * Setter for public.download_event.namespace. + */ + public void setNamespace(String value) { + set(3, value); + } + + /** + * Getter for public.download_event.namespace. + */ + public String getNamespace() { + return (String) get(3); + } + + /** + * Setter for public.download_event.extension_name. + */ + public void setExtensionName(String value) { + set(4, value); + } + + /** + * Getter for public.download_event.extension_name. + */ + public String getExtensionName() { + return (String) get(4); + } + + /** + * Setter for public.download_event.version. + */ + public void setVersion(String value) { + set(5, value); + } + + /** + * Getter for public.download_event.version. + */ + public String getVersion() { + return (String) get(5); + } + + /** + * Setter for public.download_event.target_platform. + */ + public void setTargetPlatform(String value) { + set(6, value); + } + + /** + * Getter for public.download_event.target_platform. + */ + public String getTargetPlatform() { + return (String) get(6); + } + + /** + * Setter for public.download_event.country. + */ + public void setCountry(String value) { + set(7, value); + } + + /** + * Getter for public.download_event.country. + */ + public String getCountry() { + return (String) get(7); + } + + /** + * Setter for public.download_event.ip. + */ + public void setIp(String value) { + set(8, value); + } + + /** + * Getter for public.download_event.ip. + */ + public String getIp() { + return (String) get(8); + } + + /** + * Setter for public.download_event.user_agent. + */ + public void setUserAgent(String value) { + set(9, value); + } + + /** + * Getter for public.download_event.user_agent. + */ + public String getUserAgent() { + return (String) get(9); + } + + /** + * Setter for public.download_event.count. + */ + public void setCount(Integer value) { + set(10, value); + } + + /** + * Getter for public.download_event.count. + */ + public Integer getCount() { + return (Integer) get(10); + } + + // ------------------------------------------------------------------------- + // Constructors + // ------------------------------------------------------------------------- + + /** + * Create a detached DownloadEventRecord + */ + public DownloadEventRecord() { + super(DownloadEvent.DOWNLOAD_EVENT); + } + + /** + * Create a detached, initialised DownloadEventRecord + */ + public DownloadEventRecord(OffsetDateTime time, Long extensionId, Long extensionVersionId, String namespace, String extensionName, String version, String targetPlatform, String country, String ip, String userAgent, Integer count) { + super(DownloadEvent.DOWNLOAD_EVENT); + + setTime(time); + setExtensionId(extensionId); + setExtensionVersionId(extensionVersionId); + setNamespace(namespace); + setExtensionName(extensionName); + setVersion(version); + setTargetPlatform(targetPlatform); + setCountry(country); + setIp(ip); + setUserAgent(userAgent); + setCount(count); + resetChangedOnNotNull(); + } +} diff --git a/server/src/main/jooq-gen/org/eclipse/openvsx/jooq/tables/records/DownloadStatsDailyRecord.java b/server/src/main/jooq-gen/org/eclipse/openvsx/jooq/tables/records/DownloadStatsDailyRecord.java new file mode 100644 index 000000000..d349602b1 --- /dev/null +++ b/server/src/main/jooq-gen/org/eclipse/openvsx/jooq/tables/records/DownloadStatsDailyRecord.java @@ -0,0 +1,145 @@ +/* + * This file is generated by jOOQ. + */ +package org.eclipse.openvsx.jooq.tables.records; + + +import java.time.OffsetDateTime; + +import org.eclipse.openvsx.jooq.tables.DownloadStatsDaily; +import org.jooq.impl.TableRecordImpl; + + +/** + * This class is generated by jOOQ. + */ +@SuppressWarnings({ "all", "unchecked", "rawtypes", "this-escape" }) +public class DownloadStatsDailyRecord extends TableRecordImpl { + + private static final long serialVersionUID = 1L; + + /** + * Setter for public.download_stats_daily.day. + */ + public void setDay(OffsetDateTime value) { + set(0, value); + } + + /** + * Getter for public.download_stats_daily.day. + */ + public OffsetDateTime getDay() { + return (OffsetDateTime) get(0); + } + + /** + * Setter for public.download_stats_daily.extension_id. + */ + public void setExtensionId(Long value) { + set(1, value); + } + + /** + * Getter for public.download_stats_daily.extension_id. + */ + public Long getExtensionId() { + return (Long) get(1); + } + + /** + * Setter for public.download_stats_daily.extension_version_id. + */ + public void setExtensionVersionId(Long value) { + set(2, value); + } + + /** + * Getter for public.download_stats_daily.extension_version_id. + */ + public Long getExtensionVersionId() { + return (Long) get(2); + } + + /** + * Setter for public.download_stats_daily.version. + */ + public void setVersion(String value) { + set(3, value); + } + + /** + * Getter for public.download_stats_daily.version. + */ + public String getVersion() { + return (String) get(3); + } + + /** + * Setter for public.download_stats_daily.target_platform. + */ + public void setTargetPlatform(String value) { + set(4, value); + } + + /** + * Getter for public.download_stats_daily.target_platform. + */ + public String getTargetPlatform() { + return (String) get(4); + } + + /** + * Setter for public.download_stats_daily.country. + */ + public void setCountry(String value) { + set(5, value); + } + + /** + * Getter for public.download_stats_daily.country. + */ + public String getCountry() { + return (String) get(5); + } + + /** + * Setter for public.download_stats_daily.downloads. + */ + public void setDownloads(Long value) { + set(6, value); + } + + /** + * Getter for public.download_stats_daily.downloads. + */ + public Long getDownloads() { + return (Long) get(6); + } + + // ------------------------------------------------------------------------- + // Constructors + // ------------------------------------------------------------------------- + + /** + * Create a detached DownloadStatsDailyRecord + */ + public DownloadStatsDailyRecord() { + super(DownloadStatsDaily.DOWNLOAD_STATS_DAILY); + } + + /** + * Create a detached, initialised DownloadStatsDailyRecord + */ + public DownloadStatsDailyRecord(OffsetDateTime day, Long extensionId, Long extensionVersionId, String version, String targetPlatform, String country, Long downloads) { + super(DownloadStatsDaily.DOWNLOAD_STATS_DAILY); + + setDay(day); + setExtensionId(extensionId); + setExtensionVersionId(extensionVersionId); + setVersion(version); + setTargetPlatform(targetPlatform); + setCountry(country); + setDownloads(downloads); + resetChangedOnNotNull(); + } +} diff --git a/server/src/main/resources/db/migration/V1_72__Download_Analytics.sql b/server/src/main/resources/db/migration/V1_72__Download_Analytics.sql new file mode 100644 index 000000000..d3eb798fe --- /dev/null +++ b/server/src/main/resources/db/migration/V1_72__Download_Analytics.sql @@ -0,0 +1,51 @@ +-- Time-series download analytics schema. Requires a PostgreSQL image with the timescaledb +-- extension available. See V1_72__Download_Analytics.sql.conf: continuous aggregates cannot +-- be created inside a transaction. + +CREATE EXTENSION IF NOT EXISTS timescaledb; + +CREATE TABLE download_event ( + time TIMESTAMPTZ NOT NULL, + extension_id BIGINT NOT NULL, + extension_version_id BIGINT NOT NULL, + namespace VARCHAR(255) NOT NULL, + extension_name VARCHAR(255) NOT NULL, + version VARCHAR(255) NOT NULL, + target_platform VARCHAR(255) NOT NULL, + country CHAR(2), + ip VARCHAR(45), + user_agent TEXT, + count INTEGER NOT NULL DEFAULT 1 +); + +SELECT create_hypertable('download_event', by_range('time', INTERVAL '7 days')); + +CREATE INDEX de_ext_time ON download_event (extension_id, time DESC); + +-- materialized_only = false keeps the not-yet-materialized tail (e.g. today) queryable +-- through real-time aggregation, which the settling-margin logic in the query service +-- relies on. +CREATE MATERIALIZED VIEW download_stats_daily +WITH (timescaledb.continuous, timescaledb.materialized_only = false) AS +SELECT time_bucket('1 day', time) AS day, + extension_id, extension_version_id, version, target_platform, country, + SUM(count) AS downloads +FROM download_event +GROUP BY time_bucket('1 day', time), extension_id, extension_version_id, version, target_platform, country +WITH NO DATA; + +SELECT add_continuous_aggregate_policy('download_stats_daily', + start_offset => INTERVAL '3 days', + end_offset => INTERVAL '1 hour', + schedule_interval => INTERVAL '1 hour'); + +-- compress raw chunks after 7 days, drop them after 90 days; the daily aggregate is kept forever +ALTER TABLE download_event SET ( + timescaledb.compress, + timescaledb.compress_segmentby = 'extension_id', + timescaledb.compress_orderby = 'time DESC' +); + +SELECT add_compression_policy('download_event', INTERVAL '7 days'); + +SELECT add_retention_policy('download_event', INTERVAL '90 days'); diff --git a/server/src/main/resources/db/migration/V1_72__Download_Analytics.sql.conf b/server/src/main/resources/db/migration/V1_72__Download_Analytics.sql.conf new file mode 100644 index 000000000..73bd53a14 --- /dev/null +++ b/server/src/main/resources/db/migration/V1_72__Download_Analytics.sql.conf @@ -0,0 +1 @@ +executeInTransaction=false diff --git a/server/src/test/java/org/eclipse/openvsx/AbstractPostgresContainerTest.java b/server/src/test/java/org/eclipse/openvsx/AbstractPostgresContainerTest.java index a030feeda..5c286d2d5 100644 --- a/server/src/test/java/org/eclipse/openvsx/AbstractPostgresContainerTest.java +++ b/server/src/test/java/org/eclipse/openvsx/AbstractPostgresContainerTest.java @@ -16,6 +16,7 @@ import org.springframework.test.context.DynamicPropertyRegistry; import org.springframework.test.context.DynamicPropertySource; import org.testcontainers.postgresql.PostgreSQLContainer; +import org.testcontainers.utility.DockerImageName; /** * Base class for tests that need a PostgreSQL database. @@ -30,11 +31,17 @@ * Because all contexts now share a single database, tests must keep cleaning up after themselves (via * transactional rollback or an explicit tear-down) and use unique identifiers, exactly as they already * had to when sharing a context. + *

+ * The image is timescale/timescaledb (PostgreSQL plus the timescaledb extension): the main + * migration chain contains the download analytics schema, which requires the extension. + * Override with {@code -Dovsx.test.postgres.image=...} if needed. */ @Tag("integration") public abstract class AbstractPostgresContainerTest { - static final PostgreSQLContainer POSTGRES = new PostgreSQLContainer("postgres:16.2"); + static final PostgreSQLContainer POSTGRES = new PostgreSQLContainer( + DockerImageName.parse(System.getProperty("ovsx.test.postgres.image", "timescale/timescaledb:2.17.2-pg16")) + .asCompatibleSubstituteFor("postgres")); static { POSTGRES.start(); diff --git a/server/src/test/java/org/eclipse/openvsx/RegistryAPITest.java b/server/src/test/java/org/eclipse/openvsx/RegistryAPITest.java index b1a0139e3..5ea301220 100644 --- a/server/src/test/java/org/eclipse/openvsx/RegistryAPITest.java +++ b/server/src/test/java/org/eclipse/openvsx/RegistryAPITest.java @@ -33,6 +33,7 @@ import org.mockito.ArgumentCaptor; import org.mockito.Mockito; import org.mockito.stubbing.Answer; +import org.springframework.beans.factory.ObjectProvider; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.TestConfiguration; import org.springframework.boot.webmvc.test.autoconfigure.WebMvcTest; @@ -55,6 +56,8 @@ import org.eclipse.openvsx.accesstoken.AccessTokenConfig; import org.eclipse.openvsx.accesstoken.AccessTokenService; import org.eclipse.openvsx.adapter.VSCodeIdService; +import org.eclipse.openvsx.analytics.ingestion.DownloadIngestionProcessor; +import org.eclipse.openvsx.analytics.ingestion.DownloadRecordSource; import org.eclipse.openvsx.cache.CacheService; import org.eclipse.openvsx.cache.ExtensionJsonCacheKeyGenerator; import org.eclipse.openvsx.cache.LatestExtensionVersionCacheKeyGenerator; @@ -78,7 +81,6 @@ import org.eclipse.openvsx.security.OAuth2UserServices; import org.eclipse.openvsx.security.SecurityConfig; import org.eclipse.openvsx.storage.*; -import org.eclipse.openvsx.storage.log.DownloadCountService; import org.eclipse.openvsx.trustedpublishing.TrustedPublishingConfig; import org.eclipse.openvsx.util.ChangesCursor; import org.eclipse.openvsx.util.LogService; @@ -115,7 +117,7 @@ AzureBlobStorageService.class, AwsStorageService.class, VSCodeIdService.class, - DownloadCountService.class, + DownloadIngestionProcessor.class, ExtensionDownloadMetrics.class, CacheService.class, EclipseService.class, @@ -3814,7 +3816,8 @@ StorageUtilService storageUtilService( AzureBlobStorageService azureStorage, LocalStorageService localStorage, AwsStorageService awsStorage, - DownloadCountService downloadCountService, + ObjectProvider ingestionSources, + DownloadIngestionProcessor ingestionProcessor, ExtensionDownloadMetrics downloadMetrics, SearchUtilService search, CacheService cache, @@ -3828,7 +3831,8 @@ StorageUtilService storageUtilService( azureStorage, localStorage, awsStorage, - downloadCountService, + ingestionSources, + ingestionProcessor, downloadMetrics, search, cache, diff --git a/server/src/test/java/org/eclipse/openvsx/adapter/VSCodeAPITest.java b/server/src/test/java/org/eclipse/openvsx/adapter/VSCodeAPITest.java index 448644a7a..debd63b5b 100644 --- a/server/src/test/java/org/eclipse/openvsx/adapter/VSCodeAPITest.java +++ b/server/src/test/java/org/eclipse/openvsx/adapter/VSCodeAPITest.java @@ -27,6 +27,7 @@ import org.junit.jupiter.api.Test; import org.mockito.Mockito; import org.mockito.stubbing.Answer; +import org.springframework.beans.factory.ObjectProvider; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.TestConfiguration; import org.springframework.boot.webmvc.test.autoconfigure.WebMvcTest; @@ -45,6 +46,8 @@ import org.eclipse.openvsx.MockMvcAsyncConfig; import org.eclipse.openvsx.MockTransactionTemplate; import org.eclipse.openvsx.UserService; +import org.eclipse.openvsx.analytics.ingestion.DownloadIngestionProcessor; +import org.eclipse.openvsx.analytics.ingestion.DownloadRecordSource; import org.eclipse.openvsx.cache.CacheService; import org.eclipse.openvsx.cache.FilesCacheKeyGenerator; import org.eclipse.openvsx.cache.LatestExtensionVersionCacheKeyGenerator; @@ -59,7 +62,6 @@ import org.eclipse.openvsx.security.OAuth2UserServices; import org.eclipse.openvsx.security.SecurityConfig; import org.eclipse.openvsx.storage.*; -import org.eclipse.openvsx.storage.log.DownloadCountService; import org.eclipse.openvsx.util.TargetPlatform; import org.eclipse.openvsx.util.VersionService; import org.eclipse.openvsx.web.JacksonConfig; @@ -81,7 +83,7 @@ GoogleCloudStorageService.class, AzureBlobStorageService.class, AwsStorageService.class, - DownloadCountService.class, + DownloadIngestionProcessor.class, ExtensionDownloadMetrics.class, CacheService.class, UpstreamVSCodeService.class, @@ -1528,7 +1530,8 @@ StorageUtilService storageUtilService( AzureBlobStorageService azureStorage, LocalStorageService localStorage, AwsStorageService awsStorage, - DownloadCountService downloadCountService, + ObjectProvider ingestionSources, + DownloadIngestionProcessor ingestionProcessor, ExtensionDownloadMetrics downloadMetrics, SearchUtilService search, CacheService cache, @@ -1542,7 +1545,8 @@ StorageUtilService storageUtilService( azureStorage, localStorage, awsStorage, - downloadCountService, + ingestionSources, + ingestionProcessor, downloadMetrics, search, cache, diff --git a/server/src/test/java/org/eclipse/openvsx/admin/AdminAPITest.java b/server/src/test/java/org/eclipse/openvsx/admin/AdminAPITest.java index 4cfe4bc37..e05cbc51e 100644 --- a/server/src/test/java/org/eclipse/openvsx/admin/AdminAPITest.java +++ b/server/src/test/java/org/eclipse/openvsx/admin/AdminAPITest.java @@ -26,6 +26,7 @@ import org.mockito.Mockito; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; +import org.springframework.beans.factory.ObjectProvider; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.TestConfiguration; import org.springframework.boot.webmvc.test.autoconfigure.WebMvcTest; @@ -52,6 +53,8 @@ import org.eclipse.openvsx.accesstoken.AccessTokenConfig; import org.eclipse.openvsx.accesstoken.AccessTokenService; import org.eclipse.openvsx.adapter.VSCodeIdService; +import org.eclipse.openvsx.analytics.ingestion.DownloadIngestionProcessor; +import org.eclipse.openvsx.analytics.ingestion.DownloadRecordSource; import org.eclipse.openvsx.cache.CacheService; import org.eclipse.openvsx.cache.LatestExtensionVersionCacheKeyGenerator; import org.eclipse.openvsx.eclipse.EclipseService; @@ -100,7 +103,6 @@ import org.eclipse.openvsx.storage.GoogleCloudStorageService; import org.eclipse.openvsx.storage.LocalStorageService; import org.eclipse.openvsx.storage.StorageUtilService; -import org.eclipse.openvsx.storage.log.DownloadCountService; import org.eclipse.openvsx.trustedpublishing.TrustedPublishingConfig; import org.eclipse.openvsx.util.LogService; import org.eclipse.openvsx.util.TargetPlatform; @@ -133,7 +135,7 @@ AzureBlobStorageService.class, AwsStorageService.class, VSCodeIdService.class, - DownloadCountService.class, + DownloadIngestionProcessor.class, ExtensionDownloadMetrics.class, CacheService.class, PublishExtensionVersionHandler.class, @@ -2699,7 +2701,8 @@ StorageUtilService storageUtilService( AzureBlobStorageService azureStorage, LocalStorageService localStorage, AwsStorageService awsStorage, - DownloadCountService downloadCountService, + ObjectProvider ingestionSources, + DownloadIngestionProcessor ingestionProcessor, ExtensionDownloadMetrics downloadMetrics, SearchUtilService search, CacheService cache, @@ -2713,7 +2716,8 @@ StorageUtilService storageUtilService( azureStorage, localStorage, awsStorage, - downloadCountService, + ingestionSources, + ingestionProcessor, downloadMetrics, search, cache, diff --git a/server/src/test/java/org/eclipse/openvsx/analytics/DownloadAnalyticsAPITest.java b/server/src/test/java/org/eclipse/openvsx/analytics/DownloadAnalyticsAPITest.java new file mode 100644 index 000000000..1f4fd440b --- /dev/null +++ b/server/src/test/java/org/eclipse/openvsx/analytics/DownloadAnalyticsAPITest.java @@ -0,0 +1,120 @@ +/****************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation. + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * https://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + *****************************************************************************/ +package org.eclipse.openvsx.analytics; + +import java.time.Clock; +import java.time.Instant; +import java.time.ZoneOffset; +import java.util.List; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.Mockito; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.setup.MockMvcBuilders; + +import org.eclipse.openvsx.entities.Extension; +import org.eclipse.openvsx.repositories.RepositoryService; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +class DownloadAnalyticsAPITest { + + private static final Instant NOW = Instant.parse("2026-07-15T10:00:00Z"); + + private final DownloadAnalyticsService service = Mockito.mock(DownloadAnalyticsService.class); + private final RepositoryService repositories = Mockito.mock(RepositoryService.class); + + private MockMvc mockMvc; + + @BeforeEach + void setUp() { + var controller = new DownloadAnalyticsAPI( + service, + repositories, + Clock.fixed(NOW, ZoneOffset.UTC)); + mockMvc = MockMvcBuilders.standaloneSetup(controller).build(); + + var extension = new Extension(); + extension.setId(42L); + Mockito.when(repositories.findActiveExtension("bar", "foo")).thenReturn(extension); + } + + @Test + void testResponseShape() throws Exception { + Mockito.when(service.getSeries(any())).thenReturn( + List.of( + new DownloadSeriesPoint(Instant.parse("2026-07-01T00:00:00Z"), null, 4321, false), + new DownloadSeriesPoint(Instant.parse("2026-07-02T00:00:00Z"), null, 10, true))); + + mockMvc.perform(get("/api/foo/bar/analytics/downloads?from=2026-07-01&to=2026-07-03")) + .andExpect(status().isOk()) + .andExpect( + content().json( + "{\"points\":[{\"t\":\"2026-07-01\",\"count\":4321},{\"t\":\"2026-07-02\",\"count\":10}]}", + true)); + } + + @Test + void testRequestParametersArePassedToService() throws Exception { + Mockito.when(service.getSeries(any())).thenReturn(List.of()); + + mockMvc.perform(get("/api/foo/bar/analytics/downloads?from=2026-06-01&to=2026-07-01&interval=week")) + .andExpect(status().isOk()); + + var captor = ArgumentCaptor.forClass(DownloadSeriesRequest.class); + Mockito.verify(service).getSeries(captor.capture()); + var request = captor.getValue(); + assertEquals(List.of(42L), request.extensionIds()); + assertEquals(Instant.parse("2026-06-01T00:00:00Z"), request.from()); + assertEquals(Instant.parse("2026-07-01T00:00:00Z"), request.to()); + assertEquals(DownloadSeriesInterval.WEEK, request.interval()); + assertEquals(DownloadSeriesGroupBy.NONE, request.groupBy()); + } + + @Test + void testDefaultRangeIsTheLastThirtyDays() throws Exception { + Mockito.when(service.getSeries(any())).thenReturn(List.of()); + + mockMvc.perform(get("/api/foo/bar/analytics/downloads")).andExpect(status().isOk()); + + var captor = ArgumentCaptor.forClass(DownloadSeriesRequest.class); + Mockito.verify(service).getSeries(captor.capture()); + // now is 2026-07-15T10:00Z: the range ends after today (partial) and spans 30 days + assertEquals(Instant.parse("2026-07-16T00:00:00Z"), captor.getValue().to()); + assertEquals(Instant.parse("2026-06-16T00:00:00Z"), captor.getValue().from()); + assertEquals(DownloadSeriesInterval.DAY, captor.getValue().interval()); + } + + @Test + void testParameterValidation() throws Exception { + mockMvc.perform(get("/api/foo/bar/analytics/downloads?interval=hour")) + .andExpect(status().isBadRequest()); + mockMvc.perform(get("/api/foo/bar/analytics/downloads?from=not-a-date")) + .andExpect(status().isBadRequest()); + mockMvc.perform(get("/api/foo/bar/analytics/downloads?from=2026-07-02&to=2026-07-01")) + .andExpect(status().isBadRequest()); + mockMvc.perform(get("/api/foo/bar/analytics/downloads?from=2000-01-01&to=2026-07-01")) + .andExpect(status().isBadRequest()); + } + + @Test + void testUnknownExtensionIsNotFound() throws Exception { + mockMvc.perform(get("/api/foo/unknown/analytics/downloads")).andExpect(status().isNotFound()); + } +} diff --git a/server/src/test/java/org/eclipse/openvsx/analytics/DownloadAnalyticsDisabledTest.java b/server/src/test/java/org/eclipse/openvsx/analytics/DownloadAnalyticsDisabledTest.java new file mode 100644 index 000000000..d3eb5966c --- /dev/null +++ b/server/src/test/java/org/eclipse/openvsx/analytics/DownloadAnalyticsDisabledTest.java @@ -0,0 +1,54 @@ +/****************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation. + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * https://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + *****************************************************************************/ +package org.eclipse.openvsx.analytics; + +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.webmvc.test.autoconfigure.AutoConfigureMockMvc; +import org.springframework.context.ApplicationContext; +import org.springframework.test.web.servlet.MockMvc; + +import org.eclipse.openvsx.AbstractPostgresContainerTest; + +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +/** + * ovsx.analytics.enabled defaults to false: no analytics beans exist and the endpoint is not + * mapped, byte-for-byte current behavior. The property is pinned so this holds even in the + * analytics-on test matrix run. + */ +@SpringBootTest(properties = "ovsx.analytics.enabled=false") +@AutoConfigureMockMvc +class DownloadAnalyticsDisabledTest extends AbstractPostgresContainerTest { + + @Autowired + MockMvc mockMvc; + + @Autowired + ApplicationContext context; + + @Test + void testEndpointIsNotFoundWhenAnalyticsIsDisabled() throws Exception { + mockMvc.perform(get("/api/foo/bar/analytics/downloads")).andExpect(status().isNotFound()); + } + + @Test + void testNoAnalyticsBeansWhenDisabled() { + assertTrue(context.getBeanNamesForType(DownloadAnalyticsRepository.class).length == 0); + assertTrue(context.getBeanNamesForType(DownloadAnalyticsService.class).length == 0); + assertTrue(context.getBeanNamesForType(DownloadAnalyticsAPI.class).length == 0); + } +} diff --git a/server/src/test/java/org/eclipse/openvsx/analytics/DownloadAnalyticsEndpointTest.java b/server/src/test/java/org/eclipse/openvsx/analytics/DownloadAnalyticsEndpointTest.java new file mode 100644 index 000000000..d3594cff1 --- /dev/null +++ b/server/src/test/java/org/eclipse/openvsx/analytics/DownloadAnalyticsEndpointTest.java @@ -0,0 +1,214 @@ +/****************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation. + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * https://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + *****************************************************************************/ +package org.eclipse.openvsx.analytics; + +import java.time.Instant; +import java.util.List; + +import jakarta.persistence.EntityManager; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.webmvc.test.autoconfigure.AutoConfigureMockMvc; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.mock.web.MockHttpServletRequest; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.support.TransactionTemplate; +import org.springframework.web.context.request.RequestContextHolder; +import org.springframework.web.context.request.ServletRequestAttributes; + +import org.eclipse.openvsx.AbstractPostgresContainerTest; +import org.eclipse.openvsx.entities.Extension; +import org.eclipse.openvsx.entities.ExtensionVersion; +import org.eclipse.openvsx.entities.FileResource; +import org.eclipse.openvsx.entities.Namespace; +import org.eclipse.openvsx.storage.StorageUtilService; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +/** + * Full-stack proof of the enabled configuration: TimescaleDB-backed defaults wired by the + * auto-configuration, queried through the public REST endpoint. + */ +@SpringBootTest(properties = "ovsx.analytics.enabled=true") +@AutoConfigureMockMvc +class DownloadAnalyticsEndpointTest extends AbstractPostgresContainerTest { + + @Autowired + MockMvc mockMvc; + + @Autowired + DownloadAnalyticsRepository repository; + + @Autowired + javax.sql.DataSource dataSource; + + @Autowired + EntityManager entityManager; + + @Autowired + PlatformTransactionManager transactionManager; + + @Autowired + StorageUtilService storageUtilService; + + private Extension extension; + + @AfterEach + void cleanUp() { + RequestContextHolder.resetRequestAttributes(); + new JdbcTemplate(dataSource).execute("TRUNCATE download_event"); + if (extension != null) { + inTransaction(() -> { + var managed = entityManager.find(Extension.class, extension.getId()); + managed.getVersions().forEach(extVersion -> { + entityManager + .createQuery("delete from FileResource fr where fr.extension = :extVersion") + .setParameter("extVersion", extVersion) + .executeUpdate(); + entityManager.remove(extVersion); + }); + var namespace = managed.getNamespace(); + entityManager.remove(managed); + entityManager.remove(namespace); + return null; + }); + extension = null; + } + } + + @Test + void testDownloadSeriesEndToEnd() throws Exception { + extension = seedExtension("e2ens", "e2e-ext"); + repository.save( + List.of( + event(Instant.parse("2026-07-01T10:00:00Z"), extension.getId(), 3), + event(Instant.parse("2026-07-01T18:00:00Z"), extension.getId(), 1), + event(Instant.parse("2026-07-03T00:00:00Z"), extension.getId(), 5))); + + mockMvc.perform(get("/api/e2ens/e2e-ext/analytics/downloads?from=2026-07-01&to=2026-07-04")) + .andExpect(status().isOk()) + .andExpect( + content().json( + "{\"points\":[{\"t\":\"2026-07-01\",\"count\":4},{\"t\":\"2026-07-02\",\"count\":0}," + + "{\"t\":\"2026-07-03\",\"count\":5}]}", + true)); + } + + @Test + void testUnknownExtensionIsNotFound() throws Exception { + mockMvc.perform(get("/api/nowhere/nothing/analytics/downloads")).andExpect(status().isNotFound()); + } + + /** + * Without a log-based source covering the file, a request-path download produces an + * analytics event in the same transaction as the counter update, with client data taken + * from the current HTTP request. + */ + @Test + void testRequestPathDownloadProducesAnalyticsEvent() throws Exception { + var resource = seedExtensionWithResource("e2ereq", "e2e-req-ext", "e2ereq.e2e-req-ext-1.0.0.vsix"); + var extVersion = resource.getExtension(); + + var request = new MockHttpServletRequest(); + request.addHeader("User-Agent", "VSCode 1.90.2 (Microsoft Visual Studio Code)"); + request.addHeader("X-Forwarded-For", "203.0.113.9, 10.0.0.1"); + RequestContextHolder.setRequestAttributes(new ServletRequestAttributes(request)); + + inTransaction(() -> { + storageUtilService.increaseDownloadCount(entityManager.find(FileResource.class, resource.getId())); + return null; + }); + + // the counter and the event committed together + var downloadCount = inTransaction( + () -> entityManager.find(Extension.class, extension.getId()).getDownloadCount()); + assertEquals(1, downloadCount); + + var jdbc = new JdbcTemplate(dataSource); + var event = jdbc.queryForMap( + "SELECT extension_id, extension_version_id, ip, user_agent, count FROM download_event"); + assertEquals(extension.getId(), event.get("extension_id")); + assertEquals(extVersion.getId(), event.get("extension_version_id")); + assertEquals("203.0.113.9", event.get("ip")); + assertEquals("VSCode 1.90.2 (Microsoft Visual Studio Code)", event.get("user_agent")); + assertEquals(1, event.get("count")); + + // and the event is visible through the endpoint: the default range ends tomorrow, + // so the last of the 30 points is today's (partial) bucket + mockMvc.perform(get("/api/e2ereq/e2e-req-ext/analytics/downloads")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.points[29].count").value(1)); + } + + private DownloadEvent event(Instant time, long extensionId, int count) { + return new DownloadEvent( + time, + extensionId, + extensionId * 100, + "e2ens", + "e2e-ext", + "1.0.0", + "universal", + "US", + "9.9.9.9", + "VSCode 1.90.2", + count); + } + + private FileResource seedExtensionWithResource(String namespaceName, String extensionName, String vsixFilename) { + extension = seedExtension(namespaceName, extensionName); + return inTransaction(() -> { + var extVersion = entityManager.find(Extension.class, extension.getId()).getVersions().get(0); + var resource = new FileResource(); + resource.setName(vsixFilename); + resource.setType(FileResource.DOWNLOAD); + resource.setStorageType(FileResource.STORAGE_LOCAL); + resource.setExtension(extVersion); + entityManager.persist(resource); + return resource; + }); + } + + private Extension seedExtension(String namespaceName, String extensionName) { + return inTransaction(() -> { + var namespace = new Namespace(); + namespace.setName(namespaceName); + entityManager.persist(namespace); + + var seeded = new Extension(); + seeded.setName(extensionName); + seeded.setNamespace(namespace); + seeded.setActive(true); + entityManager.persist(seeded); + + var extVersion = new ExtensionVersion(); + extVersion.setVersion("1.0.0"); + extVersion.setTargetPlatform("universal"); + extVersion.setExtension(seeded); + extVersion.setActive(true); + entityManager.persist(extVersion); + return seeded; + }); + } + + private T inTransaction(java.util.function.Supplier action) { + return new TransactionTemplate(transactionManager).execute(status -> action.get()); + } +} diff --git a/server/src/test/java/org/eclipse/openvsx/analytics/DownloadAnalyticsServiceTest.java b/server/src/test/java/org/eclipse/openvsx/analytics/DownloadAnalyticsServiceTest.java new file mode 100644 index 000000000..788e7ee75 --- /dev/null +++ b/server/src/test/java/org/eclipse/openvsx/analytics/DownloadAnalyticsServiceTest.java @@ -0,0 +1,213 @@ +/****************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation. + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * https://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + *****************************************************************************/ +package org.eclipse.openvsx.analytics; + +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.time.ZoneOffset; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class DownloadAnalyticsServiceTest { + + private static final Instant NOW = Instant.parse("2026-07-15T10:00:00Z"); + private static final Duration SETTLING_MARGIN = Duration.ofHours(2); + + private final FakeRepository repository = new FakeRepository(); + private final DownloadAnalyticsService service = new DownloadAnalyticsService( + repository, + SETTLING_MARGIN, + Clock.fixed(NOW, ZoneOffset.UTC)); + + @Test + void testDenseZeroFilledSeries() { + repository.rows = List.of( + new DownloadSeriesRow(Instant.parse("2026-07-11T00:00:00Z"), null, 5), + new DownloadSeriesRow(Instant.parse("2026-07-13T00:00:00Z"), null, 2)); + + var points = service.getSeries(dayRequest("2026-07-10T00:00:00Z", "2026-07-15T00:00:00Z")); + + assertEquals(5, points.size()); + assertEquals(point("2026-07-10T00:00:00Z", 0, false), points.get(0)); + assertEquals(point("2026-07-11T00:00:00Z", 5, false), points.get(1)); + assertEquals(point("2026-07-12T00:00:00Z", 0, false), points.get(2)); + assertEquals(point("2026-07-13T00:00:00Z", 2, false), points.get(3)); + assertEquals(point("2026-07-14T00:00:00Z", 0, false), points.get(4)); + } + + @Test + void testBucketsStartAtUtcBoundaries() { + var points = service.getSeries(dayRequest("2026-07-10T15:30:00Z", "2026-07-12T01:00:00Z")); + + assertEquals( + List.of( + Instant.parse("2026-07-10T00:00:00Z"), + Instant.parse("2026-07-11T00:00:00Z"), + Instant.parse("2026-07-12T00:00:00Z")), + points.stream().map(DownloadSeriesPoint::bucketStart).toList()); + } + + @Test + void testTrailingPointsAreMarkedPartial() { + var points = service.getSeries(dayRequest("2026-07-13T00:00:00Z", "2026-07-16T00:00:00Z")); + + assertEquals(3, points.size()); + // 2026-07-13 ended at 07-14T00:00; well past the settling margin + assertFalse(points.get(0).partial()); + // 2026-07-14 ended at 07-15T00:00 + 2h margin = 07-15T02:00 <= now, settled + assertFalse(points.get(1).partial()); + // 2026-07-15 is still running + assertTrue(points.get(2).partial()); + } + + @Test + void testLastCompletedDayStaysPartialWithinSettlingMargin() { + var earlyMorning = Instant.parse("2026-07-15T01:00:00Z"); + var service = new DownloadAnalyticsService( + repository, + SETTLING_MARGIN, + Clock.fixed(earlyMorning, ZoneOffset.UTC)); + + var points = service.getSeries(dayRequest("2026-07-13T00:00:00Z", "2026-07-15T00:00:00Z")); + + assertFalse(points.get(0).partial()); + // 2026-07-14 ended at 07-15T00:00, but the settling margin has not passed yet + assertTrue(points.get(1).partial()); + } + + @Test + void testSettledRangesAreCached() { + var request = dayRequest("2026-07-01T00:00:00Z", "2026-07-10T00:00:00Z"); + service.getSeries(request); + service.getSeries(request); + + assertEquals(1, repository.calls.get()); + } + + @Test + void testUnsettledTailIsNotCached() { + // the settled part [07-13, 07-15) is cached, the live part [07-15, 07-16) is re-queried + var request = dayRequest("2026-07-13T00:00:00Z", "2026-07-16T00:00:00Z"); + service.getSeries(request); + assertEquals(2, repository.calls.get()); + assertEquals( + List.of(Instant.parse("2026-07-13T00:00:00Z"), Instant.parse("2026-07-15T00:00:00Z")), + repository.requests.stream().map(DownloadSeriesRequest::from).toList()); + + service.getSeries(request); + assertEquals(3, repository.calls.get()); + assertEquals(Instant.parse("2026-07-15T00:00:00Z"), repository.requests.get(2).from()); + } + + @Test + void testGroupedSeriesIsZeroFilledPerGroup() { + repository.rows = List.of( + new DownloadSeriesRow(Instant.parse("2026-07-10T00:00:00Z"), "US", 3), + new DownloadSeriesRow(Instant.parse("2026-07-11T00:00:00Z"), "DE", 2)); + + var points = service.getSeries( + new DownloadSeriesRequest( + List.of(1L), + Instant.parse("2026-07-10T00:00:00Z"), + Instant.parse("2026-07-12T00:00:00Z"), + DownloadSeriesInterval.DAY, + DownloadSeriesGroupBy.COUNTRY)); + + assertEquals( + List.of( + point("2026-07-10T00:00:00Z", "DE", 0, false), + point("2026-07-10T00:00:00Z", "US", 3, false), + point("2026-07-11T00:00:00Z", "DE", 2, false), + point("2026-07-11T00:00:00Z", "US", 0, false)), + points); + } + + @Test + void testWeeklyBucketsStartOnUtcMondays() { + repository.rows = List.of(new DownloadSeriesRow(Instant.parse("2026-06-08T00:00:00Z"), null, 4)); + + var points = service.getSeries( + new DownloadSeriesRequest( + List.of(1L), + Instant.parse("2026-06-03T00:00:00Z"), + Instant.parse("2026-06-22T00:00:00Z"), + DownloadSeriesInterval.WEEK, + DownloadSeriesGroupBy.NONE)); + + // 2026-06-03 is a Wednesday; its bucket starts Monday 2026-06-01 + assertEquals( + List.of( + point("2026-06-01T00:00:00Z", 0, false), + point("2026-06-08T00:00:00Z", 4, false), + point("2026-06-15T00:00:00Z", 0, false)), + points); + } + + @Test + void testMonthlyBucketsStartOnFirstOfMonth() { + repository.rows = List.of(new DownloadSeriesRow(Instant.parse("2026-06-01T00:00:00Z"), null, 9)); + + var points = service.getSeries( + new DownloadSeriesRequest( + List.of(1L), + Instant.parse("2026-05-15T00:00:00Z"), + Instant.parse("2026-07-01T00:00:00Z"), + DownloadSeriesInterval.MONTH, + DownloadSeriesGroupBy.NONE)); + + assertEquals( + List.of(point("2026-05-01T00:00:00Z", 0, false), point("2026-06-01T00:00:00Z", 9, false)), + points); + } + + private DownloadSeriesRequest dayRequest(String from, String to) { + return DownloadSeriesRequest.of(1L, Instant.parse(from), Instant.parse(to), DownloadSeriesInterval.DAY); + } + + private DownloadSeriesPoint point(String bucketStart, long count, boolean partial) { + return point(bucketStart, null, count, partial); + } + + private DownloadSeriesPoint point(String bucketStart, String group, long count, boolean partial) { + return new DownloadSeriesPoint(Instant.parse(bucketStart), group, count, partial); + } + + private static class FakeRepository implements DownloadAnalyticsRepository { + List rows = List.of(); + final AtomicInteger calls = new AtomicInteger(); + final List requests = new ArrayList<>(); + + @Override + public void save(List events) { + } + + @Override + public List findSeries(DownloadSeriesRequest request) { + calls.incrementAndGet(); + requests.add(request); + return rows.stream() + .filter( + row -> !row.bucketStart().isBefore(request.from()) + && row.bucketStart().isBefore(request.to())) + .toList(); + } + } +} diff --git a/server/src/test/java/org/eclipse/openvsx/analytics/DownloadEventTest.java b/server/src/test/java/org/eclipse/openvsx/analytics/DownloadEventTest.java new file mode 100644 index 000000000..159f7dacc --- /dev/null +++ b/server/src/test/java/org/eclipse/openvsx/analytics/DownloadEventTest.java @@ -0,0 +1,113 @@ +/****************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation. + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * https://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + *****************************************************************************/ +package org.eclipse.openvsx.analytics; + +import java.time.Instant; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + +public class DownloadEventTest { + + private static final Instant TIME = Instant.parse("2026-07-01T14:00:00Z"); + + @Test + public void testValidEvent() { + var event = new DownloadEvent( + TIME, + 42L, + 7L, + "redhat", + "java", + "1.2.3", + "universal", + "US", + "9.9.9.9", + "VSCode 1.90.2", + 7); + assertEquals(TIME, event.time()); + assertEquals(42L, event.extensionId()); + assertEquals(7L, event.extensionVersionId()); + assertEquals("redhat", event.namespace()); + assertEquals("java", event.extensionName()); + assertEquals("1.2.3", event.version()); + assertEquals("universal", event.targetPlatform()); + assertEquals("US", event.country()); + assertEquals("9.9.9.9", event.ip()); + assertEquals("VSCode 1.90.2", event.userAgent()); + assertEquals(7, event.count()); + } + + @Test + public void testIpAndUserAgentAreOptional() { + var event = new DownloadEvent(TIME, 42L, 7L, "redhat", "java", "1.2.3", "universal", null, null, null, 1); + assertNull(event.ip()); + assertNull(event.userAgent()); + // blank values are normalized to null + var blank = new DownloadEvent(TIME, 42L, 7L, "redhat", "java", "1.2.3", "universal", null, " ", " ", 1); + assertNull(blank.ip()); + assertNull(blank.userAgent()); + } + + @Test + public void testCountMustBeAdditive() { + assertThrows(IllegalArgumentException.class, () -> event("US", 0)); + assertThrows(IllegalArgumentException.class, () -> event("US", -1)); + assertEquals(1, event("US", 1).count()); + } + + @Test + public void testCountryIsOptionalAndNormalized() { + assertNull(event(null, 1).country()); + assertEquals("DE", event("de", 1).country()); + assertThrows(IllegalArgumentException.class, () -> event("DEU", 1)); + assertThrows(IllegalArgumentException.class, () -> event("1!", 1)); + } + + @Test + public void testRequiredFields() { + assertThrows( + NullPointerException.class, + () -> new DownloadEvent(null, 42L, 7L, "n", "e", "1.0.0", "universal", null, null, null, 1)); + assertThrows( + NullPointerException.class, + () -> new DownloadEvent(TIME, 42L, 7L, null, "e", "1.0.0", "universal", null, null, null, 1)); + assertThrows( + NullPointerException.class, + () -> new DownloadEvent(TIME, 42L, 7L, "n", null, "1.0.0", "universal", null, null, null, 1)); + assertThrows( + NullPointerException.class, + () -> new DownloadEvent(TIME, 42L, 7L, "n", "e", null, "universal", null, null, null, 1)); + assertThrows( + NullPointerException.class, + () -> new DownloadEvent(TIME, 42L, 7L, "n", "e", "1.0.0", null, null, null, null, 1)); + } + + private DownloadEvent event(String country, int count) { + return new DownloadEvent( + TIME, + 42L, + 7L, + "redhat", + "java", + "1.2.3", + "universal", + country, + "9.9.9.9", + "agent", + count); + } +} diff --git a/server/src/test/java/org/eclipse/openvsx/analytics/ingestion/CountryCodesTest.java b/server/src/test/java/org/eclipse/openvsx/analytics/ingestion/CountryCodesTest.java new file mode 100644 index 000000000..6521ae2b8 --- /dev/null +++ b/server/src/test/java/org/eclipse/openvsx/analytics/ingestion/CountryCodesTest.java @@ -0,0 +1,43 @@ +/****************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation. + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * https://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + *****************************************************************************/ +package org.eclipse.openvsx.analytics.ingestion; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +public class CountryCodesTest { + + @Test + public void testIsoCodesPassThrough() { + assertEquals("US", CountryCodes.toIsoCode("US")); + assertEquals("US", CountryCodes.toIsoCode("us")); + assertEquals("DE", CountryCodes.toIsoCode("de")); + } + + @Test + public void testEnglishCountryNames() { + assertEquals("US", CountryCodes.toIsoCode("united states")); + assertEquals("DE", CountryCodes.toIsoCode("Germany")); + assertEquals("NL", CountryCodes.toIsoCode("netherlands")); + } + + @Test + public void testUnknownValues() { + assertNull(CountryCodes.toIsoCode(null)); + assertNull(CountryCodes.toIsoCode("")); + assertNull(CountryCodes.toIsoCode("atlantis")); + assertNull(CountryCodes.toIsoCode("zz")); + } +} diff --git a/server/src/test/java/org/eclipse/openvsx/analytics/ingestion/DownloadIngestionMetricsTest.java b/server/src/test/java/org/eclipse/openvsx/analytics/ingestion/DownloadIngestionMetricsTest.java new file mode 100644 index 000000000..8e0fbf0fc --- /dev/null +++ b/server/src/test/java/org/eclipse/openvsx/analytics/ingestion/DownloadIngestionMetricsTest.java @@ -0,0 +1,65 @@ +/****************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation. + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * https://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + *****************************************************************************/ +package org.eclipse.openvsx.analytics.ingestion; + +import java.time.Duration; + +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import org.eclipse.openvsx.repositories.RepositoryService; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class DownloadIngestionMetricsTest { + + private final SimpleMeterRegistry registry = new SimpleMeterRegistry(); + private final RepositoryService repositories = Mockito.mock(RepositoryService.class); + private final DownloadIngestionMetrics metrics = new DownloadIngestionMetrics(registry, repositories); + + @Test + void testParseCounters() { + metrics.recordParsedLines(100, 3); + metrics.recordParsedLines(50, 0); + + assertEquals(150, registry.counter(DownloadIngestionMetrics.LINES_METRIC).count()); + assertEquals(3, registry.counter(DownloadIngestionMetrics.SKIPPED_LINES_METRIC).count()); + } + + @Test + void testLoadVolumeCounters() { + metrics.recordLoaded(4, 25); + metrics.recordLoaded(1, 5); + + assertEquals(5, registry.counter(DownloadIngestionMetrics.EVENTS_METRIC).count()); + assertEquals(30, registry.counter(DownloadIngestionMetrics.DOWNLOADS_METRIC).count()); + } + + @Test + void testExtractLagTimer() { + metrics.recordExtractLag(Duration.ofMinutes(10)); + + var timer = registry.timer(DownloadIngestionMetrics.EXTRACT_LAG_METRIC); + assertEquals(1, timer.count()); + assertEquals(600, timer.totalTime(java.util.concurrent.TimeUnit.SECONDS)); + } + + @Test + void testDeadLetterDepthGauge() { + Mockito.when(repositories.countFailedDownloadIngestions()).thenReturn(7L); + + var gauge = registry.get(DownloadIngestionMetrics.DEAD_LETTER_METRIC).gauge(); + assertEquals(7, gauge.value()); + } +} diff --git a/server/src/test/java/org/eclipse/openvsx/analytics/ingestion/DownloadIngestionProcessorTest.java b/server/src/test/java/org/eclipse/openvsx/analytics/ingestion/DownloadIngestionProcessorTest.java new file mode 100644 index 000000000..446effa52 --- /dev/null +++ b/server/src/test/java/org/eclipse/openvsx/analytics/ingestion/DownloadIngestionProcessorTest.java @@ -0,0 +1,270 @@ +/****************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation. + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * https://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + *****************************************************************************/ +package org.eclipse.openvsx.analytics.ingestion; + +import java.time.Instant; +import java.time.LocalDateTime; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; + +import jakarta.persistence.EntityManager; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.context.TestConfiguration; +import org.springframework.context.annotation.Bean; +import org.springframework.test.context.bean.override.mockito.MockitoSpyBean; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.support.TransactionTemplate; + +import org.eclipse.openvsx.AbstractPostgresContainerTest; +import org.eclipse.openvsx.analytics.DownloadAnalyticsRepository; +import org.eclipse.openvsx.analytics.DownloadEvent; +import org.eclipse.openvsx.analytics.DownloadSeriesRequest; +import org.eclipse.openvsx.analytics.DownloadSeriesRow; +import org.eclipse.openvsx.entities.Extension; +import org.eclipse.openvsx.entities.ExtensionVersion; +import org.eclipse.openvsx.entities.FileResource; +import org.eclipse.openvsx.entities.Namespace; +import org.eclipse.openvsx.repositories.RepositoryService; + +import static org.junit.jupiter.api.Assertions.*; + +@SpringBootTest +class DownloadIngestionProcessorTest extends AbstractPostgresContainerTest { + + private static final LocalDateTime PROCESSED_ON = LocalDateTime.of(2026, 7, 1, 15, 0); + + @Autowired + DownloadIngestionProcessor processor; + + @Autowired + RecordingAnalyticsRepository analyticsRepository; + + @Autowired + EntityManager entityManager; + + @Autowired + PlatformTransactionManager transactionManager; + + @MockitoSpyBean + RepositoryService repositories; + + private final List seededEntities = new CopyOnWriteArrayList<>(); + + private long seededVersionId; + + @AfterEach + void cleanUp() { + analyticsRepository.saved.clear(); + runInTransaction(() -> { + seededEntities.reversed().forEach(entity -> { + var merged = entityManager.merge(entity); + entityManager.remove(merged); + }); + entityManager.createQuery("delete from DownloadIngestion i where i.name like 'analytics-test%'") + .executeUpdate(); + }); + seededEntities.clear(); + } + + @Test + void testProcessAggregatesSavesAndCommitsAtomically() { + var extension = seedExtension("proc1", "proc1.ext-1.0.0.vsix"); + + var hour1 = Instant.parse("2026-07-01T14:00:00Z"); + var records = List.of( + new RawDownloadRecord(hour1.plusSeconds(60), "PROC1.EXT-1.0.0.VSIX", "US", "9.9.9.9", "VSCode 1.90.2"), + new RawDownloadRecord(hour1.plusSeconds(120), "PROC1.EXT-1.0.0.VSIX", "US", "9.9.9.9", "VSCode 1.90.2"), + new RawDownloadRecord(hour1.plusSeconds(180), "PROC1.EXT-1.0.0.VSIX", null, null, null), + new RawDownloadRecord( + hour1.plusSeconds(3660), + "PROC1.EXT-1.0.0.VSIX", + "US", + "9.9.9.9", + "VSCode 1.90.2")); + + var updated = processor.process(FileResource.STORAGE_AWS, "analytics-test-1.gz", PROCESSED_ON, 5, records); + + assertEquals(1, updated.size()); + assertEquals(extension.getId(), updated.get(0).getId()); + + // micro-batch aggregation by (hour, extension-version, country, ip, user agent) + assertEquals(3, analyticsRepository.saved.size()); + var aggregated = findEvent(hour1, "US", "VSCode 1.90.2"); + assertEquals(2, aggregated.count()); + assertEquals(extension.getId(), aggregated.extensionId()); + assertEquals(seededVersionId, aggregated.extensionVersionId()); + assertEquals("proc1", aggregated.namespace()); + assertEquals("proc1-ext", aggregated.extensionName()); + assertEquals("1.0.0", aggregated.version()); + assertEquals("universal", aggregated.targetPlatform()); + assertEquals("9.9.9.9", aggregated.ip()); + assertEquals("VSCode 1.90.2", aggregated.userAgent()); + var withoutUserAgent = findEvent(hour1, null, null); + assertEquals(1, withoutUserAgent.count()); + var laterHour = findEvent(hour1.plusSeconds(3600), "US", "VSCode 1.90.2"); + assertEquals(1, laterHour.count()); + + // the download counter is incremented by the total record count in the same transaction + assertEquals(4, freshDownloadCount(extension.getId())); + + // and the ingestion entry is written + assertEquals( + List.of("analytics-test-1.gz"), + repositories.findAllSucceededDownloadIngestionsByStorageTypeAndNameIn( + FileResource.STORAGE_AWS, + List.of("analytics-test-1.gz"))); + } + + @Test + void testUnknownFileIsSkipped() { + var extension = seedExtension("proc2", "proc2.ext-1.0.0.vsix"); + + var records = List.of( + new RawDownloadRecord(Instant.parse("2026-07-01T14:00:00Z"), "NO.SUCH-1.0.0.VSIX", null, null, null)); + processor.process(FileResource.STORAGE_AWS, "analytics-test-2.gz", PROCESSED_ON, 5, records); + + assertTrue(analyticsRepository.saved.isEmpty()); + assertEquals(0, freshDownloadCount(extension.getId())); + // the file is still marked as processed + assertEquals( + List.of("analytics-test-2.gz"), + repositories.findAllSucceededDownloadIngestionsByStorageTypeAndNameIn( + FileResource.STORAGE_AWS, + List.of("analytics-test-2.gz"))); + } + + @Test + void testFilenameResolutionIsCached() { + seedExtension("proc3", "proc3.ext-1.0.0.vsix"); + + var record = new RawDownloadRecord( + Instant.parse("2026-07-01T14:00:00Z"), + "PROC3.EXT-1.0.0.VSIX", + null, + null, + null); + processor.process(FileResource.STORAGE_AWS, "analytics-test-3a.gz", PROCESSED_ON, 5, List.of(record)); + processor.process(FileResource.STORAGE_AWS, "analytics-test-3b.gz", PROCESSED_ON, 5, List.of(record)); + + Mockito.verify(repositories, Mockito.times(1)) + .findDownloadsByStorageTypeAndName(FileResource.STORAGE_AWS, List.of("PROC3.EXT-1.0.0.VSIX")); + } + + @Test + void testInducedFailureRollsBackWholeTransaction() { + var extension = seedExtension("proc4", "proc4.ext-1.0.0.vsix"); + + var records = List.of( + new RawDownloadRecord( + Instant.parse("2026-07-01T14:00:00Z"), + "PROC4.EXT-1.0.0.VSIX", + "US", + "9.9.9.9", + null)); + // the download ingestion entry's name column is varchar(255); an overlong name fails the transaction + // after the events were saved and the counter was incremented + var overlongName = "analytics-test-" + "x".repeat(300); + assertThrows( + Exception.class, + () -> processor.process(FileResource.STORAGE_AWS, overlongName, PROCESSED_ON, 5, records)); + + assertEquals(0, freshDownloadCount(extension.getId())); + assertTrue( + repositories.findAllSucceededDownloadIngestionsByStorageTypeAndNameIn( + FileResource.STORAGE_AWS, + List.of(overlongName)).isEmpty()); + } + + private DownloadEvent findEvent(Instant time, String country, String userAgent) { + return analyticsRepository.saved.stream() + .filter(event -> event.time().equals(time)) + .filter(event -> userAgent == null ? event.userAgent() == null : userAgent.equals(event.userAgent())) + .filter(event -> country == null ? event.country() == null : country.equals(event.country())) + .findFirst() + .orElseThrow(() -> new AssertionError("no event for " + time + "/" + country + "/" + userAgent)); + } + + private int freshDownloadCount(long extensionId) { + return inTransaction(() -> entityManager.find(Extension.class, extensionId).getDownloadCount()); + } + + private Extension seedExtension(String namespaceName, String vsixFilename) { + return inTransaction(() -> { + var namespace = new Namespace(); + namespace.setName(namespaceName); + entityManager.persist(namespace); + + var extension = new Extension(); + extension.setName(namespaceName + "-ext"); + extension.setNamespace(namespace); + extension.setActive(true); + entityManager.persist(extension); + + var extVersion = new ExtensionVersion(); + extVersion.setVersion("1.0.0"); + extVersion.setTargetPlatform("universal"); + extVersion.setExtension(extension); + extVersion.setActive(true); + entityManager.persist(extVersion); + seededVersionId = extVersion.getId(); + + var resource = new FileResource(); + resource.setName(vsixFilename); + resource.setType(FileResource.DOWNLOAD); + resource.setStorageType(FileResource.STORAGE_AWS); + resource.setExtension(extVersion); + entityManager.persist(resource); + + seededEntities.addAll(List.of(namespace, extension, extVersion, resource)); + return extension; + }); + } + + private void runInTransaction(Runnable action) { + inTransaction(() -> { + action.run(); + return null; + }); + } + + private T inTransaction(java.util.function.Supplier action) { + return new TransactionTemplate(transactionManager).execute(status -> action.get()); + } + + @TestConfiguration + static class RecordingRepositoryConfig { + @Bean + @org.springframework.context.annotation.Primary + RecordingAnalyticsRepository recordingAnalyticsRepository() { + return new RecordingAnalyticsRepository(); + } + } + + static class RecordingAnalyticsRepository implements DownloadAnalyticsRepository { + final List saved = new CopyOnWriteArrayList<>(); + + @Override + public void save(List events) { + saved.addAll(events); + } + + @Override + public List findSeries(DownloadSeriesRequest request) { + return List.of(); + } + } +} diff --git a/server/src/test/java/org/eclipse/openvsx/analytics/ingestion/RawDownloadRecordTest.java b/server/src/test/java/org/eclipse/openvsx/analytics/ingestion/RawDownloadRecordTest.java new file mode 100644 index 000000000..9d592fa3c --- /dev/null +++ b/server/src/test/java/org/eclipse/openvsx/analytics/ingestion/RawDownloadRecordTest.java @@ -0,0 +1,35 @@ +/****************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation. + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * https://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + *****************************************************************************/ +package org.eclipse.openvsx.analytics.ingestion; + +import java.time.Instant; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +public class RawDownloadRecordTest { + + private static final Instant TIME = Instant.parse("2026-07-01T14:23:45Z"); + + @Test + public void testRequiredFields() { + var record = new RawDownloadRecord(TIME, "FOO.BAR-1.2.3.VSIX", "US", "9.9.9.9", "VSCode 1.90.2"); + assertEquals(TIME, record.time()); + assertEquals("FOO.BAR-1.2.3.VSIX", record.vsixFilename()); + assertThrows(NullPointerException.class, () -> new RawDownloadRecord(null, "A.VSIX", null, null, null)); + assertThrows(NullPointerException.class, () -> new RawDownloadRecord(TIME, null, null, null, null)); + assertThrows(IllegalArgumentException.class, () -> new RawDownloadRecord(TIME, " ", null, null, null)); + } +} diff --git a/server/src/test/java/org/eclipse/openvsx/analytics/ingestion/aws/AccessLogRecordTest.java b/server/src/test/java/org/eclipse/openvsx/analytics/ingestion/aws/AccessLogRecordTest.java new file mode 100644 index 000000000..5dc00e244 --- /dev/null +++ b/server/src/test/java/org/eclipse/openvsx/analytics/ingestion/aws/AccessLogRecordTest.java @@ -0,0 +1,71 @@ +/****************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation. + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * https://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + *****************************************************************************/ +package org.eclipse.openvsx.analytics.ingestion.aws; + +import java.time.Instant; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +public class AccessLogRecordTest { + + private static final Instant TIME = Instant.parse("2026-07-01T14:23:45Z"); + private static final Instant FALLBACK = Instant.parse("2026-07-01T00:00:00Z"); + + @Test + public void testToDownloadRecordExtractsDownload() { + var log = new AccessLogRecord( + "GET", + 200, + "/vscjava/vscode-java-pack/0.30.4/file/vscjava.vscode-java-pack-0.30.4.vsix", + TIME, + "US", + "9.9.9.9", + "VSCode 1.90.2"); + var record = log.toDownloadRecord(FALLBACK); + assertEquals(TIME, record.time()); + assertEquals("VSCJAVA.VSCODE-JAVA-PACK-0.30.4.VSIX", record.vsixFilename()); + assertEquals("US", record.country()); + assertEquals("9.9.9.9", record.ip()); + assertEquals("VSCode 1.90.2", record.rawUserAgent()); + } + + @Test + public void testToDownloadRecordDecodesFilename() { + var log = new AccessLogRecord("GET", 200, "/ns/ext/1.0.0/file/ns.ext%2B1-1.0.0.vsix", TIME, null, null, null); + var record = log.toDownloadRecord(FALLBACK); + assertEquals("NS.EXT+1-1.0.0.VSIX", record.vsixFilename()); + } + + @Test + public void testToDownloadRecordFallsBackToFileTime() { + var log = new AccessLogRecord("GET", 200, "/ns/ext/file/ns.ext-1.0.0.vsix", null, null, null, null); + var record = log.toDownloadRecord(FALLBACK); + assertEquals(FALLBACK, record.time()); + } + + @Test + public void testToDownloadRecordFiltersNonDownloads() { + assertNull( + new AccessLogRecord("OPTIONS", 200, "/ns/ext/file/a.vsix", TIME, null, null, null) + .toDownloadRecord(FALLBACK)); + assertNull( + new AccessLogRecord("GET", 404, "/ns/ext/file/a.vsix", TIME, null, null, null) + .toDownloadRecord(FALLBACK)); + assertNull( + new AccessLogRecord("GET", 200, "/favicon.ico", TIME, null, null, null) + .toDownloadRecord(FALLBACK)); + } +} diff --git a/server/src/test/java/org/eclipse/openvsx/analytics/ingestion/aws/CloudFrontLogFileParserTest.java b/server/src/test/java/org/eclipse/openvsx/analytics/ingestion/aws/CloudFrontLogFileParserTest.java new file mode 100644 index 000000000..599e5d392 --- /dev/null +++ b/server/src/test/java/org/eclipse/openvsx/analytics/ingestion/aws/CloudFrontLogFileParserTest.java @@ -0,0 +1,89 @@ +/****************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation. + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * https://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + *****************************************************************************/ +package org.eclipse.openvsx.analytics.ingestion.aws; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.*; + +public class CloudFrontLogFileParserTest { + + @Test + public void testHeaderLinesAreSkipped() throws IOException { + var records = readFixture(); + assertNull(records.get(0)); + assertNull(records.get(1)); + } + + @Test + public void testParse() throws IOException { + var record = readFixture().get(2); + assertNotNull(record); + assertEquals("OPTIONS", record.method()); + assertEquals(200, record.status()); + assertEquals("/vscjava/vscode-java-pack/0.30.4/package.json", record.url()); + assertEquals(Instant.parse("2025-12-03T13:17:20Z"), record.timestamp()); + // CloudFront standard logs carry no country information + assertNull(record.country()); + assertEquals("1.1.1.1", record.ip()); + assertEquals("Mozilla/5.0", record.userAgent()); + } + + @Test + public void testParseDownloadLine() throws IOException { + var record = readFixture().get(3); + assertNotNull(record); + assertEquals("GET", record.method()); + assertEquals(200, record.status()); + assertEquals("/vscjava/vscode-java-pack/0.30.4/file/vscjava.vscode-java-pack-0.30.4.vsix", record.url()); + assertEquals(Instant.parse("2025-12-03T13:20:01Z"), record.timestamp()); + assertNull(record.country()); + assertEquals("VSCode 1.90.2 (Microsoft Visual Studio Code)", record.userAgent()); + } + + @Test + public void testMissingTimestampAndUserAgent() throws IOException { + var record = readFixture().get(4); + assertNotNull(record); + assertEquals("GET", record.method()); + assertNull(record.timestamp()); + assertNull(record.userAgent()); + } + + @Test + public void testMalformedLineIsSkipped() throws IOException { + assertNull(readFixture().get(5)); + } + + private List readFixture() throws IOException { + LogFileParser parser = new CloudFrontLogFileParser(); + var records = new ArrayList(); + try (var is = CloudFrontLogFileParser.class.getResourceAsStream("cloudfront.log")) { + assertNotNull(is); + try (var reader = new BufferedReader(new InputStreamReader(is))) { + String line; + while ((line = reader.readLine()) != null) { + records.add(parser.parse(line)); + } + } + } + return records; + } +} diff --git a/server/src/test/java/org/eclipse/openvsx/analytics/ingestion/aws/FastlyLogFileParserTest.java b/server/src/test/java/org/eclipse/openvsx/analytics/ingestion/aws/FastlyLogFileParserTest.java new file mode 100644 index 000000000..41400709e --- /dev/null +++ b/server/src/test/java/org/eclipse/openvsx/analytics/ingestion/aws/FastlyLogFileParserTest.java @@ -0,0 +1,91 @@ +/****************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation. + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * https://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + *****************************************************************************/ +package org.eclipse.openvsx.analytics.ingestion.aws; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; + +public class FastlyLogFileParserTest { + + @Test + public void testParse() throws IOException { + var record = readFixture().get(0); + assertNotNull(record); + assertEquals("GET", record.method()); + assertEquals(301, record.status()); + assertEquals("/favicon.ico", record.url()); + assertEquals(Instant.parse("2026-02-09T04:20:50Z"), record.timestamp()); + assertEquals("united states", record.country()); + assertEquals("1.1.1.1", record.ip()); + assertEquals("Mozilla/5.0", record.userAgent()); + } + + @Test + public void testParseDownloadLine() throws IOException { + var record = readFixture().get(1); + assertNotNull(record); + assertEquals("GET", record.method()); + assertEquals(200, record.status()); + assertEquals("/vscjava/vscode-java-pack/0.30.4/file/vscjava.vscode-java-pack-0.30.4.vsix", record.url()); + assertEquals("united states", record.country()); + assertEquals("1.1.1.1", record.ip()); + assertEquals("VSCode 1.90.2 (Microsoft Visual Studio Code)", record.userAgent()); + } + + @Test + public void testMalformedLineIsSkipped() throws IOException { + assertNull(readFixture().get(2)); + } + + @Test + public void testMissingOptionalFields() throws IOException { + var record = readFixture().get(3); + assertNotNull(record); + assertEquals("GET", record.method()); + assertEquals(200, record.status()); + assertNull(record.timestamp()); + assertNull(record.country()); + assertNull(record.ip()); + assertNull(record.userAgent()); + } + + @Test + public void testLineWithoutJsonIsSkipped() throws IOException { + assertNull(readFixture().get(4)); + } + + private List readFixture() throws IOException { + LogFileParser parser = new FastlyLogFileParser(); + var records = new ArrayList(); + try (var is = FastlyLogFileParser.class.getResourceAsStream("fastly.log")) { + assertNotNull(is); + try (var reader = new BufferedReader(new InputStreamReader(is))) { + String line; + while ((line = reader.readLine()) != null) { + records.add(parser.parse(line)); + } + } + } + return records; + } +} diff --git a/server/src/test/java/org/eclipse/openvsx/analytics/ingestion/jobs/AwsLogIngestionHandlerTest.java b/server/src/test/java/org/eclipse/openvsx/analytics/ingestion/jobs/AwsLogIngestionHandlerTest.java new file mode 100644 index 000000000..23ee0008c --- /dev/null +++ b/server/src/test/java/org/eclipse/openvsx/analytics/ingestion/jobs/AwsLogIngestionHandlerTest.java @@ -0,0 +1,313 @@ +/****************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation. + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * https://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + *****************************************************************************/ +package org.eclipse.openvsx.analytics.ingestion.jobs; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.zip.GZIPOutputStream; + +import jakarta.persistence.EntityManager; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; +import org.springframework.test.util.ReflectionTestUtils; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.support.TransactionTemplate; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.localstack.LocalStackContainer; +import org.testcontainers.utility.DockerImageName; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.CreateBucketRequest; +import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; +import software.amazon.awssdk.services.s3.model.HeadObjectRequest; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; +import software.amazon.awssdk.services.s3.model.NoSuchKeyException; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.S3Object; + +import org.eclipse.openvsx.AbstractPostgresContainerTest; +import org.eclipse.openvsx.entities.Extension; +import org.eclipse.openvsx.entities.ExtensionVersion; +import org.eclipse.openvsx.entities.FileResource; +import org.eclipse.openvsx.entities.Namespace; + +import static org.junit.jupiter.api.Assertions.*; + +@SpringBootTest +@Testcontainers +class AwsLogIngestionHandlerTest extends AbstractPostgresContainerTest { + + private static final String LOGS_BUCKET = "openvsx-logs-test"; + private static final String LOGS_PREFIX = "AWSLogs/"; + private static final String ARCHIVE_PREFIX = "processed/"; + + @Container + static LocalStackContainer localstack = new LocalStackContainer(DockerImageName.parse("localstack/localstack:4.7")) + .withServices("s3"); + + @Autowired + LogIngestionJob handler; + + @Autowired + org.eclipse.openvsx.analytics.ingestion.aws.AwsDownloadRecordSource source; + + @Autowired + EntityManager entityManager; + + @Autowired + PlatformTransactionManager transactionManager; + + S3Client s3; + + private final List seededEntities = new CopyOnWriteArrayList<>(); + + @DynamicPropertySource + static void awsProperties(DynamicPropertyRegistry registry) { + registry.add("ovsx.storage.aws.service-endpoint", () -> localstack.getEndpoint().toString()); + registry.add("ovsx.storage.aws.access-key-id", () -> "test"); + registry.add("ovsx.storage.aws.secret-access-key", () -> "test"); + registry.add("ovsx.storage.aws.region", () -> "us-east-1"); + registry.add("ovsx.storage.aws.bucket", () -> "openvsx-storage-test"); + registry.add("ovsx.storage.aws.path-style-access", () -> "true"); + registry.add("ovsx.logs.aws.bucket", () -> LOGS_BUCKET); + registry.add("ovsx.logs.aws.format", () -> "fastly"); + } + + @BeforeEach + void setUp() { + s3 = S3Client.builder() + .endpointOverride(localstack.getEndpoint()) + .credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create("test", "test"))) + .region(Region.of("us-east-1")) + .forcePathStyle(true) + .build(); + s3.createBucket(CreateBucketRequest.builder().bucket(LOGS_BUCKET).build()); + } + + @AfterEach + void cleanUp() { + for (var prefix : List.of(LOGS_PREFIX, ARCHIVE_PREFIX)) { + listKeys(prefix).forEach( + key -> s3.deleteObject(DeleteObjectRequest.builder().bucket(LOGS_BUCKET).key(key).build())); + } + runInTransaction(() -> { + seededEntities.reversed().forEach(entity -> entityManager.remove(entityManager.merge(entity))); + entityManager.createQuery("delete from DownloadIngestion i where i.name like 'AWSLogs/%'") + .executeUpdate(); + }); + seededEntities.clear(); + } + + @Test + void testProcessesLogFileUpdatesCountsAndDeletesIt() throws Exception { + var extension = seedExtension("awsone", "awsone.ext-1.0.0.vsix"); + putLogFile( + "AWSLogs/awsone-file.gz", + gzip( + String.join( + "\n", + downloadLine("/awsone/ext/1.0.0/file/awsone.ext-1.0.0.vsix", "VSCode 1.90.2"), + downloadLine("/awsone/ext/1.0.0/file/awsone.ext-1.0.0.vsix", "VSCode 1.90.2"), + downloadLine( + "/awsone/ext/1.0.0/file/awsone.ext-1.0.0.vsix", + "Mozilla/5.0 Chrome/126.0.0.0")))); + + handler.run(new IngestionJobRequest<>(LogIngestionJob.class, FileResource.STORAGE_AWS)); + + assertEquals(3, freshDownloadCount(extension.getId())); + assertEquals(1, succeededIngestions("AWSLogs/awsone-file.gz")); + assertFalse(objectExists("AWSLogs/awsone-file.gz")); + } + + @Test + void testAlreadyProcessedFileIsCountedOnlyOnce() throws Exception { + var extension = seedExtension("awstwo", "awstwo.ext-1.0.0.vsix"); + var content = gzip(downloadLine("/awstwo/ext/1.0.0/file/awstwo.ext-1.0.0.vsix", "VSCode 1.90.2")); + + putLogFile("AWSLogs/awstwo-file.gz", content); + handler.run(new IngestionJobRequest<>(LogIngestionJob.class, FileResource.STORAGE_AWS)); + assertEquals(1, freshDownloadCount(extension.getId())); + + // the same log file is presented again, e.g. after a partial cleanup failure + putLogFile("AWSLogs/awstwo-file.gz", content); + handler.run(new IngestionJobRequest<>(LogIngestionJob.class, FileResource.STORAGE_AWS)); + + assertEquals(1, freshDownloadCount(extension.getId())); + assertEquals(1, succeededIngestions("AWSLogs/awstwo-file.gz")); + // the already-processed file is cleaned up without re-counting + assertFalse(objectExists("AWSLogs/awstwo-file.gz")); + } + + @Test + void testFailingFileIsRetainedExcludedAndReprocessedAfterClearing() throws Exception { + var extension = seedExtension("awsthree", "awsthree.ext-1.0.0.vsix"); + putLogFile("AWSLogs/awsthree-file.gz", "this is not gzip".getBytes()); + + handler.run(new IngestionJobRequest<>(LogIngestionJob.class, FileResource.STORAGE_AWS)); + assertEquals(0, freshDownloadCount(extension.getId())); + assertEquals(1, failedIngestions("AWSLogs/awsthree-file.gz")); + // the file is retained for analysis + assertTrue(objectExists("AWSLogs/awsthree-file.gz")); + + // and is excluded from the next run instead of failing again + handler.run(new IngestionJobRequest<>(LogIngestionJob.class, FileResource.STORAGE_AWS)); + assertEquals(1, failedIngestions("AWSLogs/awsthree-file.gz")); + assertTrue(objectExists("AWSLogs/awsthree-file.gz")); + + // clearing the ingestion entry makes the file eligible again + runInTransaction( + () -> entityManager + .createQuery("delete from DownloadIngestion i where i.name = 'AWSLogs/awsthree-file.gz'") + .executeUpdate()); + putLogFile( + "AWSLogs/awsthree-file.gz", + gzip(downloadLine("/awsthree/ext/1.0.0/file/awsthree.ext-1.0.0.vsix", "VSCode 1.90.2"))); + handler.run(new IngestionJobRequest<>(LogIngestionJob.class, FileResource.STORAGE_AWS)); + + assertEquals(1, freshDownloadCount(extension.getId())); + assertEquals(1, succeededIngestions("AWSLogs/awsthree-file.gz")); + assertFalse(objectExists("AWSLogs/awsthree-file.gz")); + } + + @Test + void testArchivePrefixMovesProcessedFilesInsteadOfDeleting() throws Exception { + ReflectionTestUtils.setField(source, "archivePrefix", ARCHIVE_PREFIX); + try { + var extension = seedExtension("awsfour", "awsfour.ext-1.0.0.vsix"); + putLogFile( + "AWSLogs/awsfour-file.gz", + gzip( + downloadLine("/awsfour/ext/1.0.0/file/awsfour.ext-1.0.0.vsix", "VSCode 1.90.2"))); + + handler.run(new IngestionJobRequest<>(LogIngestionJob.class, FileResource.STORAGE_AWS)); + + assertEquals(1, freshDownloadCount(extension.getId())); + assertFalse(objectExists("AWSLogs/awsfour-file.gz")); + assertTrue(objectExists(ARCHIVE_PREFIX + "AWSLogs/awsfour-file.gz")); + } finally { + ReflectionTestUtils.setField(source, "archivePrefix", ""); + } + } + + private String downloadLine(String url, String userAgent) { + return "<134>2026-07-01T13:52:42Z cache-fra-x S3-Log-Stream[1]: {\"timestamp\": \"2026-07-01T12:20:50+0000\", " + + "\"geo_country\": \"united states\", \"client_ip\": \"1.1.1.1\", \"url\": \"" + url + + "\", \"request_method\": \"GET\", " + + "\"request_user_agent\": \"" + userAgent + "\", \"response_status\": 200}"; + } + + private byte[] gzip(String content) throws IOException { + var buffer = new ByteArrayOutputStream(); + try (var gzipStream = new GZIPOutputStream(buffer)) { + gzipStream.write(content.getBytes()); + } + return buffer.toByteArray(); + } + + private void putLogFile(String key, byte[] content) { + s3.putObject( + PutObjectRequest.builder().bucket(LOGS_BUCKET).key(key).build(), + RequestBody.fromBytes(content)); + } + + private boolean objectExists(String key) { + try { + s3.headObject(HeadObjectRequest.builder().bucket(LOGS_BUCKET).key(key).build()); + return true; + } catch (NoSuchKeyException e) { + return false; + } + } + + private List listKeys(String prefix) { + return s3.listObjectsV2(ListObjectsV2Request.builder().bucket(LOGS_BUCKET).prefix(prefix).build()) + .contents().stream().map(S3Object::key).toList(); + } + + private int succeededIngestions(String name) { + return countIngestions(name, true); + } + + private int failedIngestions(String name) { + return countIngestions(name, false); + } + + private int countIngestions(String name, boolean success) { + return inTransaction( + () -> entityManager + .createQuery( + "select count(i) from DownloadIngestion i where i.name = :name and i.success = :success", + Long.class) + .setParameter("name", name) + .setParameter("success", success) + .getSingleResult() + .intValue()); + } + + private int freshDownloadCount(long extensionId) { + return inTransaction(() -> entityManager.find(Extension.class, extensionId).getDownloadCount()); + } + + private Extension seedExtension(String namespaceName, String vsixFilename) { + return inTransaction(() -> { + var namespace = new Namespace(); + namespace.setName(namespaceName); + entityManager.persist(namespace); + + var extension = new Extension(); + extension.setName(namespaceName + "-ext"); + extension.setNamespace(namespace); + extension.setActive(true); + entityManager.persist(extension); + + var extVersion = new ExtensionVersion(); + extVersion.setVersion("1.0.0"); + extVersion.setTargetPlatform("universal"); + extVersion.setExtension(extension); + extVersion.setActive(true); + entityManager.persist(extVersion); + + var resource = new FileResource(); + resource.setName(vsixFilename); + resource.setType(FileResource.DOWNLOAD); + resource.setStorageType(FileResource.STORAGE_AWS); + resource.setExtension(extVersion); + entityManager.persist(resource); + + seededEntities.addAll(List.of(namespace, extension, extVersion, resource)); + return extension; + }); + } + + private void runInTransaction(Runnable action) { + inTransaction(() -> { + action.run(); + return null; + }); + } + + private T inTransaction(java.util.function.Supplier action) { + return new TransactionTemplate(transactionManager).execute(status -> action.get()); + } +} diff --git a/server/src/test/java/org/eclipse/openvsx/analytics/ingestion/jobs/IngestionJobsTest.java b/server/src/test/java/org/eclipse/openvsx/analytics/ingestion/jobs/IngestionJobsTest.java new file mode 100644 index 000000000..85df44506 --- /dev/null +++ b/server/src/test/java/org/eclipse/openvsx/analytics/ingestion/jobs/IngestionJobsTest.java @@ -0,0 +1,161 @@ +/****************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation. + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * https://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + *****************************************************************************/ +package org.eclipse.openvsx.analytics.ingestion.jobs; + +import java.time.ZoneId; + +import org.jobrunr.jobs.lambdas.JobRequest; +import org.jobrunr.scheduling.JobRequestScheduler; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; +import org.springframework.boot.test.context.runner.ApplicationContextRunner; + +import org.eclipse.openvsx.analytics.ingestion.DownloadIngestionMetrics; +import org.eclipse.openvsx.analytics.ingestion.DownloadIngestionRunner; +import org.eclipse.openvsx.analytics.ingestion.DownloadRecordSource; +import org.eclipse.openvsx.analytics.ingestion.aws.AwsDownloadRecordSource; +import org.eclipse.openvsx.analytics.ingestion.azure.AzureDownloadRecordSource; +import org.eclipse.openvsx.entities.FileResource; +import org.eclipse.openvsx.storage.AwsStorageService; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +class IngestionJobsTest { + + private static final String AWS_JOB_ID = "update-aws-download-counts"; + private static final String AZURE_JOB_ID = "update-azure-blob-download-counts"; + + private final AwsStorageService awsStorage = Mockito.mock(AwsStorageService.class); + private final DownloadIngestionRunner ingestionRunner = Mockito.mock(DownloadIngestionRunner.class); + private final JobRequestScheduler scheduler = Mockito.mock(JobRequestScheduler.class); + private final DownloadIngestionMetrics metrics = Mockito.mock(DownloadIngestionMetrics.class); + + private ApplicationContextRunner runner() { + return new ApplicationContextRunner() + .withBean(AwsStorageService.class, () -> awsStorage) + .withBean(DownloadIngestionMetrics.class, () -> metrics) + .withBean(DownloadIngestionRunner.class, () -> ingestionRunner) + .withBean(JobRequestScheduler.class, () -> scheduler) + .withUserConfiguration( + AwsDownloadRecordSource.class, + AzureDownloadRecordSource.class, + LogIngestionJob.class); + } + + @Test + void testNoSourceBeansWithoutConfiguration() { + runner().run(context -> assertThat(context).doesNotHaveBean(DownloadRecordSource.class)); + } + + @Test + void testAwsSourceExistsWhenBucketIsConfigured() { + runner().withPropertyValues("ovsx.logs.aws.bucket=my-logs").run(context -> { + assertThat(context).hasSingleBean(AwsDownloadRecordSource.class); + assertThat(context).doesNotHaveBean(AzureDownloadRecordSource.class); + }); + } + + @Test + void testAzureSourceExistsWhenLogsEndpointIsConfigured() { + runner().withPropertyValues("ovsx.logs.azure.service-endpoint=https://logs.blob.core.windows.net") + .run(context -> { + assertThat(context).hasSingleBean(AzureDownloadRecordSource.class); + assertThat(context).doesNotHaveBean(AwsDownloadRecordSource.class); + }); + } + + @Test + void testAwsSourceCoversAwsDownloadsOnly() { + when(awsStorage.isEnabled()).thenReturn(true); + runner().withPropertyValues("ovsx.logs.aws.bucket=my-logs").run(context -> { + var source = context.getBean(DownloadRecordSource.class); + assertTrue(source.covers(resource(FileResource.STORAGE_AWS))); + assertFalse(source.covers(resource(FileResource.STORAGE_AZURE))); + assertFalse(source.covers(resource(FileResource.STORAGE_LOCAL))); + }); + } + + @Test + void testAwsSourceDoesNotCoverWhenStorageServiceIsDisabled() { + when(awsStorage.isEnabled()).thenReturn(false); + runner().withPropertyValues("ovsx.logs.aws.bucket=my-logs").run(context -> { + var source = context.getBean(DownloadRecordSource.class); + assertFalse(source.covers(resource(FileResource.STORAGE_AWS))); + }); + } + + @Test + void testSchedulesRecurringJobForEnabledSource() { + when(awsStorage.isEnabled()).thenReturn(true); + runner().withPropertyValues("ovsx.logs.aws.bucket=my-logs").run(context -> { + context.getBean(LogIngestionJob.class).scheduleJobs(null); + verify(scheduler).scheduleRecurrently( + eq(AWS_JOB_ID), + eq("0 10 * * * *"), + eq(ZoneId.of("UTC")), + any(JobRequest.class)); + // the unconfigured azure job is cleaned up in the same pass + verify(scheduler).deleteRecurringJob(AZURE_JOB_ID); + }); + } + + @Test + void testDeletesRecurringJobWhenSourceIsDisabled() { + when(awsStorage.isEnabled()).thenReturn(false); + runner().withPropertyValues("ovsx.logs.aws.bucket=my-logs").run(context -> { + context.getBean(LogIngestionJob.class).scheduleJobs(null); + verify(scheduler).deleteRecurringJob(AWS_JOB_ID); + }); + } + + @Test + void testDeletesAllRecurringJobsWhenNoSourceConfigured() { + runner().run(context -> { + context.getBean(LogIngestionJob.class).scheduleJobs(null); + verify(scheduler).deleteRecurringJob(AWS_JOB_ID); + verify(scheduler).deleteRecurringJob(AZURE_JOB_ID); + }); + } + + @Test + void testHandlerRunsTheResolvedSourceThroughTheRunner() { + when(awsStorage.isEnabled()).thenReturn(true); + runner().withPropertyValues("ovsx.logs.aws.bucket=my-logs").run(context -> { + context.getBean(LogIngestionJob.class) + .run(new IngestionJobRequest<>(LogIngestionJob.class, FileResource.STORAGE_AWS)); + verify(ingestionRunner).run(context.getBean(AwsDownloadRecordSource.class)); + }); + } + + @Test + void testHandlerSkipsWhenSourceIsDisabled() { + when(awsStorage.isEnabled()).thenReturn(false); + runner().withPropertyValues("ovsx.logs.aws.bucket=my-logs").run(context -> { + context.getBean(LogIngestionJob.class) + .run(new IngestionJobRequest<>(LogIngestionJob.class, FileResource.STORAGE_AWS)); + verify(ingestionRunner, Mockito.never()).run(any()); + }); + } + + private FileResource resource(String storageType) { + var resource = new FileResource(); + resource.setStorageType(storageType); + return resource; + } +} diff --git a/server/src/test/java/org/eclipse/openvsx/analytics/timescale/TimescaleDownloadAnalyticsRepositoryTest.java b/server/src/test/java/org/eclipse/openvsx/analytics/timescale/TimescaleDownloadAnalyticsRepositoryTest.java new file mode 100644 index 000000000..513e5c075 --- /dev/null +++ b/server/src/test/java/org/eclipse/openvsx/analytics/timescale/TimescaleDownloadAnalyticsRepositoryTest.java @@ -0,0 +1,315 @@ +/****************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation. + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * https://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + *****************************************************************************/ +package org.eclipse.openvsx.analytics.timescale; + +import java.time.Instant; +import java.util.List; +import java.util.stream.IntStream; +import javax.sql.DataSource; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.support.TransactionTemplate; + +import org.eclipse.openvsx.AbstractPostgresContainerTest; +import org.eclipse.openvsx.analytics.DownloadAnalyticsRepository; +import org.eclipse.openvsx.analytics.DownloadEvent; +import org.eclipse.openvsx.analytics.DownloadSeriesGroupBy; +import org.eclipse.openvsx.analytics.DownloadSeriesInterval; +import org.eclipse.openvsx.analytics.DownloadSeriesRequest; +import org.eclipse.openvsx.analytics.DownloadSeriesRow; + +import static org.junit.jupiter.api.Assertions.*; + +@SpringBootTest(properties = "ovsx.analytics.enabled=true") +class TimescaleDownloadAnalyticsRepositoryTest extends AbstractPostgresContainerTest { + + @Autowired + DownloadAnalyticsRepository repository; + + @Autowired + DataSource dataSource; + + @Autowired + PlatformTransactionManager transactionManager; + + JdbcTemplate jdbc; + + @Autowired + void initJdbc(DataSource dataSource) { + this.jdbc = new JdbcTemplate(dataSource); + } + + @AfterEach + void cleanUp() { + jdbc.execute("TRUNCATE download_event"); + } + + @Test + void testMigrationApplied() { + assertEquals( + 1, + jdbc.queryForObject( + "SELECT COUNT(*) FROM timescaledb_information.hypertables WHERE hypertable_name = 'download_event'", + Integer.class)); + assertEquals( + 1, + jdbc.queryForObject( + "SELECT COUNT(*) FROM timescaledb_information.continuous_aggregates WHERE view_name = 'download_stats_daily'", + Integer.class)); + // the analytics schema is part of the main migration chain + assertEquals( + 1, + jdbc.queryForObject( + "SELECT COUNT(*) FROM flyway_schema_history WHERE version = '1.71' AND success", + Integer.class)); + } + + @Test + void testSaveBatches() { + var events = IntStream.range(0, 1500) + .mapToObj( + i -> event( + Instant.parse("2026-07-01T00:00:00Z").plusSeconds(i * 3600L), + 1L, + "1.0.0", + "US", + 2)) + .toList(); + repository.save(events); + + assertEquals(1500, jdbc.queryForObject("SELECT COUNT(*) FROM download_event", Integer.class)); + assertEquals( + 1500, + jdbc.queryForObject( + "SELECT COUNT(*) FROM download_event WHERE extension_version_id = 100", + Integer.class)); + assertEquals(3000, jdbc.queryForObject("SELECT SUM(count) FROM download_event", Integer.class)); + // the raw client ip and user agent are persisted as found in the logs + assertEquals( + 1500, + jdbc.queryForObject( + "SELECT COUNT(*) FROM download_event WHERE ip = '9.9.9.9' AND user_agent = 'VSCode 1.90.2'", + Integer.class)); + } + + @Test + void testSaveJoinsCallerTransaction() { + var transaction = new TransactionTemplate(transactionManager); + assertThrows(IllegalStateException.class, () -> transaction.execute(status -> { + repository.save( + List.of( + event( + Instant.parse("2026-07-01T10:00:00Z"), + 1L, + "1.0.0", + "US", + 1))); + throw new IllegalStateException("induced failure after save"); + })); + + assertEquals(0, jdbc.queryForObject("SELECT COUNT(*) FROM download_event", Integer.class)); + } + + @Test + void testFindSeriesByDay() { + repository.save( + List.of( + event(Instant.parse("2026-06-30T10:00:00Z"), 1L, "1.0.0", "US", 3), + event(Instant.parse("2026-06-30T23:00:00Z"), 1L, "1.0.0", "DE", 2), + event(Instant.parse("2026-07-01T00:00:00Z"), 1L, "1.0.0", "US", 5), + // different extension, not requested + event(Instant.parse("2026-07-01T00:00:00Z"), 2L, "1.0.0", "US", 100))); + + var rows = repository.findSeries( + DownloadSeriesRequest.of( + 1L, + Instant.parse("2026-06-29T00:00:00Z"), + Instant.parse("2026-07-02T00:00:00Z"), + DownloadSeriesInterval.DAY)); + + assertEquals( + List.of( + new DownloadSeriesRow(Instant.parse("2026-06-30T00:00:00Z"), null, 5), + new DownloadSeriesRow(Instant.parse("2026-07-01T00:00:00Z"), null, 5)), + rows); + } + + @Test + void testFindSeriesRangeFilter() { + repository.save( + List.of( + event(Instant.parse("2026-06-28T10:00:00Z"), 1L, "1.0.0", "US", 1), + event(Instant.parse("2026-06-29T10:00:00Z"), 1L, "1.0.0", "US", 2), + event(Instant.parse("2026-06-30T10:00:00Z"), 1L, "1.0.0", "US", 4))); + + // from is inclusive, to is exclusive + var rows = repository.findSeries( + DownloadSeriesRequest.of( + 1L, + Instant.parse("2026-06-29T00:00:00Z"), + Instant.parse("2026-06-30T00:00:00Z"), + DownloadSeriesInterval.DAY)); + + assertEquals(List.of(new DownloadSeriesRow(Instant.parse("2026-06-29T00:00:00Z"), null, 2)), rows); + } + + @Test + void testFindSeriesByWeekAndMonth() { + repository.save( + List.of( + // Sunday of the week starting Monday 2026-06-22, and June + event(Instant.parse("2026-06-28T10:00:00Z"), 1L, "1.0.0", "US", 1), + // Monday 2026-06-29 week, June + event(Instant.parse("2026-06-29T10:00:00Z"), 1L, "1.0.0", "US", 2), + // Wednesday of the same week, but July + event(Instant.parse("2026-07-01T10:00:00Z"), 1L, "1.0.0", "US", 4))); + + var weekly = repository.findSeries( + DownloadSeriesRequest.of( + 1L, + Instant.parse("2026-06-01T00:00:00Z"), + Instant.parse("2026-08-01T00:00:00Z"), + DownloadSeriesInterval.WEEK)); + assertEquals( + List.of( + new DownloadSeriesRow(Instant.parse("2026-06-22T00:00:00Z"), null, 1), + new DownloadSeriesRow(Instant.parse("2026-06-29T00:00:00Z"), null, 6)), + weekly); + + var monthly = repository.findSeries( + DownloadSeriesRequest.of( + 1L, + Instant.parse("2026-06-01T00:00:00Z"), + Instant.parse("2026-08-01T00:00:00Z"), + DownloadSeriesInterval.MONTH)); + assertEquals( + List.of( + new DownloadSeriesRow(Instant.parse("2026-06-01T00:00:00Z"), null, 3), + new DownloadSeriesRow(Instant.parse("2026-07-01T00:00:00Z"), null, 4)), + monthly); + } + + @Test + void testOutOfOrderSavesLandInCorrectBuckets() { + repository.save( + List.of( + event(Instant.parse("2026-07-02T10:00:00Z"), 1L, "1.0.0", "US", 1))); + // a late-arriving event for an earlier day + repository.save( + List.of( + event(Instant.parse("2026-06-30T10:00:00Z"), 1L, "1.0.0", "US", 7))); + + var rows = repository.findSeries( + DownloadSeriesRequest.of( + 1L, + Instant.parse("2026-06-29T00:00:00Z"), + Instant.parse("2026-07-03T00:00:00Z"), + DownloadSeriesInterval.DAY)); + + assertEquals( + List.of( + new DownloadSeriesRow(Instant.parse("2026-06-30T00:00:00Z"), null, 7), + new DownloadSeriesRow(Instant.parse("2026-07-02T00:00:00Z"), null, 1)), + rows); + } + + @Test + void testFindSeriesGroupBy() { + repository.save( + List.of( + event(Instant.parse("2026-07-01T08:00:00Z"), 1L, "1.0.0", "US", 1), + event(Instant.parse("2026-07-01T09:00:00Z"), 1L, "2.0.0", "DE", 2), + event(Instant.parse("2026-07-01T10:00:00Z"), 1L, "2.0.0", null, 4))); + + var from = Instant.parse("2026-07-01T00:00:00Z"); + var to = Instant.parse("2026-07-02T00:00:00Z"); + + var byVersion = repository.findSeries( + new DownloadSeriesRequest( + List.of(1L), + from, + to, + DownloadSeriesInterval.DAY, + DownloadSeriesGroupBy.VERSION)); + assertEquals( + List.of(new DownloadSeriesRow(from, "1.0.0", 1), new DownloadSeriesRow(from, "2.0.0", 6)), + byVersion); + + var byCountry = repository.findSeries( + new DownloadSeriesRequest( + List.of(1L), + from, + to, + DownloadSeriesInterval.DAY, + DownloadSeriesGroupBy.COUNTRY)); + assertEquals(3, byCountry.size()); + assertTrue(byCountry.contains(new DownloadSeriesRow(from, "US", 1))); + assertTrue(byCountry.contains(new DownloadSeriesRow(from, "DE", 2))); + assertTrue(byCountry.contains(new DownloadSeriesRow(from, null, 4))); + + var byTargetPlatform = repository.findSeries( + new DownloadSeriesRequest( + List.of(1L), + from, + to, + DownloadSeriesInterval.DAY, + DownloadSeriesGroupBy.TARGET_PLATFORM)); + assertEquals(List.of(new DownloadSeriesRow(from, "universal", 7)), byTargetPlatform); + } + + @Test + void testFindSeriesForMultipleExtensions() { + repository.save( + List.of( + event(Instant.parse("2026-07-01T08:00:00Z"), 1L, "1.0.0", "US", 1), + event(Instant.parse("2026-07-01T09:00:00Z"), 2L, "1.0.0", "US", 2), + event(Instant.parse("2026-07-01T09:00:00Z"), 3L, "1.0.0", "US", 100))); + + var rows = repository.findSeries( + new DownloadSeriesRequest( + List.of(1L, 2L), + Instant.parse("2026-07-01T00:00:00Z"), + Instant.parse("2026-07-02T00:00:00Z"), + DownloadSeriesInterval.DAY, + DownloadSeriesGroupBy.NONE)); + + assertEquals(List.of(new DownloadSeriesRow(Instant.parse("2026-07-01T00:00:00Z"), null, 3)), rows); + } + + private DownloadEvent event( + Instant time, + long extensionId, + String version, + String country, + int count + ) { + return new DownloadEvent( + time, + extensionId, + extensionId * 100, + "ns", + "ext", + version, + "universal", + country, + "9.9.9.9", + "VSCode 1.90.2", + count); + } + +} diff --git a/server/src/test/java/org/eclipse/openvsx/eclipse/EclipseServiceTest.java b/server/src/test/java/org/eclipse/openvsx/eclipse/EclipseServiceTest.java index 03bc6b483..92a5c9a9f 100644 --- a/server/src/test/java/org/eclipse/openvsx/eclipse/EclipseServiceTest.java +++ b/server/src/test/java/org/eclipse/openvsx/eclipse/EclipseServiceTest.java @@ -22,6 +22,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mockito; +import org.springframework.beans.factory.ObjectProvider; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.TestConfiguration; import org.springframework.context.annotation.Bean; @@ -38,6 +39,8 @@ import org.eclipse.openvsx.MockTransactionTemplate; import org.eclipse.openvsx.UserService; import org.eclipse.openvsx.adapter.VSCodeIdService; +import org.eclipse.openvsx.analytics.ingestion.DownloadIngestionProcessor; +import org.eclipse.openvsx.analytics.ingestion.DownloadRecordSource; import org.eclipse.openvsx.cache.CacheService; import org.eclipse.openvsx.cache.LatestExtensionVersionCacheKeyGenerator; import org.eclipse.openvsx.entities.*; @@ -50,7 +53,6 @@ import org.eclipse.openvsx.scanning.ExtensionScanService; import org.eclipse.openvsx.search.SearchUtilService; import org.eclipse.openvsx.storage.*; -import org.eclipse.openvsx.storage.log.DownloadCountService; import org.eclipse.openvsx.util.ErrorResultException; import org.eclipse.openvsx.util.LogService; import org.eclipse.openvsx.util.TargetPlatform; @@ -70,7 +72,7 @@ AzureBlobStorageService.class, AwsStorageService.class, VSCodeIdService.class, - DownloadCountService.class, + DownloadIngestionProcessor.class, ExtensionDownloadMetrics.class, CacheService.class, UserService.class, @@ -583,7 +585,8 @@ StorageUtilService storageUtilService( AzureBlobStorageService azureStorage, LocalStorageService localStorage, AwsStorageService awsStorage, - DownloadCountService downloadCountService, + ObjectProvider ingestionSources, + DownloadIngestionProcessor ingestionProcessor, ExtensionDownloadMetrics downloadMetrics, SearchUtilService search, CacheService cache, @@ -597,7 +600,8 @@ StorageUtilService storageUtilService( azureStorage, localStorage, awsStorage, - downloadCountService, + ingestionSources, + ingestionProcessor, downloadMetrics, search, cache, diff --git a/server/src/test/java/org/eclipse/openvsx/repositories/RepositoryServiceSmokeTest.java b/server/src/test/java/org/eclipse/openvsx/repositories/RepositoryServiceSmokeTest.java index dcbcceaa9..932116d1b 100644 --- a/server/src/test/java/org/eclipse/openvsx/repositories/RepositoryServiceSmokeTest.java +++ b/server/src/test/java/org/eclipse/openvsx/repositories/RepositoryServiceSmokeTest.java @@ -223,9 +223,10 @@ void testExecuteQueries() { () -> repositories.countPersistedLogs(userData), () -> repositories.findAllReviews(extension), () -> repositories - .findAllSucceededDownloadCountProcessedItemsByStorageTypeAndNameIn("storageType", STRING_LIST), + .findAllSucceededDownloadIngestionsByStorageTypeAndNameIn("storageType", STRING_LIST), () -> repositories - .findAllFailedDownloadCountProcessedItemsByStorageTypeAndNameIn("storageType", STRING_LIST), + .findAllFailedDownloadIngestionsByStorageTypeAndNameIn("storageType", STRING_LIST), + () -> repositories.countFailedDownloadIngestions(), () -> repositories.findBundledExtensionsReference(extension), () -> repositories.findDependenciesReference(extension), () -> repositories.findDownloadsByStorageTypeAndName("storageType", STRING_LIST), diff --git a/server/src/test/java/org/eclipse/openvsx/storage/StorageUtilServiceTest.java b/server/src/test/java/org/eclipse/openvsx/storage/StorageUtilServiceTest.java index ec9885eb5..acc421372 100644 --- a/server/src/test/java/org/eclipse/openvsx/storage/StorageUtilServiceTest.java +++ b/server/src/test/java/org/eclipse/openvsx/storage/StorageUtilServiceTest.java @@ -17,6 +17,8 @@ import jakarta.persistence.EntityManager; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mockito; +import org.springframework.beans.factory.ObjectProvider; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.TestConfiguration; import org.springframework.context.annotation.Bean; @@ -25,13 +27,14 @@ import org.springframework.test.context.junit.jupiter.SpringExtension; import org.eclipse.openvsx.UserService; +import org.eclipse.openvsx.analytics.ingestion.DownloadIngestionProcessor; +import org.eclipse.openvsx.analytics.ingestion.DownloadRecordSource; import org.eclipse.openvsx.cache.CacheService; import org.eclipse.openvsx.cache.FilesCacheKeyGenerator; import org.eclipse.openvsx.entities.*; import org.eclipse.openvsx.metrics.ExtensionDownloadMetrics; import org.eclipse.openvsx.repositories.RepositoryService; import org.eclipse.openvsx.search.SearchUtilService; -import org.eclipse.openvsx.storage.log.DownloadCountService; import static org.eclipse.openvsx.entities.FileResource.README; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -42,7 +45,7 @@ EntityManager.class, SearchUtilService.class, GoogleCloudStorageService.class, - DownloadCountService.class, + DownloadIngestionProcessor.class, ExtensionDownloadMetrics.class, CacheService.class, UserService.class, @@ -64,6 +67,15 @@ public class StorageUtilServiceTest { @Autowired StorageUtilService storageUtilService; + @Autowired + EntityManager entityManager; + + @Autowired + TestIngestionSource ingestionSource; + + @Autowired + DownloadIngestionProcessor ingestionProcessor; + @Test public void testCdnEnabled() { cdnServiceConfig.setEnabled(true); @@ -109,6 +121,48 @@ public void testCdnEnabledWithPlusSignInVersion() { } } + /** + * When a {@link DownloadRecordSource} covers a resource, its downloads are counted from + * access logs, so the request path must not count them a second time. This replaces the + * removed {@code DownloadCountService.isEnabled(resource)} check. + */ + @Test + public void testIncreaseDownloadCountSkipsResourcesCoveredByIngestionSource() { + var extension = mockExtension(); + var extensionVersion = mockExtensionVersion(extension, 1, "1.0.0", "universal"); + var resource = mockFileResource( + 1, + extensionVersion, + "ext.vsix", + FileResource.DOWNLOAD, + FileResource.STORAGE_AWS); + Mockito.when(entityManager.find(FileResource.class, 1L)).thenReturn(resource); + + ingestionSource.coveredStorageType = FileResource.STORAGE_AWS; + storageUtilService.increaseDownloadCount(resource); + assertEquals(100, extension.getDownloadCount()); + Mockito.verify(ingestionProcessor, Mockito.never()).captureDownload(resource); + } + + @Test + public void testIncreaseDownloadCountCountsUncoveredResources() { + var extension = mockExtension(); + var extensionVersion = mockExtensionVersion(extension, 1, "1.0.0", "universal"); + var resource = mockFileResource( + 1, + extensionVersion, + "ext.vsix", + FileResource.DOWNLOAD, + FileResource.STORAGE_AWS); + Mockito.when(entityManager.find(FileResource.class, 1L)).thenReturn(resource); + + ingestionSource.coveredStorageType = null; + storageUtilService.increaseDownloadCount(resource); + assertEquals(101, extension.getDownloadCount()); + // the request-path download also produces an analytics event + Mockito.verify(ingestionProcessor).captureDownload(resource); + } + @Test public void testCdnDisabled() { // Test a file resource for which no cdn prefix is enabled @@ -185,9 +239,55 @@ private FileResource mockFileResource( return resource; } + /** + * A minimal ingestion source stub whose coverage can be flipped per test. + */ + static class TestIngestionSource implements DownloadRecordSource { + String coveredStorageType; + + @Override + public String getStorageType() { + return coveredStorageType; + } + + @Override + public boolean isEnabled() { + return coveredStorageType != null; + } + + @Override + public String getCronSchedule() { + return "0 0 * * * *"; + } + + @Override + public boolean covers(FileResource resource) { + return coveredStorageType != null && coveredStorageType.equals(resource.getStorageType()); + } + + @Override + public java.util.Iterator> listBatches() { + return java.util.Collections.emptyIterator(); + } + + @Override + public java.util.List read(String name) { + return java.util.List.of(); + } + + @Override + public void finish(String name) { + } + } + @TestConfiguration static class TestConfig { + @Bean + TestIngestionSource testIngestionSource() { + return new TestIngestionSource(); + } + @Bean public CdnServiceConfig cdnServiceConfiguration() { var config = new CdnServiceConfig(); @@ -219,7 +319,8 @@ StorageUtilService storageUtilService( AzureBlobStorageService azureStorage, LocalStorageService localStorage, AwsStorageService awsStorage, - DownloadCountService downloadCountService, + ObjectProvider ingestionSources, + DownloadIngestionProcessor ingestionProcessor, ExtensionDownloadMetrics downloadMetrics, SearchUtilService search, CacheService cache, @@ -233,7 +334,8 @@ StorageUtilService storageUtilService( azureStorage, localStorage, awsStorage, - downloadCountService, + ingestionSources, + ingestionProcessor, downloadMetrics, search, cache, diff --git a/server/src/test/java/org/eclipse/openvsx/storage/StorageUtilServiceUploadFileTest.java b/server/src/test/java/org/eclipse/openvsx/storage/StorageUtilServiceUploadFileTest.java index a02a4209a..dd3333516 100644 --- a/server/src/test/java/org/eclipse/openvsx/storage/StorageUtilServiceUploadFileTest.java +++ b/server/src/test/java/org/eclipse/openvsx/storage/StorageUtilServiceUploadFileTest.java @@ -19,12 +19,14 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.beans.factory.ObjectProvider; +import org.eclipse.openvsx.analytics.ingestion.DownloadIngestionProcessor; +import org.eclipse.openvsx.analytics.ingestion.DownloadRecordSource; import org.eclipse.openvsx.entities.FileResource; import org.eclipse.openvsx.metrics.ExtensionDownloadMetrics; import org.eclipse.openvsx.repositories.RepositoryService; import org.eclipse.openvsx.search.SearchUtilService; -import org.eclipse.openvsx.storage.log.DownloadCountService; import org.eclipse.openvsx.util.TempFile; import static org.assertj.core.api.Assertions.assertThat; @@ -49,7 +51,9 @@ class StorageUtilServiceUploadFileTest { @Mock AwsStorageService awsStorage; @Mock - DownloadCountService downloadCountService; + ObjectProvider ingestionSources; + @Mock + DownloadIngestionProcessor ingestionProcessor; @Mock ExtensionDownloadMetrics downloadMetrics; @Mock @@ -104,7 +108,8 @@ private StorageUtilService newService() { azureStorage, localStorage, awsStorage, - downloadCountService, + ingestionSources, + ingestionProcessor, downloadMetrics, search, cache, diff --git a/server/src/test/java/org/eclipse/openvsx/storage/log/CloudFrontLogFileParserTest.java b/server/src/test/java/org/eclipse/openvsx/storage/log/CloudFrontLogFileParserTest.java deleted file mode 100644 index 3191efd59..000000000 --- a/server/src/test/java/org/eclipse/openvsx/storage/log/CloudFrontLogFileParserTest.java +++ /dev/null @@ -1,45 +0,0 @@ -/****************************************************************************** - * Copyright (c) 2026 Contributors to the Eclipse Foundation. - * - * See the NOTICE file(s) distributed with this work for additional - * information regarding copyright ownership. - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Public License 2.0 which is available at - * https://www.eclipse.org/legal/epl-2.0. - * - * SPDX-License-Identifier: EPL-2.0 - *****************************************************************************/ -package org.eclipse.openvsx.storage.log; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; - -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.*; - -public class CloudFrontLogFileParserTest { - - @Test - public void testParse() throws IOException { - LogFileParser parser = new CloudFrontLogFileParser(); - - try (var is = CloudFrontLogFileParser.class.getResourceAsStream("cloudfront.log")) { - assertNotNull(is); - try (var reader = new BufferedReader(new InputStreamReader(is))) { - var record = parser.parse(reader.readLine()); - assertNull(record); - - record = parser.parse(reader.readLine()); - assertNull(record); - - record = parser.parse(reader.readLine()); - assertEquals("OPTIONS", record.method()); - assertEquals(200, record.status()); - assertEquals("/vscjava/vscode-java-pack/0.30.4/package.json", record.url()); - } - } - } -} diff --git a/server/src/test/java/org/eclipse/openvsx/storage/log/FastlyLogFileParserTest.java b/server/src/test/java/org/eclipse/openvsx/storage/log/FastlyLogFileParserTest.java deleted file mode 100644 index a571e7298..000000000 --- a/server/src/test/java/org/eclipse/openvsx/storage/log/FastlyLogFileParserTest.java +++ /dev/null @@ -1,41 +0,0 @@ -/****************************************************************************** - * Copyright (c) 2026 Contributors to the Eclipse Foundation. - * - * See the NOTICE file(s) distributed with this work for additional - * information regarding copyright ownership. - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Public License 2.0 which is available at - * https://www.eclipse.org/legal/epl-2.0. - * - * SPDX-License-Identifier: EPL-2.0 - *****************************************************************************/ -package org.eclipse.openvsx.storage.log; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; - -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; - -public class FastlyLogFileParserTest { - - @Test - public void testParse() throws IOException { - LogFileParser parser = new FastlyLogFileParser(); - - try (var is = CloudFrontLogFileParser.class.getResourceAsStream("fastly.log")) { - assertNotNull(is); - try (var reader = new BufferedReader(new InputStreamReader(is))) { - var record = parser.parse(reader.readLine()); - assertNotNull(record); - assertEquals("GET", record.method()); - assertEquals(301, record.status()); - assertEquals("/favicon.ico", record.url()); - } - } - } -} diff --git a/server/src/test/resources/org/eclipse/openvsx/storage/log/cloudfront.log b/server/src/test/resources/org/eclipse/openvsx/analytics/ingestion/aws/cloudfront.log similarity index 55% rename from server/src/test/resources/org/eclipse/openvsx/storage/log/cloudfront.log rename to server/src/test/resources/org/eclipse/openvsx/analytics/ingestion/aws/cloudfront.log index eccb9d85a..46d9d0aed 100644 --- a/server/src/test/resources/org/eclipse/openvsx/storage/log/cloudfront.log +++ b/server/src/test/resources/org/eclipse/openvsx/analytics/ingestion/aws/cloudfront.log @@ -1,3 +1,6 @@ #Version: 1.0 #Fields: date time x-edge-location sc-bytes c-ip cs-method cs(Host) cs-uri-stem sc-status cs(Referer) cs(User-Agent) cs-uri-query cs(Cookie) x-edge-result-type x-edge-request-id x-host-header cs-protocol cs-bytes time-taken x-forwarded-for ssl-protocol ssl-cipher x-edge-response-result-type cs-protocol-version fle-status fle-encrypted-fields c-port time-to-first-byte x-edge-detailed-result-type sc-content-type sc-content-len sc-range-start sc-range-end 2025-12-03 13:17:20 LHR61-P5 380 1.1.1.1 OPTIONS abcd.cloudfront.net /vscjava/vscode-java-pack/0.30.4/package.json 200 - Mozilla/5.0 - - Miss bM-uMATMV5S8LlTbiQQL747aI5hQPlj7mOkvr4eC9xn_wp9Cbvz9bg== openvsx.somewhere.org https 66 0.044 - TLSv1.3 TLS_AES_128_GCM_SHA256 Miss HTTP/2.0 - - 61198 0.044 Miss - 0 - - +2025-12-03 13:20:01 LHR61-P5 1234567 1.1.1.1 GET abcd.cloudfront.net /vscjava/vscode-java-pack/0.30.4/file/vscjava.vscode-java-pack-0.30.4.vsix 200 - VSCode%201.90.2%20(Microsoft%20Visual%20Studio%20Code) - - Hit req2 openvsx.somewhere.org https 66 0.944 - TLSv1.3 TLS_AES_128_GCM_SHA256 Hit HTTP/2.0 - - 61198 0.144 Hit application/octet-stream 1234567 - - +- - LHR61-P5 1234567 1.1.1.1 GET abcd.cloudfront.net /foo/bar/1.0.0/file/foo.bar-1.0.0.vsix 200 - - - - Hit req3 openvsx.somewhere.org https 66 0.944 - TLSv1.3 TLS_AES_128_GCM_SHA256 Hit HTTP/2.0 - - 61198 0.144 Hit application/octet-stream 1234567 - - +garbage line diff --git a/server/src/test/resources/org/eclipse/openvsx/analytics/ingestion/aws/fastly.log b/server/src/test/resources/org/eclipse/openvsx/analytics/ingestion/aws/fastly.log new file mode 100644 index 000000000..9e1d34f2b --- /dev/null +++ b/server/src/test/resources/org/eclipse/openvsx/analytics/ingestion/aws/fastly.log @@ -0,0 +1,5 @@ +<134>2026-02-09T13:52:42Z cache-fra-eddf8230176 S3-Log-Stream-test[332221]: {"timestamp": "2026-02-09T04:20:50+0000", "client_ip": "1.1.1.1", "geo_country": "united states", "geo_city": "the dalles","host": "openvsx.somewhere.org","url": "/favicon.ico","request_method": "GET", "request_protocol": "HTTP/1.1", "request_referer": "", "request_user_agent": "Mozilla/5.0", "response_state": "HIT-SYNTH", "response_status": 301, "response_reason": "Moved Permanently", "response_body_size": 0,"fastly_server": "xxxx", "fastly_is_edge": true} +<134>2026-02-09T13:52:42Z cache-fra-eddf8230176 S3-Log-Stream-test[332221]: {"timestamp": "2026-02-09T04:20:50+0000", "client_ip": "1.1.1.1", "geo_country": "united states", "geo_city": "the dalles","host": "openvsx.somewhere.org","url": "/vscjava/vscode-java-pack/0.30.4/file/vscjava.vscode-java-pack-0.30.4.vsix","request_method": "GET", "request_protocol": "HTTP/1.1", "request_referer": "", "request_user_agent": "VSCode 1.90.2 (Microsoft Visual Studio Code)", "response_state": "HIT", "response_status": 200, "response_reason": "OK", "response_body_size": 1234567,"fastly_server": "xxxx", "fastly_is_edge": true} +<134>2026-02-09T13:52:43Z cache-fra-eddf8230176 S3-Log-Stream-test[332221]: {"timestamp": "2026-02-09T04: +<134>2026-02-09T13:52:44Z cache-fra-eddf8230176 S3-Log-Stream-test[332221]: {"host": "openvsx.somewhere.org","url": "/foo/bar/1.0.0/file/foo.bar-1.0.0.vsix","request_method": "GET", "response_status": 200} +plain text line without any json payload diff --git a/server/src/test/resources/org/eclipse/openvsx/storage/log/fastly.log b/server/src/test/resources/org/eclipse/openvsx/storage/log/fastly.log deleted file mode 100644 index dae924856..000000000 --- a/server/src/test/resources/org/eclipse/openvsx/storage/log/fastly.log +++ /dev/null @@ -1 +0,0 @@ -<134>2026-02-09T13:52:42Z cache-fra-eddf8230176 S3-Log-Stream-test[332221]: {"timestamp": "2026-02-09T04:20:50+0000", "client_ip": "1.1.1.1", "geo_country": "united states", "geo_city": "the dalles","host": "openvsx.somewhere.org","url": "/favicon.ico","request_method": "GET", "request_protocol": "HTTP/1.1", "request_referer": "", "request_user_agent": "Mozilla/5.0", "response_state": "HIT-SYNTH", "response_status": 301, "response_reason": "Moved Permanently", "response_body_size": 0,"fastly_server": "xxxx", "fastly_is_edge": true} From d10acaf7d3d8599e241037d95a95fb4e45abe62d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jordi=20G=C3=B3mez?= Date: Fri, 31 Jul 2026 13:33:12 +0200 Subject: [PATCH 02/20] feat: exposing analytics enabled flag on version endpoint --- .../org/eclipse/openvsx/LocalRegistryService.java | 4 ++++ .../org/eclipse/openvsx/json/RegistryVersionJson.java | 11 +++++++++++ 2 files changed, 15 insertions(+) diff --git a/server/src/main/java/org/eclipse/openvsx/LocalRegistryService.java b/server/src/main/java/org/eclipse/openvsx/LocalRegistryService.java index 5abbfd514..1361199c2 100644 --- a/server/src/main/java/org/eclipse/openvsx/LocalRegistryService.java +++ b/server/src/main/java/org/eclipse/openvsx/LocalRegistryService.java @@ -135,6 +135,9 @@ public LocalRegistryService( @Value("${ovsx.registry.version:}") String registryVersion; + @Value("${ovsx.analytics.enabled:false}") + boolean analyticsEnabled; + @Override public NamespaceJson getNamespace(String namespaceName) { return getNamespace(namespaceName, false); @@ -1378,6 +1381,7 @@ public RegistryVersionJson getRegistryVersion() { json.setMaxExtensionSize(publishingConfig.getMaxContentSize()); json.setTrustedPublishingAudience( trustedPublishingConfig.isEnabled() ? trustedPublishingConfig.getAudience() : null); + json.setAnalyticsEnabled(analyticsEnabled); return json; } diff --git a/server/src/main/java/org/eclipse/openvsx/json/RegistryVersionJson.java b/server/src/main/java/org/eclipse/openvsx/json/RegistryVersionJson.java index 10ff9c373..f093f7769 100644 --- a/server/src/main/java/org/eclipse/openvsx/json/RegistryVersionJson.java +++ b/server/src/main/java/org/eclipse/openvsx/json/RegistryVersionJson.java @@ -37,6 +37,9 @@ public static RegistryVersionJson error(String message) { @Nullable private String trustedPublishingAudience; + @Schema(description = "Whether download analytics are enabled and the analytics endpoints are available") + private boolean analyticsEnabled; + public String getVersion() { return version; } @@ -60,4 +63,12 @@ public String getTrustedPublishingAudience() { public void setTrustedPublishingAudience(String trustedPublishingAudience) { this.trustedPublishingAudience = trustedPublishingAudience; } + + public boolean isAnalyticsEnabled() { + return analyticsEnabled; + } + + public void setAnalyticsEnabled(boolean analyticsEnabled) { + this.analyticsEnabled = analyticsEnabled; + } } From c1e7648161d24e796783eeebe013be4285528a99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jordi=20G=C3=B3mez?= Date: Fri, 31 Jul 2026 13:34:16 +0200 Subject: [PATCH 03/20] feat: download analytics on extension details page --- webui/src/extension-registry-service.ts | 24 +++++ webui/src/extension-registry-types.ts | 13 +++ .../extension-detail-overview.tsx | 2 + .../use-extension-download-series.ts | 53 ++++++++++ .../extension-detail/weekly-downloads.tsx | 99 +++++++++++++++++++ .../unit/components/weekly-downloads.spec.tsx | 76 ++++++++++++++ 6 files changed, 267 insertions(+) create mode 100644 webui/src/pages/extension-detail/use-extension-download-series.ts create mode 100644 webui/src/pages/extension-detail/weekly-downloads.tsx create mode 100644 webui/test/unit/components/weekly-downloads.spec.tsx diff --git a/webui/src/extension-registry-service.ts b/webui/src/extension-registry-service.ts index 1ac229d21..abf5c7a06 100644 --- a/webui/src/extension-registry-service.ts +++ b/webui/src/extension-registry-service.ts @@ -28,6 +28,8 @@ import { NamespaceMembershipList, PublisherInfo, RegistryVersion, + DownloadSeries, + DownloadSeriesInterval, SearchEntry, LoginProviders, ScanResultJson, @@ -93,6 +95,28 @@ export class ExtensionRegistryService { return createAbsoluteURL(arr); } + /** + * Fetches the download time series for an extension from the analytics endpoint. The endpoint + * only exists when download analytics are enabled server-side (otherwise it responds 404), so + * callers should gate on {@link RegistryVersion.analyticsEnabled}. `from`/`to` are UTC dates + * (yyyy-MM-dd); `from` is inclusive and `to` is exclusive. + */ + async getExtensionDownloadSeries( + abortController: AbortController, + params: { namespace: string; name: string; from?: string; to?: string; interval?: DownloadSeriesInterval } + ): Promise> { + const endpoint = createAbsoluteURL( + [this.serverUrl, 'api', params.namespace, params.name, 'analytics', 'downloads'], + [ + { key: 'from', value: params.from }, + { key: 'to', value: params.to }, + { key: 'interval', value: params.interval } + ] + ); + // Non-retriable: retries are owned by the TanStack query that calls this. + return sendNonRetriableRequest({ abortController, endpoint }); + } + async getNamespaceDetails(abortController: AbortController, name: string): Promise> { const endpoint = createAbsoluteURL([this.serverUrl, 'api', name, 'details']); return sendStrictRequest({ abortController, endpoint }); diff --git a/webui/src/extension-registry-types.ts b/webui/src/extension-registry-types.ts index c03a7bcec..b6eab5d67 100644 --- a/webui/src/extension-registry-types.ts +++ b/webui/src/extension-registry-types.ts @@ -336,8 +336,21 @@ export interface TargetPlatformVersion { export interface RegistryVersion { version: string; maxExtensionSize?: number; + analyticsEnabled?: boolean; } +/** One bucket of the download time series: `t` is the UTC bucket start (yyyy-MM-dd). */ +export interface DownloadSeriesPoint { + t: string; + count: number; +} + +export interface DownloadSeries { + points: DownloadSeriesPoint[]; +} + +export type DownloadSeriesInterval = 'day' | 'week' | 'month'; + export interface LoginProviders { loginProviders: Record; } diff --git a/webui/src/pages/extension-detail/extension-detail-overview.tsx b/webui/src/pages/extension-detail/extension-detail-overview.tsx index adf8f6538..1f2b11b64 100644 --- a/webui/src/pages/extension-detail/extension-detail-overview.tsx +++ b/webui/src/pages/extension-detail/extension-detail-overview.tsx @@ -24,6 +24,7 @@ import { Extension, ExtensionReference, VERSION_ALIASES } from '../../extension- import { ExtensionListRoutes } from '../extension-list/extension-list-routes'; import { ExtensionDetailRoutes } from './extension-detail-routes'; import { ExtensionDetailDownloadsMenu } from './extension-detail-downloads-menu'; +import { WeeklyDownloads } from './weekly-downloads'; export const ExtensionDetailOverview: FunctionComponent = props => { const [loading, setLoading] = useState(true); @@ -350,6 +351,7 @@ export const ExtensionDetailOverview: FunctionComponent + {renderVersionSection()} {otherAliases.length || extension.versionAlias.length ? ( diff --git a/webui/src/pages/extension-detail/use-extension-download-series.ts b/webui/src/pages/extension-detail/use-extension-download-series.ts new file mode 100644 index 000000000..4451eeb3e --- /dev/null +++ b/webui/src/pages/extension-detail/use-extension-download-series.ts @@ -0,0 +1,53 @@ +/****************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation. + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * https://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + *****************************************************************************/ + +import { useContext } from 'react'; +import { useQuery } from '@tanstack/react-query'; +import { DateTime } from 'luxon'; +import { MainContext } from '../../context'; +import { controllerFromSignal } from '../../query-client'; +import { DownloadSeriesPoint } from '../../extension-registry-types'; + +const WEEKS = 52; +// 6 extra leading days so the first plotted point already has a full trailing-7-day window. +const LEAD_IN_DAYS = 6; + +/** + * Loads roughly the last {@link WEEKS} weeks of *daily* downloads for an extension, up to and + * including today, as the react-query result (`data` is the ordered {@link DownloadSeriesPoint} + * array). Callers turn this into a trailing 7-day ("weekly downloads") view; daily granularity is + * what lets that window end on today rather than the last complete calendar week. Gate with + * `options.enabled` on `RegistryVersion.analyticsEnabled`, since the endpoint 404s when analytics + * is disabled. + */ +export const useExtensionDownloadSeries = (namespace: string, name: string, options?: { enabled?: boolean }) => { + const { service } = useContext(MainContext); + return useQuery({ + queryKey: ['extension-downloads', namespace, name], + queryFn: async ({ signal }): Promise => { + const today = DateTime.utc().startOf('day'); + // `to` is exclusive, so today + 1 day includes today's (still-accruing) bucket. + const to = today.plus({ days: 1 }); + const from = to.minus({ weeks: WEEKS }).minus({ days: LEAD_IN_DAYS }); + const series = await service.getExtensionDownloadSeries(controllerFromSignal(signal), { + namespace, + name, + from: from.toFormat('yyyy-MM-dd'), + to: to.toFormat('yyyy-MM-dd'), + interval: 'day' + }); + return series.points; + }, + ...options + }); +}; diff --git a/webui/src/pages/extension-detail/weekly-downloads.tsx b/webui/src/pages/extension-detail/weekly-downloads.tsx new file mode 100644 index 000000000..91f95bbcb --- /dev/null +++ b/webui/src/pages/extension-detail/weekly-downloads.tsx @@ -0,0 +1,99 @@ +/****************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation. + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * https://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + *****************************************************************************/ + +import { FunctionComponent, useContext, useMemo } from 'react'; +import { Box, Typography, styled, useTheme } from '@mui/material'; +import { SparkLineChart } from '@mui/x-charts/SparkLineChart'; +import { MainContext } from '../../context'; +import { Eyebrow, cardSurface } from '../../components/page-primitives'; +import { Extension } from '../../extension-registry-types'; +import { useExtensionDownloadSeries } from './use-extension-download-series'; + +const DownloadsCard = styled(Box)(({ theme }) => ({ + ...cardSurface(theme), + padding: '0.75rem 1rem' +})); + +const DownloadsCount = styled(Typography)(({ theme }) => ({ + fontSize: '1.75rem', + lineHeight: 1.1, + fontWeight: 700, + color: theme.palette.text.primary, + fontVariantNumeric: 'tabular-nums' +})) as typeof Typography; + +const WINDOW_DAYS = 7; + +/** Trailing {@link WINDOW_DAYS}-day rolling sums of a daily series (each point covers that day and + * the previous six), so the last value is the downloads of the last week ending today. */ +function trailingWeeklySums(daily: number[]): number[] { + const rolling: number[] = []; + let windowSum = 0; + for (let i = 0; i < daily.length; i++) { + windowSum += daily[i]; + if (i >= WINDOW_DAYS) { + windowSum -= daily[i - WINDOW_DAYS]; + } + if (i >= WINDOW_DAYS - 1) { + rolling.push(windowSum); + } + } + return rolling; +} + +/** + * "Weekly downloads" sidebar card: the downloads of the last 7 days (ending today) plus a trailing + * 7-day trend sparkline over the last year. Renders nothing when download analytics are disabled + * server-side (the endpoint 404s) or when the extension has no downloads in the window, so it stays + * out of the way on registries without data. + */ +export const WeeklyDownloads: FunctionComponent<{ extension: Extension }> = ({ extension }) => { + const theme = useTheme(); + const { version } = useContext(MainContext); + const analyticsEnabled = version?.analyticsEnabled ?? false; + + const { data: points } = useExtensionDownloadSeries(extension.namespace, extension.name, { + enabled: analyticsEnabled + }); + + const counts = useMemo(() => trailingWeeklySums(points?.map(point => point.count) ?? []), [points]); + const hasDownloads = counts.some(count => count > 0); + if (!analyticsEnabled || counts.length === 0 || !hasDownloads) { + return null; + } + + const latestWeek = counts[counts.length - 1]; + + return ( + + + Weekly downloads + + {latestWeek.toLocaleString()} + + (value === null ? '' : `${value.toLocaleString()} downloads`)} + /> + + + + + ); +}; diff --git a/webui/test/unit/components/weekly-downloads.spec.tsx b/webui/test/unit/components/weekly-downloads.spec.tsx new file mode 100644 index 000000000..7f3554ed7 --- /dev/null +++ b/webui/test/unit/components/weekly-downloads.spec.tsx @@ -0,0 +1,76 @@ +/****************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation. + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * https://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + *****************************************************************************/ + +import { describe, it, expect, vi } from 'vitest'; +import { screen, waitFor } from '@testing-library/react'; +import { renderWithProviders } from '../support/test-providers'; +import { WeeklyDownloads } from '../../../src/pages/extension-detail/weekly-downloads'; +import { DownloadSeriesPoint, Extension, RegistryVersion } from '../../../src/extension-registry-types'; +import { ExtensionRegistryService } from '../../../src/extension-registry-service'; + +// The real chart pulls in SVG measurement APIs jsdom lacks; stub it so the test exercises +// the component's own logic (gating, the headline number, and the series it feeds the chart). +vi.mock('@mui/x-charts/SparkLineChart', () => ({ + SparkLineChart: ({ data }: { data: number[] }) =>
+})); + +const extension = { namespace: 'redhat', name: 'java' } as unknown as Extension; +const analyticsEnabled: RegistryVersion = { version: '1.0.0', analyticsEnabled: true }; + +function points(counts: number[]): DownloadSeriesPoint[] { + return counts.map((count, i) => ({ t: `2026-01-${String(i + 1).padStart(2, '0')}`, count })); +} + +function serviceReturning(series: DownloadSeriesPoint[]): ExtensionRegistryService { + return { + getExtensionDownloadSeries: vi.fn().mockResolvedValue({ points: series }) + } as unknown as ExtensionRegistryService; +} + +describe('WeeklyDownloads', () => { + it('shows the last-7-days total and the trailing-week trend when analytics is enabled', async () => { + // 14 daily points of 1000 → every trailing-7-day sum is 7000; rolling length = 14 - 6 = 8. + const service = serviceReturning(points(Array(14).fill(1000))); + renderWithProviders(, { + mainContext: { service, version: analyticsEnabled } + }); + + expect(await screen.findByText((7000).toLocaleString())).toBeInTheDocument(); + expect(screen.getByText(/weekly downloads/i)).toBeInTheDocument(); + expect(screen.getByTestId('sparkline')).toHaveAttribute('data-length', '8'); + expect(service.getExtensionDownloadSeries).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ namespace: 'redhat', name: 'java', interval: 'day' }) + ); + }); + + it('renders nothing (and never calls the endpoint) when analytics is disabled', () => { + const service = serviceReturning(points(Array(14).fill(1))); + renderWithProviders(, { + mainContext: { service, version: { version: '1.0.0', analyticsEnabled: false } } + }); + + expect(screen.queryByText(/weekly downloads/i)).not.toBeInTheDocument(); + expect(service.getExtensionDownloadSeries).not.toHaveBeenCalled(); + }); + + it('renders nothing when the extension has no downloads in the window', async () => { + const service = serviceReturning(points(Array(14).fill(0))); + renderWithProviders(, { + mainContext: { service, version: analyticsEnabled } + }); + + await waitFor(() => expect(service.getExtensionDownloadSeries).toHaveBeenCalled()); + expect(screen.queryByText(/weekly downloads/i)).not.toBeInTheDocument(); + }); +}); From 2f5528277fa81221f108af1848bbf3cbd9669ee7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jordi=20G=C3=B3mez?= Date: Fri, 31 Jul 2026 14:31:18 +0200 Subject: [PATCH 04/20] fix: bad fill color for downloads chart --- webui/src/pages/extension-detail/weekly-downloads.tsx | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/webui/src/pages/extension-detail/weekly-downloads.tsx b/webui/src/pages/extension-detail/weekly-downloads.tsx index 91f95bbcb..fd6fdd4ff 100644 --- a/webui/src/pages/extension-detail/weekly-downloads.tsx +++ b/webui/src/pages/extension-detail/weekly-downloads.tsx @@ -66,7 +66,8 @@ export const WeeklyDownloads: FunctionComponent<{ extension: Extension }> = ({ e enabled: analyticsEnabled }); - const counts = useMemo(() => trailingWeeklySums(points?.map(point => point.count) ?? []), [points]); + const daily = useMemo(() => points ?? [], [points]); + const counts = useMemo(() => trailingWeeklySums(daily.map(point => point.count)), [daily]); const hasDownloads = counts.some(count => count > 0); if (!analyticsEnabled || counts.length === 0 || !hasDownloads) { return null; @@ -90,6 +91,8 @@ export const WeeklyDownloads: FunctionComponent<{ extension: Extension }> = ({ e showHighlight color={theme.palette.secondary.main} valueFormatter={value => (value === null ? '' : `${value.toLocaleString()} downloads`)} + // `.MuiAreaElement-root` is the sparkline's area path; lighten its fill to a wash. + sx={{ '& .MuiAreaElement-root': { fillOpacity: 0.14 } }} /> From e0a48591517fb83faa0f8efeedec791d0280815c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jordi=20G=C3=B3mez?= Date: Mon, 24 Aug 2026 10:24:43 +0200 Subject: [PATCH 05/20] refactor: rework the navbar chrome and scroll restoration --- webui/CHANGELOG.md | 5 ++ webui/src/app-providers.tsx | 2 +- webui/src/context/extension-tint-context.tsx | 50 ----------- webui/src/context/navbar-chrome-context.tsx | 83 +++++++++++++++++++ webui/src/layout/app-layout.tsx | 12 +-- webui/src/layout/app-navbar.tsx | 17 +++- ...roll-to-top.tsx => scroll-restoration.tsx} | 13 ++- .../extension-detail/extension-detail.tsx | 4 +- 8 files changed, 116 insertions(+), 70 deletions(-) delete mode 100644 webui/src/context/extension-tint-context.tsx create mode 100644 webui/src/context/navbar-chrome-context.tsx rename webui/src/layout/{scroll-to-top.tsx => scroll-restoration.tsx} (66%) diff --git a/webui/CHANGELOG.md b/webui/CHANGELOG.md index ec406213c..72df54b55 100644 --- a/webui/CHANGELOG.md +++ b/webui/CHANGELOG.md @@ -20,6 +20,11 @@ This change log covers only the frontend library (webui) of Open VSX. - **Breaking:** `elements.claimNamespace` now receives `{ namespace, extension?, sx? }` instead of `{ extension, sx? }`. The namespace settings page offers the same claim action and has no extension to pass, so implementations must read the namespace from `namespace` rather than `extension.namespace` - `ExtensionCard` accepts an `Extension` as well as a `SearchEntry`, and takes optional `to`, `linkState`, `overlay`, `footerStart` and `dimmed` props so other surfaces can reuse it instead of copying it +### Changed + +- Rename `ScrollToTop` to `ScrollRestoration`, matching what it does on back/forward navigation +- Rename the extension tint context to `navbar-chrome-context` and add a second channel to it: a page with sections pinned under the navbar can extend the navbar's blur fan down to back them (`useExtendNavbarBlur`) + ### Fixed - Fix a React warning ("Received `true` for a non-boolean attribute `notched`") from the admin dashboard's publisher role filter, whose custom `InputBase` doesn't consume the `notched` prop MUI's `Select` injects for the (unused) outlined variant diff --git a/webui/src/app-providers.tsx b/webui/src/app-providers.tsx index 1ee78d5b1..bc54c2f25 100644 --- a/webui/src/app-providers.tsx +++ b/webui/src/app-providers.tsx @@ -32,7 +32,7 @@ interface AppProvidersProps { * bottom in one place. Ordered outer→inner; keyboard shortcuts and search wrap * every route (admin included). Router, theme, and Helmet stay at the app entry — * they're supplied by whoever mounts the library. Lower-tier, feature-scoped - * providers (e.g. extension tint) stay with their feature. + * providers (e.g. the navbar chrome) stay with their feature. */ export const AppProviders: FunctionComponent = ({ mainContext, diff --git a/webui/src/context/extension-tint-context.tsx b/webui/src/context/extension-tint-context.tsx deleted file mode 100644 index 6b48ff499..000000000 --- a/webui/src/context/extension-tint-context.tsx +++ /dev/null @@ -1,50 +0,0 @@ -/******************************************************************************** - * Copyright (c) 2026 Contributors to the Eclipse Foundation - * - * See the NOTICE file(s) distributed with this work for additional - * information regarding copyright ownership. - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Public License 2.0 which is available at - * https://www.eclipse.org/legal/epl-2.0 - * - * SPDX-License-Identifier: EPL-2.0 - ********************************************************************************/ - -import { createContext, FunctionComponent, ReactNode, useContext, useMemo, useState } from 'react'; - -/** - * The tint region an extension detail page declares while mounted. The page - * only describes it; the nav bar compares the depth against its own scroll - * position to decide when to wear the color. - */ -export interface ExtensionTint { - // Gallery color the nav wears while the region backs it, flipping its - // content to the contrast color. Null for default-colored bands, which - // keep the nav on theme colors. - color: string | null; - // Document offset where the region ends; scrolled past it the nav returns - // to theme colors. - depth: number; -} - -const ExtensionTintContext = createContext<{ - tint: ExtensionTint | null; - setTint: (tint: ExtensionTint | null) => void; -}>({ tint: null, setTint: () => {} }); - -// eslint-disable-next-line react-refresh/only-export-components -export function useExtensionTint(): ExtensionTint | null { - return useContext(ExtensionTintContext).tint; -} - -// eslint-disable-next-line react-refresh/only-export-components -export function useSetExtensionTint(): (tint: ExtensionTint | null) => void { - return useContext(ExtensionTintContext).setTint; -} - -export const ExtensionTintProvider: FunctionComponent<{ children: ReactNode }> = ({ children }) => { - const [tint, setTint] = useState(null); - const value = useMemo(() => ({ tint, setTint }), [tint]); - return {children}; -}; diff --git a/webui/src/context/navbar-chrome-context.tsx b/webui/src/context/navbar-chrome-context.tsx new file mode 100644 index 000000000..92d7b7423 --- /dev/null +++ b/webui/src/context/navbar-chrome-context.tsx @@ -0,0 +1,83 @@ +/******************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * https://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + ********************************************************************************/ + +import { createContext, FunctionComponent, ReactNode, useContext, useEffect, useMemo, useState } from 'react'; + +/** + * The tint region an extension detail page declares while mounted. The page + * only describes it; the nav bar compares the depth against its own scroll + * position to decide when to wear the color. + */ +export interface ExtensionTint { + // Gallery color the nav wears while the region backs it, flipping its + // content to the contrast color. Null for default-colored bands, which + // keep the nav on theme colors. + color: string | null; + // Document offset where the region ends; scrolled past it the nav returns + // to theme colors. + depth: number; +} + +// What the current page asks of the navbar chrome, declared while mounted: +// a tint over the page's gallery band, and extra depth for the blur fan to +// back sections pinned under the bar. +interface NavbarChrome { + tint: ExtensionTint | null; + setTint: (tint: ExtensionTint | null) => void; + blurDepth: number; + setBlurDepth: (depth: number) => void; +} + +const NavbarChromeContext = createContext({ + tint: null, + setTint: () => {}, + blurDepth: 0, + setBlurDepth: () => {} +}); + +export const NavbarChromeProvider: FunctionComponent<{ children: ReactNode }> = ({ children }) => { + const [tint, setTint] = useState(null); + const [blurDepth, setBlurDepth] = useState(0); + const value = useMemo(() => ({ tint, setTint, blurDepth, setBlurDepth }), [tint, blurDepth]); + return {children}; +}; + +// eslint-disable-next-line react-refresh/only-export-components +export function useExtensionTint(): ExtensionTint | null { + return useContext(NavbarChromeContext).tint; +} + +// eslint-disable-next-line react-refresh/only-export-components +export function useSetExtensionTint(): (tint: ExtensionTint | null) => void { + return useContext(NavbarChromeContext).setTint; +} + +/** The navbar reads how far pages want the blur fan extended. */ +// eslint-disable-next-line react-refresh/only-export-components +export function useNavbarBlurExtent(): number { + return useContext(NavbarChromeContext).blurDepth; +} + +/** + * Extends the navbar's blur fan by the given depth (px) while the calling component is mounted — + * for page sections pinned under the navbar (sticky headers, tab rows) that float on the fan. + * Last writer wins: at most one mounted component may extend the fan at a time. + */ +// eslint-disable-next-line react-refresh/only-export-components +export function useExtendNavbarBlur(depth: number): void { + const { setBlurDepth } = useContext(NavbarChromeContext); + useEffect(() => { + setBlurDepth(depth); + return () => setBlurDepth(0); + }, [depth, setBlurDepth]); +} diff --git a/webui/src/layout/app-layout.tsx b/webui/src/layout/app-layout.tsx index 137b7e43e..0d51b312a 100644 --- a/webui/src/layout/app-layout.tsx +++ b/webui/src/layout/app-layout.tsx @@ -19,7 +19,7 @@ import { Banner } from '../components/banner'; import { ShortcutsModal } from '../components/shortcuts-modal'; import { MainContext } from '../context'; import { useSearch } from '../hooks/use-search'; -import { ExtensionTintProvider } from '../context/extension-tint-context'; +import { NavbarChromeProvider } from '../context/navbar-chrome-context'; import { useShortcut } from '../hooks/use-shortcut'; import { getCookieValueByKey, setCookie } from '../utils'; import { ExtensionListRoutes } from '../pages/extension-list/extension-list-routes'; @@ -34,7 +34,7 @@ import { NotFound } from '../not-found'; import { NAVBAR_HEIGHT } from '../default/theme'; import { AppNavbar } from './app-navbar'; import { AppFooter } from './app-footer'; -import { ScrollToTop } from './scroll-to-top'; +import { ScrollRestoration } from './scroll-restoration'; const UserSettings = lazy(() => import('../pages/user/user-settings').then(m => ({ default: m.UserSettings }))); @@ -92,7 +92,7 @@ const AppLayoutContent: FunctionComponent = props => { return ( - + {BannerComponent ? ( = props => { }; // Keyboard shortcuts and search now live app-wide in AppProviders; this keeps -// only the feature-scoped tint provider. +// only the navbar-chrome channel the pages below declare into. export const AppLayout: FunctionComponent = props => ( - + - + ); export interface AppLayoutProps { diff --git a/webui/src/layout/app-navbar.tsx b/webui/src/layout/app-navbar.tsx index 430405dae..bd3f2ed9f 100644 --- a/webui/src/layout/app-navbar.tsx +++ b/webui/src/layout/app-navbar.tsx @@ -18,7 +18,7 @@ import { Link as RouteLink } from 'react-router'; import { HeaderMenu } from '../header-menu'; import { MainContext } from '../context'; import { usePageSearchBar } from '../context/search/page-search-bar-context'; -import { useExtensionTint } from '../context/extension-tint-context'; +import { useExtensionTint, useNavbarBlurExtent } from '../context/navbar-chrome-context'; import { OpenVsxMark } from '../components/openvsx-mark'; import { NavSearchField } from './nav-search-field'; import { NAVBAR_HEIGHT_PX } from '../default/theme'; @@ -101,8 +101,17 @@ export const AppNavbar: FunctionComponent = () => { const { navTint, washColor } = useNavTint(); - const fanBottom = { xs: '-48px', sm: '-80px' }; - const tintBottom = { xs: '-48px', sm: '-150px' }; + // Pages with pinned sections can deepen the fan so it backs them too — desktop only: + // on mobile the stretched blur reads as a smear, so pinned rows bring their own glass. + // Only the lightest layer stretches (backdrop-filter cost scales with area and re-runs + // every scroll frame); the tint washes are plain gradients and stretch fully. + const extraBlurDepth = useNavbarBlurExtent(); + const fanBottom = (stretched: boolean) => ({ + xs: '-48px', + sm: '-80px', + md: `-${80 + (stretched ? extraBlurDepth : 0)}px` + }); + const tintBottom = { xs: '-48px', sm: '-150px', md: `-${150 + extraBlurDepth}px` }; // While a gallery band backs the nav, the chrome wears its color and the // content flips to the contrast color, so the two surfaces read as one. @@ -150,7 +159,7 @@ export const AppNavbar: FunctionComponent = () => { left: 0, right: 0, top: 0, - bottom: fanBottom, + bottom: fanBottom(i === BLUR_LAYERS.length - 1), pointerEvents: 'none', zIndex: 0, opacity: showFan ? 1 : 0, diff --git a/webui/src/layout/scroll-to-top.tsx b/webui/src/layout/scroll-restoration.tsx similarity index 66% rename from webui/src/layout/scroll-to-top.tsx rename to webui/src/layout/scroll-restoration.tsx index 9b2cdc82b..e91095f69 100644 --- a/webui/src/layout/scroll-to-top.tsx +++ b/webui/src/layout/scroll-restoration.tsx @@ -14,13 +14,12 @@ import { FunctionComponent, useLayoutEffect } from 'react'; import { useLocation, useNavigationType } from 'react-router'; -// BrowserRouter leaves window scroll untouched on navigation, so a page opened -// from deep in a long list would start at that old offset. Reset to the top on -// forward navigations only: POP (back/forward) keeps the browser's native -// scroll restoration, and same-path param updates (e.g. search filters, which -// replace in place) must not jump either — hence keying on pathname alone. -// Links that swap content in place opt out via state `{ preserveScroll: true }`. -export const ScrollToTop: FunctionComponent = () => { +// BrowserRouter never scrolls on push, so reset to the top on forward navigations only: POP +// keeps the browser's restoration, and same-path param updates must not jump (hence keying on +// pathname alone). Links that swap content in place opt out via state `{ preserveScroll: true }`. +// TODO Use react-router's once on a data router — also fixes the pop +// landing clamped to the outgoing page's height: https://github.com/eclipse-openvsx/openvsx/issues/2079 +export const ScrollRestoration: FunctionComponent = () => { const { pathname, state } = useLocation(); const navigationType = useNavigationType(); const preserveScroll = (state as { preserveScroll?: boolean } | null)?.preserveScroll; diff --git a/webui/src/pages/extension-detail/extension-detail.tsx b/webui/src/pages/extension-detail/extension-detail.tsx index a53eed728..25137af9f 100644 --- a/webui/src/pages/extension-detail/extension-detail.tsx +++ b/webui/src/pages/extension-detail/extension-detail.tsx @@ -42,7 +42,7 @@ import { useExtensionDetail } from './use-extension-details'; import { KbdKey } from '../../components/kbd-key'; import { useShortcut } from '../../hooks/use-shortcut'; import { NAVBAR_HEIGHT, NAVBAR_HEIGHT_PX } from '../../default/theme'; -import { useSetExtensionTint } from '../../context/extension-tint-context'; +import { useSetExtensionTint } from '../../context/navbar-chrome-context'; import { PageContainer } from '../../components/page-container'; import { PillTab, PillTabs } from '../../components/pill-tabs'; import SaveAltIcon from '@mui/icons-material/SaveAlt'; @@ -399,7 +399,7 @@ export const ExtensionDetail: FunctionComponent = () => { const effectiveVersion = isTabSegment(version) ? undefined : version; const activeTab = parseTab(version); - // Tab switches preserve scroll (see ScrollToTop); when scrolled deep, glide + // Tab switches preserve scroll (see ScrollRestoration); when scrolled deep, glide // up so the new panel starts under the pinned pills. const bandRef = useRef(null); const prevTab = useRef(activeTab); From 2f96cbc38914db4e7ecea20e92b0426bd589c232 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jordi=20G=C3=B3mez?= Date: Mon, 24 Aug 2026 10:25:21 +0200 Subject: [PATCH 06/20] feat: add a Pill component and lift shared page primitives --- webui/CHANGELOG.md | 5 +++ webui/src/components/category-pill.tsx | 33 +++----------- .../src/components/extension-searchfield.tsx | 11 +---- webui/src/components/page-primitives.tsx | 36 ++++++++++++++++ webui/src/components/pill.tsx | 43 +++++++++++++++++++ webui/src/default/theme.tsx | 9 ++++ webui/src/pages/search/search-header.tsx | 12 +----- 7 files changed, 101 insertions(+), 48 deletions(-) create mode 100644 webui/src/components/pill.tsx diff --git a/webui/CHANGELOG.md b/webui/CHANGELOG.md index 72df54b55..9a7459991 100644 --- a/webui/CHANGELOG.md +++ b/webui/CHANGELOG.md @@ -20,10 +20,15 @@ This change log covers only the frontend library (webui) of Open VSX. - **Breaking:** `elements.claimNamespace` now receives `{ namespace, extension?, sx? }` instead of `{ extension, sx? }`. The namespace settings page offers the same claim action and has no extension to pass, so implementations must read the namespace from `namespace` rather than `extension.namespace` - `ExtensionCard` accepts an `Extension` as well as a `SearchEntry`, and takes optional `to`, `linkState`, `overlay`, `footerStart` and `dimmed` props so other surfaces can reuse it instead of copying it +### Added + +- Add a `Pill` component — the clickable glass pill the category pills are built on, now usable on its own — and extract the `MonoSlash`, `glassSurface` and `compactControl` page primitives out of the search field, the pills and the search header + ### Changed - Rename `ScrollToTop` to `ScrollRestoration`, matching what it does on back/forward navigation - Rename the extension tint context to `navbar-chrome-context` and add a second channel to it: a page with sections pinned under the navbar can extend the navbar's blur fan down to back them (`useExtendNavbarBlur`) +- Give Popover and Autocomplete popups the same floating-paper treatment as the other menus, and stop Popovers locking body scroll — the lock jumps the scroll position on mobile and shifts the pinned chrome ### Fixed diff --git a/webui/src/components/category-pill.tsx b/webui/src/components/category-pill.tsx index 61802996d..9f896bda8 100644 --- a/webui/src/components/category-pill.tsx +++ b/webui/src/components/category-pill.tsx @@ -12,31 +12,8 @@ ********************************************************************************/ import { FunctionComponent, useEffect, useRef } from 'react'; -import { ButtonBase, SvgIconProps } from '@mui/material'; -import { styled } from '@mui/material/styles'; -import { accentHover, focusOutline } from './page-primitives'; - -const Root = styled(ButtonBase, { - shouldForwardProp: prop => prop !== 'isSelected' -})<{ isSelected?: boolean }>(({ theme, isSelected }) => ({ - display: 'inline-flex', - alignItems: 'center', - gap: '0.4375rem', - flexShrink: 0, - overflow: 'hidden', - backgroundColor: isSelected ? theme.palette.accentSoft : theme.palette.surface2, - border: `1px solid ${isSelected ? theme.palette.secondary.main : theme.palette.divider}`, - color: isSelected ? theme.palette.secondary.light : theme.palette.text.secondary, - fontSize: '0.8125rem', - fontWeight: isSelected ? 600 : 500, - padding: '0.4375rem 0.8125rem', - borderRadius: theme.shape.borderRadiusPill, - whiteSpace: 'nowrap', - fontFamily: 'inherit', - transition: 'border-color 0.14s, color 0.14s', - ...(isSelected ? {} : accentHover(theme)), - ...focusOutline(theme) -})); +import { SvgIconProps } from '@mui/material'; +import { Pill } from './pill'; export interface CategoryPillProps { label: string; @@ -56,9 +33,9 @@ export const CategoryPill: FunctionComponent = ({ label, icon }, [isSelected]); return ( - - + + {label} - + ); }; diff --git a/webui/src/components/extension-searchfield.tsx b/webui/src/components/extension-searchfield.tsx index 20d41cbe1..a2f442b9b 100644 --- a/webui/src/components/extension-searchfield.tsx +++ b/webui/src/components/extension-searchfield.tsx @@ -17,7 +17,7 @@ import { alpha, styled } from '@mui/material/styles'; import { MONO_FONT } from '../default/theme'; import SearchIcon from '@mui/icons-material/Search'; import ClearIcon from '@mui/icons-material/Close'; -import { focusRing } from './page-primitives'; +import { focusRing, MonoSlash } from './page-primitives'; interface ExtensionSearchfieldProps { onSearchChanged: (s: string) => void; @@ -49,15 +49,6 @@ const SearchWrap = styled(Box, { '&:focus-within': focusRing(theme) })); -const MonoSlash = styled('span')(({ theme }) => ({ - fontFamily: MONO_FONT, - color: theme.palette.secondary.light, - fontSize: '1.0625rem', - lineHeight: 1, - flexShrink: 0, - userSelect: 'none' -})); - const SearchInput = styled(InputBase)(({ theme }) => ({ flex: 1, fontFamily: MONO_FONT, diff --git a/webui/src/components/page-primitives.tsx b/webui/src/components/page-primitives.tsx index 24325177d..8504c6053 100644 --- a/webui/src/components/page-primitives.tsx +++ b/webui/src/components/page-primitives.tsx @@ -13,6 +13,7 @@ import { Box, Typography } from '@mui/material'; import { alpha, CSSObject, styled, Theme } from '@mui/material/styles'; +import { MONO_FONT } from '../default/theme'; /** Normalized gap between stacked sections; the owl selector skips the first (and any null) child. */ export const SectionStack = styled(Box)(({ theme }) => ({ @@ -42,6 +43,16 @@ export const ExtensionGrid = styled(Box)({ } }); +/** Leading '/' glyph of the search fields — the field's mark and the shortcut hint in one. */ +export const MonoSlash = styled('span')(({ theme }) => ({ + fontFamily: MONO_FONT, + color: theme.palette.secondary.light, + fontSize: '1.0625rem', + lineHeight: 1, + flexShrink: 0, + userSelect: 'none' +})); + /** Small uppercase label used to head sections, columns and sidebars. */ export const Eyebrow = styled(Typography)(({ theme }) => ({ fontSize: '0.75rem', @@ -107,6 +118,31 @@ export const TagChip = styled('span', { shouldForwardProp: prop => prop !== 'acc }) ); +/** + * Translucent glass fill for controls floating over scrolled content (pills, pinned toolbars). + * Backdrop filters re-blur on every scrolled frame, so keep the frosted surfaces small. + */ +export const glassSurface = (theme: Theme, opacity = 0.7) => ({ + backgroundColor: alpha(theme.palette.surface2, opacity), + backdropFilter: 'blur(2px) saturate(1.8)' +}); + +/** + * Compact toolbar control (the "Sort by" select, filter autocompletes): one 30px text row on a + * paper fill. Selects take it directly via sx; composite inputs spread it into their outlined + * root. Plain CSS properties only, so it also works outside sx. + */ +export const compactControl = (theme: Theme) => ({ + fontSize: '0.8125rem', + fontWeight: 500, + color: theme.palette.text.primary, + height: '1.875rem', + backgroundColor: theme.palette.background.paper, + borderRadius: `${theme.shape.borderRadius}px`, + '& .MuiSelect-select': { padding: '0.25rem 2rem 0.25rem 0.625rem' }, + '& .MuiSelect-icon': { color: theme.palette.text.disabled, fontSize: '1.125rem' } +}); + /** Hover treatment for chips and pills: accent border and text color. Suppressed on touch devices. */ export const accentHover = (theme: Theme) => ({ '@media (hover: hover)': { diff --git a/webui/src/components/pill.tsx b/webui/src/components/pill.tsx new file mode 100644 index 000000000..6168b0065 --- /dev/null +++ b/webui/src/components/pill.tsx @@ -0,0 +1,43 @@ +/******************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * https://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + ********************************************************************************/ + +import { ButtonBase } from '@mui/material'; +import { alpha, styled } from '@mui/material/styles'; +import { accentHover, focusOutline, glassSurface } from './page-primitives'; + +/** + * Clickable pill primitive: a translucent glass surface flipping to an accent fill when + * selected — the same treatment as the extension detail page's sticky tabs. + */ +export const Pill = styled(ButtonBase, { + shouldForwardProp: prop => prop !== 'isSelected' +})<{ isSelected?: boolean }>(({ theme, isSelected }) => ({ + display: 'inline-flex', + alignItems: 'center', + gap: '0.4375rem', + flexShrink: 0, + overflow: 'hidden', + ...glassSurface(theme), + ...(isSelected ? { backgroundColor: alpha(theme.palette.secondary.main, 0.7) } : {}), + border: `1px solid ${isSelected ? theme.palette.secondary.main : theme.palette.divider}`, + color: isSelected ? theme.palette.secondary.contrastText : theme.palette.text.secondary, + fontSize: '0.8125rem', + fontWeight: isSelected ? 600 : 500, + padding: '0.4375rem 0.8125rem', + borderRadius: theme.shape.borderRadiusPill, + whiteSpace: 'nowrap', + fontFamily: 'inherit', + transition: 'border-color 0.14s, color 0.14s, background 0.14s', + ...(isSelected ? {} : accentHover(theme)), + ...focusOutline(theme) +})); diff --git a/webui/src/default/theme.tsx b/webui/src/default/theme.tsx index 06102434c..d9dec2799 100644 --- a/webui/src/default/theme.tsx +++ b/webui/src/default/theme.tsx @@ -372,6 +372,15 @@ export default function createDefaultTheme(themeType: 'light' | 'dark'): Theme { } }, MuiPopover: { + // no body scroll lock: toggling it jumps the scroll position on mobile and + // shifts the pinned chrome; dialogs (MuiModal directly) keep theirs + defaultProps: { disableScrollLock: true }, + styleOverrides: { + paper: ({ theme }) => floatingPaper(theme) + } + }, + // Autocomplete popups (e.g. the publisher filter) render their own paper, not a popover. + MuiAutocomplete: { styleOverrides: { paper: ({ theme }) => floatingPaper(theme) } diff --git a/webui/src/pages/search/search-header.tsx b/webui/src/pages/search/search-header.tsx index 2aa5de40d..4434b44b2 100644 --- a/webui/src/pages/search/search-header.tsx +++ b/webui/src/pages/search/search-header.tsx @@ -16,6 +16,7 @@ import { Box, IconButton, Select, MenuItem, Typography, SelectChangeEvent } from import { SortBy, SortOrder } from '../../extension-registry-types'; import { ExtensionCategory } from '../../extension-registry-types'; import { Theme } from '@mui/material/styles'; +import { compactControl } from '../../components/page-primitives'; import ArrowUpwardIcon from '@mui/icons-material/ArrowUpward'; import ArrowDownwardIcon from '@mui/icons-material/ArrowDownward'; @@ -81,16 +82,7 @@ export const SearchHeader: FunctionComponent = props => { onChange={handleSortByChange} size='small' inputProps={{ 'aria-label': 'Sort by' }} - sx={{ - fontSize: '0.8125rem', - fontWeight: 500, - color: 'text.primary', - height: '1.875rem', - bgcolor: 'background.paper', - borderRadius: 1, - '& .MuiSelect-select': { py: '0.25rem', pl: '0.625rem' }, - '& .MuiSelect-icon': { color: 'text.disabled', fontSize: '1.125rem' } - }}> + sx={compactControl}> Relevance Date Downloads From 1208a19ef3c16f44f4bb060bc3d9ebbb02e820f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jordi=20G=C3=B3mez?= Date: Mon, 24 Aug 2026 10:25:42 +0200 Subject: [PATCH 07/20] feat: expose userLoading and account-menu entries to consumers --- webui/CHANGELOG.md | 2 + webui/src/context.ts | 2 + webui/src/default/menu-content.tsx | 32 +++++- webui/src/main.tsx | 1 + webui/src/page-settings.ts | 24 +++++ webui/src/pages/user/avatar.tsx | 18 +++- webui/test/unit/default/menu-content.spec.tsx | 98 +++++++++++++++++++ webui/test/unit/main.spec.tsx | 67 +++++++++++++ webui/test/unit/pages/user/avatar.spec.tsx | 93 ++++++++++++++++++ webui/test/unit/support/menu-queries.ts | 22 +++++ webui/test/unit/support/test-providers.tsx | 42 ++++++-- webui/vite.config.mts | 8 +- 12 files changed, 395 insertions(+), 14 deletions(-) create mode 100644 webui/test/unit/default/menu-content.spec.tsx create mode 100644 webui/test/unit/main.spec.tsx create mode 100644 webui/test/unit/pages/user/avatar.spec.tsx create mode 100644 webui/test/unit/support/menu-queries.ts diff --git a/webui/CHANGELOG.md b/webui/CHANGELOG.md index 9a7459991..0e1399292 100644 --- a/webui/CHANGELOG.md +++ b/webui/CHANGELOG.md @@ -23,6 +23,8 @@ This change log covers only the frontend library (webui) of Open VSX. ### Added - Add a `Pill` component — the clickable glass pill the category pills are built on, now usable on its own — and extract the `MonoSlash`, `glassSurface` and `compactControl` page primitives out of the search field, the pills and the search header +- Add `userLoading` to `MainContext`, so custom pages can tell "not logged in" from "still resolving the user" +- Add a `userMenuContent` slot to `PageSettings.elements`: extra entries for the logged-in account menu, rendered above the admin entry. The slot receives a `MenuEntry` component to build entries with, so each entry is styled by the menu it appears in — the desktop and mobile menus style theirs differently, and a consumer cannot match both on its own ### Changed diff --git a/webui/src/context.ts b/webui/src/context.ts index aebd540c2..77153220e 100644 --- a/webui/src/context.ts +++ b/webui/src/context.ts @@ -19,6 +19,8 @@ export interface MainContext { pageSettings: PageSettings; handleError: (err: Error | Partial, options?: { onClose?: () => void }) => void; user?: UserData; + /** The initial user fetch is still in flight, so `user` being undefined doesn't yet mean logged out. */ + userLoading: boolean; updateUser: () => void; loginProviders?: Record; version?: RegistryVersion; diff --git a/webui/src/default/menu-content.tsx b/webui/src/default/menu-content.tsx index 75d6d6bfe..a2769bc3b 100644 --- a/webui/src/default/menu-content.tsx +++ b/webui/src/default/menu-content.tsx @@ -8,13 +8,23 @@ * SPDX-License-Identifier: EPL-2.0 ********************************************************************************/ -import { FunctionComponent, PropsWithChildren, useContext, useRef, useState } from 'react'; +import { + ComponentType, + FunctionComponent, + PropsWithChildren, + useCallback, + useContext, + useMemo, + useRef, + useState +} from 'react'; import { Avatar, Button, IconButton, Link, Menu, MenuItem, Typography } from '@mui/material'; import { useLocation, useNavigate, Link as RouteLink } from 'react-router'; import { UserAvatar } from '../pages/user/avatar'; import { UserSettingsRoutes } from '../pages/user/user-settings-routes'; import { alpha, styled, Theme } from '@mui/material/styles'; import { MainContext } from '../context'; +import { UserMenuEntryProps } from '../page-settings'; import { KbdKey } from '../components/kbd-key'; import { useShortcut } from '../hooks/use-shortcut'; import { focusOutline } from '../components/page-primitives'; @@ -55,11 +65,26 @@ export const MenuItemText: FunctionComponent = ({ children }) }; export const MobileUserAvatar: FunctionComponent = () => { - const { user } = useContext(MainContext); + const { user, pageSettings } = useContext(MainContext); + const { userMenuContent: UserMenuContent } = pageSettings.elements; const logoutFormRef = useRef(null); const anchorRef = useRef(null); const [open, setOpen] = useState(false); - const close = () => setOpen(false); + const close = useCallback(() => setOpen(false), []); + // Stable identity: a new component type each render would remount contributed entries, + // refetching whatever they load. + const MenuEntry = useMemo>(() => { + const Entry = ({ to, icon: Icon, children }: UserMenuEntryProps) => ( + + + + {children} + + + ); + Entry.displayName = 'MobileUserMenuEntry'; + return Entry; + }, [close]); if (!user) { return null; } @@ -91,6 +116,7 @@ export const MobileUserAvatar: FunctionComponent = () => { Settings + {UserMenuContent ? : null} {user.role === 'admin' ? ( diff --git a/webui/src/main.tsx b/webui/src/main.tsx index 41676ac48..b5037277a 100644 --- a/webui/src/main.tsx +++ b/webui/src/main.tsx @@ -148,6 +148,7 @@ export const Main: FunctionComponent = props => { service: props.service, pageSettings: props.pageSettings, user, + userLoading, updateUser, loginProviders, handleError: onError, diff --git a/webui/src/page-settings.ts b/webui/src/page-settings.ts index 84d089cbe..53eceddd4 100644 --- a/webui/src/page-settings.ts +++ b/webui/src/page-settings.ts @@ -13,6 +13,23 @@ import { SxProps, Theme } from '@mui/material/styles'; import { Extension, NamespaceDetails, SortBy } from './extension-registry-types'; import { Cookie } from './utils'; +/** One entry contributed to the account menu, rendered by the menu's own styling. */ +export interface UserMenuEntryProps { + /** Route the entry navigates to; the menu closes on click. */ + to: string; + /** Icon component (e.g. a MUI icon), sized and coloured by the menu. */ + icon: ComponentType<{ sx?: SxProps }>; + /** The entry's label. */ + children: ReactNode; +} + +export interface UserMenuContentProps { + /** Dismisses the menu. `MenuEntry` already does this on click. */ + close: () => void; + /** Renders one entry styled to match the menu it appears in. */ + MenuEntry: ComponentType; +} + export interface FooterLink { label: ReactNode; href: string; @@ -116,6 +133,13 @@ export interface PageSettings { toolbarContent?: ComponentType; defaultMenuContent?: ComponentType; mobileMenuContent?: ComponentType; + /** + * Extra entries for the logged-in account menu, rendered above the admin entry. The same + * component is used by the desktop and the mobile menu, which style their entries + * differently, so build entries with the supplied {@link UserMenuContentProps.MenuEntry} + * rather than a `MenuItem` of your own. + */ + userMenuContent?: ComponentType; footer?: FooterSettings; home?: HomePageSettings; searchHeader?: ComponentType; diff --git a/webui/src/pages/user/avatar.tsx b/webui/src/pages/user/avatar.tsx index 5a2c8fe18..334f9dc43 100644 --- a/webui/src/pages/user/avatar.tsx +++ b/webui/src/pages/user/avatar.tsx @@ -8,7 +8,7 @@ * SPDX-License-Identifier: EPL-2.0 ********************************************************************************/ -import { FunctionComponent, useContext, useRef, useState } from 'react'; +import { ComponentType, FunctionComponent, useCallback, useContext, useMemo, useRef, useState } from 'react'; import { Avatar, Box, IconButton, Link, Menu, MenuItem, Typography } from '@mui/material'; import { Link as RouteLink } from 'react-router'; import SettingsIcon from '@mui/icons-material/Settings'; @@ -17,6 +17,7 @@ import LogoutIcon from '@mui/icons-material/Logout'; import { UserSettingsRoutes } from './user-settings-routes'; import { AdminDashboardRoutes } from '../admin-dashboard/admin-dashboard-routes'; import { MainContext } from '../../context'; +import { UserMenuEntryProps } from '../../page-settings'; import { LogoutForm } from './logout'; // Radius, font and min-height come from the MuiMenuItem theme override. @@ -33,7 +34,21 @@ const iconSx = { fontSize: '1.0625rem', color: 'text.disabled', flexShrink: 0 }; export const UserAvatar: FunctionComponent = () => { const [open, setOpen] = useState(false); + const close = useCallback(() => setOpen(false), []); + // Stable identity: a new component type each render would remount contributed entries, + // refetching whatever they load. + const MenuEntry = useMemo>(() => { + const Entry = ({ to, icon: Icon, children }: UserMenuEntryProps) => ( + + + {children} + + ); + Entry.displayName = 'UserMenuEntry'; + return Entry; + }, [close]); const context = useContext(MainContext); + const { userMenuContent: UserMenuContent } = context.pageSettings.elements; const anchorRef = useRef(null); const logoutFormRef = useRef(null); @@ -123,6 +138,7 @@ export const UserAvatar: FunctionComponent = () => { Settings + {UserMenuContent ? : null} {user.role === 'admin' && ( '/logout', + getCsrfToken: vi.fn().mockResolvedValue({ value: 'csrf' }) +} as unknown as ExtensionRegistryService; + +// Records the sx the host hands the icon, which is how we assert the mobile menu — not the +// consumer — owns the entry's presentation. +let iconSx: SxProps | undefined; +const ProbeIcon: FunctionComponent<{ sx?: SxProps }> = ({ sx }) => { + iconSx = sx; + return ; +}; + +const ConsumerEntry: FunctionComponent = ({ MenuEntry }) => ( + + Analytics + +); + +function renderAvatar(userMenuContent?: PageSettings['elements']['userMenuContent']) { + return renderWithProviders(, { + mainContext: { service, user: admin, pageSettings: { elements: { userMenuContent } } as PageSettings } + }); +} + +/** The avatar itself is a menu entry; opening its menu adds the rest. */ +async function openMenu(): Promise { + await userEvent.click(screen.getByText(admin.loginName)); +} + +describe('MobileUserAvatar', () => { + it('renders the pageSettings userMenuContent entries above the admin entry', async () => { + renderAvatar(ConsumerEntry); + await openMenu(); + + expect(menuItemLabels()).toEqual([ + admin.loginName, + admin.loginName, + 'Settings', + 'Analytics', + 'Admin Dashboard', + 'Log Out' + ]); + }); + + it('closes the menu when a contributed entry is clicked', async () => { + renderAvatar(ConsumerEntry); + await openMenu(); + + await userEvent.click(screen.getByText('Analytics')); + + await waitFor(() => expect(menuItemLabels()).toEqual([admin.loginName])); + }); + + it('styles a contributed entry like its own', async () => { + iconSx = undefined; + renderAvatar(ConsumerEntry); + await openMenu(); + + // The mobile menu's own icon styling, which differs from the desktop menu's. + expect(iconSx).toEqual({ mr: 1, width: '16px', height: '16px' }); + }); + + it('renders only the built-in entries when no userMenuContent is configured', async () => { + renderAvatar(); + await openMenu(); + + expect(menuItemLabels()).toEqual([admin.loginName, admin.loginName, 'Settings', 'Admin Dashboard', 'Log Out']); + }); +}); diff --git a/webui/test/unit/main.spec.tsx b/webui/test/unit/main.spec.tsx new file mode 100644 index 000000000..cd5048c12 --- /dev/null +++ b/webui/test/unit/main.spec.tsx @@ -0,0 +1,67 @@ +/******************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation. + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * https://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + ********************************************************************************/ + +import { FunctionComponent, useContext } from 'react'; +import { describe, it, expect, vi } from 'vitest'; +import { screen, waitFor } from '@testing-library/react'; +import { renderInEntryShell } from './support/test-providers'; +import { Main } from '../../src/main'; +import { MainContext } from '../../src/context'; +import { PageSettings } from '../../src/page-settings'; +import { ErrorResult, UserData } from '../../src/extension-registry-types'; +import { ExtensionRegistryService } from '../../src/extension-registry-service'; + +// Stands in for a consumer page: the only way to observe what the context carries. +const ContextProbe: FunctionComponent = () => { + const { userLoading, user } = useContext(MainContext); + return
{`loading=${userLoading} user=${user?.loginName ?? 'none'}`}
; +}; + +const pageSettings = { + pageTitle: 'test', + // A custom home component replaces the built-in landing page, so the probe is what '/' renders. + elements: { home: ContextProbe }, + urls: { extensionDefaultIcon: '', namespaceAccessInfo: '' } +} as PageSettings; + +function renderMain() { + let resolveUser: (result: UserData | ErrorResult) => void = () => {}; + const service = { + getUser: vi.fn(() => new Promise(resolve => (resolveUser = resolve))), + getRegistryVersion: vi.fn().mockResolvedValue({ version: '1.0.0' }) + } as unknown as ExtensionRegistryService; + // `loginProviders` supplied so Main skips its own login-providers fetch. + renderInEntryShell(
); + return { resolveUser: (result: UserData | ErrorResult) => resolveUser(result) }; +} + +describe('Main', () => { + it('exposes userLoading on MainContext while the initial user fetch is in flight', async () => { + const { resolveUser } = renderMain(); + + expect(screen.getByTestId('probe')).toHaveTextContent('loading=true user=none'); + + resolveUser({ loginName: 'testuser', tokensUrl: '', createTokenUrl: '' }); + + await waitFor(() => expect(screen.getByTestId('probe')).toHaveTextContent('loading=false user=testuser')); + }); + + it('clears userLoading when the user turns out not to be logged in', async () => { + const { resolveUser } = renderMain(); + + // An error result with HTTP OK is how the server reports "not logged in". + resolveUser({ error: 'Not logged in' }); + + await waitFor(() => expect(screen.getByTestId('probe')).toHaveTextContent('loading=false user=none')); + }); +}); diff --git a/webui/test/unit/pages/user/avatar.spec.tsx b/webui/test/unit/pages/user/avatar.spec.tsx new file mode 100644 index 000000000..7d0a149e9 --- /dev/null +++ b/webui/test/unit/pages/user/avatar.spec.tsx @@ -0,0 +1,93 @@ +/******************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation. + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * https://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + ********************************************************************************/ + +import { FunctionComponent } from 'react'; +import { describe, it, expect, vi } from 'vitest'; +import { screen, waitFor } from '@testing-library/react'; +import userEvent from '@testing-library/user-event'; +import { SxProps, Theme } from '@mui/material/styles'; +import { menuItemLabels } from '../../support/menu-queries'; +import { renderWithProviders } from '../../support/test-providers'; +import { UserAvatar } from '../../../../src/pages/user/avatar'; +import { PageSettings, UserMenuContentProps } from '../../../../src/page-settings'; +import { UserData } from '../../../../src/extension-registry-types'; +import { ExtensionRegistryService } from '../../../../src/extension-registry-service'; + +const admin: UserData = { loginName: 'testuser', tokensUrl: '', createTokenUrl: '', role: 'admin' }; + +// The logout entry wraps a real form that reads both of these while rendering. +const service = { + getLogoutUrl: () => '/logout', + getCsrfToken: vi.fn().mockResolvedValue({ value: 'csrf' }) +} as unknown as ExtensionRegistryService; + +// Records the sx the host hands the icon, which is how we assert the desktop menu — not the +// consumer — owns the entry's presentation. +let iconSx: SxProps | undefined; +const ProbeIcon: FunctionComponent<{ sx?: SxProps }> = ({ sx }) => { + iconSx = sx; + return ; +}; + +const ConsumerEntry: FunctionComponent = ({ MenuEntry }) => ( + + Analytics + +); + +function renderAvatar(userMenuContent?: PageSettings['elements']['userMenuContent']) { + return renderWithProviders(, { + mainContext: { service, user: admin, pageSettings: { elements: { userMenuContent } } as PageSettings } + }); +} + +async function openMenu(): Promise { + await userEvent.click(screen.getByRole('button', { name: 'User menu' })); +} + +describe('UserAvatar', () => { + it('renders the pageSettings userMenuContent entries above the admin entry', async () => { + renderAvatar(ConsumerEntry); + await openMenu(); + + expect(menuItemLabels()).toEqual(['Settings', 'Analytics', 'Admin Dashboard', 'Log out']); + }); + + it('closes the menu when a contributed entry is clicked', async () => { + renderAvatar(ConsumerEntry); + await openMenu(); + + await userEvent.click(screen.getByText('Analytics')); + + await waitFor(() => expect(menuItemLabels()).toEqual([])); + }); + + it('styles a contributed entry like its own, and links it to the given route', async () => { + iconSx = undefined; + renderAvatar(ConsumerEntry); + await openMenu(); + + // The desktop menu's own icon styling, not anything the consumer chose. + expect(iconSx).toEqual({ fontSize: '1.0625rem', color: 'text.disabled', flexShrink: 0 }); + // The label is the link's only text, so this is the anchor itself (role queries over + // a MUI menu crash under jsdom 30). + expect(screen.getByText('Analytics')).toHaveAttribute('href', '/analytics'); + }); + + it('renders only the built-in entries when no userMenuContent is configured', async () => { + renderAvatar(); + await openMenu(); + + expect(menuItemLabels()).toEqual(['Settings', 'Admin Dashboard', 'Log out']); + }); +}); diff --git a/webui/test/unit/support/menu-queries.ts b/webui/test/unit/support/menu-queries.ts new file mode 100644 index 000000000..ffd34a496 --- /dev/null +++ b/webui/test/unit/support/menu-queries.ts @@ -0,0 +1,22 @@ +/******************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation. + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * https://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + ********************************************************************************/ + +/** + * Labels of the currently rendered menu entries, in DOM order. A plain selector rather + * than `getAllByRole`: jsdom 30.0.0 throws on the second `getComputedStyle` of an element + * carrying a percentage `calc()` — MUI's menu paper has one — and role queries compute + * styles per candidate to filter out inaccessible nodes. + */ +export function menuItemLabels(): string[] { + return Array.from(document.querySelectorAll('[role="menuitem"]')).map(item => item.textContent ?? ''); +} diff --git a/webui/test/unit/support/test-providers.tsx b/webui/test/unit/support/test-providers.tsx index 97c70ebbc..80155fb21 100644 --- a/webui/test/unit/support/test-providers.tsx +++ b/webui/test/unit/support/test-providers.tsx @@ -66,11 +66,25 @@ function mainContextValue(overrides?: Partial): MainContext { // well-typed rather than relying on `as PageSettings` to paper over a missing required field. pageSettings: { elements: {} } as PageSettings, handleError: () => {}, + userLoading: false, updateUser: () => {}, ...overrides }; } +/** + * Only the entry-shell providers the library expects from whoever mounts it (theme, + * router). Use it for components that mount `AppProviders` themselves — `Main` above + * all; everything else wants `TestProviders`. + */ +export function TestEntryShell({ children, route = '/' }: { children: ReactNode; route?: string }) { + return ( + + {children} + + ); +} + export function TestProviders({ children, route = '/', @@ -78,18 +92,28 @@ export function TestProviders({ mainContext }: ProviderOptions & { children: ReactNode }) { return ( - - - - {children} - - - + + + {children} + + ); } +/** `render` with only the entry shell — the counterpart of {@link TestEntryShell}. */ +export function renderInEntryShell( + ui: ReactElement, + options: { route?: string } & Omit = {} +) { + const { route, ...rtl } = options; + return render(ui, { + wrapper: ({ children }) => {children}, + ...rtl + }); +} + /** `render` with the app providers around it. Extra RTL options pass through. */ export function renderWithProviders(ui: ReactElement, options: ProviderOptions & Omit = {}) { const { route, queryClient, mainContext, ...rtl } = options; diff --git a/webui/vite.config.mts b/webui/vite.config.mts index 35c34f51b..fae3c36b4 100644 --- a/webui/vite.config.mts +++ b/webui/vite.config.mts @@ -24,7 +24,13 @@ export default defineConfig(() => ({ test: { include: ['test/unit/**/*.spec.{ts,tsx}'], environment: 'jsdom', - setupFiles: ['./test/setup.ts'] + setupFiles: ['./test/setup.ts'], + server: { + deps: { + // their ESM builds import directory paths Node's resolver rejects; let vite bundle them + inline: ['@mui/x-charts'] + } + } }, // lightningcss (Vite 8's default CSS transformer) ships no prebuilt binary for ppc64le, // and its minifier isn't needed once postcss is handling transforms - see From 5bba1076e88fd06c0e97a270a97421f987297906 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jordi=20G=C3=B3mez?= Date: Mon, 24 Aug 2026 10:26:01 +0200 Subject: [PATCH 08/20] feat: make the admin dashboard extensible --- webui/CHANGELOG.md | 1 + webui/src/page-settings.ts | 6 ++ .../pages/admin-dashboard/admin-dashboard.tsx | 80 ++++++++++++++---- webui/src/pages/admin-dashboard/nav-types.ts | 32 ++++++++ .../admin-dashboard/admin-dashboard.spec.tsx | 81 +++++++++++++++++++ webui/vite.config.mts | 2 +- 6 files changed, 185 insertions(+), 17 deletions(-) create mode 100644 webui/test/unit/pages/admin-dashboard/admin-dashboard.spec.tsx diff --git a/webui/CHANGELOG.md b/webui/CHANGELOG.md index 0e1399292..3bde6f181 100644 --- a/webui/CHANGELOG.md +++ b/webui/CHANGELOG.md @@ -25,6 +25,7 @@ This change log covers only the frontend library (webui) of Open VSX. - Add a `Pill` component — the clickable glass pill the category pills are built on, now usable on its own — and extract the `MonoSlash`, `glassSurface` and `compactControl` page primitives out of the search field, the pills and the search header - Add `userLoading` to `MainContext`, so custom pages can tell "not logged in" from "still resolving the user" - Add a `userMenuContent` slot to `PageSettings.elements`: extra entries for the logged-in account menu, rendered above the admin entry. The slot receives a `MenuEntry` component to build entries with, so each entry is styled by the menu it appears in — the desktop and mobile menus style theirs differently, and a consumer cannot match both on its own +- Add an `adminPages` slot to `PageSettings.elements`: extra admin dashboard pages, each declaring a name, icon, optional description and optional category, and each appearing in the side panel, as a card on the dashboard overview and as a route. Contributions are additive — a category name matching a built-in group appends to it, and a page whose path would shadow a built-in one is ignored ### Changed diff --git a/webui/src/page-settings.ts b/webui/src/page-settings.ts index 53eceddd4..9100b9ecd 100644 --- a/webui/src/page-settings.ts +++ b/webui/src/page-settings.ts @@ -11,6 +11,7 @@ import { ComponentType, ReactNode } from 'react'; import { SxProps, Theme } from '@mui/material/styles'; import { Extension, NamespaceDetails, SortBy } from './extension-registry-types'; +import { AdminPage } from './pages/admin-dashboard/nav-types'; import { Cookie } from './utils'; /** One entry contributed to the account menu, rendered by the menu's own styling. */ @@ -147,6 +148,11 @@ export interface PageSettings { claimNamespace?: ComponentType<{ namespace: string; extension?: Extension; sx?: SxProps }>; downloadTerms?: ComponentType; additionalRoutes?: ReactNode; + /** + * Extra pages for the admin dashboard, each appearing in the side panel, as a card on the + * dashboard overview and as a route. Additive only — see {@link AdminPage}. + */ + adminPages?: AdminPage[]; banner?: { content: ComponentType; props?: { diff --git a/webui/src/pages/admin-dashboard/admin-dashboard.tsx b/webui/src/pages/admin-dashboard/admin-dashboard.tsx index e1ab33664..2246c90e2 100644 --- a/webui/src/pages/admin-dashboard/admin-dashboard.tsx +++ b/webui/src/pages/admin-dashboard/admin-dashboard.tsx @@ -8,7 +8,7 @@ * SPDX-License-Identifier: EPL-2.0 ********************************************************************************/ -import { FunctionComponent, ReactNode, useContext, lazy, Suspense } from 'react'; +import { FunctionComponent, ReactNode, useContext, useMemo, lazy, Suspense } from 'react'; import { Box, Container, CssBaseline, Typography, IconButton } from '@mui/material'; import { styled } from '@mui/material/styles'; import { Route, Routes, useNavigate } from 'react-router'; @@ -26,10 +26,11 @@ import SpeedIcon from '@mui/icons-material/Speed'; import StarIcon from '@mui/icons-material/Star'; import { LoginComponent } from '../../default/login'; import { MainContext } from '../../context'; +import { createRoute } from '../../utils'; import { AdminDashboardRoutes } from './admin-dashboard-routes'; import { AdminSidepanel } from './admin-sidepanel'; import { AdminHeader } from './admin-header'; -import { isNavGroup, NavEntry } from './nav-types'; +import { AdminPage, isNavGroup, NavEntry, NavGroup, RouteEntry } from './nav-types'; import { NamespaceAdmin } from './namespace-admin'; import { PublisherAdmin } from './publisher-admin'; @@ -109,19 +110,54 @@ const navConfig: NavEntry[] = [ } ]; -const routeNames: { [key: string]: string } = { - [AdminDashboardRoutes.MAIN]: 'Admin Dashboard', - ...navConfig.reduce<{ [key: string]: string }>((acc, entry) => { - if (isNavGroup(entry)) { - entry.children.forEach(child => { - acc[child.path] = child.name; - }); +/** First path segment of every built-in page, so a contributed page cannot shadow one. */ +const builtInSegments = new Set( + navConfig + .flatMap(entry => (isNavGroup(entry) ? entry.children : [entry])) + .map(entry => entry.path.slice(AdminDashboardRoutes.MAIN.length + 1).split('/')[0]) +); + +const toRouteEntry = (page: AdminPage): RouteEntry => ({ + path: createRoute([AdminDashboardRoutes.ROOT, page.path]), + name: page.name, + icon: page.icon, + description: page.description +}); + +/** Appends contributed pages, merging each category into a group of that name if one already exists. */ +function withContributedPages(pages: AdminPage[]): NavEntry[] { + const entries = navConfig.map(entry => (isNavGroup(entry) ? { ...entry, children: [...entry.children] } : entry)); + for (const page of pages) { + const entry = toRouteEntry(page); + if (!page.category) { + entries.push(entry); + continue; + } + const group = entries.find((e): e is NavGroup => isNavGroup(e) && e.name === page.category!.name); + if (group) { + group.children.push(entry); } else { - acc[entry.path] = entry.name; + entries.push({ name: page.category.name, icon: page.category.icon, children: [entry] }); } - return acc; - }, {}) -}; + } + return entries; +} + +function buildRouteNames(items: NavEntry[]): { [key: string]: string } { + return { + [AdminDashboardRoutes.MAIN]: 'Admin Dashboard', + ...items.reduce<{ [key: string]: string }>((acc, entry) => { + if (isNavGroup(entry)) { + entry.children.forEach(child => { + acc[child.path] = child.name; + }); + } else { + acc[entry.path] = entry.name; + } + return acc; + }, {}) + }; +} const ScrollableContent = styled(Box)(({ theme }) => ({ flex: 1, @@ -156,7 +192,15 @@ const Message: FunctionComponent<{ message: string }> = ({ message }) => { }; export const AdminDashboard: FunctionComponent = props => { - const { user, loginProviders } = useContext(MainContext); + const { user, loginProviders, pageSettings } = useContext(MainContext); + + const adminPages = pageSettings.elements.adminPages; + const contributed = useMemo( + () => (adminPages ?? []).filter(page => !builtInSegments.has(page.path.split('/')[0])), + [adminPages] + ); + const navItems = useMemo(() => withContributedPages(contributed), [contributed]); + const routeNames = useMemo(() => buildRouteNames(navItems), [navItems]); const navigate = useNavigate(); const toMainPage = () => navigate('/'); @@ -166,7 +210,7 @@ export const AdminDashboard: FunctionComponent = props => { content = ( - + @@ -188,7 +232,11 @@ export const AdminDashboard: FunctionComponent = props => { } /> } /> } /> - } /> + {/* Splat so a contributed page can render nested routes; it also matches the bare path. */} + {contributed.map(page => ( + + ))} + } /> diff --git a/webui/src/pages/admin-dashboard/nav-types.ts b/webui/src/pages/admin-dashboard/nav-types.ts index 1f7ea1b6c..ebb02f174 100644 --- a/webui/src/pages/admin-dashboard/nav-types.ts +++ b/webui/src/pages/admin-dashboard/nav-types.ts @@ -29,3 +29,35 @@ export interface NavGroup { export type NavEntry = RouteEntry | NavGroup; export const isNavGroup = (entry: NavEntry): entry is NavGroup => 'children' in entry; + +/** Side panel and overview grouping for contributed admin pages. */ +export interface AdminPageCategory { + name: string; + icon: ReactNode; +} + +/** + * An admin dashboard page contributed by a consumer through + * `PageSettings.elements.adminPages`. It shows up in the side panel, as a card on the + * dashboard overview, and as a route under the admin dashboard. + */ +export interface AdminPage { + /** + * Path below the admin dashboard root, without a leading slash (e.g. `'analytics/agents'`). + * The page also receives everything below it, so it may render nested routes of its own. + * A page whose first segment is one the built-in pages already use is ignored — contributed + * pages can only be added, never override a built-in one. + */ + path: string; + name: string; + icon: ReactNode; + /** Shown under the page name on the dashboard overview. */ + description?: string; + /** + * Groups the page in the side panel and on the overview. Pages sharing a category name are + * merged into one group, and a name matching a built-in group appends to that group. + * Without a category the page sits at the top level. + */ + category?: AdminPageCategory; + element: ReactNode; +} diff --git a/webui/test/unit/pages/admin-dashboard/admin-dashboard.spec.tsx b/webui/test/unit/pages/admin-dashboard/admin-dashboard.spec.tsx new file mode 100644 index 000000000..f876c743e --- /dev/null +++ b/webui/test/unit/pages/admin-dashboard/admin-dashboard.spec.tsx @@ -0,0 +1,81 @@ +/****************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation. + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * https://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + *****************************************************************************/ + +import { describe, expect, it } from 'vitest'; +import { screen } from '@testing-library/react'; +import { AdminDashboard } from '../../../../src/pages/admin-dashboard/admin-dashboard'; +import { AdminPage } from '../../../../src/pages/admin-dashboard/nav-types'; +import { MainContext } from '../../../../src/context'; +import { PageSettings } from '../../../../src/page-settings'; +import { UserData } from '../../../../src/extension-registry-types'; +import { renderWithProviders } from '../../support/test-providers'; + +const admin = { loginName: 'root', role: 'admin' } as UserData; + +const agentsPage: AdminPage = { + path: 'analytics/agents', + name: 'Agents', + icon: , + description: 'Link customers to analytics user agents', + element:
contributed page body
+}; + +/** + * `AdminDashboard` owns a nested `Routes`, and in the app it is mounted under + * `/admin-dashboard/*` so those paths resolve relative to that match. Rendering it + * directly means the same route table resolves from the root instead, so a route here + * is the contributed `path` without the dashboard prefix. + */ +function renderDashboard(adminPages: AdminPage[] | undefined, route = '/') { + const pageSettings = { elements: { adminPages } } as PageSettings; + const mainContext: Partial = { user: admin, pageSettings }; + return renderWithProviders(, { route, mainContext }); +} + +describe('AdminDashboard contributed pages', () => { + it('shows a contributed page as an overview card with its description', () => { + renderDashboard([agentsPage]); + + expect(screen.getByText('Link customers to analytics user agents')).toBeInTheDocument(); + // Once in the side panel, once as the overview card. + expect(screen.getAllByText('Agents')).toHaveLength(2); + }); + + it('renders a contributed page at its own route', () => { + renderDashboard([agentsPage], '/analytics/agents'); + + expect(screen.getByText('contributed page body')).toBeInTheDocument(); + // The overview fallback must not also match. + expect(screen.queryByText('Welcome to the Admin Dashboard')).not.toBeInTheDocument(); + }); + + it('merges a category into the built-in group of the same name instead of adding a second one', () => { + const { unmount } = renderDashboard(undefined); + const withoutContribution = screen.getAllByText('Rate Limiting').length; + unmount(); + + renderDashboard([{ ...agentsPage, category: { name: 'Rate Limiting', icon: } }]); + + expect(screen.getAllByText('Rate Limiting')).toHaveLength(withoutContribution); + expect(screen.getByText('Link customers to analytics user agents')).toBeInTheDocument(); + }); + + it('ignores a contributed page that would shadow a built-in one', () => { + renderDashboard([{ ...agentsPage, path: 'customers', name: 'Not Customers' }]); + + expect(screen.queryByText('Not Customers')).not.toBeInTheDocument(); + expect(screen.queryByText('Link customers to analytics user agents')).not.toBeInTheDocument(); + // The built-in entry is untouched. + expect(screen.getAllByText('Customers').length).toBeGreaterThan(0); + }); +}); diff --git a/webui/vite.config.mts b/webui/vite.config.mts index fae3c36b4..339f7ccb1 100644 --- a/webui/vite.config.mts +++ b/webui/vite.config.mts @@ -28,7 +28,7 @@ export default defineConfig(() => ({ server: { deps: { // their ESM builds import directory paths Node's resolver rejects; let vite bundle them - inline: ['@mui/x-charts'] + inline: ['@mui/x-charts', '@mui/x-data-grid', '@mui/x-date-pickers'] } } }, From ad058b37f01f08c9610626e541a909197b201539 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jordi=20G=C3=B3mez?= Date: Mon, 24 Aug 2026 10:26:22 +0200 Subject: [PATCH 09/20] feat: widen the published API for consumers building their own pages --- webui/CHANGELOG.md | 1 + webui/src/index.ts | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/webui/CHANGELOG.md b/webui/CHANGELOG.md index 3bde6f181..60caedb66 100644 --- a/webui/CHANGELOG.md +++ b/webui/CHANGELOG.md @@ -26,6 +26,7 @@ This change log covers only the frontend library (webui) of Open VSX. - Add `userLoading` to `MainContext`, so custom pages can tell "not logged in" from "still resolving the user" - Add a `userMenuContent` slot to `PageSettings.elements`: extra entries for the logged-in account menu, rendered above the admin entry. The slot receives a `MenuEntry` component to build entries with, so each entry is styled by the menu it appears in — the desktop and mobile menus style theirs differently, and a consumer cannot match both on its own - Add an `adminPages` slot to `PageSettings.elements`: extra admin dashboard pages, each declaring a name, icon, optional description and optional category, and each appearing in the side panel, as a card on the dashboard overview and as a route. Contributions are additive — a category name matching a built-in group appends to it, and a page whose path would shadow a built-in one is ignored +- Widen the published API for consumers building their own pages: the request layer (`sendRequest`, `sendNonRetriableRequest`, `ErrorResponse`, `controllerFromSignal`), `MainContext`, `AppProviders`, `NotFound`, `createDefaultTheme` with the `MONO_FONT`/`NAVBAR_HEIGHT` tokens, the `createRoute`/`createAbsoluteURL`/`addQuery`/`formatCompactNumber`/`toRelativeTime` utils, the `useDebouncedCallback` and `useGridCursor` hooks, the navbar-chrome, search-focus and page-search-bar hooks, the category icon helpers, `ExtensionDetailRoutes`, and the `itemIcon`/`MenuItemText` building blocks for `userMenuContent` entries ### Changed diff --git a/webui/src/index.ts b/webui/src/index.ts index 3834d0014..ad8b4b8b3 100644 --- a/webui/src/index.ts +++ b/webui/src/index.ts @@ -32,6 +32,8 @@ export { DEFAULT_CURATED_SECTIONS } from './pages/home/use-home-data'; export { ExtensionCard, type ExtensionCardProps } from './components/extension-card'; +export { CategoryPill, type CategoryPillProps } from './components/category-pill'; +export { Pill } from './components/pill'; export * from './components/page-primitives'; export * from './components/page-container'; // Leaf hook modules keep their helpers private, so `export *` exposes only the @@ -53,3 +55,38 @@ export { // useSignal, subscribe with useSignalEffect. export * from './hooks/use-signal'; export * from './hooks/use-signal-effect'; + +// Request layer, for consumers whose registry exposes endpoints this library +// doesn't know about: they build their own service on top of these. +export { sendRequest, sendNonRetriableRequest, type ErrorResponse } from './server-request'; +export { controllerFromSignal } from './query-client'; + +// The app-wide context and the provider stack that supplies it — needed both by +// custom pages reading `service`/`user` and by consumer-side test harnesses. +export { MainContext } from './context'; +export { AppProviders } from './app-providers'; + +// `createAbsoluteURL` and `addQuery` are the other half of the request layer above: +// building an endpoint against `service.serverUrl` needs them. +export { createRoute, createAbsoluteURL, addQuery, formatCompactNumber, toRelativeTime } from './utils'; +export { NotFound } from './not-found'; + +// Theme tokens shared with the library chrome, so custom pages line up with it. +export { default as createDefaultTheme, MONO_FONT, NAVBAR_HEIGHT, NAVBAR_HEIGHT_PX } from './default/theme'; + +export * from './hooks/use-debounced-callback'; +export * from './hooks/use-grid-cursor'; + +// Navbar chrome: what a mounted page asks of the nav bar — a tint over its +// gallery band, and extra blur depth to back sections pinned under the bar. +export { useSetExtensionTint, useExtendNavbarBlur, type ExtensionTint } from './context/navbar-chrome-context'; +export { useSearchFocus, type ResultsNavAction } from './context/search/search-focus-context'; +export { usePageSearchBar, type PageSearchBarValue } from './context/search/page-search-bar-context'; + +export { useCategories, CATEGORY_ICONS, DefaultCategoryIcon } from './components/categories'; + +// Route paths, for linking into the built-in pages. +export { ExtensionDetailRoutes } from './pages/extension-detail/extension-detail-routes'; + +// Shape of the pages contributed through `PageSettings.elements.adminPages`. +export type { AdminPage, AdminPageCategory } from './pages/admin-dashboard/nav-types'; From f72f64f73161e91b43d9fbab4af54370853e7717 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jordi=20G=C3=B3mez?= Date: Mon, 24 Aug 2026 10:26:27 +0200 Subject: [PATCH 10/20] refactor: rework the weekly downloads card --- webui/CHANGELOG.md | 1 + .../use-extension-download-series.ts | 16 +- .../extension-detail/weekly-downloads.tsx | 197 +++++++++++++----- .../unit/components/weekly-downloads.spec.tsx | 104 ++++++++- 4 files changed, 247 insertions(+), 71 deletions(-) diff --git a/webui/CHANGELOG.md b/webui/CHANGELOG.md index 60caedb66..abe19eaf2 100644 --- a/webui/CHANGELOG.md +++ b/webui/CHANGELOG.md @@ -27,6 +27,7 @@ This change log covers only the frontend library (webui) of Open VSX. - Add a `userMenuContent` slot to `PageSettings.elements`: extra entries for the logged-in account menu, rendered above the admin entry. The slot receives a `MenuEntry` component to build entries with, so each entry is styled by the menu it appears in — the desktop and mobile menus style theirs differently, and a consumer cannot match both on its own - Add an `adminPages` slot to `PageSettings.elements`: extra admin dashboard pages, each declaring a name, icon, optional description and optional category, and each appearing in the side panel, as a card on the dashboard overview and as a route. Contributions are additive — a category name matching a built-in group appends to it, and a page whose path would shadow a built-in one is ignored - Widen the published API for consumers building their own pages: the request layer (`sendRequest`, `sendNonRetriableRequest`, `ErrorResponse`, `controllerFromSignal`), `MainContext`, `AppProviders`, `NotFound`, `createDefaultTheme` with the `MONO_FONT`/`NAVBAR_HEIGHT` tokens, the `createRoute`/`createAbsoluteURL`/`addQuery`/`formatCompactNumber`/`toRelativeTime` utils, the `useDebouncedCallback` and `useGridCursor` hooks, the navbar-chrome, search-focus and page-search-bar hooks, the category icon helpers, `ExtensionDetailRoutes`, and the `itemIcon`/`MenuItemText` building blocks for `userMenuContent` entries +- Add a weekly downloads card to the extension detail page, shown only when the registry reports download analytics as enabled: the last 7 days' downloads, a sparkline of the weekly totals for the year behind it, and the period the headline covers. Hovering moves a marker line and reads out that week instead, and the card shows a skeleton in the same shape while the series loads ### Changed diff --git a/webui/src/pages/extension-detail/use-extension-download-series.ts b/webui/src/pages/extension-detail/use-extension-download-series.ts index 4451eeb3e..270960c8b 100644 --- a/webui/src/pages/extension-detail/use-extension-download-series.ts +++ b/webui/src/pages/extension-detail/use-extension-download-series.ts @@ -19,16 +19,14 @@ import { controllerFromSignal } from '../../query-client'; import { DownloadSeriesPoint } from '../../extension-registry-types'; const WEEKS = 52; -// 6 extra leading days so the first plotted point already has a full trailing-7-day window. -const LEAD_IN_DAYS = 6; /** - * Loads roughly the last {@link WEEKS} weeks of *daily* downloads for an extension, up to and - * including today, as the react-query result (`data` is the ordered {@link DownloadSeriesPoint} - * array). Callers turn this into a trailing 7-day ("weekly downloads") view; daily granularity is - * what lets that window end on today rather than the last complete calendar week. Gate with - * `options.enabled` on `RegistryVersion.analyticsEnabled`, since the endpoint 404s when analytics - * is disabled. + * Loads the last {@link WEEKS} whole weeks of *daily* downloads for an extension, ending today, as + * the react-query result (`data` is the ordered {@link DownloadSeriesPoint} array). The range is an + * exact multiple of 7 days so callers can fold it into whole weeks with nothing left over; daily + * granularity is what lets those weeks end on today rather than on the last complete calendar week. + * Gate with `options.enabled` on `RegistryVersion.analyticsEnabled`, since the endpoint 404s when + * analytics is disabled. */ export const useExtensionDownloadSeries = (namespace: string, name: string, options?: { enabled?: boolean }) => { const { service } = useContext(MainContext); @@ -38,7 +36,7 @@ export const useExtensionDownloadSeries = (namespace: string, name: string, opti const today = DateTime.utc().startOf('day'); // `to` is exclusive, so today + 1 day includes today's (still-accruing) bucket. const to = today.plus({ days: 1 }); - const from = to.minus({ weeks: WEEKS }).minus({ days: LEAD_IN_DAYS }); + const from = to.minus({ weeks: WEEKS }); const series = await service.getExtensionDownloadSeries(controllerFromSignal(signal), { namespace, name, diff --git a/webui/src/pages/extension-detail/weekly-downloads.tsx b/webui/src/pages/extension-detail/weekly-downloads.tsx index fd6fdd4ff..39883afb7 100644 --- a/webui/src/pages/extension-detail/weekly-downloads.tsx +++ b/webui/src/pages/extension-detail/weekly-downloads.tsx @@ -11,92 +11,179 @@ * SPDX-License-Identifier: EPL-2.0 *****************************************************************************/ -import { FunctionComponent, useContext, useMemo } from 'react'; -import { Box, Typography, styled, useTheme } from '@mui/material'; +import { FunctionComponent, useContext, useMemo, useState } from 'react'; +import { Box, Skeleton, Typography, alpha, styled, useTheme } from '@mui/material'; import { SparkLineChart } from '@mui/x-charts/SparkLineChart'; +import { lineClasses } from '@mui/x-charts/LineChart'; +import { chartsAxisHighlightClasses } from '@mui/x-charts/ChartsAxisHighlight'; +import { DateTime } from 'luxon'; import { MainContext } from '../../context'; -import { Eyebrow, cardSurface } from '../../components/page-primitives'; -import { Extension } from '../../extension-registry-types'; +import { Eyebrow } from '../../components/page-primitives'; +import { DownloadSeriesPoint, Extension } from '../../extension-registry-types'; import { useExtensionDownloadSeries } from './use-extension-download-series'; -const DownloadsCard = styled(Box)(({ theme }) => ({ - ...cardSurface(theme), - padding: '0.75rem 1rem' -})); - +/** Kept modest on purpose: the headline shares its row with the sparkline, which needs the width. */ const DownloadsCount = styled(Typography)(({ theme }) => ({ - fontSize: '1.75rem', - lineHeight: 1.1, + fontSize: '1.25rem', + lineHeight: 1.2, fontWeight: 700, color: theme.palette.text.primary, fontVariantNumeric: 'tabular-nums' })) as typeof Typography; -const WINDOW_DAYS = 7; - -/** Trailing {@link WINDOW_DAYS}-day rolling sums of a daily series (each point covers that day and - * the previous six), so the last value is the downloads of the last week ending today. */ -function trailingWeeklySums(daily: number[]): number[] { - const rolling: number[] = []; - let windowSum = 0; - for (let i = 0; i < daily.length; i++) { - windowSum += daily[i]; - if (i >= WINDOW_DAYS) { - windowSum -= daily[i - WINDOW_DAYS]; - } - if (i >= WINDOW_DAYS - 1) { - rolling.push(windowSum); +const Period = styled(Typography)(({ theme }) => ({ + fontSize: '0.75rem', + lineHeight: 1.4, + color: theme.palette.text.secondary, + fontVariantNumeric: 'tabular-nums' +})) as typeof Typography; + +const DAY_AND_MONTH = { month: 'short', day: 'numeric' } as const; + +const WEEK_DAYS = 7; + +/** + * Tuned against the headline beside it: the row is bottom-aligned, so its height is the chart's and + * anything much taller leaves dead space above the number rather than a bigger curve. + */ +const CHART_HEIGHT_PX = 48; + +const asDate = (point: DownloadSeriesPoint): DateTime => DateTime.fromISO(point.t, { zone: 'utc' }); + +/** "Aug 21, 2026" for a single day, or "Aug 15 – Aug 21, 2026" across a week. */ +function formatPeriod(from: DownloadSeriesPoint, to: DownloadSeriesPoint): string | undefined { + const start = asDate(from); + const end = asDate(to); + if (!start.isValid || !end.isValid) { + return undefined; + } + + const endLabel = end.toLocaleString({ ...DAY_AND_MONTH, year: 'numeric' }); + return start.hasSame(end, 'day') ? endLabel : `${start.toLocaleString(DAY_AND_MONTH)} – ${endLabel}`; +} + +/** + * Folds a daily series into consecutive {@link WEEK_DAYS}-day totals, aligned so the last week ends + * on the most recent day. Any leading remainder shorter than a whole week is dropped, so every + * plotted point is a full week sharing no days with its neighbours — a rise or fall on the curve is + * a real week-on-week change. Oldest first. + */ +function weeklyTotals(daily: number[]): number[] { + const weeks: number[] = []; + for (let end = daily.length; end >= WEEK_DAYS; end -= WEEK_DAYS) { + let total = 0; + for (let day = end - WEEK_DAYS; day < end; day++) { + total += daily[day]; } + weeks.unshift(total); } - return rolling; + return weeks; } /** - * "Weekly downloads" sidebar card: the downloads of the last 7 days (ending today) plus a trailing - * 7-day trend sparkline over the last year. Renders nothing when download analytics are disabled - * server-side (the endpoint 404s) or when the extension has no downloads in the window, so it stays - * out of the way on registries without data. + * The sidebar slot, shared by the loaded and loading states so neither shifts. Deliberately not a + * card: it sits flush with the resources group below it, which is styled the same way. + */ +const sectionSx = { + display: 'flex', + flexDirection: 'column', + flex: { xs: 'none', sm: 'none', md: 1, lg: 1, xl: 'none' }, + mb: { xs: 2, sm: 2, md: 0, lg: 0, xl: 2 } +} as const; + +/** Same shape and heights as the loaded section, so the sidebar does not jump when the series lands. */ +const LoadingCard: FunctionComponent = () => ( + + Weekly downloads + + + + + + +); + +/** + * "Weekly downloads" sidebar card: the downloads of the last 7 days, with a sparkline of the weekly + * totals for the year behind it — one point per week, so the headline is simply its last point. + * Hovering reads out that week instead. Renders nothing when download analytics are disabled + * server-side (the endpoint 404s) or when the extension has no downloads in the year. */ export const WeeklyDownloads: FunctionComponent<{ extension: Extension }> = ({ extension }) => { const theme = useTheme(); const { version } = useContext(MainContext); const analyticsEnabled = version?.analyticsEnabled ?? false; + const [hovered, setHovered] = useState(undefined); - const { data: points } = useExtensionDownloadSeries(extension.namespace, extension.name, { + const { data: points, isLoading } = useExtensionDownloadSeries(extension.namespace, extension.name, { enabled: analyticsEnabled }); const daily = useMemo(() => points ?? [], [points]); - const counts = useMemo(() => trailingWeeklySums(daily.map(point => point.count)), [daily]); - const hasDownloads = counts.some(count => count > 0); - if (!analyticsEnabled || counts.length === 0 || !hasDownloads) { + const counts = useMemo(() => weeklyTotals(daily.map(point => point.count)), [daily]); + if (!analyticsEnabled) { + return null; + } + // `isLoading` is the first fetch only, and stays false while the query is disabled + if (isLoading) { + return ; + } + if (counts.length === 0 || !counts.some(count => count > 0)) { return null; } - const latestWeek = counts[counts.length - 1]; + // the last week by default; whole weeks are taken from the end, so a short first week is dropped + const selected = hovered !== undefined && hovered < counts.length ? hovered : counts.length - 1; + const first = daily.length - counts.length * WEEK_DAYS + selected * WEEK_DAYS; + const period = formatPeriod(daily[first], daily[first + WEEK_DAYS - 1]); + // Reserve room for the busiest week, so the headline's width does not track its digit count and + // resize the sparkline beside it as the pointer moves. Data-derived, so it cannot be a class. + const reserved = `${Math.max(...counts).toLocaleString().length}ch`; return ( - - - Weekly downloads - - {latestWeek.toLocaleString()} - - (value === null ? '' : `${value.toLocaleString()} downloads`)} - // `.MuiAreaElement-root` is the sparkline's area path; lighten its fill to a wash. - sx={{ '& .MuiAreaElement-root': { fillOpacity: 0.14 } }} - /> - + + Weekly downloads + {period && {period}} + + {counts[selected].toLocaleString()} + + ({ min: -maxValue / 6, max: maxValue }) }} + clipAreaOffset={{ top: 2, bottom: 2 }} + showHighlight + // A non-'none' axis highlight is also what enables the axis listener, so + // the readout above tracks the pointer anywhere along the curve. + axisHighlight={{ x: 'line' }} + onHighlightedAxisChange={items => setHovered(items[0]?.dataIndex)} + slotProps={{ lineHighlight: { r: 4 } }} + color={theme.palette.secondary.main} + sx={{ + [`& .${lineClasses.area}`]: { opacity: 0.2 }, + [`& .${lineClasses.line}`]: { strokeWidth: 3 }, + [`& .${chartsAxisHighlightClasses.root}`]: { + stroke: theme.palette.secondary.main, + strokeDasharray: 'none', + strokeWidth: 2 + } + }} + /> - + ); }; diff --git a/webui/test/unit/components/weekly-downloads.spec.tsx b/webui/test/unit/components/weekly-downloads.spec.tsx index 7f3554ed7..263bc3084 100644 --- a/webui/test/unit/components/weekly-downloads.spec.tsx +++ b/webui/test/unit/components/weekly-downloads.spec.tsx @@ -13,15 +13,35 @@ import { describe, it, expect, vi } from 'vitest'; import { screen, waitFor } from '@testing-library/react'; +import userEvent from '@testing-library/user-event'; import { renderWithProviders } from '../support/test-providers'; import { WeeklyDownloads } from '../../../src/pages/extension-detail/weekly-downloads'; import { DownloadSeriesPoint, Extension, RegistryVersion } from '../../../src/extension-registry-types'; import { ExtensionRegistryService } from '../../../src/extension-registry-service'; -// The real chart pulls in SVG measurement APIs jsdom lacks; stub it so the test exercises -// the component's own logic (gating, the headline number, and the series it feeds the chart). +// The real chart pulls in SVG measurement APIs jsdom lacks; stub it so the test exercises the +// component's own logic (gating, the headline readout, and the series it feeds the chart). The +// buttons stand in for pointer movement along the axis, which is what the real chart reports +// through onHighlightedAxisChange. vi.mock('@mui/x-charts/SparkLineChart', () => ({ - SparkLineChart: ({ data }: { data: number[] }) =>
+ SparkLineChart: ({ + data, + onHighlightedAxisChange + }: { + data: number[]; + onHighlightedAxisChange?: (items: { axisId: string; dataIndex: number }[]) => void; + }) => ( +
+ {data.map((_, index) => ( +
+ ) })); const extension = { namespace: 'redhat', name: 'java' } as unknown as Extension; @@ -37,9 +57,13 @@ function serviceReturning(series: DownloadSeriesPoint[]): ExtensionRegistryServi } as unknown as ExtensionRegistryService; } +// Two whole weeks, 1..14 downloads per day: week 0 covers Jan 1-7 (28) and week 1, the latest, +// covers Jan 8-14 (77). +const ascending = points(Array.from({ length: 14 }, (_, i) => i + 1)); + describe('WeeklyDownloads', () => { it('shows the last-7-days total and the trailing-week trend when analytics is enabled', async () => { - // 14 daily points of 1000 → every trailing-7-day sum is 7000; rolling length = 14 - 6 = 8. + // 14 daily points of 1000 → two whole weeks of 7000 each const service = serviceReturning(points(Array(14).fill(1000))); renderWithProviders(, { mainContext: { service, version: analyticsEnabled } @@ -47,13 +71,78 @@ describe('WeeklyDownloads', () => { expect(await screen.findByText((7000).toLocaleString())).toBeInTheDocument(); expect(screen.getByText(/weekly downloads/i)).toBeInTheDocument(); - expect(screen.getByTestId('sparkline')).toHaveAttribute('data-length', '8'); + expect(screen.getByTestId('sparkline')).toHaveAttribute('data-length', '2'); expect(service.getExtensionDownloadSeries).toHaveBeenCalledWith( expect.anything(), expect.objectContaining({ namespace: 'redhat', name: 'java', interval: 'day' }) ); }); + it('headlines the last week and labels the period it covers', async () => { + renderWithProviders(, { + mainContext: { service: serviceReturning(ascending), version: analyticsEnabled } + }); + + expect(await screen.findByText((77).toLocaleString())).toBeInTheDocument(); + expect(screen.getByText(/Jan 8.*Jan 14, 2026/)).toBeInTheDocument(); + }); + + it('reads out the hovered week, and returns to the last week when the pointer leaves', async () => { + renderWithProviders(, { + mainContext: { service: serviceReturning(ascending), version: analyticsEnabled } + }); + await screen.findByText((77).toLocaleString()); + + await userEvent.click(screen.getByRole('button', { name: 'hover 0' })); + + expect(screen.getByText((28).toLocaleString())).toBeInTheDocument(); + expect(screen.getByText(/Jan 1.*Jan 7, 2026/)).toBeInTheDocument(); + expect(screen.queryByText((77).toLocaleString())).not.toBeInTheDocument(); + + await userEvent.click(screen.getByRole('button', { name: 'hover out' })); + + expect(screen.getByText((77).toLocaleString())).toBeInTheDocument(); + expect(screen.getByText(/Jan 8.*Jan 14, 2026/)).toBeInTheDocument(); + }); + + it('reserves headline width for the busiest week, so the sparkline does not resize on hover', async () => { + // a quiet week of 7 then a busy one of 12,257,000 (ten chars with separators) + const uneven = points([...Array(7).fill(1), ...Array(7).fill(1751000)]); + renderWithProviders(, { + mainContext: { service: serviceReturning(uneven), version: analyticsEnabled } + }); + + // asserted on the style attribute: jsdom drops `ch` from the computed style + const reserved = `min-width: ${(12257000).toLocaleString().length}ch`; + const headline = await screen.findByText((12257000).toLocaleString()); + expect(headline.getAttribute('style')).toContain(reserved); + + // the reservation is unchanged while the single-digit week is being read out + await userEvent.click(screen.getByRole('button', { name: 'hover 0' })); + expect(screen.getByText('7').getAttribute('style')).toContain(reserved); + }); + + it('shows a skeleton while the first request is in flight, then the figures', async () => { + let resolve!: (value: { points: DownloadSeriesPoint[] }) => void; + const service = { + getExtensionDownloadSeries: vi.fn().mockReturnValue(new Promise(done => (resolve = done))) + } as unknown as ExtensionRegistryService; + renderWithProviders(, { + mainContext: { service, version: analyticsEnabled } + }); + + // the card's shell is already there, so the sidebar does not shift when the data lands + expect(screen.getByRole('status', { name: 'Loading weekly downloads' })).toBeInTheDocument(); + expect(screen.getByText(/weekly downloads/i)).toBeInTheDocument(); + expect(screen.queryByTestId('sparkline')).not.toBeInTheDocument(); + + resolve({ points: ascending }); + + expect(await screen.findByText((77).toLocaleString())).toBeInTheDocument(); + expect(screen.queryByRole('status')).not.toBeInTheDocument(); + expect(screen.getByTestId('sparkline')).toBeInTheDocument(); + }); + it('renders nothing (and never calls the endpoint) when analytics is disabled', () => { const service = serviceReturning(points(Array(14).fill(1))); renderWithProviders(, { @@ -70,7 +159,8 @@ describe('WeeklyDownloads', () => { mainContext: { service, version: analyticsEnabled } }); - await waitFor(() => expect(service.getExtensionDownloadSeries).toHaveBeenCalled()); - expect(screen.queryByText(/weekly downloads/i)).not.toBeInTheDocument(); + // the skeleton shows first, so wait for the card to settle on rendering nothing + await waitFor(() => expect(screen.queryByText(/weekly downloads/i)).not.toBeInTheDocument()); + expect(service.getExtensionDownloadSeries).toHaveBeenCalled(); }); }); From bd193c627910765acc66971432eb2abe91eaf9a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jordi=20G=C3=B3mez?= Date: Mon, 24 Aug 2026 10:34:15 +0200 Subject: [PATCH 11/20] feat: give download analytics its own database --- deploy/docker/configuration/application.yml | 7 + deploy/kubernetes/configmap.yaml | 7 + deploy/openshift/application.yml | 7 + doc/development.md | 2 +- docker-compose.yml | 28 +- server/build.gradle | 17 +- server/scripts/generate-properties.sh | 5 + server/src/dev/resources/application.yml | 5 + .../DownloadAnalyticsConfiguration.java | 7 +- .../TimescaleDownloadAnalyticsRepository.java | 101 +++++-- .../TimeseriesDatabaseConfiguration.java | 90 ++++++ .../org/eclipse/openvsx/jooq/Indexes.java | 3 - .../org/eclipse/openvsx/jooq/Public.java | 14 - .../org/eclipse/openvsx/jooq/Tables.java | 12 - .../openvsx/jooq/tables/DownloadEvent.java | 270 ------------------ .../jooq/tables/DownloadStatsDaily.java | 264 ----------------- .../tables/records/DownloadEventRecord.java | 205 ------------- .../records/DownloadStatsDailyRecord.java | 145 ---------- .../V1__Download_Analytics.sql} | 7 +- .../V1__Download_Analytics.sql.conf} | 0 .../AbstractPostgresContainerTest.java | 6 +- .../AbstractTimeseriesContainerTest.java | 48 ++++ .../DownloadAnalyticsDisabledTest.java | 13 + .../DownloadAnalyticsEndpointTest.java | 19 +- ...escaleDownloadAnalyticsRepositoryTest.java | 34 ++- 25 files changed, 326 insertions(+), 990 deletions(-) create mode 100644 server/src/main/java/org/eclipse/openvsx/analytics/timescale/TimeseriesDatabaseConfiguration.java delete mode 100644 server/src/main/jooq-gen/org/eclipse/openvsx/jooq/tables/DownloadEvent.java delete mode 100644 server/src/main/jooq-gen/org/eclipse/openvsx/jooq/tables/DownloadStatsDaily.java delete mode 100644 server/src/main/jooq-gen/org/eclipse/openvsx/jooq/tables/records/DownloadEventRecord.java delete mode 100644 server/src/main/jooq-gen/org/eclipse/openvsx/jooq/tables/records/DownloadStatsDailyRecord.java rename server/src/main/resources/db/{migration/V1_72__Download_Analytics.sql => migration-timeseries/V1__Download_Analytics.sql} (83%) rename server/src/main/resources/db/{migration/V1_72__Download_Analytics.sql.conf => migration-timeseries/V1__Download_Analytics.sql.conf} (100%) create mode 100644 server/src/test/java/org/eclipse/openvsx/AbstractTimeseriesContainerTest.java diff --git a/deploy/docker/configuration/application.yml b/deploy/docker/configuration/application.yml index 96cd0c951..57fb9ba6c 100644 --- a/deploy/docker/configuration/application.yml +++ b/deploy/docker/configuration/application.yml @@ -23,6 +23,13 @@ spring: url: jdbc:postgresql://localhost:5432/openvsx username: openvsx password: openvsx + # Download analytics is disabled by default. When enabled it keeps its time-series schema in a + # separate database, migrated on its own and requiring the timescaledb extension: + # ovsx.analytics.enabled: true + # ovsx.analytics.datasource.url: jdbc:postgresql://localhost:5433/openvsx_timeseries + # ovsx.analytics.datasource.username: openvsx + # ovsx.analytics.datasource.password: openvsx + # ovsx.analytics.datasource.maximum-pool-size: 5 flyway: baseline-on-migrate: true baseline-version: 0.1.0 diff --git a/deploy/kubernetes/configmap.yaml b/deploy/kubernetes/configmap.yaml index fe35c0ad2..569b6eac9 100644 --- a/deploy/kubernetes/configmap.yaml +++ b/deploy/kubernetes/configmap.yaml @@ -33,6 +33,13 @@ data: url: jdbc:postgresql://postgresql:5432/openvsx username: ${DB_USERNAME} password: ${DB_PASSWORD} + # Download analytics is disabled by default. When enabled it keeps its time-series schema + # in a separate database, migrated on its own and requiring the timescaledb extension: + # ovsx.analytics.enabled: true + # ovsx.analytics.datasource.url: jdbc:postgresql://postgresql-timeseries:5432/openvsx_timeseries + # ovsx.analytics.datasource.username: ${TIMESERIES_DB_USERNAME} + # ovsx.analytics.datasource.password: ${TIMESERIES_DB_PASSWORD} + # ovsx.analytics.datasource.maximum-pool-size: 5 flyway: baseline-on-migrate: true baseline-version: 0.1.0 diff --git a/deploy/openshift/application.yml b/deploy/openshift/application.yml index 43ef3e9ca..ea0e2b2df 100644 --- a/deploy/openshift/application.yml +++ b/deploy/openshift/application.yml @@ -23,6 +23,13 @@ spring: url: jdbc:postgresql://postgresql:5432/openvsx username: openvsx password: openvsx + # Download analytics is disabled by default. When enabled it keeps its time-series schema in a + # separate database, migrated on its own and requiring the timescaledb extension: + # ovsx.analytics.enabled: true + # ovsx.analytics.datasource.url: jdbc:postgresql://postgresql-timeseries:5432/openvsx_timeseries + # ovsx.analytics.datasource.username: openvsx + # ovsx.analytics.datasource.password: openvsx + # ovsx.analytics.datasource.maximum-pool-size: 5 flyway: baseline-on-migrate: true baseline-version: 0.1.0 diff --git a/doc/development.md b/doc/development.md index 8628530eb..593ee31bc 100644 --- a/doc/development.md +++ b/doc/development.md @@ -28,7 +28,7 @@ To run the Open VSX registry in a development environment, you can use `docker c * Verify Docker Compose is installed by running `docker compose version`. If an error occurs, you may need to [install docker compose](https://docs.docker.com/compose/install/) on your machine. * Decide which profile(s) to run based on your needs. The [docker-compose.yml] file defines profiles for specific components: - * `db`: Starts the PostgreSQL container. + * `db`: Starts the PostgreSQL containers: the registry database, and the separate TimescaleDB one used by download analytics. * `es`: Starts the Elasticsearch container. * `debug`: Starts the PostgreSQL and Elasticsearch containers, which suits running the OpenVSX server and web UI locally for easier debugging. * `backend`: Starts the OpenVSX server container (java). diff --git a/docker-compose.yml b/docker-compose.yml index d7506eca2..91351b008 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,9 +1,7 @@ services: postgres: - # PostgreSQL with the timescaledb extension: the main migration chain contains the - # download analytics schema, which requires the extension - image: timescale/timescaledb:2.17.2-pg16 + image: postgres:16.2 environment: - POSTGRES_USER=openvsx - POSTGRES_PASSWORD=openvsx @@ -17,6 +15,26 @@ services: - db - debug + postgres-timeseries: + # download analytics only: PostgreSQL plus the timescaledb extension, kept apart from the + # registry database so the registry never needs the extension + image: timescale/timescaledb:2.17.2-pg16 + environment: + - POSTGRES_USER=openvsx + - POSTGRES_PASSWORD=openvsx + - POSTGRES_DB=openvsx_timeseries + logging: + options: + max-size: 10m + max-file: "3" + ports: + - '5433:5432' + volumes: + - postgres-timeseries-data:/var/lib/postgresql/data + profiles: + - db + - debug + elasticsearch: image: elasticsearch:9.2.8 environment: @@ -215,6 +233,7 @@ services: - 8080:8080 depends_on: - postgres + - postgres-timeseries - elasticsearch healthcheck: test: "curl --fail --silent localhost:8081/actuator/health | grep UP || exit 1" @@ -287,3 +306,6 @@ services: " profiles: - minio + +volumes: + postgres-timeseries-data: diff --git a/server/build.gradle b/server/build.gradle index 10ca436c7..92e141186 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -200,12 +200,7 @@ jooq { name = 'org.jooq.meta.postgres.PostgresDatabase' inputSchema = 'public' includes = '.*' - // jobrunr manages its own tables; the timescaledb extension contributes - // public-schema (table-valued) functions that we never call through jOOQ - excludes = 'jobrunr.*' + - '|add_dimension|alter_job|create_hypertable|disable_chunk_skipping|drop_chunks' + - '|enable_chunk_skipping|show_chunks|show_tablespaces' + - '|chunk_compression_stats|chunks_detailed_size|hypertable_.*' + excludes = 'jobrunr.*' includeRoutines = false } target { @@ -287,10 +282,12 @@ test { jvmArgs = ['--enable-native-access=ALL-UNNAMED', '-Xmx6144m', '-Xshare:off'] // due to https://github.com/netty/netty/issues/15161 useJUnitPlatform() - // the test database image is timescale/timescaledb by default (the main migration chain - // requires the extension); override with -Dovsx.test.postgres.image=... if needed - if (System.getProperty('ovsx.test.postgres.image') != null) { - systemProperty 'ovsx.test.postgres.image', System.getProperty('ovsx.test.postgres.image') + // registry tests run on plain postgres, analytics tests on timescale/timescaledb; override + // either image with -Dovsx.test.postgres.image=... / -Dovsx.test.timeseries.image=... + ['ovsx.test.postgres.image', 'ovsx.test.timeseries.image'].each { property -> + if (System.getProperty(property) != null) { + systemProperty property, System.getProperty(property) + } } } diff --git a/server/scripts/generate-properties.sh b/server/scripts/generate-properties.sh index f07c2d180..c1178a27c 100755 --- a/server/scripts/generate-properties.sh +++ b/server/scripts/generate-properties.sh @@ -27,6 +27,11 @@ then echo "spring.datasource.url=jdbc:postgresql://postgres:5432/postgres" echo "spring.datasource.username=openvsx" echo "spring.datasource.password=openvsx" + + # Set the download analytics (timeseries) Postgres host + echo "ovsx.analytics.datasource.url=jdbc:postgresql://postgres-timeseries:5432/openvsx_timeseries" + echo "ovsx.analytics.datasource.username=openvsx" + echo "ovsx.analytics.datasource.password=openvsx" } >> "${OVSX_APP_PROFILE}" else # Set the Elasticsearch host diff --git a/server/src/dev/resources/application.yml b/server/src/dev/resources/application.yml index 11b55392e..5d0e405c9 100644 --- a/server/src/dev/resources/application.yml +++ b/server/src/dev/resources/application.yml @@ -151,6 +151,11 @@ ovsx: directory: /tmp/ovsx analytics: enabled: true + # the postgres-timeseries service of docker-compose.yml + datasource: + url: jdbc:postgresql://localhost:5433/openvsx_timeseries + username: openvsx + password: openvsx access-token: prefix: dev_ovsxat_ # use a token prefix that clearly indicates that it's for development expiration: 0 # do not expire tokens in a dev environment diff --git a/server/src/main/java/org/eclipse/openvsx/analytics/DownloadAnalyticsConfiguration.java b/server/src/main/java/org/eclipse/openvsx/analytics/DownloadAnalyticsConfiguration.java index a76383416..ae5fd34cb 100644 --- a/server/src/main/java/org/eclipse/openvsx/analytics/DownloadAnalyticsConfiguration.java +++ b/server/src/main/java/org/eclipse/openvsx/analytics/DownloadAnalyticsConfiguration.java @@ -16,6 +16,7 @@ import java.time.Duration; import org.jooq.DSLContext; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -25,15 +26,15 @@ /** * Wires download analytics when {@code ovsx.analytics.enabled=true}. The download_event schema - * is part of the main migration chain, so the database image must provide the timescaledb - * extension. + * lives in its own database, migrated and pooled separately from the registry, so the registry + * database image needs nothing beyond plain PostgreSQL. */ @Configuration @ConditionalOnProperty(name = "ovsx.analytics.enabled", havingValue = "true") class DownloadAnalyticsConfiguration { @Bean - DownloadAnalyticsRepository downloadAnalyticsRepository(DSLContext dsl) { + DownloadAnalyticsRepository downloadAnalyticsRepository(@Qualifier("timeseriesDsl") DSLContext dsl) { return new TimescaleDownloadAnalyticsRepository(dsl); } diff --git a/server/src/main/java/org/eclipse/openvsx/analytics/timescale/TimescaleDownloadAnalyticsRepository.java b/server/src/main/java/org/eclipse/openvsx/analytics/timescale/TimescaleDownloadAnalyticsRepository.java index 15de4528b..1bb9c83a8 100644 --- a/server/src/main/java/org/eclipse/openvsx/analytics/timescale/TimescaleDownloadAnalyticsRepository.java +++ b/server/src/main/java/org/eclipse/openvsx/analytics/timescale/TimescaleDownloadAnalyticsRepository.java @@ -19,6 +19,8 @@ import com.google.common.collect.Lists; import org.jooq.DSLContext; import org.jooq.Field; +import org.jooq.Record; +import org.jooq.Table; import org.jooq.impl.DSL; import org.eclipse.openvsx.analytics.DownloadAnalyticsRepository; @@ -28,19 +30,56 @@ import org.eclipse.openvsx.analytics.DownloadSeriesRequest; import org.eclipse.openvsx.analytics.DownloadSeriesRow; -import static org.eclipse.openvsx.jooq.Tables.DOWNLOAD_EVENT; -import static org.eclipse.openvsx.jooq.Tables.DOWNLOAD_STATS_DAILY; - /** * {@link DownloadAnalyticsRepository} backed by TimescaleDB: writes to the download_event - * hypertable and reads from the download_stats_daily continuous aggregate. Queries run through - * the application's transaction-aware {@link DSLContext}, so writes commit atomically with the - * download counter and the ingestion entry. + * hypertable and reads from the download_stats_daily continuous aggregate. Both live in the + * separate time-series database, addressed by name rather than through generated jOOQ classes + * (codegen runs against the registry database, which no longer holds these tables). + *

+ * Writes run on the time-series connection pool, so they cannot join a caller's registry + * transaction: an event is persisted independently of whatever the registry does afterwards. */ public class TimescaleDownloadAnalyticsRepository implements DownloadAnalyticsRepository { private static final int BATCH_SIZE = 500; + private static final Table EVENT = DSL.table(DSL.name("download_event")); + private static final Field EVENT_TIME = DSL + .field(DSL.name("download_event", "time"), OffsetDateTime.class); + private static final Field EVENT_EXTENSION_ID = DSL + .field(DSL.name("download_event", "extension_id"), Long.class); + private static final Field EVENT_EXTENSION_VERSION_ID = DSL + .field(DSL.name("download_event", "extension_version_id"), Long.class); + private static final Field EVENT_NAMESPACE = DSL + .field(DSL.name("download_event", "namespace"), String.class); + private static final Field EVENT_EXTENSION_NAME = DSL + .field(DSL.name("download_event", "extension_name"), String.class); + private static final Field EVENT_VERSION = DSL + .field(DSL.name("download_event", "version"), String.class); + private static final Field EVENT_TARGET_PLATFORM = DSL + .field(DSL.name("download_event", "target_platform"), String.class); + private static final Field EVENT_COUNTRY = DSL + .field(DSL.name("download_event", "country"), String.class); + private static final Field EVENT_IP = DSL.field(DSL.name("download_event", "ip"), String.class); + private static final Field EVENT_USER_AGENT = DSL + .field(DSL.name("download_event", "user_agent"), String.class); + private static final Field EVENT_COUNT = DSL + .field(DSL.name("download_event", "count"), Integer.class); + + private static final Table STATS = DSL.table(DSL.name("download_stats_daily")); + private static final Field STATS_DAY = DSL + .field(DSL.name("download_stats_daily", "day"), OffsetDateTime.class); + private static final Field STATS_EXTENSION_ID = DSL + .field(DSL.name("download_stats_daily", "extension_id"), Long.class); + private static final Field STATS_VERSION = DSL + .field(DSL.name("download_stats_daily", "version"), String.class); + private static final Field STATS_TARGET_PLATFORM = DSL + .field(DSL.name("download_stats_daily", "target_platform"), String.class); + private static final Field STATS_COUNTRY = DSL + .field(DSL.name("download_stats_daily", "country"), String.class); + private static final Field STATS_DOWNLOADS = DSL + .field(DSL.name("download_stats_daily", "downloads"), Long.class); + private final DSLContext dsl; public TimescaleDownloadAnalyticsRepository(DSLContext dsl) { @@ -50,19 +89,20 @@ public TimescaleDownloadAnalyticsRepository(DSLContext dsl) { @Override public void save(List events) { for (var batch : Lists.partition(events, BATCH_SIZE)) { - var insert = dsl.insertInto( - DOWNLOAD_EVENT, - DOWNLOAD_EVENT.TIME, - DOWNLOAD_EVENT.EXTENSION_ID, - DOWNLOAD_EVENT.EXTENSION_VERSION_ID, - DOWNLOAD_EVENT.NAMESPACE, - DOWNLOAD_EVENT.EXTENSION_NAME, - DOWNLOAD_EVENT.VERSION, - DOWNLOAD_EVENT.TARGET_PLATFORM, - DOWNLOAD_EVENT.COUNTRY, - DOWNLOAD_EVENT.IP, - DOWNLOAD_EVENT.USER_AGENT, - DOWNLOAD_EVENT.COUNT); + var insert = dsl + .insertInto( + EVENT, + EVENT_TIME, + EVENT_EXTENSION_ID, + EVENT_EXTENSION_VERSION_ID, + EVENT_NAMESPACE, + EVENT_EXTENSION_NAME, + EVENT_VERSION, + EVENT_TARGET_PLATFORM, + EVENT_COUNTRY, + EVENT_IP, + EVENT_USER_AGENT, + EVENT_COUNT); for (var event : batch) { insert = insert.values( OffsetDateTime.ofInstant(event.time(), ZoneOffset.UTC), @@ -85,18 +125,17 @@ public void save(List events) { public List findSeries(DownloadSeriesRequest request) { var bucket = bucketField(request.interval()); var group = groupField(request.groupBy()); - var total = DSL.sum(DOWNLOAD_STATS_DAILY.DOWNLOADS).cast(Long.class); + var total = DSL.sum(STATS_DOWNLOADS).cast(Long.class); List> groupByFields = request.groupBy() == DownloadSeriesGroupBy.NONE ? List.>of(bucket) : List.>of(bucket, group); return dsl.select(bucket, group, total) - .from(DOWNLOAD_STATS_DAILY) + .from(STATS) .where( - DOWNLOAD_STATS_DAILY.EXTENSION_ID.in(request.extensionIds()), - DOWNLOAD_STATS_DAILY.DAY - .greaterOrEqual(OffsetDateTime.ofInstant(request.from(), ZoneOffset.UTC)), - DOWNLOAD_STATS_DAILY.DAY.lessThan(OffsetDateTime.ofInstant(request.to(), ZoneOffset.UTC))) + STATS_EXTENSION_ID.in(request.extensionIds()), + STATS_DAY.greaterOrEqual(OffsetDateTime.ofInstant(request.from(), ZoneOffset.UTC)), + STATS_DAY.lessThan(OffsetDateTime.ofInstant(request.to(), ZoneOffset.UTC))) .groupBy(groupByFields) .orderBy(groupByFields) .fetch(record -> new DownloadSeriesRow(record.value1().toInstant(), record.value2(), record.value3())); @@ -106,24 +145,24 @@ private Field bucketField(DownloadSeriesInterval interval) { // `day` holds UTC-aligned buckets; date_trunc must not depend on the session time zone, // hence the AT TIME ZONE round-trip return switch (interval) { - case DAY -> DOWNLOAD_STATS_DAILY.DAY; + case DAY -> STATS_DAY; case WEEK -> DSL.field( "(date_trunc('week', {0} AT TIME ZONE 'UTC') AT TIME ZONE 'UTC')", OffsetDateTime.class, - DOWNLOAD_STATS_DAILY.DAY); + STATS_DAY); case MONTH -> DSL.field( "(date_trunc('month', {0} AT TIME ZONE 'UTC') AT TIME ZONE 'UTC')", OffsetDateTime.class, - DOWNLOAD_STATS_DAILY.DAY); + STATS_DAY); }; } private Field groupField(DownloadSeriesGroupBy groupBy) { return switch (groupBy) { case NONE -> DSL.inline(null, String.class); - case VERSION -> DOWNLOAD_STATS_DAILY.VERSION; - case TARGET_PLATFORM -> DOWNLOAD_STATS_DAILY.TARGET_PLATFORM; - case COUNTRY -> DOWNLOAD_STATS_DAILY.COUNTRY; + case VERSION -> STATS_VERSION; + case TARGET_PLATFORM -> STATS_TARGET_PLATFORM; + case COUNTRY -> STATS_COUNTRY; }; } } diff --git a/server/src/main/java/org/eclipse/openvsx/analytics/timescale/TimeseriesDatabaseConfiguration.java b/server/src/main/java/org/eclipse/openvsx/analytics/timescale/TimeseriesDatabaseConfiguration.java new file mode 100644 index 000000000..c7e38e991 --- /dev/null +++ b/server/src/main/java/org/eclipse/openvsx/analytics/timescale/TimeseriesDatabaseConfiguration.java @@ -0,0 +1,90 @@ +/****************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation. + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * https://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + *****************************************************************************/ +package org.eclipse.openvsx.analytics.timescale; + +import javax.sql.DataSource; + +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; +import org.flywaydb.core.Flyway; +import org.jooq.DSLContext; +import org.jooq.SQLDialect; +import org.jooq.impl.DSL; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.core.env.Environment; + +/** + * The time-series database behind download analytics: its own PostgreSQL instance (with the + * timescaledb extension), its own connection pool, its own Flyway migration set and its own + * jOOQ context, all configured from {@code ovsx.analytics.datasource.*}. Nothing is created + * unless {@code ovsx.analytics.enabled=true}. + */ +@Configuration +@ConditionalOnProperty(name = "ovsx.analytics.enabled", havingValue = "true") +class TimeseriesDatabaseConfiguration { + + private static final int DEFAULT_POOL_SIZE = 5; + + // defaultCandidate = false keeps these beans invisible to @ConditionalOnMissingBean and to + // plain by-type injection, so Boot still auto-configures the primary DataSource, the main + // Flyway chain and the primary DSLContext; only an explicit @Qualifier reaches them. + @Bean(destroyMethod = "close", defaultCandidate = false) + DataSource timeseriesDataSource(Environment environment) { + var config = new HikariConfig(); + config.setPoolName("timeseries"); + config.setJdbcUrl(environment.getRequiredProperty("ovsx.analytics.datasource.url")); + config.setUsername(environment.getProperty("ovsx.analytics.datasource.username")); + config.setPassword(environment.getProperty("ovsx.analytics.datasource.password")); + config.setMaximumPoolSize( + environment.getProperty( + "ovsx.analytics.datasource.maximum-pool-size", + Integer.class, + DEFAULT_POOL_SIZE)); + return new HikariDataSource(config); + } + + /** + * Migrates the time-series schema. Configured programmatically rather than through + * {@code spring.flyway.*} so that the main and the time-series migration settings cannot leak + * into each other; in particular there is no baseline, because this database starts empty and + * an unexpected schema must fail the startup. + */ + @Bean(defaultCandidate = false) + Flyway timeseriesFlyway(@Qualifier("timeseriesDataSource") DataSource dataSource) { + var flyway = Flyway.configure() + .dataSource(dataSource) + // a sibling of db/migration, never a child: Flyway scans locations recursively, + // so a child would be swept into the registry's migration chain as well + .locations("classpath:db/migration-timeseries") + .load(); + flyway.migrate(); + return flyway; + } + + /** + * Standalone jOOQ context on the time-series pool. Being outside Spring's transaction and + * exception-translation infrastructure, queries throw jOOQ's {@code DataAccessException} + * rather than Spring's, and never join a caller's registry transaction. + */ + @Bean(defaultCandidate = false) + DSLContext timeseriesDsl( + @Qualifier("timeseriesDataSource") DataSource dataSource, + // depended upon so the schema exists before the first query + @Qualifier("timeseriesFlyway") Flyway flyway + ) { + return DSL.using(dataSource, SQLDialect.POSTGRES); + } +} diff --git a/server/src/main/jooq-gen/org/eclipse/openvsx/jooq/Indexes.java b/server/src/main/jooq-gen/org/eclipse/openvsx/jooq/Indexes.java index b60338a95..7bab676fc 100644 --- a/server/src/main/jooq-gen/org/eclipse/openvsx/jooq/Indexes.java +++ b/server/src/main/jooq-gen/org/eclipse/openvsx/jooq/Indexes.java @@ -8,7 +8,6 @@ import org.eclipse.openvsx.jooq.tables.AdminStatistics; import org.eclipse.openvsx.jooq.tables.CustomerMembership; import org.eclipse.openvsx.jooq.tables.DownloadCountProcessedItem; -import org.eclipse.openvsx.jooq.tables.DownloadEvent; import org.eclipse.openvsx.jooq.tables.Extension; import org.eclipse.openvsx.jooq.tables.ExtensionReview; import org.eclipse.openvsx.jooq.tables.ExtensionScan; @@ -46,10 +45,8 @@ public class Indexes { public static final Index CUSTOMER_MEMBERSHIP_NAMESPACE_IDX = Internal.createIndex(DSL.name("customer_membership_namespace_idx"), CustomerMembership.CUSTOMER_MEMBERSHIP, new OrderField[] { CustomerMembership.CUSTOMER_MEMBERSHIP.CUSTOMER }, false); public static final Index CUSTOMER_MEMBERSHIP_USER_DATA_IDX = Internal.createIndex(DSL.name("customer_membership_user_data_idx"), CustomerMembership.CUSTOMER_MEMBERSHIP, new OrderField[] { CustomerMembership.CUSTOMER_MEMBERSHIP.USER_DATA }, false); - public static final Index DE_EXT_TIME = Internal.createIndex(DSL.name("de_ext_time"), DownloadEvent.DOWNLOAD_EVENT, new OrderField[] { DownloadEvent.DOWNLOAD_EVENT.EXTENSION_ID, DownloadEvent.DOWNLOAD_EVENT.TIME.desc() }, false); public static final Index DOWNLOAD_COUNT_PROCESSED_ITEM_NAME = Internal.createIndex(DSL.name("download_count_processed_item_name"), DownloadCountProcessedItem.DOWNLOAD_COUNT_PROCESSED_ITEM, new OrderField[] { DownloadCountProcessedItem.DOWNLOAD_COUNT_PROCESSED_ITEM.NAME }, false); public static final Index DOWNLOAD_COUNT_PROCESSED_ITEM_STORAGE_TYPE = Internal.createIndex(DSL.name("download_count_processed_item_storage_type"), DownloadCountProcessedItem.DOWNLOAD_COUNT_PROCESSED_ITEM, new OrderField[] { DownloadCountProcessedItem.DOWNLOAD_COUNT_PROCESSED_ITEM.STORAGE_TYPE }, false); - public static final Index DOWNLOAD_EVENT_TIME_IDX = Internal.createIndex(DSL.name("download_event_time_idx"), DownloadEvent.DOWNLOAD_EVENT, new OrderField[] { DownloadEvent.DOWNLOAD_EVENT.TIME.desc() }, false); public static final Index EXTENSION__NAMESPACE_ID__IDX = Internal.createIndex(DSL.name("extension__namespace_id__idx"), Extension.EXTENSION, new OrderField[] { Extension.EXTENSION.NAMESPACE_ID }, false); public static final Index EXTENSION_REVIEW__EXTENSION_ID__IDX = Internal.createIndex(DSL.name("extension_review__extension_id__idx"), ExtensionReview.EXTENSION_REVIEW, new OrderField[] { ExtensionReview.EXTENSION_REVIEW.EXTENSION_ID }, false); public static final Index EXTENSION_REVIEW__USER_ID__IDX = Internal.createIndex(DSL.name("extension_review__user_id__idx"), ExtensionReview.EXTENSION_REVIEW, new OrderField[] { ExtensionReview.EXTENSION_REVIEW.USER_ID }, false); diff --git a/server/src/main/jooq-gen/org/eclipse/openvsx/jooq/Public.java b/server/src/main/jooq-gen/org/eclipse/openvsx/jooq/Public.java index b7188100a..649d54b9a 100644 --- a/server/src/main/jooq-gen/org/eclipse/openvsx/jooq/Public.java +++ b/server/src/main/jooq-gen/org/eclipse/openvsx/jooq/Public.java @@ -19,8 +19,6 @@ import org.eclipse.openvsx.jooq.tables.CustomerMembership; import org.eclipse.openvsx.jooq.tables.DailyUsageStats; import org.eclipse.openvsx.jooq.tables.DownloadCountProcessedItem; -import org.eclipse.openvsx.jooq.tables.DownloadEvent; -import org.eclipse.openvsx.jooq.tables.DownloadStatsDaily; import org.eclipse.openvsx.jooq.tables.Extension; import org.eclipse.openvsx.jooq.tables.ExtensionReview; import org.eclipse.openvsx.jooq.tables.ExtensionScan; @@ -131,16 +129,6 @@ public class Public extends SchemaImpl { */ public final DownloadCountProcessedItem DOWNLOAD_COUNT_PROCESSED_ITEM = DownloadCountProcessedItem.DOWNLOAD_COUNT_PROCESSED_ITEM; - /** - * The table public.download_event. - */ - public final DownloadEvent DOWNLOAD_EVENT = DownloadEvent.DOWNLOAD_EVENT; - - /** - * The table public.download_stats_daily. - */ - public final DownloadStatsDaily DOWNLOAD_STATS_DAILY = DownloadStatsDaily.DOWNLOAD_STATS_DAILY; - /** * The table public.extension. */ @@ -340,8 +328,6 @@ public final List> getTables() { CustomerMembership.CUSTOMER_MEMBERSHIP, DailyUsageStats.DAILY_USAGE_STATS, DownloadCountProcessedItem.DOWNLOAD_COUNT_PROCESSED_ITEM, - DownloadEvent.DOWNLOAD_EVENT, - DownloadStatsDaily.DOWNLOAD_STATS_DAILY, Extension.EXTENSION, ExtensionReview.EXTENSION_REVIEW, ExtensionScan.EXTENSION_SCAN, diff --git a/server/src/main/jooq-gen/org/eclipse/openvsx/jooq/Tables.java b/server/src/main/jooq-gen/org/eclipse/openvsx/jooq/Tables.java index f3acab068..d965794e6 100644 --- a/server/src/main/jooq-gen/org/eclipse/openvsx/jooq/Tables.java +++ b/server/src/main/jooq-gen/org/eclipse/openvsx/jooq/Tables.java @@ -16,8 +16,6 @@ import org.eclipse.openvsx.jooq.tables.CustomerMembership; import org.eclipse.openvsx.jooq.tables.DailyUsageStats; import org.eclipse.openvsx.jooq.tables.DownloadCountProcessedItem; -import org.eclipse.openvsx.jooq.tables.DownloadEvent; -import org.eclipse.openvsx.jooq.tables.DownloadStatsDaily; import org.eclipse.openvsx.jooq.tables.Extension; import org.eclipse.openvsx.jooq.tables.ExtensionReview; import org.eclipse.openvsx.jooq.tables.ExtensionScan; @@ -117,16 +115,6 @@ public class Tables { */ public static final DownloadCountProcessedItem DOWNLOAD_COUNT_PROCESSED_ITEM = DownloadCountProcessedItem.DOWNLOAD_COUNT_PROCESSED_ITEM; - /** - * The table public.download_event. - */ - public static final DownloadEvent DOWNLOAD_EVENT = DownloadEvent.DOWNLOAD_EVENT; - - /** - * The table public.download_stats_daily. - */ - public static final DownloadStatsDaily DOWNLOAD_STATS_DAILY = DownloadStatsDaily.DOWNLOAD_STATS_DAILY; - /** * The table public.extension. */ diff --git a/server/src/main/jooq-gen/org/eclipse/openvsx/jooq/tables/DownloadEvent.java b/server/src/main/jooq-gen/org/eclipse/openvsx/jooq/tables/DownloadEvent.java deleted file mode 100644 index 3aff1eb51..000000000 --- a/server/src/main/jooq-gen/org/eclipse/openvsx/jooq/tables/DownloadEvent.java +++ /dev/null @@ -1,270 +0,0 @@ -/* - * This file is generated by jOOQ. - */ -package org.eclipse.openvsx.jooq.tables; - - -import java.time.OffsetDateTime; -import java.util.Arrays; -import java.util.Collection; -import java.util.List; - -import org.eclipse.openvsx.jooq.Indexes; -import org.eclipse.openvsx.jooq.Public; -import org.eclipse.openvsx.jooq.tables.records.DownloadEventRecord; -import org.jooq.Condition; -import org.jooq.Field; -import org.jooq.Index; -import org.jooq.Name; -import org.jooq.PlainSQL; -import org.jooq.QueryPart; -import org.jooq.SQL; -import org.jooq.Schema; -import org.jooq.Select; -import org.jooq.Stringly; -import org.jooq.Table; -import org.jooq.TableField; -import org.jooq.TableOptions; -import org.jooq.impl.DSL; -import org.jooq.impl.SQLDataType; -import org.jooq.impl.TableImpl; - - -/** - * This class is generated by jOOQ. - */ -@SuppressWarnings({ "all", "unchecked", "rawtypes", "this-escape" }) -public class DownloadEvent extends TableImpl { - - private static final long serialVersionUID = 1L; - - /** - * The reference instance of public.download_event - */ - public static final DownloadEvent DOWNLOAD_EVENT = new DownloadEvent(); - - /** - * The class holding records for this type - */ - @Override - public Class getRecordType() { - return DownloadEventRecord.class; - } - - /** - * The column public.download_event.time. - */ - public final TableField TIME = createField(DSL.name("time"), SQLDataType.TIMESTAMPWITHTIMEZONE(6).nullable(false), this, ""); - - /** - * The column public.download_event.extension_id. - */ - public final TableField EXTENSION_ID = createField(DSL.name("extension_id"), SQLDataType.BIGINT.nullable(false), this, ""); - - /** - * The column public.download_event.extension_version_id. - */ - public final TableField EXTENSION_VERSION_ID = createField(DSL.name("extension_version_id"), SQLDataType.BIGINT.nullable(false), this, ""); - - /** - * The column public.download_event.namespace. - */ - public final TableField NAMESPACE = createField(DSL.name("namespace"), SQLDataType.VARCHAR(255).nullable(false), this, ""); - - /** - * The column public.download_event.extension_name. - */ - public final TableField EXTENSION_NAME = createField(DSL.name("extension_name"), SQLDataType.VARCHAR(255).nullable(false), this, ""); - - /** - * The column public.download_event.version. - */ - public final TableField VERSION = createField(DSL.name("version"), SQLDataType.VARCHAR(255).nullable(false), this, ""); - - /** - * The column public.download_event.target_platform. - */ - public final TableField TARGET_PLATFORM = createField(DSL.name("target_platform"), SQLDataType.VARCHAR(255).nullable(false), this, ""); - - /** - * The column public.download_event.country. - */ - public final TableField COUNTRY = createField(DSL.name("country"), SQLDataType.CHAR(2), this, ""); - - /** - * The column public.download_event.ip. - */ - public final TableField IP = createField(DSL.name("ip"), SQLDataType.VARCHAR(45), this, ""); - - /** - * The column public.download_event.user_agent. - */ - public final TableField USER_AGENT = createField(DSL.name("user_agent"), SQLDataType.CLOB, this, ""); - - /** - * The column public.download_event.count. - */ - public final TableField COUNT = createField(DSL.name("count"), SQLDataType.INTEGER.nullable(false).defaultValue(DSL.field(DSL.raw("1"), SQLDataType.INTEGER)), this, ""); - - private DownloadEvent(Name alias, Table aliased) { - this(alias, aliased, (Field[]) null, null); - } - - private DownloadEvent(Name alias, Table aliased, Field[] parameters, Condition where) { - super(alias, null, aliased, parameters, DSL.comment(""), TableOptions.table(), where); - } - - /** - * Create an aliased public.download_event table reference - */ - public DownloadEvent(String alias) { - this(DSL.name(alias), DOWNLOAD_EVENT); - } - - /** - * Create an aliased public.download_event table reference - */ - public DownloadEvent(Name alias) { - this(alias, DOWNLOAD_EVENT); - } - - /** - * Create a public.download_event table reference - */ - public DownloadEvent() { - this(DSL.name("download_event"), null); - } - - @Override - public Schema getSchema() { - return aliased() ? null : Public.PUBLIC; - } - - @Override - public List getIndexes() { - return Arrays.asList(Indexes.DE_EXT_TIME, Indexes.DOWNLOAD_EVENT_TIME_IDX); - } - - @Override - public DownloadEvent as(String alias) { - return new DownloadEvent(DSL.name(alias), this); - } - - @Override - public DownloadEvent as(Name alias) { - return new DownloadEvent(alias, this); - } - - @Override - public DownloadEvent as(Table alias) { - return new DownloadEvent(alias.getQualifiedName(), this); - } - - /** - * Rename this table - */ - @Override - public DownloadEvent rename(String name) { - return new DownloadEvent(DSL.name(name), null); - } - - /** - * Rename this table - */ - @Override - public DownloadEvent rename(Name name) { - return new DownloadEvent(name, null); - } - - /** - * Rename this table - */ - @Override - public DownloadEvent rename(Table name) { - return new DownloadEvent(name.getQualifiedName(), null); - } - - /** - * Create an inline derived table from this table - */ - @Override - public DownloadEvent where(Condition condition) { - return new DownloadEvent(getQualifiedName(), aliased() ? this : null, null, condition); - } - - /** - * Create an inline derived table from this table - */ - @Override - public DownloadEvent where(Collection conditions) { - return where(DSL.and(conditions)); - } - - /** - * Create an inline derived table from this table - */ - @Override - public DownloadEvent where(Condition... conditions) { - return where(DSL.and(conditions)); - } - - /** - * Create an inline derived table from this table - */ - @Override - public DownloadEvent where(Field condition) { - return where(DSL.condition(condition)); - } - - /** - * Create an inline derived table from this table - */ - @Override - @PlainSQL - public DownloadEvent where(SQL condition) { - return where(DSL.condition(condition)); - } - - /** - * Create an inline derived table from this table - */ - @Override - @PlainSQL - public DownloadEvent where(@Stringly.SQL String condition) { - return where(DSL.condition(condition)); - } - - /** - * Create an inline derived table from this table - */ - @Override - @PlainSQL - public DownloadEvent where(@Stringly.SQL String condition, Object... binds) { - return where(DSL.condition(condition, binds)); - } - - /** - * Create an inline derived table from this table - */ - @Override - @PlainSQL - public DownloadEvent where(@Stringly.SQL String condition, QueryPart... parts) { - return where(DSL.condition(condition, parts)); - } - - /** - * Create an inline derived table from this table - */ - @Override - public DownloadEvent whereExists(Select select) { - return where(DSL.exists(select)); - } - - /** - * Create an inline derived table from this table - */ - @Override - public DownloadEvent whereNotExists(Select select) { - return where(DSL.notExists(select)); - } -} diff --git a/server/src/main/jooq-gen/org/eclipse/openvsx/jooq/tables/DownloadStatsDaily.java b/server/src/main/jooq-gen/org/eclipse/openvsx/jooq/tables/DownloadStatsDaily.java deleted file mode 100644 index c9900bf28..000000000 --- a/server/src/main/jooq-gen/org/eclipse/openvsx/jooq/tables/DownloadStatsDaily.java +++ /dev/null @@ -1,264 +0,0 @@ -/* - * This file is generated by jOOQ. - */ -package org.eclipse.openvsx.jooq.tables; - - -import java.time.OffsetDateTime; -import java.util.Collection; - -import org.eclipse.openvsx.jooq.Public; -import org.eclipse.openvsx.jooq.tables.records.DownloadStatsDailyRecord; -import org.jooq.Condition; -import org.jooq.Field; -import org.jooq.Name; -import org.jooq.PlainSQL; -import org.jooq.QueryPart; -import org.jooq.SQL; -import org.jooq.Schema; -import org.jooq.Select; -import org.jooq.Stringly; -import org.jooq.Table; -import org.jooq.TableField; -import org.jooq.TableOptions; -import org.jooq.impl.DSL; -import org.jooq.impl.SQLDataType; -import org.jooq.impl.TableImpl; - - -/** - * This class is generated by jOOQ. - */ -@SuppressWarnings({ "all", "unchecked", "rawtypes", "this-escape" }) -public class DownloadStatsDaily extends TableImpl { - - private static final long serialVersionUID = 1L; - - /** - * The reference instance of public.download_stats_daily - */ - public static final DownloadStatsDaily DOWNLOAD_STATS_DAILY = new DownloadStatsDaily(); - - /** - * The class holding records for this type - */ - @Override - public Class getRecordType() { - return DownloadStatsDailyRecord.class; - } - - /** - * The column public.download_stats_daily.day. - */ - public final TableField DAY = createField(DSL.name("day"), SQLDataType.TIMESTAMPWITHTIMEZONE(6), this, ""); - - /** - * The column public.download_stats_daily.extension_id. - */ - public final TableField EXTENSION_ID = createField(DSL.name("extension_id"), SQLDataType.BIGINT, this, ""); - - /** - * The column public.download_stats_daily.extension_version_id. - */ - public final TableField EXTENSION_VERSION_ID = createField(DSL.name("extension_version_id"), SQLDataType.BIGINT, this, ""); - - /** - * The column public.download_stats_daily.version. - */ - public final TableField VERSION = createField(DSL.name("version"), SQLDataType.VARCHAR(255), this, ""); - - /** - * The column public.download_stats_daily.target_platform. - */ - public final TableField TARGET_PLATFORM = createField(DSL.name("target_platform"), SQLDataType.VARCHAR(255), this, ""); - - /** - * The column public.download_stats_daily.country. - */ - public final TableField COUNTRY = createField(DSL.name("country"), SQLDataType.CHAR(2), this, ""); - - /** - * The column public.download_stats_daily.downloads. - */ - public final TableField DOWNLOADS = createField(DSL.name("downloads"), SQLDataType.BIGINT, this, ""); - - private DownloadStatsDaily(Name alias, Table aliased) { - this(alias, aliased, (Field[]) null, null); - } - - private DownloadStatsDaily(Name alias, Table aliased, Field[] parameters, Condition where) { - super(alias, null, aliased, parameters, DSL.comment(""), TableOptions.view(""" - create view "download_stats_daily" as SELECT _materialized_hypertable_2.day, - _materialized_hypertable_2.extension_id, - _materialized_hypertable_2.extension_version_id, - _materialized_hypertable_2.version, - _materialized_hypertable_2.target_platform, - _materialized_hypertable_2.country, - _materialized_hypertable_2.downloads - FROM _timescaledb_internal._materialized_hypertable_2 - WHERE (_materialized_hypertable_2.day < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(2)), '-infinity'::timestamp with time zone)) - UNION ALL - SELECT time_bucket('1 day'::interval, download_event."time") AS day, - download_event.extension_id, - download_event.extension_version_id, - download_event.version, - download_event.target_platform, - download_event.country, - sum(download_event.count) AS downloads - FROM download_event - WHERE (download_event."time" >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(2)), '-infinity'::timestamp with time zone)) - GROUP BY (time_bucket('1 day'::interval, download_event."time")), download_event.extension_id, download_event.extension_version_id, download_event.version, download_event.target_platform, download_event.country; - """), where); - } - - /** - * Create an aliased public.download_stats_daily table - * reference - */ - public DownloadStatsDaily(String alias) { - this(DSL.name(alias), DOWNLOAD_STATS_DAILY); - } - - /** - * Create an aliased public.download_stats_daily table - * reference - */ - public DownloadStatsDaily(Name alias) { - this(alias, DOWNLOAD_STATS_DAILY); - } - - /** - * Create a public.download_stats_daily table reference - */ - public DownloadStatsDaily() { - this(DSL.name("download_stats_daily"), null); - } - - @Override - public Schema getSchema() { - return aliased() ? null : Public.PUBLIC; - } - - @Override - public DownloadStatsDaily as(String alias) { - return new DownloadStatsDaily(DSL.name(alias), this); - } - - @Override - public DownloadStatsDaily as(Name alias) { - return new DownloadStatsDaily(alias, this); - } - - @Override - public DownloadStatsDaily as(Table alias) { - return new DownloadStatsDaily(alias.getQualifiedName(), this); - } - - /** - * Rename this table - */ - @Override - public DownloadStatsDaily rename(String name) { - return new DownloadStatsDaily(DSL.name(name), null); - } - - /** - * Rename this table - */ - @Override - public DownloadStatsDaily rename(Name name) { - return new DownloadStatsDaily(name, null); - } - - /** - * Rename this table - */ - @Override - public DownloadStatsDaily rename(Table name) { - return new DownloadStatsDaily(name.getQualifiedName(), null); - } - - /** - * Create an inline derived table from this table - */ - @Override - public DownloadStatsDaily where(Condition condition) { - return new DownloadStatsDaily(getQualifiedName(), aliased() ? this : null, null, condition); - } - - /** - * Create an inline derived table from this table - */ - @Override - public DownloadStatsDaily where(Collection conditions) { - return where(DSL.and(conditions)); - } - - /** - * Create an inline derived table from this table - */ - @Override - public DownloadStatsDaily where(Condition... conditions) { - return where(DSL.and(conditions)); - } - - /** - * Create an inline derived table from this table - */ - @Override - public DownloadStatsDaily where(Field condition) { - return where(DSL.condition(condition)); - } - - /** - * Create an inline derived table from this table - */ - @Override - @PlainSQL - public DownloadStatsDaily where(SQL condition) { - return where(DSL.condition(condition)); - } - - /** - * Create an inline derived table from this table - */ - @Override - @PlainSQL - public DownloadStatsDaily where(@Stringly.SQL String condition) { - return where(DSL.condition(condition)); - } - - /** - * Create an inline derived table from this table - */ - @Override - @PlainSQL - public DownloadStatsDaily where(@Stringly.SQL String condition, Object... binds) { - return where(DSL.condition(condition, binds)); - } - - /** - * Create an inline derived table from this table - */ - @Override - @PlainSQL - public DownloadStatsDaily where(@Stringly.SQL String condition, QueryPart... parts) { - return where(DSL.condition(condition, parts)); - } - - /** - * Create an inline derived table from this table - */ - @Override - public DownloadStatsDaily whereExists(Select select) { - return where(DSL.exists(select)); - } - - /** - * Create an inline derived table from this table - */ - @Override - public DownloadStatsDaily whereNotExists(Select select) { - return where(DSL.notExists(select)); - } -} diff --git a/server/src/main/jooq-gen/org/eclipse/openvsx/jooq/tables/records/DownloadEventRecord.java b/server/src/main/jooq-gen/org/eclipse/openvsx/jooq/tables/records/DownloadEventRecord.java deleted file mode 100644 index 26d7ee63f..000000000 --- a/server/src/main/jooq-gen/org/eclipse/openvsx/jooq/tables/records/DownloadEventRecord.java +++ /dev/null @@ -1,205 +0,0 @@ -/* - * This file is generated by jOOQ. - */ -package org.eclipse.openvsx.jooq.tables.records; - - -import java.time.OffsetDateTime; - -import org.eclipse.openvsx.jooq.tables.DownloadEvent; -import org.jooq.impl.TableRecordImpl; - - -/** - * This class is generated by jOOQ. - */ -@SuppressWarnings({ "all", "unchecked", "rawtypes", "this-escape" }) -public class DownloadEventRecord extends TableRecordImpl { - - private static final long serialVersionUID = 1L; - - /** - * Setter for public.download_event.time. - */ - public void setTime(OffsetDateTime value) { - set(0, value); - } - - /** - * Getter for public.download_event.time. - */ - public OffsetDateTime getTime() { - return (OffsetDateTime) get(0); - } - - /** - * Setter for public.download_event.extension_id. - */ - public void setExtensionId(Long value) { - set(1, value); - } - - /** - * Getter for public.download_event.extension_id. - */ - public Long getExtensionId() { - return (Long) get(1); - } - - /** - * Setter for public.download_event.extension_version_id. - */ - public void setExtensionVersionId(Long value) { - set(2, value); - } - - /** - * Getter for public.download_event.extension_version_id. - */ - public Long getExtensionVersionId() { - return (Long) get(2); - } - - /** - * Setter for public.download_event.namespace. - */ - public void setNamespace(String value) { - set(3, value); - } - - /** - * Getter for public.download_event.namespace. - */ - public String getNamespace() { - return (String) get(3); - } - - /** - * Setter for public.download_event.extension_name. - */ - public void setExtensionName(String value) { - set(4, value); - } - - /** - * Getter for public.download_event.extension_name. - */ - public String getExtensionName() { - return (String) get(4); - } - - /** - * Setter for public.download_event.version. - */ - public void setVersion(String value) { - set(5, value); - } - - /** - * Getter for public.download_event.version. - */ - public String getVersion() { - return (String) get(5); - } - - /** - * Setter for public.download_event.target_platform. - */ - public void setTargetPlatform(String value) { - set(6, value); - } - - /** - * Getter for public.download_event.target_platform. - */ - public String getTargetPlatform() { - return (String) get(6); - } - - /** - * Setter for public.download_event.country. - */ - public void setCountry(String value) { - set(7, value); - } - - /** - * Getter for public.download_event.country. - */ - public String getCountry() { - return (String) get(7); - } - - /** - * Setter for public.download_event.ip. - */ - public void setIp(String value) { - set(8, value); - } - - /** - * Getter for public.download_event.ip. - */ - public String getIp() { - return (String) get(8); - } - - /** - * Setter for public.download_event.user_agent. - */ - public void setUserAgent(String value) { - set(9, value); - } - - /** - * Getter for public.download_event.user_agent. - */ - public String getUserAgent() { - return (String) get(9); - } - - /** - * Setter for public.download_event.count. - */ - public void setCount(Integer value) { - set(10, value); - } - - /** - * Getter for public.download_event.count. - */ - public Integer getCount() { - return (Integer) get(10); - } - - // ------------------------------------------------------------------------- - // Constructors - // ------------------------------------------------------------------------- - - /** - * Create a detached DownloadEventRecord - */ - public DownloadEventRecord() { - super(DownloadEvent.DOWNLOAD_EVENT); - } - - /** - * Create a detached, initialised DownloadEventRecord - */ - public DownloadEventRecord(OffsetDateTime time, Long extensionId, Long extensionVersionId, String namespace, String extensionName, String version, String targetPlatform, String country, String ip, String userAgent, Integer count) { - super(DownloadEvent.DOWNLOAD_EVENT); - - setTime(time); - setExtensionId(extensionId); - setExtensionVersionId(extensionVersionId); - setNamespace(namespace); - setExtensionName(extensionName); - setVersion(version); - setTargetPlatform(targetPlatform); - setCountry(country); - setIp(ip); - setUserAgent(userAgent); - setCount(count); - resetChangedOnNotNull(); - } -} diff --git a/server/src/main/jooq-gen/org/eclipse/openvsx/jooq/tables/records/DownloadStatsDailyRecord.java b/server/src/main/jooq-gen/org/eclipse/openvsx/jooq/tables/records/DownloadStatsDailyRecord.java deleted file mode 100644 index d349602b1..000000000 --- a/server/src/main/jooq-gen/org/eclipse/openvsx/jooq/tables/records/DownloadStatsDailyRecord.java +++ /dev/null @@ -1,145 +0,0 @@ -/* - * This file is generated by jOOQ. - */ -package org.eclipse.openvsx.jooq.tables.records; - - -import java.time.OffsetDateTime; - -import org.eclipse.openvsx.jooq.tables.DownloadStatsDaily; -import org.jooq.impl.TableRecordImpl; - - -/** - * This class is generated by jOOQ. - */ -@SuppressWarnings({ "all", "unchecked", "rawtypes", "this-escape" }) -public class DownloadStatsDailyRecord extends TableRecordImpl { - - private static final long serialVersionUID = 1L; - - /** - * Setter for public.download_stats_daily.day. - */ - public void setDay(OffsetDateTime value) { - set(0, value); - } - - /** - * Getter for public.download_stats_daily.day. - */ - public OffsetDateTime getDay() { - return (OffsetDateTime) get(0); - } - - /** - * Setter for public.download_stats_daily.extension_id. - */ - public void setExtensionId(Long value) { - set(1, value); - } - - /** - * Getter for public.download_stats_daily.extension_id. - */ - public Long getExtensionId() { - return (Long) get(1); - } - - /** - * Setter for public.download_stats_daily.extension_version_id. - */ - public void setExtensionVersionId(Long value) { - set(2, value); - } - - /** - * Getter for public.download_stats_daily.extension_version_id. - */ - public Long getExtensionVersionId() { - return (Long) get(2); - } - - /** - * Setter for public.download_stats_daily.version. - */ - public void setVersion(String value) { - set(3, value); - } - - /** - * Getter for public.download_stats_daily.version. - */ - public String getVersion() { - return (String) get(3); - } - - /** - * Setter for public.download_stats_daily.target_platform. - */ - public void setTargetPlatform(String value) { - set(4, value); - } - - /** - * Getter for public.download_stats_daily.target_platform. - */ - public String getTargetPlatform() { - return (String) get(4); - } - - /** - * Setter for public.download_stats_daily.country. - */ - public void setCountry(String value) { - set(5, value); - } - - /** - * Getter for public.download_stats_daily.country. - */ - public String getCountry() { - return (String) get(5); - } - - /** - * Setter for public.download_stats_daily.downloads. - */ - public void setDownloads(Long value) { - set(6, value); - } - - /** - * Getter for public.download_stats_daily.downloads. - */ - public Long getDownloads() { - return (Long) get(6); - } - - // ------------------------------------------------------------------------- - // Constructors - // ------------------------------------------------------------------------- - - /** - * Create a detached DownloadStatsDailyRecord - */ - public DownloadStatsDailyRecord() { - super(DownloadStatsDaily.DOWNLOAD_STATS_DAILY); - } - - /** - * Create a detached, initialised DownloadStatsDailyRecord - */ - public DownloadStatsDailyRecord(OffsetDateTime day, Long extensionId, Long extensionVersionId, String version, String targetPlatform, String country, Long downloads) { - super(DownloadStatsDaily.DOWNLOAD_STATS_DAILY); - - setDay(day); - setExtensionId(extensionId); - setExtensionVersionId(extensionVersionId); - setVersion(version); - setTargetPlatform(targetPlatform); - setCountry(country); - setDownloads(downloads); - resetChangedOnNotNull(); - } -} diff --git a/server/src/main/resources/db/migration/V1_72__Download_Analytics.sql b/server/src/main/resources/db/migration-timeseries/V1__Download_Analytics.sql similarity index 83% rename from server/src/main/resources/db/migration/V1_72__Download_Analytics.sql rename to server/src/main/resources/db/migration-timeseries/V1__Download_Analytics.sql index d3eb798fe..42441d0ac 100644 --- a/server/src/main/resources/db/migration/V1_72__Download_Analytics.sql +++ b/server/src/main/resources/db/migration-timeseries/V1__Download_Analytics.sql @@ -1,6 +1,7 @@ --- Time-series download analytics schema. Requires a PostgreSQL image with the timescaledb --- extension available. See V1_72__Download_Analytics.sql.conf: continuous aggregates cannot --- be created inside a transaction. +-- Time-series download analytics schema, applied to the separate timeseries database. Requires +-- a PostgreSQL image with the timescaledb extension available. Every migration in this set that +-- creates a hypertable, a continuous aggregate or one of their policies needs an +-- executeInTransaction=false sidecar, as those cannot run inside a transaction block. CREATE EXTENSION IF NOT EXISTS timescaledb; diff --git a/server/src/main/resources/db/migration/V1_72__Download_Analytics.sql.conf b/server/src/main/resources/db/migration-timeseries/V1__Download_Analytics.sql.conf similarity index 100% rename from server/src/main/resources/db/migration/V1_72__Download_Analytics.sql.conf rename to server/src/main/resources/db/migration-timeseries/V1__Download_Analytics.sql.conf diff --git a/server/src/test/java/org/eclipse/openvsx/AbstractPostgresContainerTest.java b/server/src/test/java/org/eclipse/openvsx/AbstractPostgresContainerTest.java index 5c286d2d5..4ed32d30c 100644 --- a/server/src/test/java/org/eclipse/openvsx/AbstractPostgresContainerTest.java +++ b/server/src/test/java/org/eclipse/openvsx/AbstractPostgresContainerTest.java @@ -31,16 +31,12 @@ * Because all contexts now share a single database, tests must keep cleaning up after themselves (via * transactional rollback or an explicit tear-down) and use unique identifiers, exactly as they already * had to when sharing a context. - *

- * The image is timescale/timescaledb (PostgreSQL plus the timescaledb extension): the main - * migration chain contains the download analytics schema, which requires the extension. - * Override with {@code -Dovsx.test.postgres.image=...} if needed. */ @Tag("integration") public abstract class AbstractPostgresContainerTest { static final PostgreSQLContainer POSTGRES = new PostgreSQLContainer( - DockerImageName.parse(System.getProperty("ovsx.test.postgres.image", "timescale/timescaledb:2.17.2-pg16")) + DockerImageName.parse(System.getProperty("ovsx.test.postgres.image", "postgres:16.2")) .asCompatibleSubstituteFor("postgres")); static { diff --git a/server/src/test/java/org/eclipse/openvsx/AbstractTimeseriesContainerTest.java b/server/src/test/java/org/eclipse/openvsx/AbstractTimeseriesContainerTest.java new file mode 100644 index 000000000..39abb55db --- /dev/null +++ b/server/src/test/java/org/eclipse/openvsx/AbstractTimeseriesContainerTest.java @@ -0,0 +1,48 @@ +/****************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation. + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * https://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + *****************************************************************************/ +package org.eclipse.openvsx; + +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; +import org.testcontainers.postgresql.PostgreSQLContainer; +import org.testcontainers.utility.DockerImageName; + +/** + * Base class for tests that need the time-series database on top of the registry database, i.e. + * download analytics. Like the registry container, this one is a JVM-wide singleton started once + * in its static initializer and reaped by Ryuk when the JVM exits. + *

+ * Two containers rather than two databases in one is deliberate: the timescale image installs the + * extension into {@code template1}, so a second database inside it would still carry timescaledb - + * exactly the coupling that keeping the two schemas apart is meant to remove. Override the image + * with {@code -Dovsx.test.timeseries.image=...} if needed. + */ +public abstract class AbstractTimeseriesContainerTest extends AbstractPostgresContainerTest { + + static final PostgreSQLContainer TIMESERIES = new PostgreSQLContainer( + DockerImageName + .parse(System.getProperty("ovsx.test.timeseries.image", "timescale/timescaledb:2.17.2-pg16")) + .asCompatibleSubstituteFor("postgres")); + + static { + TIMESERIES.start(); + } + + @DynamicPropertySource + static void timeseriesProperties(DynamicPropertyRegistry registry) { + registry.add("ovsx.analytics.enabled", () -> true); + registry.add("ovsx.analytics.datasource.url", TIMESERIES::getJdbcUrl); + registry.add("ovsx.analytics.datasource.username", TIMESERIES::getUsername); + registry.add("ovsx.analytics.datasource.password", TIMESERIES::getPassword); + } +} diff --git a/server/src/test/java/org/eclipse/openvsx/analytics/DownloadAnalyticsDisabledTest.java b/server/src/test/java/org/eclipse/openvsx/analytics/DownloadAnalyticsDisabledTest.java index d3eb5966c..0e9d26419 100644 --- a/server/src/test/java/org/eclipse/openvsx/analytics/DownloadAnalyticsDisabledTest.java +++ b/server/src/test/java/org/eclipse/openvsx/analytics/DownloadAnalyticsDisabledTest.java @@ -12,6 +12,9 @@ *****************************************************************************/ package org.eclipse.openvsx.analytics; +import java.util.List; +import javax.sql.DataSource; + import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; @@ -21,6 +24,8 @@ import org.eclipse.openvsx.AbstractPostgresContainerTest; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @@ -51,4 +56,12 @@ void testNoAnalyticsBeansWhenDisabled() { assertTrue(context.getBeanNamesForType(DownloadAnalyticsService.class).length == 0); assertTrue(context.getBeanNamesForType(DownloadAnalyticsAPI.class).length == 0); } + + @Test + void testNoTimeseriesDatabaseWhenDisabled() { + // no second pool, and none of the ovsx.analytics.datasource.* properties are read + var dataSources = List.of(context.getBeanNamesForType(DataSource.class)); + assertEquals(1, dataSources.size()); + assertFalse(dataSources.contains("timeseriesDataSource")); + } } diff --git a/server/src/test/java/org/eclipse/openvsx/analytics/DownloadAnalyticsEndpointTest.java b/server/src/test/java/org/eclipse/openvsx/analytics/DownloadAnalyticsEndpointTest.java index d3594cff1..d9edfb0df 100644 --- a/server/src/test/java/org/eclipse/openvsx/analytics/DownloadAnalyticsEndpointTest.java +++ b/server/src/test/java/org/eclipse/openvsx/analytics/DownloadAnalyticsEndpointTest.java @@ -19,6 +19,7 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.webmvc.test.autoconfigure.AutoConfigureMockMvc; import org.springframework.jdbc.core.JdbcTemplate; @@ -29,7 +30,7 @@ import org.springframework.web.context.request.RequestContextHolder; import org.springframework.web.context.request.ServletRequestAttributes; -import org.eclipse.openvsx.AbstractPostgresContainerTest; +import org.eclipse.openvsx.AbstractTimeseriesContainerTest; import org.eclipse.openvsx.entities.Extension; import org.eclipse.openvsx.entities.ExtensionVersion; import org.eclipse.openvsx.entities.FileResource; @@ -43,12 +44,12 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; /** - * Full-stack proof of the enabled configuration: TimescaleDB-backed defaults wired by the - * auto-configuration, queried through the public REST endpoint. + * Full-stack proof of the enabled configuration: the TimescaleDB-backed repository on its own + * database, queried through the public REST endpoint. */ -@SpringBootTest(properties = "ovsx.analytics.enabled=true") +@SpringBootTest @AutoConfigureMockMvc -class DownloadAnalyticsEndpointTest extends AbstractPostgresContainerTest { +class DownloadAnalyticsEndpointTest extends AbstractTimeseriesContainerTest { @Autowired MockMvc mockMvc; @@ -57,6 +58,7 @@ class DownloadAnalyticsEndpointTest extends AbstractPostgresContainerTest { DownloadAnalyticsRepository repository; @Autowired + @Qualifier("timeseriesDataSource") javax.sql.DataSource dataSource; @Autowired @@ -117,9 +119,8 @@ void testUnknownExtensionIsNotFound() throws Exception { } /** - * Without a log-based source covering the file, a request-path download produces an - * analytics event in the same transaction as the counter update, with client data taken - * from the current HTTP request. + * Without a log-based source covering the file, a request-path download produces an analytics + * event alongside the counter update, with client data taken from the current HTTP request. */ @Test void testRequestPathDownloadProducesAnalyticsEvent() throws Exception { @@ -136,7 +137,7 @@ void testRequestPathDownloadProducesAnalyticsEvent() throws Exception { return null; }); - // the counter and the event committed together + // the counter committed in the registry database var downloadCount = inTransaction( () -> entityManager.find(Extension.class, extension.getId()).getDownloadCount()); assertEquals(1, downloadCount); diff --git a/server/src/test/java/org/eclipse/openvsx/analytics/timescale/TimescaleDownloadAnalyticsRepositoryTest.java b/server/src/test/java/org/eclipse/openvsx/analytics/timescale/TimescaleDownloadAnalyticsRepositoryTest.java index 513e5c075..d4d2ffe6f 100644 --- a/server/src/test/java/org/eclipse/openvsx/analytics/timescale/TimescaleDownloadAnalyticsRepositoryTest.java +++ b/server/src/test/java/org/eclipse/openvsx/analytics/timescale/TimescaleDownloadAnalyticsRepositoryTest.java @@ -17,15 +17,17 @@ import java.util.stream.IntStream; import javax.sql.DataSource; +import org.jooq.exception.DataAccessException; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.transaction.PlatformTransactionManager; import org.springframework.transaction.support.TransactionTemplate; -import org.eclipse.openvsx.AbstractPostgresContainerTest; +import org.eclipse.openvsx.AbstractTimeseriesContainerTest; import org.eclipse.openvsx.analytics.DownloadAnalyticsRepository; import org.eclipse.openvsx.analytics.DownloadEvent; import org.eclipse.openvsx.analytics.DownloadSeriesGroupBy; @@ -35,23 +37,24 @@ import static org.junit.jupiter.api.Assertions.*; -@SpringBootTest(properties = "ovsx.analytics.enabled=true") -class TimescaleDownloadAnalyticsRepositoryTest extends AbstractPostgresContainerTest { +@SpringBootTest +class TimescaleDownloadAnalyticsRepositoryTest extends AbstractTimeseriesContainerTest { @Autowired DownloadAnalyticsRepository repository; - @Autowired - DataSource dataSource; - @Autowired PlatformTransactionManager transactionManager; JdbcTemplate jdbc; + JdbcTemplate registryJdbc; + + // the time-series pool is not a default autowiring candidate; only the qualifier reaches it @Autowired - void initJdbc(DataSource dataSource) { - this.jdbc = new JdbcTemplate(dataSource); + void initJdbc(@Qualifier("timeseriesDataSource") DataSource timeseries, DataSource registry) { + this.jdbc = new JdbcTemplate(timeseries); + this.registryJdbc = new JdbcTemplate(registry); } @AfterEach @@ -71,14 +74,20 @@ void testMigrationApplied() { jdbc.queryForObject( "SELECT COUNT(*) FROM timescaledb_information.continuous_aggregates WHERE view_name = 'download_stats_daily'", Integer.class)); - // the analytics schema is part of the main migration chain + // the time-series database has its own migration chain, starting over at version 1 assertEquals( 1, jdbc.queryForObject( - "SELECT COUNT(*) FROM flyway_schema_history WHERE version = '1.71' AND success", + "SELECT COUNT(*) FROM flyway_schema_history WHERE version = '1' AND success", Integer.class)); } + @Test + void testAnalyticsSchemaIsAbsentFromRegistryDatabase() { + assertNull(registryJdbc.queryForObject("SELECT to_regclass('download_event')::text", String.class)); + assertNull(registryJdbc.queryForObject("SELECT to_regclass('download_stats_daily')::text", String.class)); + } + @Test void testSaveBatches() { var events = IntStream.range(0, 1500) @@ -108,7 +117,7 @@ void testSaveBatches() { } @Test - void testSaveJoinsCallerTransaction() { + void testSaveSurvivesRolledBackCallerTransaction() { var transaction = new TransactionTemplate(transactionManager); assertThrows(IllegalStateException.class, () -> transaction.execute(status -> { repository.save( @@ -122,7 +131,8 @@ void testSaveJoinsCallerTransaction() { throw new IllegalStateException("induced failure after save"); })); - assertEquals(0, jdbc.queryForObject("SELECT COUNT(*) FROM download_event", Integer.class)); + // the time-series database is not part of the registry transaction + assertEquals(1, jdbc.queryForObject("SELECT COUNT(*) FROM download_event", Integer.class)); } @Test From b84a70cd34fda795d5a0ef40e25dd07c2a492874 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jordi=20G=C3=B3mez?= Date: Mon, 24 Aug 2026 10:34:48 +0200 Subject: [PATCH 12/20] fix: save download events outside the registry transaction --- .../DownloadAnalyticsRepository.java | 6 +- .../ingestion/DownloadIngestionMetrics.java | 9 +++ .../ingestion/DownloadIngestionProcessor.java | 57 ++++++++++++---- .../ingestion/DownloadIngestionRunner.java | 11 +-- .../TimescaleDownloadAnalyticsRepository.java | 67 ++++++++++--------- .../DownloadIngestionMetricsTest.java | 8 +++ .../DownloadIngestionProcessorTest.java | 52 ++++++++++++-- ...escaleDownloadAnalyticsRepositoryTest.java | 19 ++++++ 8 files changed, 172 insertions(+), 57 deletions(-) diff --git a/server/src/main/java/org/eclipse/openvsx/analytics/DownloadAnalyticsRepository.java b/server/src/main/java/org/eclipse/openvsx/analytics/DownloadAnalyticsRepository.java index c54dc08eb..aa299191b 100644 --- a/server/src/main/java/org/eclipse/openvsx/analytics/DownloadAnalyticsRepository.java +++ b/server/src/main/java/org/eclipse/openvsx/analytics/DownloadAnalyticsRepository.java @@ -20,8 +20,10 @@ public interface DownloadAnalyticsRepository { /** - * Persists the given events. Implementations must participate in the caller's transaction, - * so that events, the extension download counter and the download ingestion entry commit atomically. + * Persists the given events atomically: either all of them are stored or none are. + * Implementations live in their own database and therefore cannot join the caller's registry + * transaction, so the extension download counter and the download ingestion entry commit + * independently of these events. */ void save(List events); diff --git a/server/src/main/java/org/eclipse/openvsx/analytics/ingestion/DownloadIngestionMetrics.java b/server/src/main/java/org/eclipse/openvsx/analytics/ingestion/DownloadIngestionMetrics.java index 7255392d9..d8af3e0c8 100644 --- a/server/src/main/java/org/eclipse/openvsx/analytics/ingestion/DownloadIngestionMetrics.java +++ b/server/src/main/java/org/eclipse/openvsx/analytics/ingestion/DownloadIngestionMetrics.java @@ -38,6 +38,7 @@ public class DownloadIngestionMetrics { public static final String SKIPPED_LINES_METRIC = "openvsx_analytics_log_lines_skipped_total"; public static final String EVENTS_METRIC = "openvsx_analytics_events_loaded_total"; public static final String DOWNLOADS_METRIC = "openvsx_analytics_downloads_loaded_total"; + public static final String FAILED_EVENTS_METRIC = "openvsx_analytics_events_failed_total"; public static final String EXTRACT_LAG_METRIC = "openvsx_analytics_extract_lag"; public static final String DEAD_LETTER_METRIC = "openvsx_analytics_dead_letter_depth"; @@ -47,6 +48,7 @@ public class DownloadIngestionMetrics { private final Counter skippedLines; private final Counter events; private final Counter downloads; + private final Counter failedEvents; private final Timer extractLag; public DownloadIngestionMetrics(MeterRegistry registry, RepositoryService repositories) { @@ -62,6 +64,9 @@ public DownloadIngestionMetrics(MeterRegistry registry, RepositoryService reposi this.downloads = Counter.builder(DOWNLOADS_METRIC) .description("Downloads counted by the ingestion pipeline") .register(registry); + this.failedEvents = Counter.builder(FAILED_EVENTS_METRIC) + .description("Aggregated download events lost because the analytics store rejected them") + .register(registry); this.extractLag = Timer.builder(EXTRACT_LAG_METRIC) .description("Delay between a download and its ingestion from access logs") .register(registry); @@ -93,6 +98,10 @@ public void recordLoaded(int eventCount, int downloadCount) { downloads.increment(downloadCount); } + public void recordFailedEvents(int eventCount) { + failedEvents.increment(eventCount); + } + public void recordExtractLag(Duration lag) { if (!lag.isNegative()) { extractLag.record(lag); diff --git a/server/src/main/java/org/eclipse/openvsx/analytics/ingestion/DownloadIngestionProcessor.java b/server/src/main/java/org/eclipse/openvsx/analytics/ingestion/DownloadIngestionProcessor.java index 4286d36f1..4e53b2ac8 100644 --- a/server/src/main/java/org/eclipse/openvsx/analytics/ingestion/DownloadIngestionProcessor.java +++ b/server/src/main/java/org/eclipse/openvsx/analytics/ingestion/DownloadIngestionProcessor.java @@ -94,14 +94,22 @@ public record ResolvedExtension( String targetPlatform ) {} + /** + * The registry-side outcome of processing one log file: the extensions whose counters changed + * (for cache eviction and search updates) and the aggregated events still to be handed to + * analytics. + */ + public record ProcessedFile(List extensions, List events) {} + /** * Processes one log file's download records: resolves vsix filenames to extension versions, - * aggregates them into hourly {@link DownloadEvent}s and, in a single transaction, saves the - * events, increments the extension download counters and writes the download ingestion entry. - * Returns the extensions whose counters changed, for cache eviction and search updates. + * aggregates them into hourly {@link DownloadEvent}s and, in a single transaction, increments + * the extension download counters and writes the download ingestion entry. The aggregated + * events are returned rather than stored, for {@link #saveEvents(List)} to persist once this + * transaction committed. */ @Transactional - public List process( + public ProcessedFile process( String storageType, String fileName, LocalDateTime processedOn, @@ -111,9 +119,6 @@ public List process( return Observation.createNotStarted("DownloadIngestionProcessor#process", observations).observe(() -> { var resolved = resolveExtensions(storageType, records); var events = aggregate(records, resolved); - if (!events.isEmpty()) { - analyticsRepository.ifAvailable(repository -> repository.save(events)); - } var extensionDownloads = events.stream().collect( Collectors.groupingBy(DownloadEvent::extensionId, Collectors.summingInt(DownloadEvent::count))); @@ -125,15 +130,35 @@ public List process( metrics.recordLoaded(events.size(), events.stream().mapToInt(DownloadEvent::count).sum()); records.stream().map(RawDownloadRecord::time).max(Instant::compareTo).ifPresent( latest -> metrics.recordExtractLag(Duration.between(latest, Instant.now()))); - return extensions; + return new ProcessedFile(extensions, events); + }); + } + + /** + * Stores the events of an already-committed {@link #process} call in the analytics database. + * Writing analytics after the registry commit under-counts if the analytics database fails - + * a visible gap, in a log file already marked processed and never retried - whereas writing it + * before would over-count on a registry rollback, with no idempotency key to dedupe on: + * silent corruption. Under-counting is the better failure. + */ + public void saveEvents(List events) { + if (events.isEmpty()) { + return; + } + + analyticsRepository.ifAvailable(repository -> { + try { + repository.save(events); + } catch (Exception e) { + logger.error("could not store {} download events for analytics", events.size(), e); + metrics.recordFailedEvents(events.size()); + } }); } /** * Records a single request-path download of a file that no {@link DownloadRecordSource} - * covers. Client IP and user agent are taken from the current HTTP request, if any. The - * event save participates in the caller's transaction, so it commits atomically with the - * download counter. + * covers. Client IP and user agent are taken from the current HTTP request, if any. */ public void captureDownload(FileResource resource) { analyticsRepository.ifAvailable(repository -> { @@ -159,8 +184,14 @@ public void captureDownload(FileResource resource) { clientIp(request), userAgent, 1); - repository.save(List.of(event)); - metrics.recordLoaded(1, 1); + try { + repository.save(List.of(event)); + metrics.recordLoaded(1, 1); + } catch (Exception e) { + // the caller is serving a download; an analytics outage must not fail it + logger.error("could not record the download of {} for analytics", resource.getName(), e); + metrics.recordFailedEvents(1); + } }); } diff --git a/server/src/main/java/org/eclipse/openvsx/analytics/ingestion/DownloadIngestionRunner.java b/server/src/main/java/org/eclipse/openvsx/analytics/ingestion/DownloadIngestionRunner.java index 026268b8c..659c57d6d 100644 --- a/server/src/main/java/org/eclipse/openvsx/analytics/ingestion/DownloadIngestionRunner.java +++ b/server/src/main/java/org/eclipse/openvsx/analytics/ingestion/DownloadIngestionRunner.java @@ -90,13 +90,16 @@ public void run(DownloadRecordSource source) { var executionTime = (int) stopWatch.lastTaskInfo().getTimeMillis(); if (records != null) { try { - // saves analytics events, increments download counters and writes the - // download ingestion entry in one transaction - var updatedExtensions = processor + // increments download counters and writes the download ingestion entry + // in one transaction + var processed = processor .process(storageType, name, processedOn, executionTime, records); - updatedExtensions + processed.extensions() .forEach(extension -> allUpdatedExtensions.put(extension.getId(), extension)); success = true; + // and only then hands the events to the analytics database, which + // cannot join that transaction + processor.saveEvents(processed.events()); } catch (Exception e) { logger.error("failed to process item: {}", name, e); } diff --git a/server/src/main/java/org/eclipse/openvsx/analytics/timescale/TimescaleDownloadAnalyticsRepository.java b/server/src/main/java/org/eclipse/openvsx/analytics/timescale/TimescaleDownloadAnalyticsRepository.java index 1bb9c83a8..efc0941d0 100644 --- a/server/src/main/java/org/eclipse/openvsx/analytics/timescale/TimescaleDownloadAnalyticsRepository.java +++ b/server/src/main/java/org/eclipse/openvsx/analytics/timescale/TimescaleDownloadAnalyticsRepository.java @@ -36,8 +36,9 @@ * separate time-series database, addressed by name rather than through generated jOOQ classes * (codegen runs against the registry database, which no longer holds these tables). *

- * Writes run on the time-series connection pool, so they cannot join a caller's registry - * transaction: an event is persisted independently of whatever the registry does afterwards. + * Writes run on the time-series connection pool and cannot join a caller's registry transaction: + * one {@link #save(List)} call is one transaction of its own, atomic across its batches and + * independent of whatever the registry does afterwards. */ public class TimescaleDownloadAnalyticsRepository implements DownloadAnalyticsRepository { @@ -88,37 +89,39 @@ public TimescaleDownloadAnalyticsRepository(DSLContext dsl) { @Override public void save(List events) { - for (var batch : Lists.partition(events, BATCH_SIZE)) { - var insert = dsl - .insertInto( - EVENT, - EVENT_TIME, - EVENT_EXTENSION_ID, - EVENT_EXTENSION_VERSION_ID, - EVENT_NAMESPACE, - EVENT_EXTENSION_NAME, - EVENT_VERSION, - EVENT_TARGET_PLATFORM, - EVENT_COUNTRY, - EVENT_IP, - EVENT_USER_AGENT, - EVENT_COUNT); - for (var event : batch) { - insert = insert.values( - OffsetDateTime.ofInstant(event.time(), ZoneOffset.UTC), - event.extensionId(), - event.extensionVersionId(), - event.namespace(), - event.extensionName(), - event.version(), - event.targetPlatform(), - event.country(), - event.ip(), - event.userAgent(), - event.count()); + dsl.transaction(configuration -> { + for (var batch : Lists.partition(events, BATCH_SIZE)) { + var insert = DSL.using(configuration) + .insertInto( + EVENT, + EVENT_TIME, + EVENT_EXTENSION_ID, + EVENT_EXTENSION_VERSION_ID, + EVENT_NAMESPACE, + EVENT_EXTENSION_NAME, + EVENT_VERSION, + EVENT_TARGET_PLATFORM, + EVENT_COUNTRY, + EVENT_IP, + EVENT_USER_AGENT, + EVENT_COUNT); + for (var event : batch) { + insert = insert.values( + OffsetDateTime.ofInstant(event.time(), ZoneOffset.UTC), + event.extensionId(), + event.extensionVersionId(), + event.namespace(), + event.extensionName(), + event.version(), + event.targetPlatform(), + event.country(), + event.ip(), + event.userAgent(), + event.count()); + } + insert.execute(); } - insert.execute(); - } + }); } @Override diff --git a/server/src/test/java/org/eclipse/openvsx/analytics/ingestion/DownloadIngestionMetricsTest.java b/server/src/test/java/org/eclipse/openvsx/analytics/ingestion/DownloadIngestionMetricsTest.java index 8e0fbf0fc..c2ce1a9ac 100644 --- a/server/src/test/java/org/eclipse/openvsx/analytics/ingestion/DownloadIngestionMetricsTest.java +++ b/server/src/test/java/org/eclipse/openvsx/analytics/ingestion/DownloadIngestionMetricsTest.java @@ -46,6 +46,14 @@ void testLoadVolumeCounters() { assertEquals(30, registry.counter(DownloadIngestionMetrics.DOWNLOADS_METRIC).count()); } + @Test + void testFailedEventsCounter() { + metrics.recordFailedEvents(3); + metrics.recordFailedEvents(1); + + assertEquals(4, registry.counter(DownloadIngestionMetrics.FAILED_EVENTS_METRIC).count()); + } + @Test void testExtractLagTimer() { metrics.recordExtractLag(Duration.ofMinutes(10)); diff --git a/server/src/test/java/org/eclipse/openvsx/analytics/ingestion/DownloadIngestionProcessorTest.java b/server/src/test/java/org/eclipse/openvsx/analytics/ingestion/DownloadIngestionProcessorTest.java index 446effa52..89a0f2117 100644 --- a/server/src/test/java/org/eclipse/openvsx/analytics/ingestion/DownloadIngestionProcessorTest.java +++ b/server/src/test/java/org/eclipse/openvsx/analytics/ingestion/DownloadIngestionProcessorTest.java @@ -68,6 +68,7 @@ class DownloadIngestionProcessorTest extends AbstractPostgresContainerTest { @AfterEach void cleanUp() { + analyticsRepository.failing = false; analyticsRepository.saved.clear(); runInTransaction(() -> { seededEntities.reversed().forEach(entity -> { @@ -81,7 +82,7 @@ void cleanUp() { } @Test - void testProcessAggregatesSavesAndCommitsAtomically() { + void testProcessAggregatesAndDefersAnalytics() { var extension = seedExtension("proc1", "proc1.ext-1.0.0.vsix"); var hour1 = Instant.parse("2026-07-01T14:00:00Z"); @@ -96,10 +97,15 @@ void testProcessAggregatesSavesAndCommitsAtomically() { "9.9.9.9", "VSCode 1.90.2")); - var updated = processor.process(FileResource.STORAGE_AWS, "analytics-test-1.gz", PROCESSED_ON, 5, records); + var processed = processor.process(FileResource.STORAGE_AWS, "analytics-test-1.gz", PROCESSED_ON, 5, records); - assertEquals(1, updated.size()); - assertEquals(extension.getId(), updated.get(0).getId()); + assertEquals(1, processed.extensions().size()); + assertEquals(extension.getId(), processed.extensions().get(0).getId()); + + // the events are handed back rather than stored, so the analytics write happens after + // the registry transaction committed + assertTrue(analyticsRepository.saved.isEmpty()); + processor.saveEvents(processed.events()); // micro-batch aggregation by (hour, extension-version, country, ip, user agent) assertEquals(3, analyticsRepository.saved.size()); @@ -118,7 +124,7 @@ void testProcessAggregatesSavesAndCommitsAtomically() { var laterHour = findEvent(hour1.plusSeconds(3600), "US", "VSCode 1.90.2"); assertEquals(1, laterHour.count()); - // the download counter is incremented by the total record count in the same transaction + // the download counter is incremented by the total record count assertEquals(4, freshDownloadCount(extension.getId())); // and the ingestion entry is written @@ -135,7 +141,9 @@ void testUnknownFileIsSkipped() { var records = List.of( new RawDownloadRecord(Instant.parse("2026-07-01T14:00:00Z"), "NO.SUCH-1.0.0.VSIX", null, null, null)); - processor.process(FileResource.STORAGE_AWS, "analytics-test-2.gz", PROCESSED_ON, 5, records); + var processed = processor + .process(FileResource.STORAGE_AWS, "analytics-test-2.gz", PROCESSED_ON, 5, records); + processor.saveEvents(processed.events()); assertTrue(analyticsRepository.saved.isEmpty()); assertEquals(0, freshDownloadCount(extension.getId())); @@ -189,6 +197,32 @@ void testInducedFailureRollsBackWholeTransaction() { List.of(overlongName)).isEmpty()); } + @Test + void testAnalyticsFailureLeavesIngestionIntact() { + var extension = seedExtension("proc5", "proc5.ext-1.0.0.vsix"); + + var records = List.of( + new RawDownloadRecord( + Instant.parse("2026-07-01T14:00:00Z"), + "PROC5.EXT-1.0.0.VSIX", + "US", + "9.9.9.9", + null)); + var processed = processor + .process(FileResource.STORAGE_AWS, "analytics-test-5.gz", PROCESSED_ON, 5, records); + + // the analytics store is a separate database; losing it under-counts, nothing more + analyticsRepository.failing = true; + assertDoesNotThrow(() -> processor.saveEvents(processed.events())); + + assertEquals(1, freshDownloadCount(extension.getId())); + assertEquals( + List.of("analytics-test-5.gz"), + repositories.findAllSucceededDownloadIngestionsByStorageTypeAndNameIn( + FileResource.STORAGE_AWS, + List.of("analytics-test-5.gz"))); + } + private DownloadEvent findEvent(Instant time, String country, String userAgent) { return analyticsRepository.saved.stream() .filter(event -> event.time().equals(time)) @@ -257,8 +291,14 @@ RecordingAnalyticsRepository recordingAnalyticsRepository() { static class RecordingAnalyticsRepository implements DownloadAnalyticsRepository { final List saved = new CopyOnWriteArrayList<>(); + volatile boolean failing; + @Override public void save(List events) { + if (failing) { + throw new IllegalStateException("analytics database is unreachable"); + } + saved.addAll(events); } diff --git a/server/src/test/java/org/eclipse/openvsx/analytics/timescale/TimescaleDownloadAnalyticsRepositoryTest.java b/server/src/test/java/org/eclipse/openvsx/analytics/timescale/TimescaleDownloadAnalyticsRepositoryTest.java index d4d2ffe6f..043943d88 100644 --- a/server/src/test/java/org/eclipse/openvsx/analytics/timescale/TimescaleDownloadAnalyticsRepositoryTest.java +++ b/server/src/test/java/org/eclipse/openvsx/analytics/timescale/TimescaleDownloadAnalyticsRepositoryTest.java @@ -135,6 +135,25 @@ void testSaveSurvivesRolledBackCallerTransaction() { assertEquals(1, jdbc.queryForObject("SELECT COUNT(*) FROM download_event", Integer.class)); } + @Test + void testSaveIsAtomicAcrossItsBatches() { + // version is VARCHAR(255), so the 501st event lands in a second batch and fails it + var overlongVersion = "1.0.0-" + "x".repeat(300); + var events = IntStream.rangeClosed(0, 500) + .mapToObj( + i -> event( + Instant.parse("2026-07-01T00:00:00Z").plusSeconds(i * 60L), + 1L, + i == 500 ? overlongVersion : "1.0.0", + "US", + 1)) + .toList(); + + assertThrows(DataAccessException.class, () -> repository.save(events)); + + assertEquals(0, jdbc.queryForObject("SELECT COUNT(*) FROM download_event", Integer.class)); + } + @Test void testFindSeriesByDay() { repository.save( From 48677728005adab73065fb0ec3180f59fd47b889 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jordi=20G=C3=B3mez?= Date: Mon, 24 Aug 2026 10:34:51 +0200 Subject: [PATCH 13/20] feat: make the download series publicly cacheable --- .../openvsx/analytics/DownloadAnalyticsAPI.java | 8 +++++++- .../openvsx/analytics/DownloadAnalyticsAPITest.java | 10 ++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/eclipse/openvsx/analytics/DownloadAnalyticsAPI.java b/server/src/main/java/org/eclipse/openvsx/analytics/DownloadAnalyticsAPI.java index 822bf9f0c..b32bcb9e4 100644 --- a/server/src/main/java/org/eclipse/openvsx/analytics/DownloadAnalyticsAPI.java +++ b/server/src/main/java/org/eclipse/openvsx/analytics/DownloadAnalyticsAPI.java @@ -16,6 +16,7 @@ import java.time.LocalDate; import java.time.ZoneOffset; import java.time.format.DateTimeParseException; +import java.util.concurrent.TimeUnit; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Parameter; @@ -24,6 +25,7 @@ import io.swagger.v3.oas.annotations.responses.ApiResponse; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.http.CacheControl; import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; @@ -116,7 +118,11 @@ public ResponseEntity getDownloads( LocalDate.ofInstant(point.bucketStart(), ZoneOffset.UTC).toString(), point.count())) .toList(); - return ResponseEntity.ok(new DownloadSeriesJson(points)); + // Aggregate, non-personal data that is identical for every caller, so it is publicly + // cacheable. Without an explicit value Spring Security defaults the response to no-store. + return ResponseEntity.ok() + .cacheControl(CacheControl.maxAge(10, TimeUnit.MINUTES).cachePublic()) + .body(new DownloadSeriesJson(points)); } private DownloadSeriesRequest buildRequest(long extensionId, String from, String to, String interval) { diff --git a/server/src/test/java/org/eclipse/openvsx/analytics/DownloadAnalyticsAPITest.java b/server/src/test/java/org/eclipse/openvsx/analytics/DownloadAnalyticsAPITest.java index 1f4fd440b..8c6416460 100644 --- a/server/src/test/java/org/eclipse/openvsx/analytics/DownloadAnalyticsAPITest.java +++ b/server/src/test/java/org/eclipse/openvsx/analytics/DownloadAnalyticsAPITest.java @@ -31,6 +31,7 @@ import static org.mockito.ArgumentMatchers.any; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.header; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; class DownloadAnalyticsAPITest { @@ -70,6 +71,15 @@ void testResponseShape() throws Exception { true)); } + @Test + void testSeriesIsPubliclyCacheable() throws Exception { + Mockito.when(service.getSeries(any())).thenReturn(List.of()); + + mockMvc.perform(get("/api/foo/bar/analytics/downloads")) + .andExpect(status().isOk()) + .andExpect(header().string("Cache-Control", "max-age=600, public")); + } + @Test void testRequestParametersArePassedToService() throws Exception { Mockito.when(service.getSeries(any())).thenReturn(List.of()); From 3485e528cd7d114d93f1aa2106d4f440bcee17ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jordi=20G=C3=B3mez?= Date: Mon, 24 Aug 2026 10:41:00 +0200 Subject: [PATCH 14/20] docs: fold the changelog entries into the existing next section --- webui/CHANGELOG.md | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/webui/CHANGELOG.md b/webui/CHANGELOG.md index abe19eaf2..26df6ff7e 100644 --- a/webui/CHANGELOG.md +++ b/webui/CHANGELOG.md @@ -9,6 +9,12 @@ This change log covers only the frontend library (webui) of Open VSX. - Add a "Data Consistency" page to the admin dashboard (#1622): a live overview of every registered consistency check's finding count, with actions to refresh it and to fix findings one at a time or all at once - Show a "Namespace not verified" state on an extension card when it can't be activated because its namespace already exists in a referenced external gallery and hasn't been verified, in both the "My Extensions" and namespace member extension lists. The card keeps its colour and takes a warning-toned frame and icon, since this is the publisher's to fix rather than an extension that is simply switched off - Show a warning notice with a claim action wherever an unverified namespace is holding something back — the extension settings page when the extension has a namespace ownership conflict, and the namespace settings page for any unverified namespace — making clear the namespace must be claimed (verified) first. The action is the deployment's configured `elements.claimNamespace`, falling back to the namespace access documentation when none is configured. The admin dashboard's extension and namespace views show the same explanation without the claim action, since claiming is the publisher's action to take, not an admin's on someone else's behalf +- Add a `Pill` component — the clickable glass pill the category pills are built on, now usable on its own — and extract the `MonoSlash`, `glassSurface` and `compactControl` page primitives out of the search field, the pills and the search header +- Add `userLoading` to `MainContext`, so custom pages can tell "not logged in" from "still resolving the user" +- Add a `userMenuContent` slot to `PageSettings.elements`: extra entries for the logged-in account menu, rendered above the admin entry. The slot receives a `MenuEntry` component to build entries with, so each entry is styled by the menu it appears in — the desktop and mobile menus style theirs differently, and a consumer cannot match both on its own +- Add an `adminPages` slot to `PageSettings.elements`: extra admin dashboard pages, each declaring a name, icon, optional description and optional category, and each appearing in the side panel, as a card on the dashboard overview and as a route. Contributions are additive — a category name matching a built-in group appends to it, and a page whose path would shadow a built-in one is ignored +- Widen the published API for consumers building their own pages: the request layer (`sendRequest`, `sendNonRetriableRequest`, `ErrorResponse`, `controllerFromSignal`), `MainContext`, `AppProviders`, `NotFound`, `createDefaultTheme` with the `MONO_FONT`/`NAVBAR_HEIGHT` tokens, the `createRoute`/`createAbsoluteURL`/`addQuery`/`formatCompactNumber`/`toRelativeTime` utils, the `useDebouncedCallback` and `useGridCursor` hooks, the navbar-chrome, search-focus and page-search-bar hooks, the category icon helpers, `ExtensionDetailRoutes`, and the `itemIcon`/`MenuItemText` building blocks for `userMenuContent` entries +- Add a weekly downloads card to the extension detail page, shown only when the registry reports download analytics as enabled: the last 7 days' downloads, a sparkline of the weekly totals for the year behind it, and the period the headline covers. Hovering moves a marker line and reads out that week instead, and the card shows a skeleton in the same shape while the series loads ### Changed @@ -19,18 +25,6 @@ This change log covers only the frontend library (webui) of Open VSX. - Add an `outlinedWarning` style to `MuiButton`, so a warning-toned outlined button follows the theme like the secondary and error ones instead of MUI's default half-opacity border - **Breaking:** `elements.claimNamespace` now receives `{ namespace, extension?, sx? }` instead of `{ extension, sx? }`. The namespace settings page offers the same claim action and has no extension to pass, so implementations must read the namespace from `namespace` rather than `extension.namespace` - `ExtensionCard` accepts an `Extension` as well as a `SearchEntry`, and takes optional `to`, `linkState`, `overlay`, `footerStart` and `dimmed` props so other surfaces can reuse it instead of copying it - -### Added - -- Add a `Pill` component — the clickable glass pill the category pills are built on, now usable on its own — and extract the `MonoSlash`, `glassSurface` and `compactControl` page primitives out of the search field, the pills and the search header -- Add `userLoading` to `MainContext`, so custom pages can tell "not logged in" from "still resolving the user" -- Add a `userMenuContent` slot to `PageSettings.elements`: extra entries for the logged-in account menu, rendered above the admin entry. The slot receives a `MenuEntry` component to build entries with, so each entry is styled by the menu it appears in — the desktop and mobile menus style theirs differently, and a consumer cannot match both on its own -- Add an `adminPages` slot to `PageSettings.elements`: extra admin dashboard pages, each declaring a name, icon, optional description and optional category, and each appearing in the side panel, as a card on the dashboard overview and as a route. Contributions are additive — a category name matching a built-in group appends to it, and a page whose path would shadow a built-in one is ignored -- Widen the published API for consumers building their own pages: the request layer (`sendRequest`, `sendNonRetriableRequest`, `ErrorResponse`, `controllerFromSignal`), `MainContext`, `AppProviders`, `NotFound`, `createDefaultTheme` with the `MONO_FONT`/`NAVBAR_HEIGHT` tokens, the `createRoute`/`createAbsoluteURL`/`addQuery`/`formatCompactNumber`/`toRelativeTime` utils, the `useDebouncedCallback` and `useGridCursor` hooks, the navbar-chrome, search-focus and page-search-bar hooks, the category icon helpers, `ExtensionDetailRoutes`, and the `itemIcon`/`MenuItemText` building blocks for `userMenuContent` entries -- Add a weekly downloads card to the extension detail page, shown only when the registry reports download analytics as enabled: the last 7 days' downloads, a sparkline of the weekly totals for the year behind it, and the period the headline covers. Hovering moves a marker line and reads out that week instead, and the card shows a skeleton in the same shape while the series loads - -### Changed - - Rename `ScrollToTop` to `ScrollRestoration`, matching what it does on back/forward navigation - Rename the extension tint context to `navbar-chrome-context` and add a second channel to it: a page with sections pinned under the navbar can extend the navbar's blur fan down to back them (`useExtendNavbarBlur`) - Give Popover and Autocomplete popups the same floating-paper treatment as the other menus, and stop Popovers locking body scroll — the lock jumps the scroll position on mobile and shifts the pinned chrome From 35e5126e36e744c6da0f8d3d25a85ec96527b03a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jordi=20G=C3=B3mez?= Date: Mon, 24 Aug 2026 10:59:54 +0200 Subject: [PATCH 15/20] test: cap datasource pools so the suite does not exhaust postgres connections --- .../org/eclipse/openvsx/AbstractTimeseriesContainerTest.java | 1 + server/src/test/resources/application.yml | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/server/src/test/java/org/eclipse/openvsx/AbstractTimeseriesContainerTest.java b/server/src/test/java/org/eclipse/openvsx/AbstractTimeseriesContainerTest.java index 39abb55db..f96bcff76 100644 --- a/server/src/test/java/org/eclipse/openvsx/AbstractTimeseriesContainerTest.java +++ b/server/src/test/java/org/eclipse/openvsx/AbstractTimeseriesContainerTest.java @@ -44,5 +44,6 @@ static void timeseriesProperties(DynamicPropertyRegistry registry) { registry.add("ovsx.analytics.datasource.url", TIMESERIES::getJdbcUrl); registry.add("ovsx.analytics.datasource.username", TIMESERIES::getUsername); registry.add("ovsx.analytics.datasource.password", TIMESERIES::getPassword); + registry.add("ovsx.analytics.datasource.maximum-pool-size", () -> 2); } } diff --git a/server/src/test/resources/application.yml b/server/src/test/resources/application.yml index ddef45cba..eb46fd574 100644 --- a/server/src/test/resources/application.yml +++ b/server/src/test/resources/application.yml @@ -1,4 +1,9 @@ spring: + # Every cached Spring context in the suite holds an idle pool against the one shared container, + # so the default of 10 exhausts its max_connections long before the tests are done. + datasource: + hikari: + maximum-pool-size: 4 jpa: properties: hibernate: From 2d2937f283c19671c43d69b3dfc3f2cfe7009ab7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jordi=20G=C3=B3mez=20Hidalgo?= <31970428+gnugomez@users.noreply.github.com> Date: Mon, 31 Aug 2026 10:36:16 +0200 Subject: [PATCH 16/20] Potential fix for pull request finding MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> Signed-off-by: Jordi Gómez Hidalgo <31970428+gnugomez@users.noreply.github.com> --- webui/src/pages/admin-dashboard/admin-dashboard.tsx | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/webui/src/pages/admin-dashboard/admin-dashboard.tsx b/webui/src/pages/admin-dashboard/admin-dashboard.tsx index 2246c90e2..0c9820e01 100644 --- a/webui/src/pages/admin-dashboard/admin-dashboard.tsx +++ b/webui/src/pages/admin-dashboard/admin-dashboard.tsx @@ -196,7 +196,11 @@ export const AdminDashboard: FunctionComponent = props => { const adminPages = pageSettings.elements.adminPages; const contributed = useMemo( - () => (adminPages ?? []).filter(page => !builtInSegments.has(page.path.split('/')[0])), + () => + (adminPages ?? []).filter(page => { + const segment = page.path.split('/')[0].toLowerCase(); + return segment.length > 0 && !builtInSegments.has(segment); + }), [adminPages] ); const navItems = useMemo(() => withContributedPages(contributed), [contributed]); From 2f859ffc798a08cb1624712793103669e097d0c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jordi=20G=C3=B3mez?= Date: Mon, 31 Aug 2026 11:05:31 +0200 Subject: [PATCH 17/20] fix: materialize download events that arrive outside the refresh window --- .../migration-timeseries/V1__Download_Analytics.sql | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/server/src/main/resources/db/migration-timeseries/V1__Download_Analytics.sql b/server/src/main/resources/db/migration-timeseries/V1__Download_Analytics.sql index 42441d0ac..cf8956920 100644 --- a/server/src/main/resources/db/migration-timeseries/V1__Download_Analytics.sql +++ b/server/src/main/resources/db/migration-timeseries/V1__Download_Analytics.sql @@ -35,8 +35,19 @@ FROM download_event GROUP BY time_bucket('1 day', time), extension_id, extension_version_id, version, target_platform, country WITH NO DATA; +-- Materialize what is already there before the policy takes over. Until its first run the +-- watermark sits at -infinity and real-time aggregation answers everything, so the gap only +-- opens once the policy advances it: from then on buckets below the watermark are served from +-- the materialization alone, and anything never materialized reads as zero. +CALL refresh_continuous_aggregate('download_stats_daily', NULL, NULL); + +-- start_offset tracks the raw retention below rather than the schedule. Log ingestion applies no +-- date filter, so a delayed or backfilled file writes events well outside a short window, and +-- once the watermark has passed them they would never materialize while the raw rows are dropped +-- at 90 days. A refresh only reprocesses invalidated ranges, so the wider window costs nothing +-- when nothing old changed. SELECT add_continuous_aggregate_policy('download_stats_daily', - start_offset => INTERVAL '3 days', + start_offset => INTERVAL '90 days', end_offset => INTERVAL '1 hour', schedule_interval => INTERVAL '1 hour'); From 4ec48b1f1db1a6557b9e2afed969ce401b3552b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jordi=20G=C3=B3mez?= Date: Mon, 31 Aug 2026 11:05:31 +0200 Subject: [PATCH 18/20] fix: apply review fixes --- .../analytics/DownloadAnalyticsAPI.java | 2 +- .../ingestion/DownloadIngestionProcessor.java | 4 ++- .../aws/AwsDownloadRecordSource.java | 27 ++++++++++--------- .../TimeseriesDatabaseConfiguration.java | 12 +++++++++ 4 files changed, 31 insertions(+), 14 deletions(-) diff --git a/server/src/main/java/org/eclipse/openvsx/analytics/DownloadAnalyticsAPI.java b/server/src/main/java/org/eclipse/openvsx/analytics/DownloadAnalyticsAPI.java index b32bcb9e4..f3391fa2f 100644 --- a/server/src/main/java/org/eclipse/openvsx/analytics/DownloadAnalyticsAPI.java +++ b/server/src/main/java/org/eclipse/openvsx/analytics/DownloadAnalyticsAPI.java @@ -92,7 +92,7 @@ public ResponseEntity getDownloads( @Parameter(description = "Extension name", example = "java") String extension, @RequestParam(required = false) @Parameter( - description = "UTC start date (inclusive), defaults to 30 buckets before 'to'", + description = "UTC start date (inclusive), defaults to 30 days before 'to' whatever the interval", example = "2026-06-16" ) String from, @RequestParam(required = false) diff --git a/server/src/main/java/org/eclipse/openvsx/analytics/ingestion/DownloadIngestionProcessor.java b/server/src/main/java/org/eclipse/openvsx/analytics/ingestion/DownloadIngestionProcessor.java index 4e53b2ac8..d1281da1c 100644 --- a/server/src/main/java/org/eclipse/openvsx/analytics/ingestion/DownloadIngestionProcessor.java +++ b/server/src/main/java/org/eclipse/openvsx/analytics/ingestion/DownloadIngestionProcessor.java @@ -127,7 +127,6 @@ public ProcessedFile process( : increaseDownloadCounts(extensionDownloads); persistIngestion(fileName, storageType, processedOn, executionTime, true); - metrics.recordLoaded(events.size(), events.stream().mapToInt(DownloadEvent::count).sum()); records.stream().map(RawDownloadRecord::time).max(Instant::compareTo).ifPresent( latest -> metrics.recordExtractLag(Duration.between(latest, Instant.now()))); return new ProcessedFile(extensions, events); @@ -149,6 +148,9 @@ public void saveEvents(List events) { analyticsRepository.ifAvailable(repository -> { try { repository.save(events); + // counted here rather than in process(): with analytics disabled or the write + // failing, nothing was loaded and the counter must not say otherwise + metrics.recordLoaded(events.size(), events.stream().mapToInt(DownloadEvent::count).sum()); } catch (Exception e) { logger.error("could not store {} download events for analytics", events.size(), e); metrics.recordFailedEvents(events.size()); diff --git a/server/src/main/java/org/eclipse/openvsx/analytics/ingestion/aws/AwsDownloadRecordSource.java b/server/src/main/java/org/eclipse/openvsx/analytics/ingestion/aws/AwsDownloadRecordSource.java index d29270e21..36eca5c57 100644 --- a/server/src/main/java/org/eclipse/openvsx/analytics/ingestion/aws/AwsDownloadRecordSource.java +++ b/server/src/main/java/org/eclipse/openvsx/analytics/ingestion/aws/AwsDownloadRecordSource.java @@ -143,18 +143,21 @@ public List next() { @Override public List read(String name) throws IOException { - var inputStream = getS3Client().getObject( - GetObjectRequest.builder() - .bucket(bucket) - .key(name) - .build(), - ResponseTransformer.toInputStream()); - - // records without their own timestamp fall back to the log file's date - var lastModified = inputStream.response().lastModified(); - var fallbackTime = lastModified != null ? lastModified : Instant.now(); - - try (var downloadsTempFile = new TempFile("aws-downloads-", ".gz")) { + try ( + // the response keeps an HTTP connection checked out until it is closed; leaking one + // per log object exhausts the S3 client's pool and stalls ingestion + var inputStream = getS3Client().getObject( + GetObjectRequest.builder() + .bucket(bucket) + .key(name) + .build(), + ResponseTransformer.toInputStream()); + var downloadsTempFile = new TempFile("aws-downloads-", ".gz"); + ) { + // records without their own timestamp fall back to the log file's date + var lastModified = inputStream.response().lastModified(); + var fallbackTime = lastModified != null ? lastModified : Instant.now(); + Files.copy(inputStream, downloadsTempFile.getPath(), StandardCopyOption.REPLACE_EXISTING); try ( var fileStream = new FileInputStream(downloadsTempFile.getPath().toFile()); diff --git a/server/src/main/java/org/eclipse/openvsx/analytics/timescale/TimeseriesDatabaseConfiguration.java b/server/src/main/java/org/eclipse/openvsx/analytics/timescale/TimeseriesDatabaseConfiguration.java index c7e38e991..b42089009 100644 --- a/server/src/main/java/org/eclipse/openvsx/analytics/timescale/TimeseriesDatabaseConfiguration.java +++ b/server/src/main/java/org/eclipse/openvsx/analytics/timescale/TimeseriesDatabaseConfiguration.java @@ -38,6 +38,11 @@ class TimeseriesDatabaseConfiguration { private static final int DEFAULT_POOL_SIZE = 5; + // A download records its event on the request path, so an unreachable time-series database has + // to surface as a fast failure the caller can swallow. Hikari's 30 second default would hold a + // request thread for that long on every download until the outage ends. + private static final long DEFAULT_CONNECTION_TIMEOUT_MS = 2_000L; + // defaultCandidate = false keeps these beans invisible to @ConditionalOnMissingBean and to // plain by-type injection, so Boot still auto-configures the primary DataSource, the main // Flyway chain and the primary DSLContext; only an explicit @Qualifier reaches them. @@ -53,6 +58,13 @@ DataSource timeseriesDataSource(Environment environment) { "ovsx.analytics.datasource.maximum-pool-size", Integer.class, DEFAULT_POOL_SIZE)); + var connectionTimeout = environment.getProperty( + "ovsx.analytics.datasource.connection-timeout", + Long.class, + DEFAULT_CONNECTION_TIMEOUT_MS); + config.setConnectionTimeout(connectionTimeout); + // Hikari rejects a validation timeout that is not below the connection timeout + config.setValidationTimeout(Math.max(250L, connectionTimeout / 2)); return new HikariDataSource(config); } From 2312570ebc9a13b66a3b3d4bc7e332eb054339e5 Mon Sep 17 00:00:00 2001 From: Thomas Neidhart Date: Mon, 31 Aug 2026 21:52:07 +0200 Subject: [PATCH 19/20] fix import --- webui/src/default/menu-content.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/webui/src/default/menu-content.tsx b/webui/src/default/menu-content.tsx index a09b656c3..75bc1435a 100644 --- a/webui/src/default/menu-content.tsx +++ b/webui/src/default/menu-content.tsx @@ -20,7 +20,6 @@ import { } from 'react'; import { Avatar, Button, IconButton, Link, Menu, MenuItem, Typography } from '@mui/material'; import { useLocation, useNavigate, Link as RouteLink } from 'react-router'; -import { FunctionComponent, PropsWithChildren, useContext, useRef, useState } from 'react'; import { UserAvatar } from '../pages/user/avatar'; import { UserSettingsRoutes } from '../pages/user/user-settings-routes'; import { PublishRoutes } from '../pages/publish/publish-routes'; From 4feac79e4649522dc5094b2cd5a90077e7d244ac Mon Sep 17 00:00:00 2001 From: Thomas Neidhart Date: Mon, 31 Aug 2026 21:59:01 +0200 Subject: [PATCH 20/20] fix unused imports --- webui/src/default/menu-content.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/webui/src/default/menu-content.tsx b/webui/src/default/menu-content.tsx index 75bc1435a..9fdb20fbf 100644 --- a/webui/src/default/menu-content.tsx +++ b/webui/src/default/menu-content.tsx @@ -18,8 +18,8 @@ import { useRef, useState } from 'react'; -import { Avatar, Button, IconButton, Link, Menu, MenuItem, Typography } from '@mui/material'; -import { useLocation, useNavigate, Link as RouteLink } from 'react-router'; +import { Avatar, IconButton, Link, Menu, MenuItem, Typography } from '@mui/material'; +import { useLocation, Link as RouteLink } from 'react-router'; import { UserAvatar } from '../pages/user/avatar'; import { UserSettingsRoutes } from '../pages/user/user-settings-routes'; import { PublishRoutes } from '../pages/publish/publish-routes';