jpo-deduplicator/sample.env at develop · CDOT-CV/jpo-deduplicator · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
### COMMON variables - START ###
# (Required) The IP address of Docker host machine which can be found by running "ifconfig"
# Hint: look for "inet addr:" within "eth0" or "en0" for OSX


# Set the HOST IP of where the containers are running
DOCKER_HOST_IP=
KAFKA_BOOTSTRAP_SERVERS=${DOCKER_HOST_IP}:9092

# GitHub properties for pulling the latest version of the JPO-ODE
# Required for building the deduplicator. Documentation: https://docs.github.com/en/packages/working-with-a-github-packages-registry/working-with-the-apache-maven-registry#authenticating-to-github-packages
MAVEN_GITHUB_TOKEN=
MAVEN_GITHUB_ORG=usdot-jpo-ode


# Optional Variables to configure which version
DOCKERHUB_HOST=usdotjpoode
DOCKERHUB_RELEASE=2025-q3

# Docker compose restart policy: https://docs.docker.com/engine/containers/start-containers-automatically/
RESTART_POLICY="on-failure:3"

# Available profiles:
# - all
#   - deduplicator_base
#       - deduplicator
#   - deduplicator_full
#       - deduplicator
#       - geojsonconverter
#       - ode
#       - adm
#
# All profiles from the jpo-utils repo are also allowed
# Example: COMPOSE_PROFILES=kafka,kafka_ui,kafka_setup, deduplicator_full
COMPOSE_PROFILES=kafka,kafka_ui,kafka_setup, deduplicator_full


# Set to "CONFLUENT" if broker is a Confluent Cloud broker
KAFKA_TYPE=

# Set to actual Confluent Cloud access key and secret values for SASL authentication
CONFLUENT_KEY=
CONFLUENT_SECRET=

KAFKA_TOPIC_CREATE_DEDUPLICATOR=true
CONNECT_CREATE_DEDUPLICATOR=true

# Deduplicator Feature Flags
ENABLE_PROCESSED_MAP_DEDUPLICATION=true
ENABLE_PROCESSED_MAP_WKT_DEDUPLICATION=true
ENABLE_ODE_MAP_DEDUPLICATION=true
ENABLE_ODE_TIM_DEDUPLICATION=true
ENABLE_PROCESSED_SPAT_DEDUPLICATION=true
ENABLE_ODE_BSM_DEDUPLICATION=true
ENABLE_PROCESSED_BSM_DEDUPLICATION=true

# RocksDB Bounded Memory Config Properties
# 128 MB = 134217728
# 64 MB = 67108864
# 16 MB = 16777216
DEDUPLICATOR_ROCKSDB_TOTAL_OFF_HEAP_MEMORY=134217728
DEDUPLICATOR_ROCKSDB_INDEX_FILTER_BLOCK_RATIO=0.1
DEDUPLICATOR_ROCKSDB_TOTAL_MEMTABLE_MEMORY=67108864
DEDUPLICATOR_ROCKSDB_BLOCK_SIZE=4096
DEDUPLICATOR_ROCKSDB_N_MEMTABLES=2
DEDUPLICATOR_ROCKSDB_MEMTABLE_SIZE=16777216