diff --git a/Gemfile b/Gemfile index 9faed820..902f6917 100644 --- a/Gemfile +++ b/Gemfile @@ -119,3 +119,6 @@ gem "cssbundling-rails", "~> 1.4" # Aborts requests that are taking too long. # Set the timeout by setting the RACK_TIMEOUT_SERVICE_TIMEOUT env var # gem "rack-timeout" + +# Http client for making API requests +gem "faraday", "~> 2.13.1" diff --git a/Gemfile.lock b/Gemfile.lock index 033da032..db9e66bd 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -134,6 +134,12 @@ GEM railties (>= 5.0.0) faker (3.4.2) i18n (>= 1.8.11, < 2) + faraday (2.13.1) + faraday-net_http (>= 2.0, < 3.5) + json + logger + faraday-net_http (3.4.1) + net-http (>= 0.5.0) ffi (1.16.3) geo_coord (0.2.0) geocoder (1.8.2) @@ -163,6 +169,7 @@ GEM listen (3.9.0) rb-fsevent (~> 0.10, >= 0.10.3) rb-inotify (~> 0.9, >= 0.9.10) + logger (1.7.0) loofah (2.22.0) crass (~> 1.0.2) nokogiri (>= 1.12.0) @@ -178,6 +185,8 @@ GEM mini_mime (1.1.5) minitest (5.25.1) msgpack (1.7.2) + net-http (0.6.0) + uri net-imap (0.4.10) date net-protocol @@ -349,6 +358,7 @@ GEM tzinfo (2.0.6) concurrent-ruby (~> 1.0) unicode-display_width (2.5.0) + uri (1.0.3) view_component (3.11.0) activesupport (>= 5.2.0, < 8.0) concurrent-ruby (~> 1.0) @@ -387,6 +397,7 @@ DEPENDENCIES dotenv-rails factory_bot_rails (~> 6.4.3) faker (~> 3.4.2) + faraday (~> 2.13.1) geo_coord geocoder (~> 1.8) haversine! diff --git a/app/components/layout/header_component/header_component.html.erb b/app/components/layout/header_component/header_component.html.erb index 21996d34..93efc94e 100644 --- a/app/components/layout/header_component/header_component.html.erb +++ b/app/components/layout/header_component/header_component.html.erb @@ -49,6 +49,12 @@
<% if helpers.user_signed_in? %> + <% if helpers.current_user.admin? %> +
+ <%= link_to 'Tools', admin_tools_path, class: 'button is-light' %> +
+ <% end %> +
<%= link_to admin_user_path(current_user), class: 'button is-light' do %> Logged in as <%= helpers.current_user.email %> diff --git a/app/controllers/admin/facilities_controller.rb b/app/controllers/admin/facilities_controller.rb index 47ffbae2..4c26fd44 100644 --- a/app/controllers/admin/facilities_controller.rb +++ b/app/controllers/admin/facilities_controller.rb @@ -85,11 +85,10 @@ def load_facilities facilities = facilities.discarded end - if params[:service_id] == "none" + if params[:service] == "none" facilities = facilities.without_services - elsif params[:service_id].present? - facilities = facilities.joins(:services) - .where(services: { id: params[:service_id] }) + elsif params[:service].present? + facilities = facilities.with_service(params[:service]) end if params[:welcome_customer] == "none" @@ -115,7 +114,7 @@ def load_facility end def load_services_dropdown - @services_dropdown = [["No Services", :none]] + Service.pluck(:name, :id) + @services_dropdown = [["No Services", :none]] + Service.pluck(:name, :key) end def load_welcomes_dropdown diff --git a/app/controllers/admin/tools_controller.rb b/app/controllers/admin/tools_controller.rb new file mode 100644 index 00000000..58b284fb --- /dev/null +++ b/app/controllers/admin/tools_controller.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +class Admin::ToolsController < Admin::BaseController + before_action :enforce_admin_user + + def index + + end + + def import_facilities + api_key = params[:api] + + # Validate that both parameters are present and supported + unless External::ApiHelper.supported_api?(api_key) + redirect_to admin_tools_path, alert: "Invalid API selected. Please choose from the supported APIs." + return + end + + result = External::VancouverCity::Syncer.call( + api_key: api_key, + api_client: External::VancouverCity.default_client + ) + + if result.success? + total_count = result.data[:total_count] || 0 + redirect_to admin_facilities_path(service: "water_fountain"), notice: "#{total_count} Facilities imported successfully from #{External::ApiHelper.api_name(api_key)}." + else + error_messages = result.errors.join(', ') + redirect_to admin_tools_path, alert: "Failed to import facilities: #{error_messages}" + end + end + + # Helper method for the view + helper_method :api_options_for_select + + private + + def api_options_for_select + External::ApiHelper.api_options + end + + def enforce_admin_user + redirect_to root_path, alert: "Access denied! You must be an admin to access tools" unless current_user&.admin? + end +end \ No newline at end of file diff --git a/app/models/facility.rb b/app/models/facility.rb index ad1d59bb..4148318c 100644 --- a/app/models/facility.rb +++ b/app/models/facility.rb @@ -39,6 +39,8 @@ class Facility < ApplicationRecord scope :with_service, ->(service_key_or_name) { joins(:services).where(services: Service.exact_search(service_key_or_name)) } scope :without_services, -> { where.not(facility_services: FacilityService.all) } scope :without_welcomes, -> { where.not(facility_welcomes: FacilityWelcome.all) } + scope :external, -> { where.not(external_id: nil) } + scope :not_external, -> { where(external_id: nil) } def managed_by?(user) f_user_id = if user.respond_to? :id @@ -68,6 +70,10 @@ def self.statuses %i[live pending_reviews discarded] end + def external? + external_id.present? + end + def status if discarded? :discarded diff --git a/app/services/external/api_helper.rb b/app/services/external/api_helper.rb new file mode 100644 index 00000000..38b0b35c --- /dev/null +++ b/app/services/external/api_helper.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +# Configuration class for supported Vancouver City APIs and facility services +class External::ApiHelper + # Available Vancouver City Facilities APIs + # Each API represents a different type of facility data available from Vancouver's Open Data portal + SUPPORTED_APIS = { + 'drinking-fountains' => 'Drinking Fountains' + }.freeze + + # Mapping of dataset IDs to service keys + # This mapping is used to associate API keys with specific service types in the system + DATASET_ID_TO_SERVICE_KEY = { + 'drinking-fountains' => 'water_fountain' + }.freeze + + class << self + # Get all supported API options for select fields + # @return [Array] Array of [display_name, api_key] pairs + def api_options + SUPPORTED_APIS.map { |key, name| [name, key] } + end + + # Get all supported API keys + # @return [Array] Array of API keys + def supported_api_keys + SUPPORTED_APIS.keys + end + + # Check if an API is supported + # @param api_key [String] The API key to check + # @return [Boolean] True if the API is supported + def supported_api?(api_key) + SUPPORTED_APIS.key?(api_key.to_s) + end + + # Get the service key for a given API key + # @param api_key [String] The API key to find the service key for + # @return [String, nil] The service key or nil if not found + def service_key_for(api_key) + DATASET_ID_TO_SERVICE_KEY.dig(api_key.to_s) + end + + # Get the display name for an API + # @param api_key [String] The API key + # @return [String, nil] The display name or nil if not found + def api_name(api_key) + SUPPORTED_APIS[api_key.to_s] + end + end +end diff --git a/app/services/external/vancouver_city.rb b/app/services/external/vancouver_city.rb new file mode 100644 index 00000000..0d97dcfa --- /dev/null +++ b/app/services/external/vancouver_city.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +# Namespace for Vancouver City API integration services +module External::VancouverCity + # Convenience method to get default API client + def self.default_client + VancouverApiClient.default_client + end +end diff --git a/app/services/external/vancouver_city/adapters/faraday_adapter.rb b/app/services/external/vancouver_city/adapters/faraday_adapter.rb new file mode 100644 index 00000000..306f368b --- /dev/null +++ b/app/services/external/vancouver_city/adapters/faraday_adapter.rb @@ -0,0 +1,146 @@ +# frozen_string_literal: true + +require 'faraday' + +module External::VancouverCity + module Adapters + # Faraday HTTP adapter for the Vancouver API client + # Uses the builder pattern for flexible configuration + class FaradayAdapter + attr_reader :connection + + def initialize(connection) + @connection = connection + end + + def self.create(vancouver_api_config) + builder(vancouver_api_config.base_url) + .timeout(vancouver_api_config.timeout) + .open_timeout(vancouver_api_config.open_timeout) + .build + end + + # Builder class for creating configured Faraday connections + class Builder + DEFAULT_TIMEOUT = 30 + DEFAULT_OPEN_TIMEOUT = 10 + DEFAULT_USER_AGENT = 'Linkvan API Client' + + def initialize(base_url) + @base_url = base_url + @timeout = DEFAULT_TIMEOUT + @open_timeout = DEFAULT_OPEN_TIMEOUT + @user_agent = DEFAULT_USER_AGENT + @headers = {} + @adapter = Faraday.default_adapter + end + # Set request timeout + # @param timeout [Integer] Request timeout in seconds + # @return [Builder] self for method chaining + def timeout(timeout) + @timeout = timeout + self + end + + # Set connection timeout + # @param open_timeout [Integer] Connection timeout in seconds + # @return [Builder] self for method chaining + def open_timeout(open_timeout) + @open_timeout = open_timeout + self + end + + # Set user agent string + # @param user_agent [String] User agent for requests + # @return [Builder] self for method chaining + def user_agent(user_agent) + @user_agent = user_agent + self + end + + # Add custom header + # @param name [String] Header name + # @param value [String] Header value + # @return [Builder] self for method chaining + def header(name, value) + @headers[name] = value + self + end + + # Set Faraday adapter + # @param adapter [Symbol, Object] Faraday adapter + # @return [Builder] self for method chaining + def adapter(adapter) + @adapter = adapter + self + end + + # Build the configured Faraday connection + # @return [FaradayAdapter] Configured adapter instance + def build + connection = Faraday.new(url: @base_url) do |config| + config.adapter @adapter + + # Set timeouts + config.options.timeout = @timeout + config.options.open_timeout = @open_timeout + + # Set default headers + config.headers['User-Agent'] = @user_agent + config.headers['Accept'] = 'application/json' + + # Add custom headers + @headers.each do |name, value| + config.headers[name] = value + end + end + + FaradayAdapter.new(connection) + end + end + + # Create a new builder for the given base URL + # @param base_url [String] The base URL for the API + # @return [Builder] A new builder instance + def self.builder(base_url) + Builder.new(base_url) + end + + # Delegate HTTP methods to the Faraday connection + def get(path, params = {}) + @connection.get(path, params) + end + + def post(path, body = nil, params = {}) + @connection.post(path, body, params) + end + + def put(path, body = nil, params = {}) + @connection.put(path, body, params) + end + + def delete(path, params = {}) + @connection.delete(path, params) + end + + def patch(path, body = nil, params = {}) + @connection.patch(path, body, params) + end + + # Access connection options for testing + def options + @connection.options + end + + # Access connection headers for testing + def headers + @connection.headers + end + + # Access connection URL prefix for testing + def url_prefix + @connection.url_prefix + end + end + end +end diff --git a/app/services/external/vancouver_city/facility_builder.rb b/app/services/external/vancouver_city/facility_builder.rb new file mode 100644 index 00000000..8c5514d8 --- /dev/null +++ b/app/services/external/vancouver_city/facility_builder.rb @@ -0,0 +1,182 @@ +# frozen_string_literal: true + +module External::VancouverCity + # Service for building facility objects from Vancouver City Open Data API records + # Inherits from ApplicationService and handles record validation and error recovery + class FacilityBuilder < ApplicationService + attr_reader :record, :api_key + + ResultData = Struct.new(:facility, keyword_init: true) do + def blank? + facility.nil? + end + end + + # Initialize the builder with required parameters + # @param record [Hash] Single API response record + # @param api_key [String] One of the supported API keys from External::ApiHelper + def initialize(record:, api_key:) + super() + @record = record + @api_key = api_key + end + + # Main method that performs the facility building operation + # @return [ApplicationService::Result] Result object with facility data and errors + def call + return Result.new(data: ResultData.new, errors: errors) if invalid? + + begin + facility = build_facility_from_record + + # Build facility services + service_builder = FacilityServiceBuilder.new(facility: facility, fields: record, api_key: api_key) + service_result = service_builder.call + unless service_result.success? + service_result.errors.each { |error| add_error(error) } + end + + # Build facility welcomes + welcome_builder = FacilityWelcomeBuilder.new(facility: facility, fields: record) + welcome_result = welcome_builder.call + unless welcome_result.success? + welcome_result.errors.each { |error| add_error(error) } + end + + # Build facility schedules + schedule_builder = FacilityScheduleBuilder.new(facility: facility, fields: record) + schedule_result = schedule_builder.call + unless schedule_result.success? + schedule_result.errors.each { |error| add_error(error) } + end + + if facility&.valid? + Result.new(data: ResultData.new(facility: facility), errors: errors) + else + add_error("Facility #{facility&.name} is invalid: #{facility&.errors&.full_messages&.join(', ')}") + Result.new(data: ResultData.new, errors: errors) + end + rescue StandardError => e + add_error("Failed to build facility from record: #{e.message}") + Rails.logger.warn "Failed to build facility from record: #{e.message}" + Rails.logger.warn "Record data: #{record.inspect}" + Result.new(data: ResultData.new, errors: errors) + end + end + + # Validates the input parameters + # @return [Array] Array of error messages + def validate + @errors = [] + + if record.blank? + add_error("Record is required") + elsif !record.is_a?(Hash) + add_error("Record must be a Hash") + elsif !valid_geometry? + add_error("Geometry should be either Array with 2 elements or Hash with 'lat' and 'lon' keys") + end + end + + private + + def valid_geometry? + coordinates.present? || geo_point_2d.present? + end + + # Build a Facility object from an API record + # @param record [Hash] Single API response record + # @return [Facility, nil] Built Facility object or nil if invalid + def build_facility_from_record + coords = coordinates.presence || geo_point_2d + + facility_data = { + name: extract_name(record), + address: extract_address(record), + phone: extract_phone(record), + website: extract_website(record), + notes: extract_notes(record), + lat: coords[:lat], + long: coords[:long], + verified: true, + external_id: record['mapid'] || "#{api_key}-unknown-id", + }.compact + + Facility.new(facility_data) + end + + # Extract facility name from fields + # @param fields [Hash] API record fields + # @return [String, nil] Facility name + def extract_name(fields) + name = fields['name'] + return nil unless name + + # Replace special characters with whitespace and clean up + name.gsub(/\\n/, ' ').tr("\n", ' ').gsub(/\s+/, ' ').strip.presence + end + + # Extract address from fields + # @param fields [Hash] API record fields + # @return [String, nil] Facility address + def extract_address(fields) + # For drinking fountains, use the location field and geo_local_area + location = fields['location'] + area = fields['geo_local_area'] + + [location, area].compact.join(', ').presence + end + + # Extract phone number from fields + # @param fields [Hash] API record fields + # @return [String, nil] Phone number + def extract_phone(fields) + fields['phone'] || fields['phone_number'] || fields['contact_phone'] + end + + # Extract website from fields + # @param fields [Hash] API record fields + # @return [String, nil] Website URL + def extract_website(fields) + fields['website'] || fields['url'] || fields['web_site'] + end + + # Extract notes/description from fields + # @param fields [Hash] API record fields + # @return [String, nil] Notes or description + def extract_notes(fields) + notes_parts = [] + + # Include maintainer info + notes_parts << "Maintained by: #{fields['maintainer']}" if fields['maintainer'].present? + + # Include operation info + notes_parts << "Operation: #{fields['in_operation']}" if fields['in_operation'].present? + + # Include pet friendly info + notes_parts << "Pet friendly: #{fields['pet_friendly']}" if fields['pet_friendly'].present? + + notes_parts.join('. ').presence + end + + # Extract coordinates from geometry + # @return [Hash] Hash with :lat and :long keys + def coordinates + coords = record.dig('geom', 'geometry', 'coordinates').presence || [] + return {} unless coords.size == 2 + + # GeoJSON coordinates are [longitude, latitude] + { lat: coords[1], long: coords[0] } + end + + # Extract coordinates from geo_point_2d field + # @return [Hash] Hash with :lat and :long keys + def geo_point_2d + geo_point = record.dig('geo_point_2d').presence || {} + return {} unless geo_point.is_a?(Hash) + return {} unless geo_point.key?('lat') && geo_point.key?('lon') + + { lat: geo_point['lat'], long: geo_point['lon'] } + end + end +end diff --git a/app/services/external/vancouver_city/facility_schedule_builder.rb b/app/services/external/vancouver_city/facility_schedule_builder.rb new file mode 100644 index 00000000..755d5ba0 --- /dev/null +++ b/app/services/external/vancouver_city/facility_schedule_builder.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +module External::VancouverCity + # Service for building facility schedule objects for Vancouver City facilities + # Creates open-all-day schedules for all weekdays as per business requirements + class FacilityScheduleBuilder < ApplicationService + attr_reader :facility, :fields + + # Initialize the builder with required parameters + # @param facility [Facility] The facility object to add schedules to + # @param fields [Hash] API record fields (currently unused but kept for future extensibility) + def initialize(facility:, fields:) + super() + @facility = facility + @fields = fields + end + + # Main method that performs the schedule building operation + # @return [ApplicationService::Result] Result object with success status and errors + def call + return Result.new(data: nil, errors: errors) if invalid? + + begin + add_facility_schedules + Result.new(data: { schedules_count: facility.schedules.size }, errors: errors) + rescue StandardError => e + add_error("Failed to build facility schedules: #{e.message}") + Result.new(data: nil, errors: errors) + end + end + + # Validates the input parameters + # @return [Array] Array of error messages + def validate + @errors = [] + + if facility.nil? + add_error("Facility is required") + elsif !facility.is_a?(Facility) + add_error("Facility must be a Facility object") + end + + if fields.nil? + add_error("Fields are required") + elsif !fields.is_a?(Hash) + add_error("Fields must be a Hash") + end + + errors + end + + private + + # Add schedules to facility based on business requirements + # Creates open-all-day schedules for all weekdays + def add_facility_schedules + FacilitySchedule.week_days.keys.each do |day| + facility.schedules.build( + week_day: day, + closed_all_day: false, + open_all_day: true + ) + end + end + end +end diff --git a/app/services/external/vancouver_city/facility_service_builder.rb b/app/services/external/vancouver_city/facility_service_builder.rb new file mode 100644 index 00000000..08b27e33 --- /dev/null +++ b/app/services/external/vancouver_city/facility_service_builder.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true + +module External::VancouverCity + # Service for building facility service associations for Vancouver City facilities + # Associates facilities with services based on API key + class FacilityServiceBuilder < ApplicationService + attr_reader :facility, :fields, :api_key + + # Initialize the builder with required parameters + # @param facility [Facility] The facility object to add services to + # @param fields [Hash] API record fields (currently unused but kept for future extensibility) + # @param api_key [String] The API key used to find the corresponding service + def initialize(facility:, fields:, api_key:) + super() + @facility = facility + @fields = fields + @api_key = api_key + end + + # Main method that performs the service association building operation + # @return [ApplicationService::Result] Result object with success status and errors + def call + return Result.new(data: nil, errors: errors) if invalid? + + begin + add_facility_services + Result.new(data: { services_count: facility.facility_services.size }, errors: errors) + rescue StandardError => e + add_error("Failed to build facility services: #{e.message}") + Result.new(data: nil, errors: errors) + end + end + + # Validates the input parameters + # @return [Array] Array of error messages + def validate + @errors = [] + + if facility.blank? + add_error("Facility is required") + elsif !facility.is_a?(Facility) + add_error("Facility must be a Facility object") + end + + if fields.blank? + add_error("Fields are required") + elsif !fields.is_a?(Hash) + add_error("Fields must be a Hash") + end + + if api_key.blank? + add_error("API key is required") + elsif !External::ApiHelper.supported_api?(api_key) + add_error("Unsupported API key: #{api_key}") + end + + errors + end + + private + + # Add services to facility based on API key + def add_facility_services + service_key = External::ApiHelper.service_key_for(api_key) + return if service_key.nil? + + service = Service.find_by(key: service_key) + return if service.blank? + + # Build FacilityService association without saving + facility.facility_services.build(service: service) + end + end +end diff --git a/app/services/external/vancouver_city/facility_syncer.rb b/app/services/external/vancouver_city/facility_syncer.rb new file mode 100644 index 00000000..6141898c --- /dev/null +++ b/app/services/external/vancouver_city/facility_syncer.rb @@ -0,0 +1,101 @@ +# frozen_string_literal: true + +module External::VancouverCity + # Service for syncing facility data from Vancouver City Open Data API + # Inherits from ApplicationService and handles pagination to fetch all facilities + class FacilitySyncer < ApplicationService + attr_reader :record, :api_key, :logger + + ResultData = Struct.new(:operation, :facility, keyword_init: true) do + delegate :present?, :blank?, to: :facility + end + + def initialize(record:, api_key:, logger: Rails.logger) + @record = record + @api_key = api_key + @logger = logger + end + + def call + builder_result = FacilityBuilder.call(record: record, api_key: api_key) + if builder_result.failed? + add_errors(builder_result.errors) + return Result.new( + data: ResultData.new(operation: nil, facility: nil), + errors: errors) + end + + built_facility = builder_result.data[:facility] + existing_facility = Facility.find_by(external_id: built_facility.external_id) + + # If no external_id match, look for name match but prefer internal facilities + if existing_facility.blank? + existing_facility = Facility.where(name: built_facility.name) + .order(Arel.sql('external_id IS NULL DESC, external_id')) + .first + end + operation = if existing_facility.blank? + :create + elsif existing_facility.external? + :external_update + else + :internal_update + end + result_facility = nil + + ApplicationRecord.transaction do + case operation + when :external_update + logger.info "Facility with external_id '#{existing_facility.external_id}' already exists, updating services" + update_external_facility(existing_facility, built_facility) + result_facility = existing_facility + when :internal_update + logger.warn "Facility with name '#{existing_facility.name}' already exists internally, adding services" + update_internal_facility(existing_facility, built_facility) + result_facility = existing_facility + when :create + logger.info "Creating new facility with external_id '#{built_facility.external_id}'" + if built_facility.invalid? + add_errors(built_facility.errors) + result_facility = nil + else + built_facility.save! + result_facility = built_facility + end + end + rescue ActiveRecord::RecordInvalid => e + add_error("Failed to save facility: #{e.message}") + result_facility = nil + rescue StandardError => e + add_error("Unexpected error during facility sync: #{e.message}") + result_facility = nil + end + + Result.new( + data: ResultData.new(operation: operation, facility: result_facility), + errors: errors + ) + end + + private + + def update_internal_facility(internal_facility, built_facility) + add_missing_services(internal_facility, built_facility) + end + + def update_external_facility(external_facility, built_facility) + add_missing_services(external_facility, built_facility) + + external_facility.update!(built_facility.attributes.slice('name', 'address', 'lat', 'long', 'verified')) + end + + def add_missing_services(existing_facility, built_facility) + built_services = built_facility.facility_services.map(&:service).uniq + existing_services = existing_facility.facility_services.map(&:service).uniq + new_services = built_services - existing_services + new_services.each do |service| + existing_facility.facility_services.create!(service: service) + end + end + end +end \ No newline at end of file diff --git a/app/services/external/vancouver_city/facility_welcome_builder.rb b/app/services/external/vancouver_city/facility_welcome_builder.rb new file mode 100644 index 00000000..6a4825d2 --- /dev/null +++ b/app/services/external/vancouver_city/facility_welcome_builder.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +module External::VancouverCity + # Service for building facility welcome objects for Vancouver City facilities + # Creates welcomes for all customer types as per business requirements + class FacilityWelcomeBuilder < ApplicationService + attr_reader :facility, :fields + + # Initialize the builder with required parameters + # @param facility [Facility] The facility object to add welcomes to + # @param fields [Hash] API record fields (currently unused but kept for future extensibility) + def initialize(facility:, fields:) + super() + @facility = facility + @fields = fields + end + + # Main method that performs the welcome building operation + # @return [ApplicationService::Result] Result object with success status and errors + def call + return Result.new(data: nil, errors: errors) if invalid? + + begin + add_facility_welcomes + Result.new(data: { welcomes_count: facility.facility_welcomes.size }, errors: errors) + rescue StandardError => e + add_error("Failed to build facility welcomes: #{e.message}") + Result.new(data: nil, errors: errors) + end + end + + # Validates the input parameters + # @return [Array] Array of error messages + def validate + @errors = [] + + if facility.nil? + add_error("Facility is required") + elsif !facility.is_a?(Facility) + add_error("Facility must be a Facility object") + end + + if fields.nil? + add_error("Fields are required") + elsif !fields.is_a?(Hash) + add_error("Fields must be a Hash") + end + + errors + end + + private + + # Add welcomes to facility for all customer types + def add_facility_welcomes + welcomes = FacilityWelcome.all_customers + + welcomes.each do |customer_type| + facility.facility_welcomes.build(customer: customer_type.value) + end + end + end +end diff --git a/app/services/external/vancouver_city/syncer.rb b/app/services/external/vancouver_city/syncer.rb new file mode 100644 index 00000000..02a4f8ce --- /dev/null +++ b/app/services/external/vancouver_city/syncer.rb @@ -0,0 +1,105 @@ +# frozen_string_literal: true + +module External::VancouverCity + # Service for syncing facility data from Vancouver City Open Data API + # Inherits from ApplicationService and handles pagination to fetch all facilities + class Syncer < ApplicationService + attr_reader :api_key, :api_client + + PAGE_SIZE = 50 # Maximum records per request allowed by the API + + # Initialize the syncer with required parameters + # @param api_key [String] One of the supported API keys from External::ApiHelper + # @param api_client [VancouverApiClient] The API client instance + def initialize(api_key:, api_client:) + super() + @api_key = api_key + @api_client = api_client + end + + # Main method that performs the sync operation + # @return [ApplicationService::Result] Result object with data and errors + def call + return Result.new(data: nil, errors: errors) if invalid? + + facilities = [] + offset = 0 + + loop do + Rails.logger.info "Fetching facilities from #{api_key} API (offset: #{offset}, limit: #{PAGE_SIZE})" + + begin + response = api_client.get_dataset_records(api_key, limit: PAGE_SIZE, offset: offset) + records = response.body.dig('results') || [] + + break if records.empty? + + # Process each record and build Facility objects + batch_facilities = process_records(records) + facilities.concat(batch_facilities) + + # If we got fewer records than the limit, we've reached the end + break if records.size < PAGE_SIZE + + offset += PAGE_SIZE + rescue VancouverApiError => e + add_error("API request failed: #{e.message}") + break + rescue StandardError => e + add_error("Unexpected error during sync: #{e.message}") + break + end + end + + Rails.logger.info "Successfully processed #{facilities.size} facilities from #{api_key} API" + + Result.new( + data: { + facilities: facilities, + total_count: facilities.size, + api_key: api_key + }, + errors: errors + ) + end + + # Validates the input parameters + # @return [Array] Array of error messages + def validate + @errors = [] + + unless External::ApiHelper.supported_api?(api_key) + add_error("Unsupported API: #{api_key}") + end + + if api_client.nil? + add_error("API client is required") + elsif !api_client.is_a?(VancouverApiClient) + add_error("API client must be an instance of VancouverApiClient") + end + + errors + end + + private + + # Process API records and convert them to Facility objects + # @param records [Array] Array of API response records + # @return [Array] Array of built Facility objects + def process_records(records) + facilities = [] + + records.each do |record| + syncer_result = FacilitySyncer.call(record: record, api_key: api_key) + + if syncer_result.success? + facilities << syncer_result.data[:facility] + else + add_errors(syncer_result.errors) + end + end + + facilities + end + end +end diff --git a/app/services/external/vancouver_city/vancouver_api_client.rb b/app/services/external/vancouver_city/vancouver_api_client.rb new file mode 100644 index 00000000..8c268b08 --- /dev/null +++ b/app/services/external/vancouver_city/vancouver_api_client.rb @@ -0,0 +1,262 @@ +# frozen_string_literal: true + +require 'faraday' +require 'json' +require_relative 'adapters/faraday_adapter' + +module External::VancouverCity + class VancouverApiConfig + BASE_URL = 'https://opendata.vancouver.ca/api/explore/v2.1' + DEFAULT_TIMEOUT = 30 # seconds + DEFAULT_OPEN_TIMEOUT = 10 # seconds + + attr_reader :base_url, :timeout, :open_timeout + + def initialize(base_url: nil, timeout: nil, open_timeout: nil) + @base_url = base_url || BASE_URL + @timeout = timeout || DEFAULT_TIMEOUT + @open_timeout = open_timeout || DEFAULT_OPEN_TIMEOUT + end + end + + DEFAULT_ADAPTER = Adapters::FaradayAdapter.builder(VancouverApiConfig::BASE_URL) + .timeout(VancouverApiConfig::DEFAULT_TIMEOUT) + .open_timeout(VancouverApiConfig::DEFAULT_OPEN_TIMEOUT) + .build + + # HTTP client for the Vancouver Open Data API (Opendatasoft Explore API v2.1) + # + # This client provides access to Vancouver's open data portal at: + # https://opendata.vancouver.ca/api/explore/v2.1/ + # + # Example usage: + # # Using the default adapter + # client = External::VancouverCity::VancouverApiClient.default_client + # + # # Using a custom configuration + # config = External::VancouverCity::VancouverApiConfig.new(timeout: 60) + # client = External::VancouverCity::VancouverApiClient.with_config(config) + # + # response = client.get_dataset_records('drinking-fountains', limit: 20) + # records = response.body + class VancouverApiClient + attr_reader :adapter + + # Create a client with the default adapter + # @return [VancouverApiClient] Client instance with default configuration + def self.default_client + new(adapter: DEFAULT_ADAPTER) + end + + # Create a client with custom configuration + # @param config [VancouverApiConfig] Custom configuration + # @return [VancouverApiClient] Client instance with custom configuration + def self.with_config(config) + adapter = Adapters::FaradayAdapter.create(config) + new(adapter: adapter) + end + + # Create a client with custom timeout values + # @param timeout [Integer] Request timeout in seconds + # @param open_timeout [Integer] Connection timeout in seconds + # @return [VancouverApiClient] Client instance with custom timeouts + def self.with_timeouts(timeout: 30, open_timeout: 10) + config = VancouverApiConfig.new(timeout: timeout, open_timeout: open_timeout) + with_config(config) + end + + # Initialize the client with a mandatory adapter + # @param adapter [FaradayAdapter] Pre-configured HTTP adapter (required) + def initialize(adapter:) + @adapter = adapter + end + + # Get records from a dataset + # + # @param dataset_id [String] The identifier of the dataset (e.g., 'drinking-fountains') + # @param options [Hash] Query parameters for the API request + # @option options [String] :select Fields to select (default: '*' for all fields) + # @option options [String] :where Filter expression using ODSQL + # @option options [String] :order_by Sort expression (e.g., 'name asc') + # @option options [Integer] :limit Number of records to return (max 100, default 10) + # @option options [Integer] :offset Starting index for pagination (default 0) + # @option options [String] :group_by Group by expression for aggregations + # @option options [String] :refine Facet refinement (e.g., 'field:value') + # @option options [String] :exclude Facet exclusion (e.g., 'field:value') + # @option options [String] :lang Language for formatting (default 'en') + # @option options [String] :timezone Timezone for date formatting + # @option options [Boolean] :include_links Whether to include navigation links + # @option options [Boolean] :include_app_metas Whether to include application metadata + # @return [Faraday::Response] The HTTP response containing the dataset records + # @raise [VancouverApiError] If the API request fails + # + # @example Get first 20 drinking fountains + # client.get_dataset_records('drinking-fountains', limit: 20) + # + # @example Get fountains with specific filters + # client.get_dataset_records('drinking-fountains', + # where: 'location_type = "Park"', + # order_by: 'name asc', + # limit: 50 + # ) + def get_dataset_records(dataset_id, **options) + # Build query parameters, filtering out nil values + params = build_query_params(options) + + # Make the API request + path = "catalog/datasets/#{dataset_id}/records" + + handle_response do + @adapter.get(path, params) + end + end + + # Get information about a specific dataset + # + # @param dataset_id [String] The identifier of the dataset + # @param options [Hash] Query parameters + # @option options [String] :lang Language for formatting (default 'en') + # @option options [Boolean] :include_links Whether to include navigation links + # @option options [Boolean] :include_app_metas Whether to include application metadata + # @return [Faraday::Response] The HTTP response containing dataset information + # @raise [VancouverApiError] If the API request fails + def get_dataset(dataset_id, **options) + params = build_query_params(options.slice(:lang, :include_links, :include_app_metas)) + path = "catalog/datasets/#{dataset_id}" + + handle_response do + @adapter.get(path, params) + end + end + + # List all available datasets + # + # @param options [Hash] Query parameters for filtering and pagination + # @option options [String] :select Fields to select + # @option options [String] :where Filter expression + # @option options [String] :order_by Sort expression + # @option options [Integer] :limit Number of datasets to return + # @option options [Integer] :offset Starting index for pagination + # @option options [String] :refine Facet refinement + # @option options [String] :exclude Facet exclusion + # @option options [String] :lang Language for formatting + # @return [Faraday::Response] The HTTP response containing available datasets + # @raise [VancouverApiError] If the API request fails + def get_datasets(**options) + params = build_query_params(options) + path = "catalog/datasets" + + handle_response do + @adapter.get(path, params) + end + end + + # Get a specific record from a dataset + # + # @param dataset_id [String] The identifier of the dataset + # @param record_id [String] The identifier of the specific record + # @param options [Hash] Query parameters + # @option options [String] :lang Language for formatting + # @option options [String] :timezone Timezone for date formatting + # @return [Faraday::Response] The HTTP response containing the specific record + # @raise [VancouverApiError] If the API request fails + def get_dataset_record(dataset_id, record_id, **options) + params = build_query_params(options.slice(:lang, :timezone)) + path = "catalog/datasets/#{dataset_id}/records/#{record_id}" + + handle_response do + @adapter.get(path, params) + end + end + + private + + # Build query parameters hash, removing nil values + # @param options [Hash] The options hash to filter + # @return [Hash] Filtered parameters hash + def build_query_params(options) + params = {} + + # Map all supported parameters + param_mapping = { + select: :select, + where: :where, + group_by: :group_by, + order_by: :order_by, + limit: :limit, + offset: :offset, + refine: :refine, + exclude: :exclude, + lang: :lang, + timezone: :timezone, + include_links: :include_links, + include_app_metas: :include_app_metas + } + + param_mapping.each do |key, param_name| + value = options[key] + params[param_name] = value unless value.nil? + end + + params + end + + # Handle API response and error checking + # @yield Block that makes the HTTP request + # @return [Faraday::Response] The successful response with parsed JSON body + # @raise [VancouverApiError] If the request fails + def handle_response + response = yield + + # Check for HTTP errors + unless response.success? + error_message = "API request failed with status #{response.status}" + + # Try to parse error response if it's JSON + if response.headers['content-type']&.include?('application/json') + begin + error_body = JSON.parse(response.body) + error_message += ": #{error_body['error'] || error_body['message'] || response.body}" + rescue JSON::ParserError + error_message += ": #{response.body}" + end + else + error_message += ": #{response.body[0..200]}#{'...' if response.body.length > 200}" + end + + raise VancouverApiError.new(error_message, response.status, response.body) + end + + # Parse JSON response body for successful responses + if response.headers['content-type']&.include?('application/json') + begin + response.env.body = JSON.parse(response.body) + rescue JSON::ParserError => e + raise VancouverApiError.new("Failed to parse JSON response: #{e.message}", response.status, response.body) + end + end + + response + rescue Faraday::TimeoutError => e + raise VancouverApiError.new("Request timeout: #{e.message}", nil, nil) + rescue Faraday::ConnectionFailed => e + raise VancouverApiError.new("Connection failed: #{e.message}", nil, nil) + rescue VancouverApiError + # Re-raise our own errors without wrapping + raise + rescue StandardError => e + raise VancouverApiError.new("Unexpected error: #{e.message}", nil, nil) + end + end + + # Custom error class for Vancouver API client errors + class VancouverApiError < StandardError + attr_reader :status_code, :response_body + + def initialize(message, status_code = nil, response_body = nil) + super(message) + @status_code = status_code + @response_body = response_body + end + end +end diff --git a/app/views/admin/facilities/index.html.erb b/app/views/admin/facilities/index.html.erb index d3a157ea..e7827b5e 100644 --- a/app/views/admin/facilities/index.html.erb +++ b/app/views/admin/facilities/index.html.erb @@ -11,7 +11,7 @@
- <%= form_with url: admin_facilities_path, method: :get, data: { controller: "auto-submit" } do |form| %> + <%= form_with url: admin_facilities_path, method: :get, data: { controller: "auto-submit" } do |form| %>
@@ -24,9 +24,9 @@
- <%= form.select(:service_id, + <%= form.select(:service, @services_dropdown, - { include_blank: "Service", selected: params[:service_id] }, + { include_blank: "Service", selected: params[:service] }, { data: { action: "change->auto-submit#change" } }) %> diff --git a/app/views/admin/tools/index.html.erb b/app/views/admin/tools/index.html.erb new file mode 100644 index 00000000..72f3707e --- /dev/null +++ b/app/views/admin/tools/index.html.erb @@ -0,0 +1,78 @@ + + +
+
+
+
+ <%= render Shared::CardComponent.new(title: "Vancouver City API") do |card| %> +
+

Choose which facilities to import from Vancouver City's Open Data API.

+ + <%= form_with url: import_facilities_admin_tools_path, method: :post, class: "form", id: "import-form" do |form| %> +
+ <%= form.label :api, "API Endpoint", class: "label" %> +
+
+ <%= form.select :api, + options_for_select(api_options_for_select), + { include_blank: 'Select an API...' }, + { class: "select", required: true } %> +
+
+
+ +
+
+ <%= form.submit "Import Facilities", class: "button is-primary", id: "import-button", data: { disable_with: "Importing..." } %> +
+
+ <% end %> + + + +
+ <% end %> +
+
+
+ + diff --git a/config/database.yml b/config/database.yml index 8fa49950..f91215a8 100644 --- a/config/database.yml +++ b/config/database.yml @@ -64,7 +64,7 @@ development: test: <<: *default # Uses linkvan-api/postgres container - url: postgres://postgres:postgres@localhost:5432 + url: <%= ENV.fetch('TEST_DATABASE_URL', 'postgres://postgres:postgres@localhost:5432') %> database: linkvan_api_test # As with config/credentials.yml, you never want to store sensitive information, diff --git a/config/puma.rb b/config/puma.rb index 16cb8e11..11aed7bc 100644 --- a/config/puma.rb +++ b/config/puma.rb @@ -30,15 +30,18 @@ # Workers do not work on JRuby or Windows (both of which do not support # processes). # -# workers ENV.fetch("WEB_CONCURRENCY") { 2 } -workers Integer(ENV.fetch("WEB_CONCURRENCY") { 2 }) +# Disable workers in development to avoid macOS forking issues +if ENV.fetch("RAILS_ENV", "development") != "development" + workers Integer(ENV.fetch("WEB_CONCURRENCY") { 2 }) -# Use the `preload_app!` method when specifying a `workers` number. -# This directive tells Puma to first boot the application and load code -# before forking the application. This takes advantage of Copy On Write -# process behavior so workers use less memory. -# -preload_app! + # Use the `preload_app!` method when specifying a `workers` number. + # This directive tells Puma to first boot the application and load code + # before forking the application. This takes advantage of Copy On Write + # process behavior so workers use less memory. + # + # Only preload app when using workers (not in development) + preload_app! +end # Allow puma to be restarted by `rails restart` command. plugin :tmp_restart diff --git a/config/routes.rb b/config/routes.rb index 3d314bd0..1f77a6f3 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -17,7 +17,7 @@ end namespace :admin do - root to: "admin/dashboard#index" + root to: "dashboard#index" resources :dashboard, only: %i[index show] @@ -25,6 +25,12 @@ # root to: "dashboard#index" # end + resources :tools do + collection do + post :import_facilities + end + end + resources :users do resources :passwords, only: %i[new create] end diff --git a/db/migrate/20161125211440_change_facility_suitability.rb b/db/migrate/20161125211440_change_facility_suitability.rb index e7db6873..5f10e19d 100644 --- a/db/migrate/20161125211440_change_facility_suitability.rb +++ b/db/migrate/20161125211440_change_facility_suitability.rb @@ -1,21 +1,16 @@ class ChangeFacilitySuitability < ActiveRecord::Migration[4.2] + # Define a local model class that only knows about existing columns + class MigrationFacility < ActiveRecord::Base + self.table_name = 'facilities' + end + def up change_column :facilities, :suitability, :string - Facility.find_each do |f| - if f.suitability == "Children" - f.suitability = "children" - f.save - end - end + MigrationFacility.where(suitability: "Children").update_all(suitability: "children") end def down change_column :facilities, :suitability, :string - Facility.find_each do |f| - if f.suitability == "children" - f.suitability = "Children" - f.save - end - end + MigrationFacility.where(suitability: "children").update_all(suitability: "Children") end end diff --git a/db/migrate/20161125232147_change_all_suitability_permutations.rb b/db/migrate/20161125232147_change_all_suitability_permutations.rb index f0fed50f..2886c5f3 100644 --- a/db/migrate/20161125232147_change_all_suitability_permutations.rb +++ b/db/migrate/20161125232147_change_all_suitability_permutations.rb @@ -1,54 +1,13 @@ class ChangeAllSuitabilityPermutations < ActiveRecord::Migration[4.2] - def change + # Define a local model class that only knows about existing columns + class MigrationFacility < ActiveRecord::Base + self.table_name = 'facilities' + end + + def up change_column :facilities, :suitability, :string - Facility.find_each do |f| - case f.suitability - when "Children Youth" - f.suitability = "children youth" - f.save - when "Children Youth Adults" - f.suitability = "children youth adults" - f.save - when "Children Youth Adults Seniors" - f.suitability = "children youth adults seniors" - f.save - when "Children Adults" - f.suitability = "children adults" - f.save - when "Children Seniors" - f.suitability = "children seniors" - f.save - when "Children Adults Seniors" - f.suitability = "children adults seniors" - f.save - when "Children Youth Seniors" - f.suitability = "children youth seniors" - f.save - when "Youth" - f.suitability = "youth" - f.save - when "Youth Adults" - f.suitability = "youth adults" - f.save - when "Youth Seniors" - f.suitability = "youth seniors" - f.save - when "Youth Adults Seniors" - f.suitability = "youth adults seniors" - f.save - when "Adults" - f.suitability = "adults" - f.save - when "Adults Seniors" - f.suitability = "adults seniors" - f.save - when "Seniors" - f.suitability = "seniors" - f.save - else - f.suitability = f.suitability - f.save - end + MigrationFacility.find_each do |f| + f.update_column(:suitability, f.suitability.downcase) end end end diff --git a/db/migrate/20170311000347_add_suitability_to_welcomes.rb b/db/migrate/20170311000347_add_suitability_to_welcomes.rb index e8a66c1c..705845bb 100644 --- a/db/migrate/20170311000347_add_suitability_to_welcomes.rb +++ b/db/migrate/20170311000347_add_suitability_to_welcomes.rb @@ -1,7 +1,11 @@ class AddSuitabilityToWelcomes < ActiveRecord::Migration[4.2] + class MigrationFacility < ActiveRecord::Base + self.table_name = 'facilities' + end + def change change_column :facilities, :welcomes, :string - Facility.find_each do |f| + MigrationFacility.find_each do |f| f.welcomes = f.welcomes.concat(" " + f.suitability) f.save end diff --git a/db/migrate/20190824231340_add_zone_to_facilities.rb b/db/migrate/20190824231340_add_zone_to_facilities.rb index 48100eb7..fa66969e 100644 --- a/db/migrate/20190824231340_add_zone_to_facilities.rb +++ b/db/migrate/20190824231340_add_zone_to_facilities.rb @@ -1,4 +1,8 @@ class AddZoneToFacilities < ActiveRecord::Migration[4.2] + class MigrationFacility < ActiveRecord::Base + self.table_name = 'facilities' + end + def change add_reference :facilities, :zone, index: true, foreign_key: true @@ -9,11 +13,11 @@ def change reversible do |dir| dir.up do zone = Zone.create(name: zone_name, description: zone_description) - Facility.update_all({ zone_id: zone.id }) + MigrationFacility.update_all({ zone_id: zone.id }) end dir.down do - Facility.update_all({ zone_id: nil }) + MigrationFacility.update_all({ zone_id: nil }) zone = Zone.find_by(name: zone_name) zone.destroy end diff --git a/db/migrate/20210523173050_import_facilities_schedule_from_facilities.rb b/db/migrate/20210523173050_import_facilities_schedule_from_facilities.rb index 6174a5e0..64e474e8 100644 --- a/db/migrate/20210523173050_import_facilities_schedule_from_facilities.rb +++ b/db/migrate/20210523173050_import_facilities_schedule_from_facilities.rb @@ -7,8 +7,12 @@ class ImportFacilitiesScheduleFromFacilities < ActiveRecord::Migration[6.1] fri: :friday, sat: :saturday }.freeze + class MigrationFacility < ActiveRecord::Base + self.table_name = 'facilities' + end + def up - Facility.all.find_each do |facility| + MigrationFacility.all.find_each do |facility| say_with_time "Facility: #{facility.id}" do create_schedules(facility) end diff --git a/db/migrate/20250630180209_add_external_id_to_facility.rb b/db/migrate/20250630180209_add_external_id_to_facility.rb new file mode 100644 index 00000000..c8bbbbe6 --- /dev/null +++ b/db/migrate/20250630180209_add_external_id_to_facility.rb @@ -0,0 +1,5 @@ +class AddExternalIdToFacility < ActiveRecord::Migration[7.0] + def change + add_column :facilities, :external_id, :string + end +end diff --git a/db/schema.rb b/db/schema.rb index 3bb6c254..29be5ad1 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.0].define(version: 2024_04_14_180502) do +ActiveRecord::Schema[7.0].define(version: 2025_06_30_180209) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -89,6 +89,7 @@ t.integer "zone_id" t.datetime "deleted_at" t.string "discard_reason" + t.string "external_id" t.index ["user_id"], name: "index_facilities_on_user_id" t.index ["zone_id"], name: "index_facilities_on_zone_id" end diff --git a/docker-compose.yml b/docker-compose.yml index 26e712c3..d1c48363 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -44,6 +44,7 @@ services: - "35432" ports: - "35432" + - "127.0.0.1:35432:35432" # Uncomment to access this containers Postgres instance via port 5432 # - "127.0.0.1:5432:5432" environment: diff --git a/env.template b/env.template index 80b35df1..f36ec0f9 100644 --- a/env.template +++ b/env.template @@ -3,6 +3,7 @@ # Database: #DATABASE_URL= +#TEST_DATABASE_URL= # Set a key for JWT tokens #JWT_KEY= diff --git a/lib/tasks/data.rake b/lib/tasks/data.rake index 145edaeb..d899223c 100644 --- a/lib/tasks/data.rake +++ b/lib/tasks/data.rake @@ -28,6 +28,7 @@ namespace :data do "#{header} #{msg}\n" end + attention_logger = ActiveSupport::Logger.new("#{Rails.root}/log/import.log") logger = Rails.logger logger.extend(ActiveSupport::Logger.broadcast(stdout_logger)) @@ -135,6 +136,7 @@ namespace :data do next if time_slot.save logger.warn "[seed_fake] Can't create #{idx + 1}#{(idx + 1).ordinal} time slot for facility (id: #{facility.id}). Errors: #{time_slot.errors.full_messages}" + attention_logger.warn "[import] Can't create #{idx + 1}#{(idx + 1).ordinal} time slot for facility '#{facility.name}' (id: #{facility.id}). Errors: #{time_slot.errors.full_messages}" failed_schedules << facility.id end end @@ -168,6 +170,7 @@ namespace :data do ApplicationRecord.transaction do unless facility.save logger.error "[seed_fake] Failed to create Facility (id: #{facility_attribs["id"]}). Errors: #{facility.errors.full_messages}" + attention_logger.error "[import] Failed to create Facility '#{facility.name}' (id: #{facility_attribs["id"]}). Errors: #{facility.errors.full_messages}" next end diff --git a/lib/tasks/fake_data/facilities.rake b/lib/tasks/fake_data/facilities.rake index 82b2d4f0..6f9cdc9f 100644 --- a/lib/tasks/fake_data/facilities.rake +++ b/lib/tasks/fake_data/facilities.rake @@ -29,7 +29,6 @@ namespace :fake_data do params[:long] = rand(*LIMITS[:long]) params[:phone] = Faker::PhoneNumber.cell_phone params[:website] = Faker::Internet.url(path: "") - params[:description] = Faker::Lorem.paragraph params[:notes] = Faker::Lorem.paragraphs.join("\n\n") params[:verified] = (rand > 0.4) params[:zone] = zones.sample diff --git a/spec/factories/services.rb b/spec/factories/services.rb index d52fc68e..75d3adbd 100644 --- a/spec/factories/services.rb +++ b/spec/factories/services.rb @@ -2,5 +2,10 @@ factory :service do sequence(:name, "aa") { |n| "service_#{n}" } key { name.parameterize.underscore } + + factory :water_fountain_service do + name { "Water Fountain" } + key { "water_fountain" } + end end end diff --git a/spec/services/external/vancouver_api/adapters/faraday_adapter_spec.rb b/spec/services/external/vancouver_api/adapters/faraday_adapter_spec.rb new file mode 100644 index 00000000..55343bc7 --- /dev/null +++ b/spec/services/external/vancouver_api/adapters/faraday_adapter_spec.rb @@ -0,0 +1,82 @@ +# frozen_string_literal: true + +require 'rails_helper' + +RSpec.describe External::VancouverCity::Adapters::FaradayAdapter, type: :service do + let(:base_url) { 'https://api.example.com' } + + describe '.builder' do + it 'returns a builder instance' do + builder = described_class.builder(base_url) + expect(builder).to be_a(described_class::Builder) + end + end + + describe 'Builder' do + let(:builder) { described_class.builder(base_url) } + + describe '#build' do + it 'creates an adapter with default configuration' do + adapter = builder.build + + expect(adapter).to be_a(described_class) + expect(adapter.options.timeout).to eq(30) + expect(adapter.options.open_timeout).to eq(10) + expect(adapter.headers['User-Agent']).to eq('Linkvan API Client') + expect(adapter.headers['Accept']).to eq('application/json') + expect(adapter.url_prefix.to_s).to eq("#{base_url}/") + end + + it 'creates an adapter with custom configuration' do + adapter = builder + .timeout(60) + .open_timeout(20) + .user_agent('Custom Agent') + .header('Custom-Header', 'custom-value') + .build + + expect(adapter.options.timeout).to eq(60) + expect(adapter.options.open_timeout).to eq(20) + expect(adapter.headers['User-Agent']).to eq('Custom Agent') + expect(adapter.headers['Custom-Header']).to eq('custom-value') + end + end + + describe 'fluent interface' do + it 'allows method chaining' do + result = builder + .timeout(45) + .open_timeout(15) + .user_agent('Test Agent') + .header('X-Test', 'value') + + expect(result).to be(builder) + end + end + end + + describe 'HTTP method delegation' do + let(:mock_connection) { instance_double(Faraday::Connection) } + let(:adapter) { described_class.new(mock_connection) } + + it 'delegates get to connection' do + allow(mock_connection).to receive(:get) + adapter.get('/path', { param: 'value' }) + expect(mock_connection).to have_received(:get).with('/path', { param: 'value' }) + end + + it 'delegates post to connection' do + allow(mock_connection).to receive(:post) + adapter.post('/path', { data: 'value' }) + expect(mock_connection).to have_received(:post).with('/path', { data: 'value' }, {}) + end + + it 'delegates other HTTP methods' do + %w[put delete patch].each do |method| + allow(mock_connection).to receive(method.to_sym) + adapter.send(method, '/path') + expect(mock_connection).to have_received(method.to_sym) + end + end + end +end diff --git a/spec/services/external/vancouver_api/integration_test.rb b/spec/services/external/vancouver_api/integration_test.rb new file mode 100644 index 00000000..5653e1a7 --- /dev/null +++ b/spec/services/external/vancouver_api/integration_test.rb @@ -0,0 +1,73 @@ +# Final integration test for the Vancouver API Client +require_relative 'vancouver_api_client' + +def test_client + client = External::VancouverCity::VancouverApiClient.new + + puts "=== Vancouver API Client Integration Test ===" + + # Test 1: Basic dataset records request + puts "\n1. Testing basic dataset records request..." + response = client.get_dataset_records('drinking-fountains', limit: 3) + if response.success? && response.body['total_count'] > 0 + puts "✓ Success: Got #{response.body['results'].length} records" + else + puts "✗ Failed: Could not fetch records" + return false + end + + # Test 2: Dataset information + puts "\n2. Testing dataset information..." + dataset_response = client.get_dataset('drinking-fountains') + if dataset_response.success? && dataset_response.body['dataset_id'] + puts "✓ Success: Got dataset info for '#{dataset_response.body['dataset_id']}'" + else + puts "✗ Failed: Could not fetch dataset info" + return false + end + + # Test 3: Datasets list + puts "\n3. Testing datasets list..." + datasets_response = client.get_datasets(limit: 5) + if datasets_response.success? && datasets_response.body['total_count'] > 0 + puts "✓ Success: Got #{datasets_response.body['results'].length} datasets" + else + puts "✗ Failed: Could not fetch datasets list" + return false + end + + # Test 4: Query with parameters + puts "\n4. Testing query with parameters..." + filtered_response = client.get_dataset_records('drinking-fountains', + select: 'mapid,name,location', + order_by: 'name asc', + limit: 5 + ) + if filtered_response.success? && filtered_response.body['results'].all? { |r| r.keys.sort == ['location', 'mapid', 'name'] } + puts "✓ Success: Got filtered results with correct fields" + else + puts "✗ Failed: Query with parameters didn't work correctly" + return false + end + + # Test 5: Error handling + puts "\n5. Testing error handling..." + begin + client.get_dataset_records('non-existent-dataset') + puts "✗ Failed: Should have raised an error for non-existent dataset" + return false + rescue VancouverAPI::VancouverApiError => e + puts "✓ Success: Properly handled error - #{e.message[0..50]}..." + end + + puts "\n=== All tests passed! The client is working correctly. ===" + return true +end + +# Run the test +if test_client + puts "\n🎉 Vancouver API Client is ready for use!" +else + puts "\n❌ Some tests failed. Please check the implementation." + exit 1 +end diff --git a/spec/services/external/vancouver_api/openapi.json b/spec/services/external/vancouver_api/openapi.json new file mode 100644 index 00000000..314fafab --- /dev/null +++ b/spec/services/external/vancouver_api/openapi.json @@ -0,0 +1,2573 @@ +{ + "openapi": "3.0.3", + "info": { + "title": "Opendatasoft's Explore API Reference Documentation", + "version": "v2.1", + "description": "# Introduction\n\nWelcome to the Opendatasoft Explore API!\n\nThe Opendatasoft Explore API v2 is organized around REST. It provides access\nto all the data available through the platform in a coherent, hierarchical way.\n\nIf you want to learn more about how to make the most out of the API, we recommend\nreading the article [Introduction to API explore](https://userguide.opendatasoft.com/l/fr/article/b3dyxp1acz-une-introduction-l-api-explore). This article will provide you\nwith a comprehensive overview of the API's capabilities and guide you through\nthe process of accessing and manipulating data.\n\n- Only the HTTP `GET` method is supported.\n- All API endpoints return JSON.\n- Endpoints are organized in a hierarchical way describing the relative\n relationship between objects.\n- All responses contain a list of links allowing easy and relevant\n navigation through the API endpoints.\n- All endpoints use the Opendatasoft Query Language (ODSQL). This means\n that most of the time, parameters work the same way for all endpoints.\n- While the `records` endpoint is subject to a [limited number of returned records](https://help.opendatasoft.com/apis/ods-explore-v2/#tag/Dataset/operation/getRecords), the `exports` endpoint has no limitations.\n\n## Status\n\nThe v2.1 version is stable and production ready: no breaking change will be introduced in the future, following our [versioning policy](#section/Versioning).\n\n\n## v2.1 Changelog\n\nFollowing our [versioning policy](#section/Versioning), some changes have been introduced in the 2.1 version of the Explore API. Some of them may not be backward compatible with the previous version:\n\n- the `group_by` clause is now available [on export endpoints](#tag/Dataset/operation/exportRecords)\n- the XLSX export format replaces the old XLS format\n- full text searches have been reworked:\n - the [search()](#section/ODSQL-predicates/search()) function has been changed,\n - two new functions are available: [suggest()](#section/ODSQL-predicates/suggest()) and [startswith()](#section/ODSQL-predicates/startswith())\n- date functions ([year()](#section/ODSQL-functions/year()), [month()](#section/ODSQL-functions/month()), [day()](#section/ODSQL-functions/day()), [hour()](#section/ODSQL-functions/hour()), [minute()](#section/ODSQL-functions/minute()), [second()](#section/ODSQL-functions/second())) now return integers, instead of strings\n- [grouping by geo fields](#section/Opendatasoft-Query-Language-(ODSQL)/Group-by-clause) is now deprecated and the `geo_cluster()` grouping function should be used instead\n- in a `group_by`, [null values are now grouped](#section/Opendatasoft-Query-Language-(ODSQL)/Language-elements)\n- [date aggregation keys are formatted as isoformat dates](#section/Opendatasoft-Query-Language-(ODSQL)/Group-by-clause), previously as timestamps\n- [CSV exports](#tag/Dataset/operation/exportRecordsCSV) output a Byte Marker Order (BOM) character by default\n- [GPX exports](#tag/Dataset/operation/exportRecordsGPX) use `` instead of `` by default to export attributes\n- datetimes in GeojSON exports are now output as isoformat strings, they were previously output as integer timestamps\n- geometry functions have been renamed (old function names are still available):\n - `distance()` becomes [`within_distance()`](#section/ODSQL-predicates/within_distance())\n - `geometry(geom1, geom2, INTERSECTS)` becomes [`intersects()`](#section/ODSQL-predicates/intersects())\n - `geometry(geom1, geom2, DISJOINT)` becomes [`disjoint()`](#section/ODSQL-predicates/disjoint())\n - `geometry(geom1, geom2, WITHIN)` becomes [`within()`](#section/ODSQL-predicates/within())\n\nRegarding endpoints:\n- `/api/explore/v2.0//datasets//aggregates` and `/api/explore/v2.0//datasets//query` are not available anymore in v2.1. Their functionalities have been merged into the unique \"records\" endpoint (`/api/explore/v2.0//datasets//records`),\n- The output schemas of the following endpoints have been changed:\n - [`/api/explore/v2.1//datasets`](#tag/Catalog/operation/getDatasets)\n - [`/api/explore/v2.1//datasets/`](#tag/Catalog/operation/getDataset)\n - [`/api/explore/v2.1//datasets//records`](#tag/Dataset/operation/getRecords)\n - [`/api/explore/v2.1//datasets//records/`](#tag/Dataset/operation/getRecord)\n\n\n## Base URL\n\nThe Explore API is accessed using a base URL that is specific to a domain. In\nthe examples provided in the documentation, we use the domain\n.\n\nURL paths start with `/api/explore/v2.1`. A path to a resource looks like this:\n`https:///api/explore/v2.1/`.\n\n# Getting Started\n\nTo try out the Explore API, you'll get the first two records from a dataset that includes\nmonthly prices for gold.\n\nYou'll get data from a portal providing public access to data.\nSo, you don't need an account or an API key.\n\n## Step 1: Find dataset information\n\nFor this example, you will use the [GET records endpoint](#operation/getRecords) to list records from a dataset.\nSo, you need the dataset identifier and the name of the field that stores the dates to build your request.\n\n1. Open a browser and go to the desired domain:\n `https://documentation-resources.opendatasoft.com/explore`.\n\n The `explore` page lists all datasets on the domain.\n\n2. Click the desired dataset.\n\nFor this example, click \"Gold Prices\".\n\n3. Open the **Information** tab and check the **dataset identifier**.\n\nIn this example, the dataset identifier is `gold-prices`.\n\n## Step 2: Build your ODSQL query\n\nIn this example, you just want to retrieve the first set of two dataset records.\nThe records returned by the request are grouped into pages. Pages are limited in size according to the number of records per page as specified by the `limit` parameter.\n\n1. To determine how many records to retrieve, start the query with a `limit` parameter. Since you only want to retrieve the first two records, use `2` as the parameter value.\n2. To determine the specific page of data to be returned, use an `offset` parameter. Since you only want to retrieve the first set of two records, use `0` as the parameter value or keep it unspecified, since `0` is the default value.\n3. Put all the elements together.\n\nThe complete query is `limit=2&offset=0`.\n\n## Step 3: Build your request\n\nTo retrieve data using the Explore API, use the `GET` HTTP method.\n\nThe path to a resource is made up of the following elements:\n\n- A domain: in this example, query the Explore API on the `documentation-resources.opendatasoft.com` domain.\n- A resource: from the domain's `catalog`, retrieve the `records` of the `gold-prices` dataset.\n- A query: use the `select=count(*)&group_by=year(date)` query built in the previous step.\n\nThe complete path is\n`https://documentation-resources.opendatasoft.com/api/explore/v2.1/catalog/datasets/gold-prices/records?limit=2`\n\nYou can access the Explore API using curl or any HTTP client.\n\nTo make a query using curl, open a terminal and paste the following command:\n\n```shell\ncurl -X GET \"https://documentation-resources.opendatasoft.com/api/explore/v2.1/catalog/datasets/gold-prices/records?limit=2\"\n```\n\nThe response should look like this:\n\n```json\n{\n \"total_count\": 384,\n \"results\": [\n {\n \"date\": \"1988-01\",\n \"price\": 477.758\n },\n {\n \"date\": \"1988-03\",\n \"price\": 443.491\n }\n ]\n}\n```\n\n`total_count` shows the total number of records returned by the query.\n\nThe `results` array contains the two records returned by the request. In case you use a `group_by`, the `total_count`\nshows the **available** total number of groups returned by the API (with a maximum of 20000).\n\n\nFor example, we can see the first record contains two fields: `date` and `price`.\nGiven the information from the first record, in January 1988 (`1988-01`), the gold price was $477.758 (`477.758`).\n\n## Next steps\n\nRead the [Opendatasoft Query Language\n(ODSQL)](<#section/Opendatasoft-Query-Language-(ODSQL)>) reference documentation. It\nwill help you build queries, search and filter data from Opendatasoft\nportals.\n\n# Authentication\n\nAn authenticated user can be granted access to restricted datasets and\nbenefit from extended quotas for API calls. The API features an\nauthentication mechanism for users to be granted their specific\nauthorizations.\n\nFor the platform to authenticate a user, you need to either:\n\n- be logged in a portal, so a session cookie authenticating your user is\n passed along your API calls, or\n- provide an API key via the Authorization header or as a query parameter.\n\n## Finding and generating API keys\n\nAPI keys are managed via your user profile page at\n`https:///account/` or by clicking on your name in the header.\n\nGo to the tab named My API keys to see your existing API keys, revoke them\nand create new ones.\n\n**Note:** By default, every API key authenticates requests as coming from your user,\nwhich means they grant the same rights (yours) to any person using them.\nTherefore, you should not share your keys. For advanced usages, API key\npermissions can be edited using the API key Automation API.\n\n## Providing API keys within requests\n\nIf you try to access a private portal's catalog without being authenticated, the API returns a `401 Unauthorized` error.\n\nAfter generating an API key, you can use it to make authenticated requests. Depending on the permissions granted to the user for which the API key has been created, the JSON response contains only data about the datasets this user can access on the portal.\n\nIt is good practice to pass the API key to the `Authorization` header in the following format:\n\n`Authorization: Apikey `\n\nAlternatively, you can pass the API key as a query parameter in the following format:\n\n`apikey=`\n\nReplace ``with your API key.\n\n**Note:** We recommend passing the API key via a header over in a query parameter because headers are not stored in your browser history or server logs, minimizing the risk of exposure of your API key.\n\n## Using OAuth2 authorization\n\nOpendatasoft implements the OAuth2 authorization flow, allowing third-party\napplication makers to access the data hosted on an Opendatasoft platform on\nbehalf of a user while never having to deal with a password, avoiding any\nuser credential being compromised.\n\nThe Opendatasoft OAuth2 authorization flow is compliant with RFC 6749 and\nuses Bearer Tokens in compliance with RFC 6750.\n\nApplication developers who want to use the Opendatasoft APIs with OAuth2\nmust go through the following steps:\n\n1. Register their application with the Opendatasoft platform.\n2. Request approval from users via an OAuth2 authorization grant.\n3. Request a bearer token that will allow them to query the Opendatasoft\n platform APIs for a limited amount of time.\n4. Refresh the Bearer Token when it expires.\n\nCurrently, applications are registered on a specific domain and can only\naccess data on this domain.\n\n### Register an application for OAuth2 authentication\n\n1. Go to the My applications tab of your account page on the domain you want\n to register the application on.\n2. Fill the registration form with the following information:\n - Application name: the name of the application\n - Type:\n - confidential: client password is kept secret from the user and only used from a trusted environment (e.g., a web service, where the client password is stored server-side and never sent to the user)\n - public: client password is embedded in a client-side application, making it potentially available to the world (e.g., a mobile or desktop application)\n - Redirection URL: the URL users will be redirected to after they have granted you permission to access their data\n3. Store the resulting client ID and client secret that will be needed to\n perform the next steps.\n\n### Getting an authorization grant\n\n1. Redirect users to /oauth2/authorize/ with the appropriate query\n parameters.\n2. The user will then be authenticated in the platform and redirected to a\n page identifying your application.\n3. From there, the user will review the information you filled in the form\n described above and the scope of the requested access and grant your\n application the right to access their data.\n4. Once the user has accepted those terms, they will be redirected to your\n application's redirection URL with query parameters describing your\n authorization grant.\n\nThe query parameters you need to supply when redirecting the user are the\nfollowing:\n\n- `client_id`: the client ID you were given during registration\n- `redirect_uri`: the redirect URI you provided during registration\n- `response_type`: this should always be set to code\n- `scopes` (optional): a list of space-separated requested scopes.\n Currently, only `all` is supported.\n- `state` (optional): a random string of your choice\n\nThe state parameter is not mandatory, but providing one is recommended for\nsecurity reasons to verify the returned value provided in the authorization\ngrant redirect.\n\n> Example of a call to `/oauth2/authorize/`:\n\n```http\nGET /oauth2/authorize/?\nclient_id=123456789&\nredirect_uri=https://example.com&\nresponse_type=code&\nstate=ilovedata&\nscope=all HTTP/1.1\n```\n\nThe authorization grant redirect will have these values:\n\n- `code`: a 30-characters-long authorization code\n- `state`: the state passed in the request described above\n\nThe 30-character authorization code must now be converted into a bearer\ntoken within 1 hour before expiring.\n\nHere is an example of redirection following a successful authorization:\n\n```http\nHTTP/1.0 302 FOUND\n\nLocation:\nhttps://example.com?state=ilovedata&code=gKnAQc2yIfdz2mY25xxgpTY2uyG5Sv\n\n```\n\n### Converting an authorization grant to a bearer token\n\nTo receive a bearer token, convert the previously obtained authorization\ngrant via a POST request to `/oauth2/token/` with the following parameters:\n\n- `client_id`: the client ID you were given during registration\n- `client_secret`: the client secret you were given during registration\n- `redirect_uri`: the redirect URI you provided during registration\n- `grant_type`: this should always be set to `authorization_code`\n- `code`: the 30-character authorization code received as an authorization\n grant\n- `scopes` _(optional)_: a list of space-separated requested scopes.\n Currently, only `all` is supported.\n- `state` _(optional)_: a random string of your choice\n\nExample call to `/oauth2/token/`:\n\n```http\nPOST /oauth2/token/ HTTP/1.1\n\nclient_id=cid&\nclient_secret=csc&\ngrant_type=authorization_code&\ncode=GokshWxRFXmW0MaLHkDv5HrG6wieGs&\nscopes=all&\nredirect_uri=https://example.com&\nstate=ilovedata\n```\n\nAlternative call with an `Authorization` header:\n\n```http\nPOST /oauth2/token/ HTTP/1.1\n\nAuthorization: Basic Y2lkOmNzYw==\n\ngrant_type=authorization_code&\ncode=GokshWxRFXmW0MaLHkDv5HrG6wieGs&\nscopes=all&\nredirect_uri=https://example.com&state=ilovedata\n```\n\nAlternatively, you can pass your client ID and client secret through the\nAuthorization header\n\nThe response to this request is a JSON representation of a bearer token,\nwhich contains the following values:\n\n- `access_token`: the token you can use to access the user's data.\n- `expires_in`: the number of seconds before token expiration\n- `token_type`: the type of the token. It will always be `Bearer`\n- `state`: the state passed in the request described above\n- `scope`: the list of scopes of this authorization code\n- `refresh_token`: a refresh token that can be used to renew this bearer\n token when expired\n\n> Unlike the access token, which can be used any number of times until\n> expiration, the refresh token doesn't expire but can only be used once.\n\nExample response for a bearer token request:\n\n```http\nHTTP/1.0 200 OK\n\nContent-Type: application/json\n```\n\n```json\n{\n \"access_token\": \"9kxoTUYvSxnAiMpv008NBqRiqk5xWt\",\n \"expires_in\": 3600,\n \"token_type\": \"Bearer\",\n \"state\": \"ilovedata\",\n \"scope\": \"all\",\n \"refresh_token\": \"jFfDUcsK9zzNMs1zwczzJxGrimPtmf\"\n}\n```\n\n### Using the bearer token\n\nThe bearer token can be passed along requests for authentication in three\ndifferent ways:\n\n- as a query parameter of the request\n\n```http\nGET /api/end/point?access_token=9kxoTUYvSxnAiMpv008NBqRiqk5xWt HTTP/1.1\n```\n\n- in the request's `Authorization` header\n\n```http\nGET /api/end/point HTTP/1.1\n\nAuthorization: Bearer 9kxoTUYvSxnAiMpv008NBqRiqk5xWt\n```\n\n- in the request body\n\n```http\nGET /api/end/point HTTP/1.1\n\naccess_token=9kxoTUYvSxnAiMpv008NBqRiqk5xWt\n```\n\n### Refreshing a bearer token\n\nTo refresh an expired bearer token, send a request to the `/oauth2/token/`\nendpoint, with the following query parameters:\n\n- `client_id`: the client ID you were given during registration\n- `client_secret`: the client secret you were given during registration\n- `refresh_token`: the refresh token returned in the bearer token response\n- `grant_type`: this should always be set to `refresh_token`\n- `scopes`: a list of space-separated requested scopes. Currently, only\n `all` is supported.\n- `state` _(optional)_: a random string of your choice\n\nThe response to this request is identical to the bearer token response.\n\nExample token refresh call:\n\n```http\nPOST /oauth2/token/ HTTP/1.1\n\nclient_id=cid&\nclient_secret=csc&\ngrant_type=refresh_token&\nrefresh_token=jFfDUcsK9zzNMs1zwczzJxGrimPtmf&\nscopes=all&\nredirect_uri=https://example.com&\nstate=ilovedata\n```\n\n# Opendatasoft Query Language (ODSQL)\n\nFiltering features are built in the core of the Opendatasoft API engine.\n\nThe Opendatasoft Query Language (ODSQL) makes it possible to express complex\nqueries as a filtering context for datasets or records and build\naggregations or computed fields.\n\nA given filtering context can simply be copied from one API to\nanother. For example, it is possible to build a user interface that allows\nthe user to visually select the records they are interested in, using\nfull-text search, facets, and geo-filtering. Then, it allows them to\ndownload these records with the same filtering context.\n\nThe ODSQL is split into five different kinds of clauses:\n\n- The [`select` clause](#section/Opendatasoft-Query-Language-(ODSQL)/Select-clause) allows choosing the returned fields, giving them an\n alias, manipulating them with functions like count, sum, min, max, etc.\n- The [`where` clause](#section/Opendatasoft-Query-Language-(ODSQL)/Where-clause) acts as a filter for the returned datasets or records,\n thanks to boolean operations, filter functions, arithmetic expressions, etc.\n- The [`group by` clause](#section/Opendatasoft-Query-Language-(ODSQL)/Group-by-clause) allows aggregating rows together based on fields,\n numeric ranges, or dates.\n- The [`order by` and `limit` clauses](#section/Opendatasoft-Query-Language-(ODSQL)/Order-by-clause) allow choosing the order and quantity\n of rows received as a response.\n\nThese clauses are used as parameters in the Explore API v2 for searching,\naggregating, and exporting datasets and records. Depending on the used\nendpoint, some features of the query language are available or not in the\nrequest.\n\n**Note:** the whole query language is case insensitive, and spaces are optional. In this documentation, the uppercase is used for language keywords, only for clarity purposes.\n\n## Language elements\n\nODSQL clauses are composed of basic language elements. These can either be\n[field names or aliases](#field-names), [literals](#literals-in-odsql-clauses) or [reserved keywords](#reserved-keywords-in-odsql-clauses).\n\n### Field names\n\nA field name is made of alphanumeric characters and underscores and refers to a field of a dataset or to a dynamically created field that only exists during the query (a.k.a. an alias).\n\n**Note:** if a field name contains only numbers or is a keyword, it must be enclosed in back quotes.\n\n> Examples of a field names:\n\n```sql\nmy_field > 10 -- my_field is a field name\n\n`12` > 10 -- without back quotes, 12 would be considered a numeric literal\n\n`and`: \"value\" -- AND is a keyword, `and` represents a field name then\n```\n\n### Literals in ODSQL clauses\n\nLiterals are fixed values of a specific type and can be used in comparison, assignments, or functions.\n\nThere are 6 types of literal:\n\n- string\n- numeric\n- date\n- boolean\n- geometry\n- null\n\n#### String literal\n\nA string literal is a literal enclosed in either single or double quotes.\n\n> Examples of a string literal:\n\n```sql\n\"Word\"\n\n\"Multiple words\"\n\n'Using single quotes'\n```\n\n**Note:** `\\` (backslash) character can be used to escape special characters. For example to escape a single quote: `'Don\\'t worry'`.\n\n#### Numeric literal\n\nA numeric literal is either an integer or a decimal value. It is not enclosed in quotes.\n\n> Examples of numeric literals:\n\n```sql\n100\n\n5.8\n\nmy_field > 108.7\n```\n\n#### Date literal\n\nA date literal is defined with a `date` keyword followed by a valid date\nformat enclosed in single quotes.\n\nA valid date can be:\n\n- an [ISO 8601 date](https://en.wikipedia.org/wiki/ISO_8601), or\n- a slash-separated date in the YYYY/MM/DD (year/month/day) format.\n\n> Examples of a date literal:\n\n```sql\ndate'2017-04-03T08:02'\n\ndate'2018/04/01'\n```\n\n#### Boolean literal\n\nA boolean literal can either be a `TRUE` or a `FALSE` keyword (case\ninsensitive). It should be used in boolean filters.\n\n> Example of a boolean literal:\n\n```sql\nmy_boolean_field is TRUE\n\nmy_boolean_field: FALSE\n```\n\n#### Geometry literal\n\nA geometry literal is defined with a `geom` keyword followed by a valid\ngeometry expression enclosed in single quotes.\n\nSupported geometry expressions are:\n\n- [WKT/WKB](https://en.wikipedia.org/wiki/Well-known_text)\n- [GeoJSON geometry](https://en.wikipedia.org/wiki/GeoJSON)\n\n> Example of a geometry literal:\n\n```sql\nwithin_distance(my_geo_field, geom'POINT(1 1)', 10km)\n\ngeometry(my_geo_field, geom'{\"type\": \"Polygon\",\"coordinates\":[[[100.0,\n0.0],[101.0, 0.0],[101.0, 1.0],[100.0, 1.0],[100.0,0.0]]]}')\n```\n#### Null literal\n\nThe `null` literal (case insensitive) is used to represent the absence of a value.\n\nIt is present in the [is null filter](#section/ODSQL-predicates/IS-NULL-filter) to test whether a field has a value or not.\n\n### Reserved keywords in ODSQL clauses\n\nReserved keywords can be used inside clauses for building ODSQL expressions.\n\nWhen used in a clause as a field literal, the reserved keyword must be\nescaped with back quotes.\n\nList of reserved keywords:\n\n- and\n- as\n- asc\n- avg\n- by\n- count\n- date_format\n- day\n- dayofweek\n- desc\n- distinct\n- equi\n- false\n- group\n- hour\n- ifnull\n- or\n- limit\n- lower\n- max\n- millisecond\n- min\n- minute\n- month\n- not\n- null\n- quarter\n- range\n- search\n- second\n- select\n- sum\n- top\n- true\n- upper\n- where\n- year\n\nFor example, `not` is a reserved keyword and must be escaped with back quotes if referred to as a field literal:\n\n```sql\nmy_field_literal is not true -- my_field_literal is not a reserved keyword, there's no need to escape it\n\n`not` is not true -- not is a reserved keyword and must be escaped\n```\n\n### Handling null values\n\nA `null` value in a dataset is used when the value in a field is unknown or missing. It means that there is no data for a field in a record.\n\nEach clause behaves differently to handle null values:\n\n- When selecting a field in a `select` clause, null values are represented as `null`.\n- When filtering with a `where` clause, a comparison involving at least one null value is false, meaning that null values are filtered out of the result.\n- When grouping with a `group_by` clause, no group exists for null values in v2.0, a null group do exist starting with v2.1\n- When sorting with an `order_by` clause, null values come after all other values, regardless of the sorting direction (i.e., ascending or descending).\n\nDefault handling of null values can be changed by filtering using the [is null filter](#section/ODSQL-predicates/IS-NULL-filter) or replacing null values by an alternative value or expression using the `ifnull` function.\n\n\n## Select clause\n\nThe select clause can be used in records Explore APIs as the parameter `select`.\n\nThe select clause allows:\n\n- choosing the fields that will be returned for each row,\n- transforming fields using arithmetic,\n- renaming fields,\n- adding computed virtual fields to fields, and\n- including or excluding fields based on a pattern.\n\nA select clause is composed of a single select expression or a list of comma-separated expressions.\n\nA select expression can be:\n\n- a field literal,\n- an include/exclude function,\n- an arithmetic expression, or\n- an aggregation function.\n\nExcept for the include/exclude function, a select expression can define a label with the keyword `AS`. This label will be used in the output of the API as `key` for the select expression result.\n\n### Select field names\n\nA select field name is the simplest form of select expression. It takes a field name that must be returned in the result.\nIt also accepts the special character `*` to select all fields (it is the default behavior).\n\nIf a select expression is used in conjunction with a `group by` clause, the selected field name must be in the `group by` clause.\n\n> Examples of a select field literal:\n\n```sql\n* -- Select all fields\n\nfield1, field2, field3 -- Only select field1, field2, and field3\n\nfield1 AS my_field, field2 -- Renaming field1 as my_field and select field2\n```\n\n\n### Select aggregation\n\nLike in the SQL language, a `select` can also express an aggregation expression.\n\nThe following aggregation functions are available:\n\n- [avg (average)](#section/ODSQL-aggregate-functions/avg())\n- [count](#section/ODSQL-aggregate-functions/count())\n- [count distinct](#section/ODSQL-aggregate-functions/count(distinct))\n- [envelope](#section/ODSQL-aggregate-functions/envelope())\n- [max (maximum)](#section/ODSQL-aggregate-functions/max())\n- [median](#section/ODSQL-aggregate-functions/median())\n- [min (minimum)](#section/ODSQL-aggregate-functions/min())\n- [percentile](#section/ODSQL-aggregate-functions/percentile())\n- [sum](#section/ODSQL-aggregate-functions/sum())\n\n> Examples of an aggregation expression:\n\n```sql\nSUM(population) as sum_population -- Will compute the sum of all values for the field `population` returned as sum_population\n\nCOUNT(*) -- Return number of elements\n```\n\n## Where clause\n\nThe where clause can be used in the whole Explore API as the parameter `where`.\n\nThe where clause allows one to filter rows with a combination of boolean expressions.\n\nA where expression can be:\n\n- a search query\n- a filter function\n- a comparison filter\n- an equality filter\n\nWhere expressions can be combined with [boolean operators](#boolean-operators) and grouped via parenthesis.\n\n> Example of a where clause with boolean operators:\n```sql\nmy_numeric_field > 10 and my_text_field like \"paris\" or within_distance(my_geo_field, geom'POINT(1 1)', 1km)\n```\n> This where clause filters results where numeric_field > 10 and (my_text_field contains the word `paris` or distance between my_geo_field and the point with 1,1 as lat,lon is under 1 kilometer)\n\n**Note**: it is generally possible to use multiple `where` clauses on an API endpoint. They are combined with a boolean `AND` in that case.\n\n### Boolean operators\n\nWhere expressions can use boolean operators to express boolean filter.\n\nThere are 3 different boolean operations:\n\n- `AND`: results must match left and right expressions\n- `OR`: results must match left or right expression\n- `NOT`: inverses the next expression\n\n`AND` has precedence over the `OR` operator. It means that, in the expression `a or b and c`, the sub-expression `b and c` is interpreted and executed first. It can also be written with parenthesis: `a or (b and c)`.\n\nIn order to change operator precedence, it is possible to use parenthesis in the expression. To give precedence to the `OR` operator, the above expression can be written `(a or b) and c`.\n\n> Examples of a boolean operator:\n\n```sql\nmy_boolean_field OR my_numeric_field > 50 and my_date_field > date'1972'\n-- Results can have my_boolean_field to true. They can also have my_numeric_field greater than 50 and my_date_field older than 1972\n\n(my_boolean_field OR my_numeric_field > 50) and my_date_field > date'1972'\n-- Results must have my_date_field older than 1972. They also must have my_boolean_field to true or my_numeric_field greater than 50\n```\n\n### Search query filter\n\nFilter search queries are queries that don’t refer to fields. They only contain quoted strings and boolean operators. Filter search queries perform full-text searches on all visible fields of each record and return matching rows.\n\nIf the string contains more than one word, the query will be an `AND` query on each tokenized word.\n\n> Examples of a search query:\n\n```sql\n\"tree\"\n\n\"tree\" AND \"flower\"\n\n\"tree\" OR \"car\"\n\nNOT \"dog\"\n\n\"dog\" AND NOT \"cat\"\n```\n\n> Examples of a search query with multiple words:\n\n```sql\n\"film\" -- returns results that contain film\n\n\"action movies\" -- returns results that contain action and movies.\n```\n\n### Filter functions\n\nFilter functions are built-in functions that can be used in a `where` clause.\n\nAvailable filter functions are:\n\n- [`search` function](#section/ODSQL-predicates/search()), to perform a full-text search\n- [`suggest` function](#section/ODSQL-predicates/suggest())\n- [`startswith` function](#section/ODSQL-predicates/startswith())\n- [`in_bbox` function](#section/ODSQL-predicates/in_bbox()), to filter in a geographical area defined by a bounding box\n- [`within_distance` function](#section/ODSQL-predicates/within_distance()), to filter in a geographical area defined by a circle\n- [`intersects`](#section/ODSQL-predicates/intersects()), [`disjoint`](#section/ODSQL-predicates/disjoint()) and [`within`](#section/ODSQL-predicates/within()) to filter in a geographical area defined by a geometry\n\n### Comparison operators\n\nThree types of comparison operators can be used in a `where` clause:\n\n- [text comparison operators](#section/ODSQL-predicates/Text-comparison-operators)\n- [numeric comparison operators](#section/ODSQL-predicates/Numeric-comparison-operators)\n- [date comparison operators](#section/ODSQL-predicates/Date-comparison-operators)\n\n## Group by clause\n\nThe group by clause can be used in the Explore API as the parameter `group_by`.\n\nThe group by clause creates groups from data depending on a group by expression. Groups of data cannot be returned directly and aggregation functions in the `select` clause have to be used to \"summarize\" and return a value for each group. An operation of \"aggregation\" can then be described by two parts: the `group_by` part that make groups of rows of data from a specific criterion and an aggregation function in the `select` clause to reduce each group to a row.\n\nA group by clause can contain:\n\n- a single group by expression, or\n- a list of comma-separated group by expressions.\n\nLike select expressions, a group by expression can have an `AS` statement to give it a label.\n\nA group by expression can be:\n\n- empty,\n- a field,\n- [static ranges](#section/ODSQL-grouping-functions/range()-group-by-static-ranges),\n- [ranges of equal widths](#section/ODSQL-grouping-functions/range()-group-by-ranges-of-equal-widths),\n- the result of a function applied on a field value (e.g. a date function, or a date format)\n\n> Example of a simple group by expression with a label:\n\n```sql\ngroup_by=my_field as myfield\n```\n\n> Example of multiple group by expressions with a label:\n\n```sql\ngroup_by=my_field1,my_field2 as my_field\n```\n\n### Empty group by\n\nWhen no `group_by` part is expressed, rows of data are implicitly grouped into an sole group and aggregation functions are computed on the whole set of records.\n\n### Group by field\n\nA group by field expression allows the grouping of specified field values. It creates a group for each different field value.\n\n**Format:** `group_by=`\n\n> Example of a simple group by field expression\n\n```sql\ngroup_by=my_field\n```\n\n**Note:**\n- Starting with v2.1, grouping by geopoint or geoshape fields is not supported directly anymore. Please use the [geo_cluster() grouping function](#section/ODSQL-grouping-functions/geo_cluster()) to make groups out of geo points.\n- Starting with v2.1, grouping by a date field now formats the key of each group as a string representing the ISO formatting of the date, when it was an integer timestamp in v2.0\n\n## Order by clause\n\nThe order by clause can be used to sort rows returned by a query.\n\nThe parameter `order_by` adds an order by clause to an API query.\nIt accepts a list of comma-separated expressions followed by a direction:\n\n- ASC for ascending\n- DESC for descending\n\n**Format:** `order_by = expression [ ASC | DESC ], ...`\n\nAn order by expression can be:\n\n- a field\n- an aggregation function\n- [a `random` function](#section/ODSQL-functions/random())\n\nThe direction, if not specified, is ASC (ascending) by default.\nThe random sorting will circumvent the default direction.\n\n**Note:** when ordering by both aggregations and fields, the aggregation order must be at the head of the list. For example, `order_by = avg(age), gender works`, but `order_by = gender, avg(age)` returns an error.\n\n> Examples of an order by clause\n\n```sql\ngroup_by=city & order_by=city ASC -- Order cities alphabetically\n\ngroup_by=city & order_by=count(*) DESC -- Order each city by its number of records\n\nselect=count(*) as population_count & group_by=city & order_by=population_count DESC -- Order each city by its number of records, using a label\n\ngroup_by=city, year(birth_date) as birth_year & order_by=city DESC, birth_year ASC -- Order by city and then by year of birth\n```\n\n# ODSQL functions\n\n## length()\n\n**Syntax:** `length(|)`\n\n**Returned type:** `integer`\n\n**Clauses where it can be used:** `select`, `where`, `order_by`\n\nReturns the string length of its parameter, i.e. the number of characters that composes the string.\n\n## now()\n\n**Syntax:** `now()`\n\n**Returned type:** `datetime`\n\n**Clauses where it can be used:** `select`, `where`, `order_by`\n\n### Parameters to the now() function\n\n> Examples, assuming the current date time is 2021-05-06 12:34:55.450500+00:00, which is a Thursday\n```sql\nnow() -- Returns '2021-05-06T12:34:55.450500+00:00'\nnow(year=2000) -- Sets the year component to return '2000-05-06T12:34:55.450500+00:00'\nnow(years=-1) -- Sets the year to one year ago which is '2020-05-06T12:34:55.450500+00:00'\nnow(year=2001, months=-1) -- Sets the year to 2001 and subtract 1 month to return '2000-04-06T12:34:55.450500+00:00'\nnow(day=31,month=2) -- Sets the day to 31, then the month to 2. The actual day part is rounded to 28 '2021-02-28T12:34:55.450500+00:00'\nnow(weekday=0) -- Sets the day to the next Monday which is '2021-05-10T12:34:55.450500+00:00'\nnow(mondays=+1) -- Sets the day to the next Monday which is also '2021-05-10T12:34:55.450500+00:00'\nnow(mondays=-1) -- Sets the day to the previous Monday which is '2021-05-03T12:34:55.450500+00:00'\n```\n\nWithout any parameters, the `now()` function returns the current date and time.\n\nThe function may also be called with named parameters to set or modify certain parts of the current date and time.\n\nWith each parameter, an integer value is required, interpreted as an absolute value or as a relative value to a part of the current date and time.\n\nParameter names in their singular form will set a certain part of the current date and time to the given value. Parameter names written in plural will add or subtract the given value to a part of the current date and time.\n\nIf a parameter is used multiple times in the call, only the last one is actually used, the others are ignored.\n\n| Parameter name | Accepted values | Description |\n| -------------- | -------------------- | --------------------------------------------------------------------------------------------------------- |\n| `year` | 1 to 9999 | Year component |\n| `years` | Any integer | Value to add to or subtract from the year component |\n| `month` | 1 to 12 | Month component |\n| `months` | Any integer | Value to add to or subtract from the month component, then the year component in case of overflow |\n| `day` | Any positive integer | Day component, rounded to the maximum valid day number for the current month |\n| `days` | Any integer | Value to add to or subtract from the day component, then the month component in case of overflow |\n| `hour` | 0 to 23 | Hour component |\n| `hours` | Any integer | Value to add to or subtract from the hour component, then the day component in case of overflow |\n| `minute` | 0 to 59 | Minute component |\n| `minutes` | Any integer | Value to add to or subtract from the minute component, then the hour component in case of overflow |\n| `second` | 0 to 59 | Second component |\n| `seconds` | Any integer | Value to add to or subtract from the second component, then the minute component in case of overflow |\n| `microsecond` | 0 to 999999 | Microsecond component |\n| `microseconds` | Any integer | Value to add to or subtract from the microsecond component, then the second component in case of overflow |\n| `weekday` | 0 to 6 | Day of the week, 0 for monday to 6 for sunday |\n| `mondays` | Any integer | Number of Mondays to add to or subtract from the current date |\n| `tuesdays` | Any integer | Number of Tuesdays to add to or subtract from the current date |\n| `wednesdays` | Any integer | Number of Wednesdays to add to or subtract from the current date |\n| `thursdays` | Any integer | Number of Thursdays to add to or subtract from the current date |\n| `fridays` | Any integer | Number of Fridays to add to or subtract from the current date |\n| `saturdays` | Any integer | Number of Saturdays to add to or subtract from the current date |\n| `sundays` | Any integer | Number of Sundays to add to or subtract from the current date |\n\n## year()\n\n**Syntax:** `year(|||)`\n\n**Returned type:** `integer`\n\n**Clauses where it can be used:** `select`, `where`, `order_by`, `group_by`\n\nReturns the year number of a date or datetime as a string.\n\n## month()\n\n**Syntax:** `month(|||)`\n\n**Returned type:** `integer`\n\n**Clauses where it can be used:** `select`, `where`, `order_by`, `group_by`\n\nReturns the month number (between 1 and 12) of a date or datetime as a string.\n\n## day()\n\n**Syntax:** `day(|||)`\n\n**Returned type:** `integer`\n\n**Clauses where it can be used:** `select`, `where`, `order_by`, `group_by`\n\nReturns the day number of the month (between 1 and 31) of a date or datetime as a string.\n\n## hour()\n\n**Syntax:** `hour(|||)`\n\n**Returned type:** `integer`\n\n**Clauses where it can be used:** `select`, `where`, `order_by`, `group_by`\n\nReturns the hour number (between 0 and 23) of a date or datetime as a string.\n\n## minute()\n\n**Syntax:** `minute(|||)`\n\n**Returned type:** `integer`\n\n**Clauses where it can be used:** `select`, `where`, `order_by`, `group_by`\n\nReturns the minute number (between 0 and 59) of a date or datetime as a string.\n\n## second()\n\n**Syntax:** `second(|||)`\n\n**Returned type:** `integer`\n\n**Clauses where it can be used:** `select`, `where`, `order_by`, `group_by`\n\nReturns the second number (between 0 and 59) of a date or datetime as a string.\n\n## date_format()\n\n**Syntax:** `date_format(, )`\n\n**Arguments:**\n\n- ``: a date field,\n- ``: a string describing how to format the date (see below)\n\n**Returned type:** `string`.\n\n**Clauses where it can be used:** `select`, `where`, `order_by`, `group_by`\n\n\n`` is a string, where each character or group of characters\nwill be replaced by parts of the date in the returned string.\n\nThe following formats are available for a date format expression:\n\n| Symbol | Description | Examples |\n| ------------ | ------------------------------------------------------ | -------- |\n| yy or YY | year on two digits | 20 |\n| yyyy or YYYY | year on four digits | 2020 |\n| xx | weekyear\\* on two digits | 96 |\n| xxxx | weekyear\\* on four digits | 1996 |\n| w | week of weekyear | 7 |\n| ww | week of weekyear, left-padded with 0 | 07 |\n| e | day of week, as a number, 1 for Monday to 7 for Sunday | 2 |\n| E | day of week, abbreviated name | sun. |\n| EEEE | day of week, full name | Sunday |\n| D | day of year | 89 |\n| DDD | day of year, left-padded with 0 | 089 |\n| M | month of year | 7 |\n| MM | month of year, left-padded with 0 | 07 |\n| MMMM | month of year, full name | July |\n| d | day of month | 8 |\n| dd | day of month, left-padded with 0 | 08 |\n| H | hour of day, 0-23 | 9 |\n| HH | hour of day, 00-23, left-padded with 0 | 09 |\n| m | minute of hour, 0-59 | 13 |\n| mm | minute of hour, 00-59, left-padded with 0 | 09 |\n| s | second of minute, 0-59 | 13 |\n| ss | second of minute, 00-59, left-padded with 0 | 09 |\n\n\\*Years and week years differ slightly. For more information, see the\n[definition](https://en.wikipedia.org/wiki/ISO_week_date) of week years.\n\nThe date format can contain free text that won't be interpreted. The free\ntext must be surrounded by single quotes '.\n\nTo insert a single quote in the final string, it must be doubled.\n\nSome special characters can also be used as delimiters between date\ncomponents: `?`, `,`, `.`, `:`, `/` and `-`.\n\n> Examples of a `date_format` function, where `date_field` = '2007-11-20T01:23:45':\n\n```sql\ndate_format(date_field, 'dd/MM/YYYY') -- Returns '20/11/2007'\n\ndate_format(date_field, \"'The date is 'dd/MM/YYYY\") -- Returns 'The date is\n20/11/2007'\n\ndate_format(date_field, \"'The date is '''dd/MM/YYYY''\") -- Returns \"The date\nis '20/11/2007'\"\n\ndate_format(date_field, 'E') -- Returns 'mar.'\n\ndate_format(date_field, 'EEEE') -- Returns 'mardi'\n\ndate_format(date_field, 'H') -- Returns '1'\n\ndate_format(date_field, 'HH') -- Returns '01'\n\ndate_format(date_field, 'yy') -- Returns '07'\n\ndate_format(date_field, 'yyyy') -- Returns '2007'\n\ndate_format(date_field, 'M') -- Returns '11'\n\ndate_format(date_field, 'MM') -- Returns '11'\n```\n\nWhen used in the `where` clause, `date_format` must be compared to string\nvalues.\n\n> Example of a `date_format` function used in a `where` clause:\n\n```sql\nwhere=date_format(date_field, 'dd') = '08'\n```\n\nYou can use the `lang` parameter to force the output language.\n\n## json_format()\n\n**Syntax:** `json_format(,[[|]])`\n\n**Returned type:** `text` or `json`\n\n**Clause where it can be used:** `select`\n\n**Description:**\n\nFormats the text field into JSON if possible. If the text can be transformed into valid JSON, it returns the formatted JSON string. If the text cannot be transformed into valid JSON, it returns either the fallback value if provided or the original string.\n\n- `` (mandatory): A text field to be formatted into JSON. It cannot be multivalued.\n\n- `` (optional): A fallback string to return if the text cannot be transformed into valid JSON. If omitted, the original string is returned in case of invalid JSON.\n\n**Note**: On `/exports`, except with the JSON export format, this function returns the original text value.\n\n> Example of a `json_format` function used in a `select` clause:\n\n```sql\nselect=json_format(text_field)\n\nselect=json_format(text_field, 'bad json')\n\nselect=json_format(text_field, null)\n```\n\n## ifnull()\n\n**Syntax:** `ifnull(, )`\n\n**Arguments:**\n\n- ``: a field or an expression\n- ``: an alternative field, expression or literal\n\n**Clauses where it can be used:** `select`, `where`, `order_by`, `group_by`\n\n**Returned type:** the type of `` when not null\n\n**Returned value:** the result of `` if `` returns a null value. The result of `` otherwise.\n\nThe returned type of `` and `` should be the same.\n\nFor `group_by` clause, expressions are restricted to fields and literals.\n\n> Examples of `ifnull` function, where `int_field` contains some null values:\n\n```sql\nifnull(int_field, 0) -- value of int_field is 0 for each row that contains a null value\n```\n\n## lower()\n\n**Syntax:** `lower(|)`\n\n**Returned type:** `string`\n\n**Clauses where it can be used:** `select`, `where`, `order_by` and `group_by`\n\nReturns a string in lowercase.\n\n> Some examples:\n```sql\nlower('JAZZ') -- returns 'jazz'\n\nlower(text_field) -- returns the lowercase representation of the field\n```\n\n## include() and exclude()\n\n**Syntax:** `include()`\n\n**Syntax:** `exclude()`\n\n**Clauses where it can be used:** `select` only\n\nInclude and exclude are functions that accept fields names.\n\nFields listed in an include function are present in the result, whereas fields listed in an exclude function are absent from the result.\n\nFields can contain a wildcard suffix (the `*` character). In that case, the inclusion/exclusion works on all field names beginning with the value preceding the wildcard.\n\n**Note**: `include()` and `exclude()` are pseudo functions: they do not return a value, but are used as a declaration to constrain the list of returned fields.\n\n> Examples of an include/exclude:\n\n```sql\ninclude(pop) -- will only include fields which name is pop\n\nexclude(pop) -- will exclude fields which name is pop\n\ninclude(pop*) -- Will include fields beginning with pop\n```\n\n## Arithmetic operators\n\nAn arithmetic expression accepts simple arithmetic operations. It accepts field names, numeric constants or text values, and scalar functions. More complex arithmetic expressions can be formed by connecting these elements with arithmetic operators:\n\n- `+`: add\n- `-`: subtract\n- `*`: multiply\n- `/`: divide\n\n**Note:** A division by zero returns a null value.\n\nArithmetic operators are only defined on numeric values.\n\n> Examples of arithmetic expressions:\n\n```sql\n2 * population -- the value of the field `population` doubled\n\n\"hello\" -- the constant string \"hello\"\n\nlength(country_name) -- the string length of the field `country_name`\n```\n\n## random()\n\n**Syntax:** `random ( )`\n\n**Clauses where it can be used:** `order_by` only\n\nThe `` is the seed of the random function.\nWhen using the random function with one same seed, it will return the same random order each time.\n\n> Examples of an order by random\n\n```sql\ngroup_by=city & order_by=random(1) -- Order cities randomly\n\ngroup_by=city & order_by=random(1) -- Order cities randomly in the exact same order as the first example\n\ngroup_by=city & order_by=random(2) -- Order cities randomly in a different order than the first example\n```\n\n## distance()\n\n**Syntax:** `distance(, )`\n\n**Clauses where it can be used:** `select`, `order_by`\n\n**Returned type:** numeric\n\n> Examples of a `distance` function:\n\n```sql\ndistance(field_name, GEOM'')\n```\n\nThe `distance` function computes arc distance between geo_point field and a point geometry as reference. Distance (in m) can be returned using `select` and/or used to sort records.\n\n## vector_similarity()\n\n**Syntax:** `vector_similarity(\"\")`\n\n**Clauses where it can be used:** only `order_by`\n\n**Returned type:** float\n\nThis function is able to compute a semantic distance between your search query and the catalog metadata, e.g the titles,\nkeywords, themes and descriptions. It can only be used for a catalog search. The results will be semantically sorted,\ni.e. the first results will have some content where the meaning should be close to your search query. For instance if\nyou search `\"funny kitty\"`, the results should have some content which contains the query terms but also `cats`, `fun`,\n`pets`, etc.\n\n> Examples of a `vector_similarity` function:\n\n```sql\nvector_similarity(\"jazz concerts in nyc\")\n```\n\n\n\n# ODSQL predicates\n\nPredicates are functions that return a boolean value (`true` or `false`). They can be used to filter results in the `where` clause.\n\n## search()\n\n**Syntax:** `search(|*, )` where:\n- first parameters are the set of fields on which the search is done:\n - `*` or empty to search on all visible fields\n - a subset of field names separated with a comma `,`\n- the string to search for as last parameter\n\n**Clauses where it can be used:** `where` only\n\n**Returned type:** boolean\n\nThe `search()` function performs a full-text query on all selected fields of each record and return matching records.\n\nIt is a fuzzy search and a prefix search: `` is first split into terms separated by a space, the first terms are searched for with a certain level of fuziness (see below), and the last term is a prefix search.\nThe level of fuziness for each term depends on the length of the term:\n- for terms with a length > 5, it matches strings with a [Levenshtein distance](https://en.wikipedia.org/wiki/Levenshtein_distance) of 2,\n- for terms with a length > 2, it matches strings with a [Levenshtein distance](https://en.wikipedia.org/wiki/Levenshtein_distance) of 1\n\nThe matching is **case insensitive**.\n\n> Examples of a `search` function:\n\n```sql\nsearch(title, \"bok of secret\") -- will match \"THE BOOK OF SECRETS\"\n```\n\n\n## suggest()\n\n**Syntax:** `suggest(|*, )` where:\n- first parameters are the set of fields on which the search is done:\n - `*` or empty to search on all visible fields\n - a subset of field names separated with a comma `,`\n- the string to search for as last parameter\n\n**Clauses where it can be used:** `where` only\n\n**Returned type:** boolean\n\n> Examples of a `suggest` function:\n\n```sql\nsuggest(*, \"film\") -- returns results that contain film, films, filmography, etc. in at least one visible field\nsuggest(\"film\") -- equivalent to the above query\nsearch(title, \"secret\") -- will match \"THE BOOK OF SECRETS\"\n\nsuggest(text_field, other_text_field, \"film\") -- same search but in text_field or other_text_field\nsuggest(text_field, \"film\") OR suggest(other_text_field, \"film\") -- equivalent to the above query\n\nsuggest(text_field, \"film\") AND suggest(other_text_field, \"film\") -- returns results that contain film, films, filmography, etc. in both fields\n```\n\nThe `suggest()` function performs a full-text query on all selected fields of each record and return matching records.\nIt is a prefix search: it matches the text fields that contain terms **beginning with** the searched string.\n\nThe matching is **case insensitive**.\n\n**Note:** this function may miss some results that match the prefix when it is used with small prefixes.\n\n## vector_similarity_threshold()\n\n**Syntax:** `vector_similarity_threshold()` where:\n- the string of your search query as unique parameter\n\n**Clauses where it can be used:** `where` only\n\n**Ony available for the catalog search**. It won't work if you want to use it to filter records from a dataset.\n\nIt can be used to carry out a **semantic search** of your catalog instead of a classic lexical search (based on the term\nand not the meaning). With this function, you can search using a sentence, synonyms, or in languages other than the original language of your catalog.\n\nThis semantic search analyzes only the following asset metadata to compute the similarity score: *title*, *keywords*, *themes* and\n*description*. When the *territory* dataset metadata is available, it can also be used.\n\n**Returned type:** boolean\n\nContrary to the `vector_similarity()` function used within `order_by` which returns all catalog results, this function\ncan automatically compute a semantic score threshold in order to avoid irrelevant assets in your search results. This\n\"threshold method\" is based on the [\"Kneedle\" algorithm](https://dl.acm.org/doi/abs/10.1109/ICDCSW.2011.20) to find the\ninflection point from the semantic scores.\n\nSource: Finding a \"Kneedle\" in a Haystack: Detecting Knee Points in System Behavior, *Satopaa, Ville and Albrecht, Jeannie and Irwin, David and Raghavan, Barath*, 2011, IEEE Computer Society -- https://doi.org/10.1109/ICDCSW.2011.20\n\n> Examples of a `vector_similarity_threshold` function:\n\n```sql\n-- can return results that contain film, movie, pulp fiction, filmography,\n-- or content about other movie directors.\nvector_similarity_threshold(\"quentin tarantino movies\")\n\n-- can also return results in another language, for instance in French or German depending on the portal.\nvector_similarity_threshold(\"content about music & jazz\")\n```\n\n**Note**: The semantic method may generate irrelevant results. This potential inaccuracy is due to several underlying issues:\n * The vector content may prove insufficient in detail or context to reliably produce accurate results.\n * A significant challenge is the inherent asymmetry between the concise nature of user search queries and the more extensive data within asset embeddings, which can impair result precision.\n\n\n\n## startswith()\n\n**Syntax:** `startswith(|*, )` where:\n- first parameters are the set of fields on which the search is done:\n - `*` or empty to search on all visible fields\n - a subset of field names separated with a comma `,`\n- the string to search for as last parameter\n\n**Clauses where it can be used:** `where` only\n\n**Returned type:** boolean\n\n> Examples of a `startswith` function:\n\n```sql\nstartswith(id, \"ID4536\") -- will match id that start with \"ID4536\"\nstartswith(title, \"SECRET\") -- will match \"SECRET DEFENSE\" but not \"THE BOOK OF SECRETS\", nor \"book of secret\"\n```\n\nThe `startswith()` function performs a text query on all selected fields of each record and return matching records.\nIt is a prefix search: it matches the text fields that contain **strings** **beginning with** the searched string.\nContrary to the `suggest()` function, the comparison is made on the whole string, without splitting it by spaces and forming **terms** before.\n\nThe matching is **case sensitive**.\n\n\n## within_distance()\n\n**Syntax:** `within_distance(, , )`\n\n**Clauses where it can be used:** `where` only\n\n**Returned type:** boolean\n\n> Examples of a `within_distance` function:\n\n```sql\nwithin_distance(field_name, GEOM'', 1km)\n\nwithin_distance(field_name, GEOM'', 100yd)\n```\n\nThe `within_distance` function limits the result set to a geographical area defined by a circle. This circle must be defined by its center and a distance.\n\n- The center of the circle is expressed as a [geometry literal](#section/Opendatasoft-Query-Language-(ODSQL)/Language-elements).\n- The distance is numeric and can have a unit in:\n\n - miles (mi)\n - yards (yd)\n - feet (ft)\n - meters (m)\n - centimeters (cm)\n - kilometers (km)\n - millimeters (mm)\n\n## in_bbox()\n\n**Syntax:** `in_bbox(, lat1, lon1, lat2, lon2)`\n\n**Clauses where it can be used:** `where` only\n\n**Returned type:** boolean\n\nThis function limits the results to records that have their `` contained in a given bounding box. The bounding box is expressed by giving its two extreme points: (lat1, lon1) for the latitude and longitude of the first point and (lat2, lon2) for the latitude and longitude of the second point.\n\n## intersects()\n\n**Syntax:** `intersects(, )`\n\n**Clauses where it can be used:** `where` only\n\n**Returned type:** boolean\n\n> Examples of a `geometry` function:\n\n```sql\nintersects(geo_shape, geom'POLYGON((2.331161 48.869762, 2.3600006 48.87574, 2.373046875 48.85101, 2.3503875 48.84209, 2.3376846 48.85451, 2.3311614 48.869762))')\n```\n\nThe `intersects` function limits the result set to a geographical area that intersects a given geometry.\n\nThis function must be defined with a [geometry literal](#section/Opendatasoft-Query-Language-(ODSQL)/Language-elements).\n\n## disjoint()\n\n**Syntax:** `disjoint(, )`\n\n**Clauses where it can be used:** `where` only\n\n**Returned type:** boolean\n\n> Examples of a `geometry` function:\n\n```sql\ndisjoint(geo_shape, geom'POLYGON((2.331161 48.869762, 2.3600006 48.87574, 2.373046875 48.85101, 2.3503875 48.84209, 2.3376846 48.85451, 2.3311614 48.869762))')\n```\n\nThe `disjoint` function limits the result set to a geographical area that is disjoint from a given geometry.\n\nThis function must be defined with a [geometry literal](#section/Opendatasoft-Query-Language-(ODSQL)/Language-elements).\n\n## within()\n\n**Syntax:** `within(, )`\n\n**Clauses where it can be used:** `where` only\n\n**Returned type:** boolean\n\n> Examples of a `within` function:\n\n```sql\nwithin(geo_shape, geom'POLYGON((2.331161 48.869762, 2.3600006 48.87574, 2.373046875 48.85101, 2.3503875 48.84209, 2.3376846 48.85451, 2.3311614 48.869762))')\n```\n\nThe `within` function limits the result set to a geographical area that lie within a given geometry.\n\nThis function must be defined with a [geometry literal](#section/Opendatasoft-Query-Language-(ODSQL)/Language-elements).\n\n\n## Text comparison operators\n\n**Clauses where it can be used:** `where` only\n\n|
Operator
| Description |\n| --------------------------------------- | -------------------------------------------------------------------------------- |\n| `=` | Perform an exact query (not tokenized and not normalized) on the specified field |\n\n## Numeric comparison operators\n\n**Clauses where it can be used:** `where` only\n\n|
Operator
| Description |\n| --------------------------------------- | ----------------------------------------------------------------------------------------------------------- |\n| `=` | Match a numeric value |\n| `>`,`<`,`>=`,`<=` | Return results whose field values are larger, smaller, larger or equal, smaller or equal to the given value |\n\n## Date comparison operators\n\n**Clauses where it can be used:** `where` only\n\n|
Operator
| Description |\n| --------------------------------------- | ------------------------------------------------------------------- |\n| `=` | Match a date |\n| `>`,`<`,`>=`,`<=` | Return results whose field date are after or before the given value |\n\n## Boolean field filter\n\n**Syntax:**\n\n- ``\n- ` is (true|false)`\n\n**Clauses where it can be used:** `where` only\n\nA boolean field filter takes a boolean field and restricts results only if the boolean value is `true`.\n\nThere are 2 ways of creating a filter expression:\n\n- with a field literal only: in that case, it filters the result where the field literal value is `true`\n- with a field literal followed by the `is` keyword, then `true` or `false` keywords\n\n> Examples of a boolean field filter:\n\n```sql\nmy_boolean_field -- Filters results where boolean_field is true\n\nmy_boolean_field is false -- Filters results where boolean_field is false\n```\n\nwhere `` must be a valid boolean field\n\n## IN filter\n\n**Syntax:**\n\n- on a numeric range: ` IN (]|[) (TO|..) (]|[)`\n- on a date range: ` (IN|:) (]|[) (TO|..) (]|[)`\n- on a list: ` IN (, *)`\n- on a multivalued field: ` IN `\n\n**Clauses where it can be used:** `where` only\n\nAn `IN` filter restricts results using a search in a list or a range of values.\n\nThere are 3 ways of using an `IN` filter:\n- to search that a field's value is present in a numeric or a date range.\n- to search that a field's value is present in a list of literals.\n- to search that a literal value is present in a multivalued field's values.\n\n> Example of an `IN` filter expression on a numeric range:\n\n```sql\nnumeric_field IN [1..10] -- Filters results such as 1 <= numeric_field <= 10\n\nnumeric_field IN ]1..10[ -- Filters results such as 1 < numeric_field < 10\n```\n\n> Example of an `IN` filter expression on a date range:\n\n```sql\ndate_field IN [date'2017'..date'2018'] -- Filters results such as date_field date is between year 2017 and 2018\n```\n\nYou can also use the `:` syntax:\n\n```sql\ndate_field:['2022-03-12' TO '2022-04-27'] -- single quotes for the date parameters\n\ndate_field:[20220312 TO 20220427] -- the date parameters as a full integer\n```\n\n> Note that in the previous example, you don't have to cast the parameter if you use quotes or a full integer. However, this does not work if you don't use quotes or a specific cast such as `date:'YYYY-MM-DD'`. For instance the following example will raise an ODSQL error.\n\n```sql\ndate_field IN [2022-03-12 TO 2022-04-27] -- this is not valid\n```\n\n> Example of an `IN` filter expression on a list of literals:\n\n```sql\nmy_field IN (\"Paris\", \"Nantes\", \"Lorient\", \"Besançon\") -- Filters results such as my_field is equal to \"Paris\", \"Nantes\", \"Lorient\" or \"Besançon\"\n```\n\n> Example of an `IN` filter expression on a multivalued field:\n\n```sql\n\"Paris\" IN multivalued_text_field -- Filters results such as the literal \"Paris\" is present in the multivalued field\n15 IN multivalued_int_field -- Same as above but with an integer literal\n12.087 IN mutlivalued_decimal_field -- Same as above but with a decimal literal\ntrue IN mutlivalued_boolean_field -- Same as above but with a boolean literal\n```\n\n\n## IS NULL filter\n\n**Syntax:**\n\n- ` is null`\n- ` is not null`\n\n**Clauses where it can be used:** `where` only\n\nA null field filter takes a field and restricts results only if the field values are null.\nThe opposite filter, `is not null`, takes a field and restricts results only if the field values are not null.\n\n> Examples of a null filter expression:\n\n```sql\nfilm_name is null -- matches records where film_name is null\n\nfilm_name is not null -- matches records where film_name is not null\n```\n\n# ODSQL aggregate functions\n\nAggregation functions are functions that perform a computation on a set of values and return one value. They are usually used in conjunction with a `group_by` clause.\n\n## avg()\n\n**Syntax:** `avg()`\n\n**Clauses where it can be used:** `select`, `order_by`\n\n**Returned type:** numeric\n\nThis function takes a numeric field. It returns the average (`avg`) of this field over a group.\n\n> Example of an `avg` aggregation:\n\n```sql\navg(population) as avg_population -- Return the average of the population\n```\n\n## count()\n\n**Syntax:** `count(|*)`\n\n**Clauses where it can be used:** `select`, `order_by`\n\n**Returned type:** integer\n\nThis function computes a number of elements.\n\nIt accepts the following parameters:\n\n- a field name: only returns the count for non-`null` values of this field\n- a `*`: returns the count of all elements\n\n> Examples of a `count` aggregation:\n\n```sql\ncount(*) -- Return number of elements\n\ncount(population) as population_count_not_empty -- Return number of elements where `population` field is not empty\n```\n\n## count(distinct)\n\n**Syntax:** `count(distinct |*)`\n\n**Clauses where it can be used:** `select`, `order_by`\n\n**Returned type:** integer\n\nThis function computes the **unique** numbers of elements, eliminating the repetitive appearance of the same data.\n\nIt accepts the following parameters:\n\n- a field name: only returns the number of unique non-`null` values of this field.\n- the function `ifnull(, )`: same as above, but replace all `null` values with an alternative expression before counting. See the documentation of the [ifnull function](#section/ODSQL-functions/ifnull()) for more details on its syntax.\n\n**Note:** For performance reasons, the count is always approximated.\n\n> Examples of a `count distinct` aggregation:\n\n```sql\ncount(distinct species) -- Return the number of unique values for the field species\n\ncount(distinct ifnull(species, \"'unknown'\")) -- Same as above, but null values will be counted as equals to 'unknown'\n```\n\n## envelope()\n\n**Syntax:** `envelope()`\n\n**Clauses where it can be used:** `select`\n\n**Returned type:** geo_shape\n\nThis function takes a geo_point field. It returns the convex hull (`envelope`) of all the points of the geo_point field.\n\n> Example of an `envelope` aggregation:\n\n```sql\nenvelope(geo_point) as convex_hull -- Return the convex_hull for the geo_point field\n```\n\n## bbox()\n\n**Syntax:** `bbox()`\n\n**Clauses where it can be used:** `select`\n\n**Returned type:** geo_shape\n\nThis function takes a geo_point or a geo_shape field. It returns the bounding box of all the geometries.\n\n> Example of an `bbox` aggregation:\n\n```sql\nbbox(geo_point) -- Return the bounding box of all the points\n```\n\n## max()\n\n**Syntax:** `max(|)`\n\n**Clauses where it can be used:** `select`, `order_by`\n\n**Returned type:** numeric or date\n\nThis function takes a numeric or a date field name. It returns the maximum value (`max`) of this field.\n\n> Example of a `max` aggregation:\n\n```sql\nmax(population) as max_population -- Return max value for population field\n```\n\n## median()\n\n**Syntax:** `median()`\n\n**Clauses where it can be used:** `select`, `order_by`\n\n**Returned type:** numeric\n\nThis function takes a numeric field name. It returns the median (`median`) of this field's values. Since the median is the 50th percentile, it is a shortcut for `percentile(field, 50)`.\n\n> Example of a `median` aggregation:\n\n```sql\nmedian(age) as med -- Return the median of the age field\n```\n\n## min()\n\n**Syntax:** `max(|)`\n\n**Clauses where it can be used:** `select`, `order_by`\n\n**Returned type:** numeric or date\n\nThis function takes a numeric or a date field name. It returns the minimum value (`min`) of this field.\n\n> Example of a `min` aggregation:\n\n```sql\nmin(population) as min_population -- Return min value for population field\n```\n\n## percentile()\n\n**Syntax:** `percentile(, )`\n\n**Clauses where it can be used:** `select`, `order_by`\n\n**Returned type:** numeric\n\nThis function takes a numeric field name and a percentile. It returns the nth percentile (`percentile`) of this field. Percentile must be a decimal value between `0` and `100`.\n\n> Example of a `percentile` aggregation:\n\n```sql\npercentile(age, 1) as first_percentile -- Return the first percentile of the age field\n```\n\n## sum()\n\n**Syntax:** `sum()`\n\n**Clauses where it can be used:** `select`, `order_by`\n\n**Returned type:** numeric\n\nThis function takes a numeric field name as an argument. It returns the sum of all values for a field.\n\n> Example of a `sum` aggregation:\n\n```sql\nsum(population) as sum_population -- Return the sum of all values for the population field\n```\n\n# ODSQL grouping functions\n\nGrouping functions are functions that can be used in the `group_by` clause to separate a set of records into different sets that share a common property. An [aggregate function](#section/ODSQL-aggregate-functions) can then be applied on each group separately.\n\n## range() - group by static ranges\n\n**Syntax for numerical ranges:** `range( [, *]?, [,]* [, *]?)`\nwhere `` must be a numeric field\n\n**Syntax for date/datetime ranges:** `range( [, *]?, [,]* [, *]?)`\nwhere `` must be a date or datetime field.\n\n**Clauses where it can be used:** `group_by` only\n\nThe static range function takes a variable number of parameters:\n\n- a field name, and\n- a variable number of parameters. Each parameter can be a numerical literal, a date literal or the special syntax `*` to denote infinity.\n\nA `*` as first step makes values lower than the lower bound included in the first group, a `*` as last step makes values greater than the upper bound included in the last group.\n\nNote that the resulting aggregation includes the lower bound and excludes the higher bound.\n\nRanges can be set on numerical fields and on date/datetime fields.\n\nFor a recall, date literals are composed of the `date` identifier followed by a date in ISO format, e.g. `date'2021-02-01'`\n\n> Examples of a group by static ranges expression:\n\n```sql\nRANGE(population, *, 10, 50, 100, *) -- Creates 4 groups: [*, 9], [10, 49], [50, 99] and [100, *]\nRANGE(population, 20.5, *) -- Creates 1 group: [20.5, *[\nRANGE(population, 1,2,3) -- Creates 2 groups: [1-1], [2, 2]\nRANGE(date, *, date'2020-11-13', date'2021-01-01') -- Creates 2 groups: [*, 2020-11-13T00:00:00.000Z[ and [2020-11-13T00:00:00.000Z, 2021-01-01T00:00:00.000Z[\n```\n\n## range() - group by ranges of equal widths\n\n**Syntax for numerical fields:** `group_by=range(, )` where `` must be a numeric field\n\n**Syntax for date/datetime fields:** `group_by=range(, )` where `` must be a date/datetime field, and `` is one of the following (case sensitive) string constants:\n\n- `ms`, `millisecond` or `milliseconds`,\n- `s`, `second` or `seconds`,\n- `m`, `minute` or `minutes`,\n- `h`, `hour` or `hours`,\n- `d`, `day` or `days`,\n- `w`, `week` or `weeks`,\n- `M`, `month` or `months`,\n- `q`, `quarter` or `quarters`,\n- `y`, `year` or `years`.\n\n**Note:** For some interval units (week, month, quarter, and year), an interval value of more than one is not supported yet.\n\n**Clauses where it can be used:** `group_by` only\n\n\nIt is possible to group values of a field by ranges of equal widths, also known as histograms.\n\nRanges of equal widths are supported for numerical fields and date/datetime fields.\n\nThe `range` function for ranges of equal widths takes for parameters:\n\n- a field name, and\n- the desired width of each group.\n\nFor date/datetime fields, the width of each group is expressed by a time interval with a special syntax (see above).\n\n**Note:** groups that do not contain any data are not returned.\n\n> Example of a group by ranges of equal widths expression:\n\n```sql\nRANGE(population, 5)\n```\n\n> `5` is the desired width of each returned group.\n> For values of a `population` field that span from 10 to 28, it creates the following groups:\n\n```markdown\n- [10, 15[\n- [15, 20[\n- [20, 25[\n- [25, 30[\n```\n\n> Example of a date histogram:\n\n```sql\nRANGE(date, 1 day)\n```\n\n> Groups created (one for each day):\n\n```markdown\n- [2020-01-01T00:00:00.000Z, 2020-01-02T00:00:00.000Z[\n- [2020-01-02T00:00:00.000Z, 2020-01-03T00:00:00.000Z[\n- [2020-01-04T00:00:00.000Z, 2020-01-05T00:00:00.000Z[\n- ...\n```\n\n> No group is created for 2020-01-03 since no data is available for this day.\n\n## geo_cluster()\n\n**Syntax:** `group_by=geo_cluster(, [, ])` where:\n- `` is an integer between 0 and 25\n- `` is an optional integer. It defaults to 40.\n\n**Clauses where it can be used:** `group_by` only\n\nThis function groups points that are close to each other.\n\nIt first groups points by their [geohash](https://en.wikipedia.org/wiki/Geohash) of a certain level. The level (or precision) of the used geohash grid is determined by both the zoom and the radius parameters:\n- `zoom_level` follows the [\"slippy map\" zoom level hierarchy](https://wiki.openstreetmap.org/wiki/Zoom_levels): at zoom level 0, one tile represents the whole planet and each sub level sub divides the tile into 4 sub tiles.\n- `radius` is expressed in pixels on a tile of 256x256 pixels at the given `zoom_level`\n- the geohash precision is the one where radius in meter is smaller than the geohash cell dimension\n\nA second step merges groups that may have points that are very close. This is to circumvent the \"grid\" effect of the first step.\ne.g. At geohash grid level 1, France is split into 4 geohash cells \"g\", \"u\", \"e\" and \"s\" that cross somewhere north-east from Bordeaux. A dense group of points that lie in a small area around Bordeaux may be split into more than 1 bucket with the first step. This second step is here to join them back.\n\nThe join step is done by:\n- computing the centroid of each group in the first step, in addition to the list of points that lie within,\n- merging groups that have centroids that are closer than the distance given by the radius parameter\n\n# Versioning\n\nThe development of the Explore API V2 continues, new features are added every month and bugs are fixed every week. To ensure that evolutions do not break any application, dashoard or usage, the team guarantees that:\n- ODSQL is backward compatible, new syntax cannot replace existing one\n- responses body are stable, new keys can be added but keys cannot be renamed or deleted\n- urls and endpoints are stable\n\nSome features require to go deeper and to introduce a breaking change. A breaking change is a violation of one or multiple of our warranties listed above (e.g., a different syntax in ODSQL, a modification of the response body structure, etc.). To ensure that it won't break any existing usage, these changes will be part of a new API version.\n\nAn API version is composed of:\n- a stable url, which is `/api/explore/v2.1` for the version ${{api_versi- an exhaustive documentation, available on the [Helphub](https://help.opendatasoft.com/apis/ods-explore-v2/)\n\nWhen a new version is available, the team will communicate the release notes widely and be available to help on migration. Previous version response contains a header `ODS-Explore-API-Deprecation`.\n\n## Deprecation warnings\n\nWhen a feature needs to be changed in a breaking way, the new behaviour is introduced in a new version and the behaviour stays the same for the current version. The API response may contain in this case dedicated HTTP headers that give information about the possible deprecation of a used feature:\n- `ODS-Explore-API-Deprecation` will contain deprecation messages (separated by `;` if there are multiple messages). A deprecation message has the following formatting: `: deprecation message`. e.g. `DATE_KEYS_AS_ISOFORMAT: Dates used in group keys are currently returned as timestamps and will be returned as standard formatted date strings in the next API version`\n- `Link` contains the URL of the version changelog\n\n", + "contact": { + "email": "support@opendatasoft.com" + }, + "license": { + "name": "Copyright Opendatasoft" + } + }, + "servers": [ + { + "url": "https://documentation-resources.opendatasoft.com/api/explore/v2.1", + "description": "Portal that hosts example datasets" + } + ], + "security": [ + { + "apikey": [] + } + ], + "tags": [ + { + "name": "Catalog", + "description": "API to enumerate datasets" + }, + { + "name": "Dataset", + "description": "API to work on records" + } + ], + "paths": { + "/catalog/datasets": { + "get": { + "summary": "Query catalog datasets", + "operationId": "getDatasets", + "tags": [ + "Catalog" + ], + "description": "Retrieve available datasets.", + "parameters": [ + { + "$ref": "#/components/parameters/select" + }, + { + "$ref": "#/components/parameters/where" + }, + { + "$ref": "#/components/parameters/order_by" + }, + { + "$ref": "#/components/parameters/limit" + }, + { + "$ref": "#/components/parameters/offset" + }, + { + "$ref": "#/components/parameters/refine" + }, + { + "$ref": "#/components/parameters/exclude" + }, + { + "$ref": "#/components/parameters/lang" + }, + { + "$ref": "#/components/parameters/timezone" + }, + { + "$ref": "#/components/parameters/group_by" + }, + { + "$ref": "#/components/parameters/include_links" + }, + { + "$ref": "#/components/parameters/include_app_metas" + } + ], + "responses": { + "200": { + "description": "A list of available datasets", + "content": { + "application/json; charset=utf-8": { + "schema": { + "$ref": "#/components/schemas/results_dataset-v2.1" + }, + "examples": { + "datasets": { + "$ref": "#/components/examples/datasets-v2.1" + } + } + } + } + }, + "400": { + "$ref": "#/components/responses/bad_request" + }, + "401": { + "description": "Unauthorized" + }, + "429": { + "$ref": "#/components/responses/quota" + }, + "500": { + "description": "Internal Server Error" + } + } + } + }, + "/catalog/exports": { + "get": { + "summary": "List export formats", + "operationId": "listExportFormats", + "tags": [ + "Catalog" + ], + "description": "List available export formats", + "responses": { + "200": { + "description": "A list of available export formats", + "content": { + "application/json; charset=utf-8": { + "schema": { + "type": "object", + "properties": { + "links": { + "type": "array", + "items": { + "$ref": "#/components/schemas/links" + } + } + } + } + } + } + }, + "400": { + "$ref": "#/components/responses/bad_request" + }, + "401": { + "description": "Unauthorized" + }, + "429": { + "$ref": "#/components/responses/quota" + }, + "500": { + "description": "Internal Server Error" + } + } + } + }, + "/catalog/exports/{format}": { + "get": { + "summary": "Export a catalog", + "operationId": "exportDatasets", + "tags": [ + "Catalog" + ], + "description": "Export a catalog in the desired format.", + "parameters": [ + { + "$ref": "#/components/parameters/format-catalog-v2.1" + }, + { + "$ref": "#/components/parameters/select" + }, + { + "$ref": "#/components/parameters/where" + }, + { + "$ref": "#/components/parameters/order_by" + }, + { + "$ref": "#/components/parameters/group_by" + }, + { + "$ref": "#/components/parameters/limit_export" + }, + { + "$ref": "#/components/parameters/offset" + }, + { + "$ref": "#/components/parameters/refine" + }, + { + "$ref": "#/components/parameters/exclude" + }, + { + "$ref": "#/components/parameters/lang" + }, + { + "$ref": "#/components/parameters/timezone" + } + ], + "responses": { + "200": { + "description": "Return a file" + }, + "400": { + "$ref": "#/components/responses/bad_request" + }, + "401": { + "description": "Unauthorized" + }, + "429": { + "$ref": "#/components/responses/quota" + }, + "500": { + "description": "Internal Server Error" + } + } + } + }, + "/catalog/exports/csv": { + "get": { + "summary": "Export a catalog in CSV", + "operationId": "exportCatalogCSV", + "tags": [ + "Catalog" + ], + "description": "Export a catalog in CSV (Comma Separated Values). Specific parameters are described here", + "parameters": [ + { + "name": "delimiter", + "in": "query", + "required": false, + "schema": { + "type": "string", + "enum": [ + ";", + ",", + "\t", + "|" + ], + "default": ";" + }, + "description": "Sets the field delimiter of the CSV export" + }, + { + "name": "list_separator", + "in": "query", + "required": false, + "schema": { + "type": "string", + "default": "," + }, + "description": "Sets the separator character used for multivalued strings" + }, + { + "name": "quote_all", + "in": "query", + "required": false, + "schema": { + "type": "boolean", + "default": false + }, + "description": "Set it to true to force quoting all strings, i.e. surrounding all strings with quote characters" + }, + { + "name": "with_bom", + "in": "query", + "required": false, + "schema": { + "type": "boolean", + "default": true + }, + "description": "Set it to true to force the first characters of the CSV file to be a Unicode Byte Order Mask (0xFEFF). It usually makes Excel correctly open the output CSV file without warning.\n**Warning:** the default value of this parameter is `false` in v2.0 and `true` starting with v2.1" + } + ], + "responses": { + "200": { + "description": "Return a file" + }, + "400": { + "$ref": "#/components/responses/bad_request" + }, + "401": { + "description": "Unauthorized" + }, + "429": { + "$ref": "#/components/responses/quota" + }, + "500": { + "description": "Internal Server Error" + } + } + } + }, + "/catalog/exports/dcat{dcat_ap_format}": { + "get": { + "summary": "Export a catalog in RDF/XML (DCAT)", + "operationId": "exportCatalogDCAT", + "tags": [ + "Catalog" + ], + "description": "Export a catalog in RDF/XML described with DCAT (Data Catalog Vocabulary). Specific parameters are described here", + "parameters": [ + { + "$ref": "#/components/parameters/dcat_format" + }, + { + "name": "include_exports", + "in": "query", + "required": false, + "schema": { + "$ref": "#/components/schemas/enum-format-datasets-v2.1" + }, + "description": "Sets the datasets exports exposed in the DCAT export. By default, all exports are exposed.", + "examples": { + "legacy": { + "summary": "Only expose csv, json and geojson datasets exports", + "value": "csv,json,geojson" + } + } + }, + { + "name": "use_labels_in_exports", + "in": "query", + "required": false, + "schema": { + "type": "boolean", + "default": true + }, + "description": "If set to `true`, this parameter will make distributions output the label of each field rather than its name. This parameter only applies on distributions that contain a list of the fields in their output (e.g., CSV, XLSX)." + } + ], + "responses": { + "200": { + "description": "Return a file" + }, + "400": { + "$ref": "#/components/responses/bad_request" + }, + "401": { + "description": "Unauthorized" + }, + "429": { + "$ref": "#/components/responses/quota" + }, + "500": { + "description": "Internal Server Error" + } + } + } + }, + "/catalog/facets": { + "get": { + "summary": "List facet values", + "operationId": "getDatasetsFacets", + "tags": [ + "Catalog" + ], + "description": "Enumerate facet values for datasets and returns a list of values for each facet.\nCan be used to implement guided navigation in large result sets.", + "parameters": [ + { + "$ref": "#/components/parameters/facet" + }, + { + "$ref": "#/components/parameters/refine" + }, + { + "$ref": "#/components/parameters/exclude" + }, + { + "$ref": "#/components/parameters/where" + }, + { + "$ref": "#/components/parameters/timezone" + } + ], + "responses": { + "200": { + "description": "An enumeration of facets", + "content": { + "application/json; charset=utf-8": { + "schema": { + "type": "object", + "properties": { + "links": { + "type": "array", + "items": { + "$ref": "#/components/schemas/links" + } + }, + "facets": { + "type": "array", + "items": { + "$ref": "#/components/schemas/facet_enumeration" + } + } + } + }, + "examples": { + "catalog_facets": { + "$ref": "#/components/examples/catalog_facets" + } + } + } + } + }, + "400": { + "$ref": "#/components/responses/bad_request" + }, + "401": { + "description": "Unauthorized" + }, + "429": { + "$ref": "#/components/responses/quota" + }, + "500": { + "description": "Internal Server Error" + } + } + } + }, + "/catalog/datasets/{dataset_id}/records": { + "get": { + "summary": "Query dataset records", + "operationId": "getRecords", + "tags": [ + "Dataset" + ], + "description": "Perform a query on dataset records.", + "parameters": [ + { + "$ref": "#/components/parameters/dataset_id" + }, + { + "$ref": "#/components/parameters/select" + }, + { + "$ref": "#/components/parameters/where" + }, + { + "$ref": "#/components/parameters/group_by" + }, + { + "$ref": "#/components/parameters/order_by" + }, + { + "$ref": "#/components/parameters/limit" + }, + { + "$ref": "#/components/parameters/offset" + }, + { + "$ref": "#/components/parameters/refine" + }, + { + "$ref": "#/components/parameters/exclude" + }, + { + "$ref": "#/components/parameters/lang" + }, + { + "$ref": "#/components/parameters/timezone" + }, + { + "$ref": "#/components/parameters/include_links" + }, + { + "$ref": "#/components/parameters/include_app_metas" + } + ], + "responses": { + "200": { + "description": "Records", + "content": { + "application/json; charset=utf-8": { + "schema": { + "$ref": "#/components/schemas/results-v2.1" + }, + "examples": { + "records": { + "$ref": "#/components/examples/records" + }, + "group_by_country": { + "$ref": "#/components/examples/group_by_country-v2.1" + } + } + } + } + }, + "400": { + "$ref": "#/components/responses/bad_request" + }, + "401": { + "description": "Unauthorized" + }, + "429": { + "$ref": "#/components/responses/quota" + }, + "500": { + "description": "Internal Server Error" + } + } + } + }, + "/catalog/datasets/{dataset_id}/exports": { + "get": { + "summary": "List export formats", + "operationId": "listDatasetExportFormats", + "tags": [ + "Dataset" + ], + "description": "List available export formats", + "parameters": [ + { + "$ref": "#/components/parameters/dataset_id" + } + ], + "responses": { + "200": { + "description": "A list of available export formats", + "content": { + "application/json; charset=utf-8": { + "schema": { + "type": "object", + "properties": { + "links": { + "type": "array", + "items": { + "$ref": "#/components/schemas/links" + } + } + } + } + } + } + }, + "400": { + "$ref": "#/components/responses/bad_request" + }, + "401": { + "description": "Unauthorized" + }, + "429": { + "$ref": "#/components/responses/quota" + }, + "500": { + "description": "Internal Server Error" + } + } + } + }, + "/catalog/datasets/{dataset_id}/exports/{format}": { + "get": { + "summary": "Export a dataset", + "operationId": "exportRecords", + "tags": [ + "Dataset" + ], + "description": "Export a dataset in the desired format.\n**Note:** The `group_by` parameter is only available on exports starting with the v2.1", + "parameters": [ + { + "$ref": "#/components/parameters/dataset_id" + }, + { + "$ref": "#/components/parameters/format-datasets-v2.1" + }, + { + "$ref": "#/components/parameters/select" + }, + { + "$ref": "#/components/parameters/where" + }, + { + "$ref": "#/components/parameters/order_by" + }, + { + "$ref": "#/components/parameters/group_by" + }, + { + "$ref": "#/components/parameters/limit_export" + }, + { + "$ref": "#/components/parameters/refine" + }, + { + "$ref": "#/components/parameters/exclude" + }, + { + "$ref": "#/components/parameters/lang" + }, + { + "$ref": "#/components/parameters/timezone" + }, + { + "$ref": "#/components/parameters/use_labels" + }, + { + "$ref": "#/components/parameters/compressed" + }, + { + "$ref": "#/components/parameters/epsg" + } + ], + "responses": { + "200": { + "description": "Return a file" + }, + "400": { + "$ref": "#/components/responses/bad_request" + }, + "401": { + "description": "Unauthorized" + }, + "429": { + "$ref": "#/components/responses/quota" + }, + "500": { + "description": "Internal Server Error" + } + } + } + }, + "/catalog/datasets/{dataset_id}/exports/csv": { + "get": { + "summary": "Export a dataset in CSV", + "operationId": "exportRecordsCSV", + "tags": [ + "Dataset" + ], + "description": "Export a dataset in CSV (Comma Separated Values). Specific parameters are described here", + "parameters": [ + { + "$ref": "#/components/parameters/dataset_id" + }, + { + "name": "delimiter", + "in": "query", + "required": false, + "schema": { + "type": "string", + "enum": [ + ";", + ",", + "\t", + "|" + ], + "default": ";" + }, + "description": "Sets the field delimiter of the CSV export" + }, + { + "name": "list_separator", + "in": "query", + "required": false, + "schema": { + "type": "string", + "default": "," + }, + "description": "Sets the separator character used for multivalued strings" + }, + { + "name": "quote_all", + "in": "query", + "required": false, + "schema": { + "type": "boolean", + "default": false + }, + "description": "Set it to true to force quoting all strings, i.e. surrounding all strings with quote characters" + }, + { + "name": "with_bom", + "in": "query", + "required": false, + "schema": { + "type": "boolean", + "default": true + }, + "description": "Set it to true to force the first characters of the CSV file to be a Unicode Byte Order Mask (0xFEFF). It usually makes Excel correctly open the output CSV file without warning.\n**Warning:** the default value of this parameter is `false` in v2.0 and `true` starting with v2.1" + } + ], + "responses": { + "200": { + "description": "Return a file" + }, + "400": { + "$ref": "#/components/responses/bad_request" + }, + "401": { + "description": "Unauthorized" + }, + "429": { + "$ref": "#/components/responses/quota" + }, + "500": { + "description": "Internal Server Error" + } + } + } + }, + "/catalog/datasets/{dataset_id}/exports/parquet": { + "get": { + "summary": "Export a dataset in Parquet", + "operationId": "exportRecordsParquet", + "tags": [ + "Dataset" + ], + "description": "Export a dataset in Parquet. Specific parameters are described here", + "parameters": [ + { + "$ref": "#/components/parameters/dataset_id" + }, + { + "name": "parquet_compression", + "in": "query", + "required": false, + "schema": { + "type": "string", + "enum": [ + "snappy", + "zstd" + ], + "default": "snappy" + }, + "description": "Sets the compression parameter for the Parquet export file" + } + ], + "responses": { + "200": { + "description": "Return a file" + }, + "400": { + "$ref": "#/components/responses/bad_request" + }, + "401": { + "description": "Unauthorized" + }, + "429": { + "$ref": "#/components/responses/quota" + }, + "500": { + "description": "Internal Server Error" + } + } + } + }, + "/catalog/datasets/{dataset_id}/exports/gpx": { + "get": { + "summary": "Export a dataset in GPX", + "operationId": "exportRecordsGPX", + "tags": [ + "Dataset" + ], + "description": "Export a dataset in GPX. Specific parameters are described here", + "parameters": [ + { + "$ref": "#/components/parameters/dataset_id" + }, + { + "name": "name_field", + "in": "query", + "required": false, + "schema": { + "type": "string" + }, + "description": "Sets the field that is used as the 'name' attribute in the GPX output" + }, + { + "name": "description_field_list", + "in": "query", + "required": false, + "schema": { + "type": "string" + }, + "description": "Sets the fields to use in the 'description' attribute of the GPX output" + }, + { + "name": "use_extension", + "in": "query", + "required": false, + "schema": { + "type": "boolean", + "default": true + }, + "description": "Set it to true to use the `` tag for attributes (as GDAL does). Set it to false to use the `` tag for attributes.\n**Warning:** the default value of this parameter is `false` in v2.0 and `true` starting with v2.1" + } + ], + "responses": { + "200": { + "description": "Return a file" + }, + "400": { + "$ref": "#/components/responses/bad_request" + }, + "401": { + "description": "Unauthorized" + }, + "429": { + "$ref": "#/components/responses/quota" + }, + "500": { + "description": "Internal Server Error" + } + } + } + }, + "/catalog/datasets/{dataset_id}": { + "get": { + "summary": "Show dataset information", + "operationId": "getDataset", + "tags": [ + "Catalog" + ], + "description": "Returns a list of available endpoints for the specified dataset, with metadata and endpoints.\n\nThe response includes the following links:\n* the attachments endpoint\n* the files endpoint\n* the records endpoint\n* the catalog endpoint.", + "parameters": [ + { + "$ref": "#/components/parameters/dataset_id" + }, + { + "$ref": "#/components/parameters/select" + }, + { + "$ref": "#/components/parameters/lang" + }, + { + "$ref": "#/components/parameters/timezone" + }, + { + "$ref": "#/components/parameters/include_links" + }, + { + "$ref": "#/components/parameters/include_app_metas" + } + ], + "responses": { + "200": { + "description": "The dataset", + "content": { + "application/json; charset=utf-8json": { + "schema": { + "$ref": "#/components/schemas/dataset-v2.1" + }, + "examples": { + "dataset": { + "$ref": "#/components/examples/dataset-v2.1" + } + } + } + } + }, + "400": { + "$ref": "#/components/responses/bad_request" + }, + "401": { + "description": "Unauthorized" + }, + "429": { + "$ref": "#/components/responses/quota" + }, + "500": { + "description": "Internal Server Error" + } + } + } + }, + "/catalog/datasets/{dataset_id}/facets": { + "get": { + "summary": "List dataset facets", + "operationId": "getRecordsFacets", + "tags": [ + "Dataset" + ], + "description": "Enumerates facet values for records and returns a list of values for each facet.\nCan be used to implement guided navigation in large result sets.\n", + "parameters": [ + { + "$ref": "#/components/parameters/dataset_id" + }, + { + "$ref": "#/components/parameters/where" + }, + { + "$ref": "#/components/parameters/refine" + }, + { + "$ref": "#/components/parameters/exclude" + }, + { + "$ref": "#/components/parameters/facet" + }, + { + "$ref": "#/components/parameters/lang" + }, + { + "$ref": "#/components/parameters/timezone" + } + ], + "responses": { + "200": { + "description": "Facets enumeration", + "content": { + "application/json; charset=utf-8": { + "schema": { + "type": "object", + "properties": { + "links": { + "type": "array", + "items": { + "$ref": "#/components/schemas/links" + } + }, + "facets": { + "type": "array", + "items": { + "$ref": "#/components/schemas/facet_enumeration" + } + } + } + }, + "examples": { + "facets": { + "$ref": "#/components/examples/facets" + } + } + } + } + }, + "400": { + "$ref": "#/components/responses/bad_request" + }, + "401": { + "description": "Unauthorized" + }, + "429": { + "$ref": "#/components/responses/quota" + }, + "500": { + "description": "Internal Server Error" + } + } + } + }, + "/catalog/datasets/{dataset_id}/attachments": { + "get": { + "summary": "List dataset attachments", + "operationId": "getDatasetAttachments", + "tags": [ + "Dataset" + ], + "description": "Returns a list of all available attachments for a dataset.\n", + "parameters": [ + { + "$ref": "#/components/parameters/dataset_id" + } + ], + "responses": { + "200": { + "description": "List of all available attachments", + "content": { + "application/json; charset=utf-8": { + "schema": { + "type": "object", + "properties": { + "links": { + "type": "array", + "items": { + "$ref": "#/components/schemas/links" + } + }, + "attachments": { + "type": "array", + "items": { + "$ref": "#/components/schemas/attachment" + } + } + } + }, + "examples": { + "attachments": { + "$ref": "#/components/examples/attachments" + } + } + } + } + }, + "400": { + "$ref": "#/components/responses/bad_request" + }, + "401": { + "description": "Unauthorized" + }, + "429": { + "$ref": "#/components/responses/quota" + }, + "500": { + "description": "Internal Server Error" + } + } + } + }, + "/catalog/datasets/{dataset_id}/records/{record_id}": { + "get": { + "summary": "Read a dataset record", + "operationId": "getRecord", + "tags": [ + "Dataset" + ], + "description": "Reads a single dataset record based on its identifier.\n", + "parameters": [ + { + "$ref": "#/components/parameters/dataset_id" + }, + { + "$ref": "#/components/parameters/record_id" + }, + { + "$ref": "#/components/parameters/select" + }, + { + "$ref": "#/components/parameters/lang" + }, + { + "$ref": "#/components/parameters/timezone" + } + ], + "responses": { + "200": { + "description": "A single record", + "content": { + "application/json; charset=utf-8": { + "schema": { + "$ref": "#/components/schemas/record" + }, + "examples": { + "record": { + "$ref": "#/components/examples/record" + } + } + } + } + }, + "400": { + "$ref": "#/components/responses/bad_request" + }, + "401": { + "description": "Unauthorized" + }, + "429": { + "$ref": "#/components/responses/quota" + }, + "500": { + "description": "Internal Server Error" + } + } + } + } + }, + "components": { + "securitySchemes": { + "apikey": { + "type": "apiKey", + "description": "API key to make authenticated requests.", + "name": "apikey", + "in": "query" + } + }, + "parameters": { + "select": { + "name": "select", + "in": "query", + "description": "Examples:\n- `select=size` - Example of select, which only return the \"size\" field.\n- `select=size * 2 as bigger_size` - Example of a complex expression with a label, which returns a new field named \"bigger_size\" and containing the double of size field value.\n- `select=dataset_id, fields` - Example of a select in catalog ODSQL query to only retrieve dataset_id and schema of datasets.\n\nA select expression can be used to add, remove or change the fields to return.\nAn expression can be:\n - a wildcard ('*'): all fields are returned.\n - A field name: only the specified field is returned.\n - An include/exclude function: All fields matching the include or exclude expression are included or excluded. This expression can contain wildcard.\n - A complex expression. The result of the expression is returned. A label can be set for this expression, and in that case, the field will be named after this label.", + "schema": { + "type": "string" + } + }, + "where": { + "name": "where", + "in": "query", + "description": "A `where` filter is a text expression performing a simple full-text search that can also include logical operations\n(NOT, AND, OR...) and lots of other functions to perform complex and precise search operations.\n\nFor more information, see [Opendatasoft Query Language (ODSQL)]() reference documentation.", + "schema": { + "type": "string" + } + }, + "order_by": { + "name": "order_by", + "in": "query", + "description": "Example: `order_by=sum(age) desc, name asc`\n\nA comma-separated list of field names or aggregations to sort on, followed by an order (`asc` or `desc`).\n\nResults are sorted in ascending order by default. To sort results in descending order, use the `desc` keyword.", + "style": "form", + "explode": false, + "schema": { + "type": "string" + } + }, + "limit": { + "name": "limit", + "in": "query", + "description": "Number of items to return.\n\nTo use with the `offset` parameter to implement pagination.\n\nThe maximum possible value depends on whether the query contains a `group_by` clause or not.\n\nFor a query **without** a `group_by`:\n - the maximum value for `limit` is 100,\n - `offset+limit` should be less than 10000\n\nFor a query **with** a `group_by`:\n - the maximum value for `limit` is 20000,\n - `offset+limit` should be less than 20000\n\n**Note:** If you need more results, please use the /exports endpoint.\n", + "schema": { + "maximum": 100, + "minimum": -1, + "type": "integer", + "default": 10 + } + }, + "offset": { + "name": "offset", + "in": "query", + "description": "Index of the first item to return (starting at 0).\n\nTo use with the `limit` parameter to implement pagination.\n\n**Note:** the maximum value depends on the type of query, see the note on `limit` for the details\n", + "schema": { + "minimum": 0, + "type": "integer", + "default": 0 + } + }, + "refine": { + "name": "refine", + "in": "query", + "description": "Example: `refine=modified:2020` - Return only the value `2020` from the `modified` facet.\n\nA facet filter used to limit the result set.\nUsing this parameter, you can refine your query to display only the selected facet value in the response.\n\nRefinement uses the following syntax: `refine=:`\n\nFor date, and other hierarchical facets, when refining on one value, all second-level values related to that entry will appear in facets enumeration. For example, after refining on the year 2019, the related second-level month will appear. And when refining on August 2019, the third-level day will appear.\n\n**`refine` must not be confused with a `where` filter. Refining with a facet is equivalent to selecting an entry in the left navigation panel.**", + "style": "form", + "explode": true, + "schema": { + "type": "string" + } + }, + "exclude": { + "name": "exclude", + "in": "query", + "description": "Examples:\n- `exclude=city:Paris` - Exclude the value `Paris` from the `city` facet. Facets enumeration will display `Paris` as `excluded` without any count information.\n- `exclude=modified:2019/12` - Exclude the value `2019/12` from the `modified` facet. Facets enumeration will display `2020` as `excluded` without any count information.\n\nA facet filter used to exclude a facet value from the result set.\nUsing this parameter, you can filter your query to exclude the selected facet value in the response.\n\n`exclude` uses the following syntax: `exclude=:`\n\n**`exclude` must not be confused with a `where` filter. Excluding a facet value is equivalent to removing an entry in the left navigation panel.**", + "style": "form", + "explode": true, + "schema": { + "type": "string" + } + }, + "lang": { + "name": "lang", + "in": "query", + "description": "A language value.\n\nIf specified, the `lang` value override the default language, which is \"fr\".\nThe language is used to format string, for example in the `date_format` function.", + "schema": { + "type": "string", + "enum": [ + "en", + "fr", + "nl", + "pt", + "it", + "ar", + "de", + "es", + "ca", + "eu", + "sv" + ] + }, + "style": "form" + }, + "timezone": { + "name": "timezone", + "in": "query", + "description": "Set the timezone for datetime fields.\n\nTimezone IDs are defined by the [Unicode CLDR project](https://github.com/unicode-org/cldr). The list of timezone IDs is available in [timezone.xml](https://github.com/unicode-org/cldr/blob/master/common/bcp47/timezone.xml).", + "schema": { + "type": "string", + "default": "UTC" + }, + "examples": { + "UTC": { + "summary": "UTC timezone", + "value": "UTC" + }, + "Europe/Paris": { + "summary": "Paris timezone", + "value": "Europe/Paris" + }, + "US/Eastern": { + "summary": "Eastern timezone", + "value": "US/Eastern" + }, + "Europe/London": { + "summary": "London timezone", + "value": "Europe/London" + }, + "Europe/Berlin": { + "summary": "Berlin timezone", + "value": "Europe/Berlin" + } + } + }, + "group_by": { + "name": "group_by", + "in": "query", + "description": "Example: `group_by=city_field as city`\n\nA group by expression defines a grouping function for an aggregation.\nIt can be:\n - a field name: group result by each value of this field\n - a range function: group result by range\n - a date function: group result by date\n\nIt is possible to specify a custom name with the 'as name' notation.", + "style": "form", + "explode": false, + "schema": { + "type": "string" + } + }, + "include_links": { + "name": "include_links", + "in": "query", + "description": "If set to `true`, this parameter will add HATEOAS links in the response.\n", + "schema": { + "type": "boolean", + "default": false + } + }, + "include_app_metas": { + "name": "include_app_metas", + "in": "query", + "description": "If set to `true`, this parameter will add application metadata to the response.\n", + "schema": { + "type": "boolean", + "default": false + } + }, + "format-catalog-v2.1": { + "name": "format", + "in": "path", + "required": true, + "schema": { + "type": "string", + "enum": [ + "csv", + "data.json", + "dcat", + "dcat_ap_ch", + "dcat_ap_de", + "dcat_ap_se", + "dcat_ap_sp", + "dcat_ap_it", + "dcat_ap_vl", + "dcat_ap_benap", + "dublin_core", + "json", + "rdf", + "rss", + "ttl", + "xlsx" + ], + "description": "Format specifier for the catalog export.\n`dcat_ap_*` formats are only available upon activation.\nSee [here](#tag/Catalog/operation/listExportFormats) to get the list of available export formats" + }, + "style": "simple" + }, + "limit_export": { + "name": "limit", + "in": "query", + "description": "Number of items to return in export.\n\nUse -1 (default) to retrieve all records\n", + "schema": { + "minimum": -1, + "type": "integer", + "default": -1 + } + }, + "dcat_format": { + "name": "dcat_ap_format", + "in": "path", + "required": true, + "schema": { + "type": "string", + "enum": [ + "_ap_ch", + "_ap_de", + "_ap_se", + "_ap_sp", + "_ap_it", + "_ap_vl", + "_ap_benap" + ], + "description": "DCAT format specifier for the catalog export.\n`dcat_ap_*` formats are only available upon activation." + }, + "style": "simple" + }, + "facet": { + "name": "facet", + "in": "query", + "description": "A facet is a field used for simple filtering (through the `refine` and `exclude` parameters) or exploration (with the `/facets` endpoint).\n\nIt can also be a function such as `facet=facet(name=\"field_name\")` which is identical to `facet=field_name`. But this `facet()` function\ncan also take some optional arguments such as `disjunctive`, `hierarchical`, `separator`, `sort` and `limit`.\n\n* `disjunctive`: a boolean `true/false`, whether multiple values can be selected for the facet\n* `hierarchical`: a boolean `true/false` if the field is hierarchical. The separator must be given as the argument.\n For instance, you can do `facet=facet(name=\"filepath\", hierarchical=true, separator=\"/\")` to retrieve facets related to this field which might look like `\"/home/user/file.txt\"`\n* `separator`: a string, e.g. `/`, `-`, `;`\n* `sort`: a string which describes how to sort the facets. Possible arguments are `count` and `-count` for all field types, `alphanum` and `-alphanum` for `date`, `datetime` and `text`, `num` and `-num` for `decimal` and `int`\n* `limit`: an integer to limit the number of results\n", + "style": "form", + "explode": true, + "schema": { + "type": "string" + } + }, + "dataset_id": { + "name": "dataset_id", + "in": "path", + "description": "The identifier of the dataset to be queried.\n\nYou can find it in the \"Information\" tab of the dataset page or in the dataset URL, right after `/datasets/`.", + "required": true, + "schema": { + "type": "string" + } + }, + "format-datasets-v2.1": { + "name": "format", + "in": "path", + "required": true, + "schema": { + "$ref": "#/components/schemas/enum-format-datasets-v2.1" + }, + "style": "simple" + }, + "use_labels": { + "name": "use_labels", + "in": "query", + "description": "If set to `true`, this parameter will make exports output the label of each field rather than its name.\n\nThis parameter only makes sense for formats that contain a list of the fields in their output.\n", + "schema": { + "type": "boolean", + "default": false + } + }, + "compressed": { + "name": "compressed", + "in": "query", + "description": "If set to `true`, this parameter can compress the output file of a specific export format with GZIP, e.g. `.csv.gzip`.\n", + "required": false, + "schema": { + "type": "boolean", + "default": false + } + }, + "epsg": { + "name": "epsg", + "in": "query", + "description": "This parameter sets the EPSG code to project shapes into for formats that support geometric features.\n", + "schema": { + "type": "integer", + "default": 4326 + } + }, + "record_id": { + "name": "record_id", + "in": "path", + "required": true, + "schema": { + "type": "string" + }, + "description": "Record identifier" + } + }, + "schemas": { + "links": { + "type": "object", + "properties": { + "href": { + "type": "string", + "format": "uri" + }, + "rel": { + "type": "string", + "enum": [ + "self", + "first", + "last", + "next", + "dataset", + "catalog" + ] + } + } + }, + "dataset-v2.1": { + "type": "object", + "properties": { + "_links": { + "type": "array", + "items": { + "$ref": "#/components/schemas/links" + } + }, + "dataset_id": { + "type": "string" + }, + "dataset_uid": { + "type": "string", + "readOnly": true + }, + "attachments": { + "type": "array", + "items": { + "type": "object", + "properties": { + "mimetype": { + "type": "string" + }, + "url": { + "type": "string" + }, + "id": { + "type": "string" + }, + "title": { + "type": "string" + } + } + } + }, + "has_records": { + "type": "boolean" + }, + "data_visible": { + "type": "boolean" + }, + "features": { + "type": "array", + "description": "A map of available features for a dataset, with the fields they apply to.\n", + "items": { + "type": "string" + } + }, + "metas": { + "type": "object" + }, + "fields": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "label": { + "type": "string" + }, + "type": { + "type": "string" + }, + "annotations": { + "type": "object" + }, + "description": { + "type": "string", + "nullable": true + } + } + } + }, + "additionalProperties": {} + } + }, + "results_dataset-v2.1": { + "type": "object", + "properties": { + "total_count": { + "type": "integer" + }, + "_links": { + "type": "array", + "items": { + "$ref": "#/components/schemas/links" + } + }, + "results": { + "type": "array", + "items": { + "$ref": "#/components/schemas/dataset-v2.1" + } + } + } + }, + "enum-format-datasets-v2.1": { + "type": "string", + "enum": [ + "csv", + "fgb", + "geojson", + "gpx", + "json", + "jsonl", + "jsonld", + "kml", + "n3", + "ov2", + "parquet", + "rdfxml", + "shp", + "turtle", + "xlsx" + ] + }, + "facet_value_enumeration": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "count": { + "type": "integer" + }, + "value": { + "type": "string" + }, + "state": { + "type": "string" + } + } + }, + "facet_enumeration": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "facets": { + "type": "array", + "items": { + "$ref": "#/components/schemas/facet_value_enumeration" + } + } + } + }, + "record-v2.1": { + "type": "object", + "properties": { + "_id": { + "type": "string" + }, + "_timestamp": { + "type": "string", + "format": "dateTime" + }, + "_size": { + "type": "integer", + "format": "int64" + }, + "_links": { + "type": "array", + "items": { + "$ref": "#/components/schemas/links" + } + }, + "field1": { + "type": "string" + }, + "field2": { + "type": "integer" + } + }, + "additionalProperties": {} + }, + "results-v2.1": { + "type": "object", + "properties": { + "total_count": { + "type": "integer" + }, + "_links": { + "type": "array", + "items": { + "$ref": "#/components/schemas/links" + } + }, + "results": { + "type": "array", + "items": { + "$ref": "#/components/schemas/record-v2.1" + } + } + } + }, + "attachment": { + "type": "object", + "properties": { + "href": { + "type": "string" + }, + "metas": { + "type": "object", + "properties": { + "mime-type": { + "type": "string" + }, + "title": { + "type": "string" + }, + "url": { + "type": "string" + }, + "id": { + "type": "string" + } + } + } + } + }, + "record": { + "type": "object", + "properties": { + "record": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "timestamp": { + "type": "string", + "format": "dateTime" + }, + "size": { + "type": "integer", + "format": "int64" + }, + "fields": { + "type": "object", + "properties": { + "field1": { + "type": "string" + }, + "field2": { + "type": "integer" + } + }, + "additionalProperties": {} + } + } + }, + "links": { + "type": "array", + "items": { + "$ref": "#/components/schemas/links" + } + } + } + } + }, + "examples": { + "datasets-v2.1": { + "value": { + "total_count": 19, + "results": [ + { + "dataset_id": "world-administrative-boundaries-countries-and-territories", + "dataset_uid": "da_6kvv9v", + "attachments": [], + "has_records": true, + "data_visible": true, + "fields": [ + { + "annotations": {}, + "description": null, + "type": "geo_point_2d", + "name": "geo_point_2d", + "label": "Geo Point" + }, + { + "annotations": {}, + "description": null, + "type": "geo_shape", + "name": "geo_shape", + "label": "Geo Shape" + }, + { + "description": null, + "label": "Status", + "type": "text", + "name": "status", + "annotations": { + "facet": [] + } + }, + { + "description": "ISO 3 code of the country to which the territory belongs", + "label": "ISO 3 country code", + "type": "text", + "name": "color_code", + "annotations": { + "facet": [] + } + }, + { + "description": null, + "label": "Region of the territory", + "type": "text", + "name": "region", + "annotations": { + "facet": [] + } + }, + { + "description": null, + "label": "ISO 3 territory code", + "type": "text", + "name": "iso3", + "annotations": { + "sortable": [] + } + }, + { + "description": null, + "label": "Continent of the territory", + "type": "text", + "name": "continent", + "annotations": { + "facet": [] + } + }, + { + "description": "Name of the territory", + "label": "English Name", + "type": "text", + "name": "name", + "annotations": { + "sortable": [] + } + }, + { + "annotations": {}, + "description": null, + "type": "text", + "name": "iso_3166_1_alpha_2_codes", + "label": "ISO 3166-1 Alpha 2-Codes" + }, + { + "annotations": {}, + "label": "French Name", + "type": "text", + "name": "french_short", + "description": "French term, when it is available in https://data.opendatasoft.com/explore/dataset/countries-territories-taxonomy-mvp-ct-taxonomy-with-hxl-tags1@public/table/, English name otherwise" + } + ], + "metas": { + "default": { + "records_count": 256, + "modified": "2021-06-23T14:59:57+00:00", + "source_domain_address": null, + "references": "https://geonode.wfp.org/layers/geonode:wld_bnd_adm0_wfp", + "keyword": [ + "United Nation", + "ISO-3 code", + "Countries", + "Territories", + "Shape", + "Boundaries" + ], + "source_domain_title": null, + "geographic_reference": [ + "world" + ], + "timezone": null, + "title": "World Administrative Boundaries - Countries and Territories", + "parent_domain": null, + "theme": [ + "Administration, Government, Public finances, Citizenship" + ], + "modified_updates_on_data_change": false, + "metadata_processed": "2021-06-23T15:00:02.656000+00:00", + "data_processed": "2019-05-15T07:49:01+00:00", + "territory": [ + "World" + ], + "description": "

This dataset displays level 0 world administrative boundaries. It contains countries as well as non-sovereign territories (like, for instance, French overseas). 

", + "modified_updates_on_metadata_change": false, + "shared_catalog": null, + "source_domain": null, + "attributions": null, + "geographic_area_mode": null, + "geographic_reference_auto": true, + "geographic_area": null, + "publisher": "World Food Programme (UN agency)", + "language": "en", + "license": "Open Government Licence v3.0", + "source_dataset": null, + "metadata_languages": [ + "en" + ], + "oauth_scope": null, + "federated": true, + "license_url": "http://www.nationalarchives.gov.uk/doc/open-government-licence/version/3/" + } + }, + "features": [ + "analyze", + "geo" + ] + }, + { + "dataset_id": "geonames-all-cities-with-a-population-1000", + "dataset_uid": "da_5m8ykr", + "attachments": [ + { + "mimetype": "application/zip", + "url": "odsfile://cities1000.zip", + "id": "cities1000_zip", + "title": "cities1000.zip" + } + ], + "has_records": true, + "data_visible": true, + "fields": [ + { + "description": null, + "label": "Geoname ID", + "type": "text", + "name": "geoname_id", + "annotations": { + "facetsort": [ + "-count" + ], + "id": [] + } + }, + { + "description": null, + "label": "Name", + "type": "text", + "name": "name", + "annotations": { + "sortable": [] + } + }, + { + "description": null, + "label": "ASCII Name", + "type": "text", + "name": "ascii_name", + "annotations": {} + }, + { + "description": null, + "label": "Alternate Names", + "type": "text", + "name": "alternate_names", + "annotations": { + "multivalued": [ + "," + ] + } + }, + { + "description": "see http://www.geonames.org/export/codes.html", + "label": "Feature Class", + "type": "text", + "name": "feature_class", + "annotations": {} + }, + { + "description": "see http://www.geonames.org/export/codes.html", + "label": "Feature Code", + "type": "text", + "name": "feature_code", + "annotations": {} + }, + { + "description": null, + "label": "Country Code", + "type": "text", + "name": "country_code", + "annotations": {} + }, + { + "description": null, + "label": "Country name EN", + "type": "text", + "name": "cou_name_en", + "annotations": { + "facet": [], + "facetsort": [ + "alphanum" + ], + "disjunctive": [] + } + }, + { + "description": null, + "label": "Country Code 2", + "type": "text", + "name": "country_code_2", + "annotations": {} + }, + { + "description": null, + "label": "Admin1 Code", + "type": "text", + "name": "admin1_code", + "annotations": {} + }, + { + "description": null, + "label": "Admin2 Code", + "type": "text", + "name": "admin2_code", + "annotations": { + "facetsort": [ + "-count" + ] + } + }, + { + "description": null, + "label": "Admin3 Code", + "type": "text", + "name": "admin3_code", + "annotations": {} + }, + { + "description": null, + "label": "Admin4 Code", + "type": "text", + "name": "admin4_code", + "annotations": {} + }, + { + "description": null, + "label": "Population", + "type": "int", + "name": "population", + "annotations": {} + }, + { + "description": null, + "label": "Elevation", + "type": "text", + "name": "elevation", + "annotations": {} + }, + { + "description": null, + "label": "DIgital Elevation Model", + "type": "int", + "name": "dem", + "annotations": {} + }, + { + "description": null, + "label": "Timezone", + "type": "text", + "name": "timezone", + "annotations": { + "facet": [], + "hierarchical": [ + "/" + ] + } + }, + { + "description": null, + "label": "Modification date", + "type": "date", + "name": "modification_date", + "annotations": {} + }, + { + "description": null, + "label": "LABEL EN", + "type": "text", + "name": "label_en", + "annotations": {} + }, + { + "description": null, + "label": "Coordinates", + "type": "geo_point_2d", + "name": "coordinates", + "annotations": { + "facetsort": [ + "-count" + ] + } + } + ], + "metas": { + "default": { + "records_count": 137609, + "modified": "2021-06-23T14:37:45+00:00", + "source_domain_address": null, + "references": null, + "keyword": null, + "source_domain_title": null, + "geographic_reference": [ + "world" + ], + "timezone": null, + "title": "Geonames - All Cities with a population > 1000", + "parent_domain": null, + "theme": null, + "modified_updates_on_data_change": false, + "metadata_processed": "2021-06-23T14:49:23.198000+00:00", + "data_processed": "2021-06-23T14:49:23+00:00", + "territory": [ + "World" + ], + "description": null, + "modified_updates_on_metadata_change": false, + "shared_catalog": null, + "source_domain": null, + "attributions": null, + "geographic_area_mode": null, + "geographic_reference_auto": true, + "geographic_area": null, + "publisher": null, + "language": "en", + "license": null, + "source_dataset": null, + "metadata_languages": [ + "en" + ], + "oauth_scope": null, + "federated": false, + "license_url": null + } + }, + "features": [ + "geo", + "analyze", + "timeserie" + ] + } + ] + } + }, + "catalog_facets": { + "value": { + "links": [], + "facets": [ + { + "name": "publisher", + "facets": [ + { + "count": 2, + "state": "displayed", + "name": "Opendatasoft", + "value": "Opendatasoft" + }, + { + "count": 2, + "state": "displayed", + "name": "Opendatasoft - Data Team", + "value": "Opendatasoft - Data Team" + } + ] + }, + { + "name": "features", + "facets": [ + { + "count": 19, + "state": "displayed", + "name": "analyze", + "value": "analyze" + }, + { + "count": 13, + "state": "displayed", + "name": "timeserie", + "value": "timeserie" + } + ] + }, + { + "name": "language", + "facets": [ + { + "count": 17, + "state": "displayed", + "name": "en", + "value": "en" + }, + { + "count": 4, + "state": "displayed", + "name": "fr", + "value": "fr" + } + ] + } + ] + } + }, + "records": { + "value": { + "total_count": 137611, + "links": [], + "records": [ + { + "links": [], + "record": { + "id": "53d4524dcb82c676bacd467cd5ace953f2e0389c", + "timestamp": "2021-06-22T08:02:59.954Z", + "size": 194, + "fields": { + "admin1_code": "27", + "elevation": null, + "name": "Saint-Leu", + "modification_date": "2019-03-26", + "alternate_names": [ + "Saint-Leu" + ], + "feature_class": "P", + "admin3_code": "711", + "cou_name_en": "France", + "coordinates": { + "lat": 46.7306, + "lon": 4.50083 + }, + "country_code_2": null, + "geoname_id": "2978771", + "feature_code": "PPL", + "label_en": "France", + "dem": 366, + "country_code": "FR", + "ascii_name": "Saint-Leu", + "timezone": "Europe/Paris", + "admin2_code": "71", + "admin4_code": "71436", + "population": 29278 + } + } + }, + { + "links": [], + "record": { + "id": "d5251445f329dc74cc5c5e30c95378eb9807a019", + "timestamp": "2021-06-22T08:02:59.954Z", + "size": 310, + "fields": { + "admin1_code": "32", + "elevation": null, + "name": "Saint-Léger-lès-Domart", + "modification_date": "2016-02-18", + "alternate_names": [ + "Saint-Leger", + "Saint-Leger-les-Domart", + "Saint-Léger", + "Saint-Léger-lès-Domart" + ], + "feature_class": "P", + "admin3_code": "802", + "cou_name_en": "France", + "coordinates": { + "lat": 50.05208, + "lon": 2.14067 + }, + "country_code_2": null, + "geoname_id": "2978817", + "feature_code": "PPL", + "label_en": "France", + "dem": 31, + "country_code": "FR", + "ascii_name": "Saint-Leger-les-Domart", + "timezone": "Europe/Paris", + "admin2_code": "80", + "admin4_code": "80706", + "population": 1781 + } + } + } + ] + } + }, + "group_by_country-v2.1": { + "value": { + "results": [ + { + "count": 16729, + "cou_name_en": "United States" + }, + { + "count": 9945, + "cou_name_en": "Italy" + }, + { + "count": 8981, + "cou_name_en": "Mexico" + } + ] + } + }, + "dataset-v2.1": { + "value": { + "dataset_id": "geonames-all-cities-with-a-population-1000", + "dataset_uid": "da_s2n5ed", + "attachments": [], + "has_records": true, + "data_visible": true, + "fields": [ + { + "description": null, + "label": "Geoname ID", + "type": "text", + "name": "geoname_id", + "annotations": { + "facetsort": [ + "-count" + ], + "id": [] + } + }, + { + "description": null, + "label": "Name", + "type": "text", + "name": "name", + "annotations": { + "sortable": [] + } + }, + { + "annotations": {}, + "description": null, + "type": "text", + "name": "ascii_name", + "label": "ASCII Name" + }, + { + "description": null, + "label": "Alternate Names", + "type": "text", + "name": "alternate_names", + "annotations": { + "multivalued": [ + "," + ] + } + }, + { + "annotations": {}, + "label": "Feature Class", + "type": "text", + "name": "feature_class", + "description": "see http://www.geonames.org/export/codes.html" + }, + { + "annotations": {}, + "label": "Feature Code", + "type": "text", + "name": "feature_code", + "description": "see http://www.geonames.org/export/codes.html" + }, + { + "annotations": {}, + "description": null, + "type": "text", + "name": "country_code", + "label": "Country Code" + }, + { + "description": null, + "label": "Country name EN", + "type": "text", + "name": "cou_name_en", + "annotations": { + "facet": [], + "facetsort": [ + "alphanum" + ], + "disjunctive": [] + } + }, + { + "annotations": {}, + "description": null, + "type": "text", + "name": "country_code_2", + "label": "Country Code 2" + }, + { + "annotations": {}, + "description": null, + "type": "text", + "name": "admin1_code", + "label": "Admin1 Code" + }, + { + "description": null, + "label": "Admin2 Code", + "type": "text", + "name": "admin2_code", + "annotations": { + "facetsort": [ + "-count" + ] + } + }, + { + "annotations": {}, + "description": null, + "type": "text", + "name": "admin3_code", + "label": "Admin3 Code" + }, + { + "annotations": {}, + "description": null, + "type": "text", + "name": "admin4_code", + "label": "Admin4 Code" + }, + { + "annotations": {}, + "description": null, + "type": "int", + "name": "population", + "label": "Population" + }, + { + "annotations": {}, + "description": null, + "type": "text", + "name": "elevation", + "label": "Elevation" + }, + { + "annotations": {}, + "description": null, + "type": "int", + "name": "dem", + "label": "DIgital Elevation Model" + }, + { + "description": null, + "label": "Timezone", + "type": "text", + "name": "timezone", + "annotations": { + "facet": [], + "hierarchical": [ + "/" + ] + } + }, + { + "annotations": {}, + "description": null, + "type": "date", + "name": "modification_date", + "label": "Modification date" + }, + { + "annotations": {}, + "description": null, + "type": "text", + "name": "label_en", + "label": "LABEL EN" + }, + { + "description": null, + "label": "Coordinates", + "type": "geo_point_2d", + "name": "coordinates", + "annotations": { + "facetsort": [ + "-count" + ] + } + } + ], + "metas": { + "default": { + "records_count": 137611, + "modified": "2021-06-23T07:50:20+00:00", + "source_domain_address": null, + "references": "https://download.geonames.org/export/dump/", + "keyword": [ + "Geonames", + "city", + "world" + ], + "source_domain_title": null, + "geographic_reference": [ + "world" + ], + "timezone": null, + "title": "Geonames - All Cities with a population > 1000", + "parent_domain": null, + "theme": [ + "Administration, Government, Public finances, Citizenship" + ], + "modified_updates_on_data_change": true, + "metadata_processed": "2021-06-23T07:50:26.162000+00:00", + "data_processed": "2021-06-22T08:47:08+00:00", + "territory": [ + "World" + ], + "description": "

All cities with a population > 1000 or seats of adm div (ca 80.000)

Sources and Contributions

Enrichment:

  • add country name
", + "modified_updates_on_metadata_change": false, + "shared_catalog": null, + "source_domain": null, + "attributions": [ + "https://www.geonames.org/about.html" + ], + "geographic_area_mode": null, + "geographic_reference_auto": true, + "geographic_area": null, + "publisher": "GeoNames", + "language": "en", + "license": "CC BY 4.0", + "source_dataset": null, + "metadata_languages": [ + "en" + ], + "oauth_scope": null, + "federated": true, + "license_url": "https://creativecommons.org/licenses/by/4.0/" + } + }, + "features": [ + "geo", + "analyze", + "timeserie" + ] + } + }, + "facets": { + "value": { + "links": [], + "facets": [ + { + "facets": [ + { + "count": 68888, + "state": "displayed", + "name": "Europe", + "value": "Europe" + }, + { + "count": 36276, + "state": "displayed", + "name": "America", + "value": "America" + } + ], + "name": "timezone" + }, + { + "facets": [ + { + "count": 313, + "state": "displayed", + "name": "Afghanistan", + "value": "Afghanistan" + }, + { + "count": 356, + "state": "displayed", + "name": "Albania", + "value": "Albania" + } + ], + "name": "cou_name_en" + } + ] + } + }, + "attachments": { + "value": { + "links": [], + "attachments": [ + { + "href": "https://documentation-resources.opendatasoft.com/api/v2/catalog/datasets/geonames-all-cities-with-a-population-1000/attachments/cities1000_zip", + "metas": { + "mime-type": "application/zip", + "title": "cities1000.zip" + } + } + ] + } + }, + "record": { + "value": { + "links": [], + "record": { + "id": "5ce430b62d47a400a495c30345fb6fdfac5550f0", + "timestamp": "2021-06-23T14:46:05.881Z", + "size": 190, + "fields": { + "timezone": "Europe/Brussels", + "elevation": null, + "name": "Fraire", + "modification_date": "2020-04-05", + "dem": 238, + "cou_name_en": "Belgium", + "feature_class": "P", + "admin3_code": "93", + "alternate_names": [ + "Fraire" + ], + "coordinates": { + "lat": 50.26127, + "lon": 4.5076 + }, + "country_code_2": null, + "geoname_id": "2798031", + "feature_code": "PPL", + "label_en": "Belgium", + "admin4_code": "93088", + "country_code": "BE", + "ascii_name": "Fraire", + "admin1_code": "WAL", + "admin2_code": "WNA", + "population": 1492 + } + } + } + } + }, + "responses": { + "bad_request": { + "description": "Bad Request", + "content": { + "application/json; charset=utf-8": { + "schema": { + "type": "object", + "properties": { + "message": { + "type": "string", + "minLength": 1 + }, + "error_code": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "message", + "error_code" + ] + }, + "examples": { + "invalid_odsql": { + "value": { + "message": "ODSQL query is malformed: invalid_function() Clause(s) containing the error(s): select.", + "error_code": "ODSQLError" + } + } + } + } + } + }, + "quota": { + "description": "Too many requests", + "content": { + "application/json; charset=utf-8": { + "schema": { + "type": "object", + "properties": { + "errorcode": { + "type": "number" + }, + "reset_time": { + "type": "string", + "minLength": 1 + }, + "limit_time_unit": { + "type": "string", + "minLength": 1 + }, + "call_limit": { + "type": "number" + }, + "error": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "errorcode", + "reset_time", + "limit_time_unit", + "call_limit", + "error" + ] + }, + "examples": { + "quota_exceeded": { + "value": { + "errorcode": 10002, + "reset_time": "2021-01-26T00:00:00Z", + "limit_time_unit": "day", + "call_limit": 10000, + "error": "Too many requests on the domain. Please contact the domain administrator." + } + } + } + } + } + } + } + } +} \ No newline at end of file diff --git a/spec/services/external/vancouver_api/vancouver_api_client/client_creation_spec.rb b/spec/services/external/vancouver_api/vancouver_api_client/client_creation_spec.rb new file mode 100644 index 00000000..7856cccf --- /dev/null +++ b/spec/services/external/vancouver_api/vancouver_api_client/client_creation_spec.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +require 'rails_helper' +require_relative 'shared_helpers' + +RSpec.describe External::VancouverCity::VancouverApiClient, 'client creation and initialization', type: :service do + include_context 'vancouver api client shared setup' + + describe '.default_client' do + it 'creates a client with the default adapter' do + client = described_class.default_client + expect(client.adapter).to eq(External::VancouverCity::DEFAULT_ADAPTER) + end + end + + describe '.with_config' do + it 'creates a client with custom configuration' do + config = External::VancouverCity::VancouverApiConfig.new(timeout: 60, open_timeout: 20) + client = described_class.with_config(config) + + adapter = client.adapter + expect(adapter.options.timeout).to eq(60) + expect(adapter.options.open_timeout).to eq(20) + end + end + + describe '.with_timeouts' do + it 'creates a client with custom timeout values' do + client = described_class.with_timeouts(timeout: 120, open_timeout: 30) + + adapter = client.adapter + expect(adapter.options.timeout).to eq(120) + expect(adapter.options.open_timeout).to eq(30) + end + end + + describe '#initialize' do + context 'with default adapter' do + it 'uses the provided adapter' do + adapter = client.adapter + expect(adapter).to eq(default_adapter) + end + end + + context 'with custom adapter' do + let(:mock_adapter) { instance_double(External::VancouverCity::Adapters::FaradayAdapter) } + let(:client) { described_class.new(adapter: mock_adapter) } + + it 'uses the provided adapter' do + expect(client.adapter).to eq(mock_adapter) + end + end + end +end diff --git a/spec/services/external/vancouver_api/vancouver_api_client/dataset_apis_spec.rb b/spec/services/external/vancouver_api/vancouver_api_client/dataset_apis_spec.rb new file mode 100644 index 00000000..bdbdfd13 --- /dev/null +++ b/spec/services/external/vancouver_api/vancouver_api_client/dataset_apis_spec.rb @@ -0,0 +1,116 @@ +# frozen_string_literal: true + +require 'rails_helper' +require_relative 'shared_helpers' + +RSpec.describe External::VancouverCity::VancouverApiClient, 'dataset APIs', type: :service do + include_context 'vancouver api client shared setup' + + describe '#get_dataset' do + let(:dataset_id) { 'drinking-fountains' } + let(:mock_adapter) { instance_double(External::VancouverCity::Adapters::FaradayAdapter) } + let(:test_client) { create_test_client_with_mock_adapter(mock_adapter) } + let(:response_body) do + { + 'dataset_id' => dataset_id, + 'metas' => { + 'default' => { + 'title' => 'Drinking fountains', + 'records_count' => 278 + } + }, + 'fields' => [ + { 'name' => 'mapid', 'type' => 'text' }, + { 'name' => 'name', 'type' => 'text' } + ] + } + end + let(:mock_response) { create_successful_mock_response(response_body.to_json) } + + before do + allow(mock_adapter).to receive(:get).and_return(mock_response) + end + + it 'calls the correct endpoint' do + test_client.get_dataset(dataset_id) + + expect(mock_adapter).to have_received(:get) + .with("catalog/datasets/#{dataset_id}", {}) + end + + it 'returns successful response' do + response = test_client.get_dataset(dataset_id) + + expect(response.success?).to be true + expect(response.status).to eq(200) + end + end + + describe '#get_datasets' do + let(:mock_adapter) { instance_double(External::VancouverCity::Adapters::FaradayAdapter) } + let(:test_client) { create_test_client_with_mock_adapter(mock_adapter) } + let(:response_body) do + { + 'total_count' => 150, + 'results' => [ + { + 'dataset_id' => 'drinking-fountains', + 'metas' => { 'default' => { 'title' => 'Drinking fountains' } } + } + ] + } + end + let(:mock_response) { create_successful_mock_response(response_body.to_json) } + + before do + allow(mock_adapter).to receive(:get).and_return(mock_response) + end + + it 'calls the correct endpoint with parameters' do + test_client.get_datasets(limit: 20) + + expect(mock_adapter).to have_received(:get) + .with("catalog/datasets", { limit: 20 }) + end + + it 'returns successful response' do + response = test_client.get_datasets(limit: 20) + + expect(response.success?).to be true + expect(response.status).to eq(200) + end + end + + describe '#get_dataset_record' do + let(:dataset_id) { 'drinking-fountains' } + let(:record_id) { 'DFPB0001' } + let(:mock_adapter) { instance_double(External::VancouverCity::Adapters::FaradayAdapter) } + let(:test_client) { create_test_client_with_mock_adapter(mock_adapter) } + let(:response_body) do + { + 'mapid' => record_id, + 'name' => 'Fountain location: Aberdeen Park', + 'location' => 'plaza' + } + end + let(:mock_response) { create_successful_mock_response(response_body.to_json) } + + before do + allow(mock_adapter).to receive(:get).and_return(mock_response) + end + + it 'calls the correct endpoint' do + test_client.get_dataset_record(dataset_id, record_id) + + expect(mock_adapter).to have_received(:get) + .with("catalog/datasets/#{dataset_id}/records/#{record_id}", {}) + end + + it 'returns successful response' do + response = test_client.get_dataset_record(dataset_id, record_id) + + expect(response.success?).to be true + expect(response.status).to eq(200) + end + end +end diff --git a/spec/services/external/vancouver_api/vancouver_api_client/dataset_records_spec.rb b/spec/services/external/vancouver_api/vancouver_api_client/dataset_records_spec.rb new file mode 100644 index 00000000..11979f7c --- /dev/null +++ b/spec/services/external/vancouver_api/vancouver_api_client/dataset_records_spec.rb @@ -0,0 +1,88 @@ +# frozen_string_literal: true + +require 'rails_helper' +require_relative 'shared_helpers' + +RSpec.describe External::VancouverCity::VancouverApiClient, '#get_dataset_records', type: :service do + include_context 'vancouver api client shared setup' + + let(:dataset_id) { 'drinking-fountains' } + let(:mock_adapter) { instance_double(External::VancouverCity::Adapters::FaradayAdapter) } + let(:test_client) { create_test_client_with_mock_adapter(mock_adapter) } + let(:response_body) do + { + 'total_count' => 278, + 'results' => [ + { + 'mapid' => 'DFPB0001', + 'name' => 'Fountain location: Aberdeen Park', + 'location' => 'plaza', + 'maintainer' => 'Parks' + } + ] + } + end + + context 'successful request' do + let(:mock_response) { create_successful_mock_response(response_body.to_json) } + + before do + allow(mock_adapter).to receive(:get) + .with("catalog/datasets/#{dataset_id}/records", { limit: 20 }) + .and_return(mock_response) + end + + it 'returns successful response with parsed body' do + response = test_client.get_dataset_records(dataset_id, limit: 20) + + expect(response.success?).to be true + expect(response.status).to eq(200) + end + + it 'calls the adapter with correct parameters' do + test_client.get_dataset_records(dataset_id, limit: 20) + + expect(mock_adapter).to have_received(:get) + .with("catalog/datasets/#{dataset_id}/records", { limit: 20 }) + end + end + + context 'with query parameters' do + let(:params) do + { + select: 'name,location', + where: 'maintainer = "Parks"', + order_by: 'name asc', + limit: 50, + offset: 10 + } + end + let(:mock_response) { create_successful_mock_response(response_body.to_json) } + + before do + allow(mock_adapter).to receive(:get).and_return(mock_response) + end + + it 'passes all query parameters correctly' do + test_client.get_dataset_records(dataset_id, **params) + + expect(mock_adapter).to have_received(:get) + .with("catalog/datasets/#{dataset_id}/records", params) + end + end + + context 'with nil parameters' do + let(:mock_response) { create_successful_mock_response(response_body.to_json) } + + before do + allow(mock_adapter).to receive(:get).and_return(mock_response) + end + + it 'filters out nil values from parameters' do + test_client.get_dataset_records(dataset_id, limit: 10, where: nil, select: nil) + + expect(mock_adapter).to have_received(:get) + .with("catalog/datasets/#{dataset_id}/records", { limit: 10 }) + end + end +end diff --git a/spec/services/external/vancouver_api/vancouver_api_client/error_handling_spec.rb b/spec/services/external/vancouver_api/vancouver_api_client/error_handling_spec.rb new file mode 100644 index 00000000..bdf89bba --- /dev/null +++ b/spec/services/external/vancouver_api/vancouver_api_client/error_handling_spec.rb @@ -0,0 +1,160 @@ +# frozen_string_literal: true + +require 'rails_helper' +require_relative 'shared_helpers' + +RSpec.describe External::VancouverCity::VancouverApiClient, 'error handling', type: :service do + include_context 'vancouver api client shared setup' + + let(:dataset_id) { 'drinking-fountains' } + let(:mock_adapter) { instance_double(External::VancouverCity::Adapters::FaradayAdapter) } + let(:test_client) { create_test_client_with_mock_adapter(mock_adapter) } + + describe 'HTTP error responses' do + context 'when dataset not found' do + let(:mock_response) do + create_error_mock_response( + status: 404, + body: 'Page not found', + content_type: 'text/html' + ) + end + + before do + allow(mock_adapter).to receive(:get).and_return(mock_response) + end + + it 'raises VancouverApiError with appropriate message' do + expect { + test_client.get_dataset_records('invalid-dataset') + }.to raise_error(External::VancouverCity::VancouverApiError) do |error| + expect(error.message).to include('API request failed with status 404') + expect(error.status_code).to eq(404) + expect(error.response_body).to include('Page not found') + end + end + end + + context 'when server error occurs with JSON response' do + let(:mock_response) do + create_error_mock_response( + status: 500, + body: { error: 'Internal Server Error' }.to_json, + content_type: 'application/json' + ) + end + + before do + allow(mock_adapter).to receive(:get).and_return(mock_response) + end + + it 'raises VancouverApiError with JSON error message' do + expect { + test_client.get_dataset_records(dataset_id) + }.to raise_error(External::VancouverCity::VancouverApiError) do |error| + expect(error.message).to include('Internal Server Error') + expect(error.status_code).to eq(500) + end + end + end + + context 'when response body is very long' do + let(:long_error_body) { 'a' * 300 } + let(:mock_response) do + create_error_mock_response( + status: 400, + body: long_error_body, + content_type: 'text/plain' + ) + end + + before do + allow(mock_adapter).to receive(:get).and_return(mock_response) + end + + it 'truncates very long error messages' do + expect { + test_client.get_dataset_records(dataset_id) + }.to raise_error(External::VancouverCity::VancouverApiError) do |error| + expect(error.message).to include('...') + expect(error.message.length).to be < 280 # Adjusted for actual truncation behavior + end + end + end + end + + describe 'network errors' do + context 'when network timeout occurs' do + before do + allow(mock_adapter).to receive(:get).and_raise(Faraday::TimeoutError.new('execution expired')) + end + + it 'raises VancouverApiError for timeout' do + expect { + test_client.get_dataset_records(dataset_id) + }.to raise_error(External::VancouverCity::VancouverApiError) do |error| + expect(error.message).to include('Request timeout') + expect(error.status_code).to be_nil + end + end + end + + context 'when connection fails' do + before do + allow(mock_adapter).to receive(:get).and_raise(Faraday::ConnectionFailed.new('Connection refused')) + end + + it 'raises VancouverApiError for connection failure' do + expect { + test_client.get_dataset_records(dataset_id) + }.to raise_error(External::VancouverCity::VancouverApiError) do |error| + expect(error.message).to include('Connection failed') + end + end + end + end + + describe 'JSON parsing errors' do + context 'when response has invalid JSON' do + let(:mock_response) do + instance_double(Faraday::Response, + success?: true, + status: 200, + body: 'invalid json {', + headers: { 'content-type' => 'application/json' }, + env: double(body: nil) + ) + end + + before do + allow(mock_response.env).to receive(:body=) + allow(mock_adapter).to receive(:get).and_return(mock_response) + end + + it 'raises VancouverApiError for JSON parsing error' do + expect { + test_client.get_dataset_records(dataset_id) + }.to raise_error(External::VancouverCity::VancouverApiError) do |error| + expect(error.message).to include('Failed to parse JSON response') + end + end + end + end + + describe 'unexpected errors' do + context 'when unexpected error occurs' do + before do + allow(mock_adapter).to receive(:get).and_raise(RuntimeError.new('Unexpected error')) + end + + it 'raises VancouverApiError for unexpected errors' do + expect { + test_client.get_dataset_records(dataset_id) + }.to raise_error(External::VancouverCity::VancouverApiError) do |error| + expect(error.message).to include('Unexpected error') + expect(error.status_code).to be_nil + end + end + end + end +end diff --git a/spec/services/external/vancouver_api/vancouver_api_client/request_structure_spec.rb b/spec/services/external/vancouver_api/vancouver_api_client/request_structure_spec.rb new file mode 100644 index 00000000..7df1f5d2 --- /dev/null +++ b/spec/services/external/vancouver_api/vancouver_api_client/request_structure_spec.rb @@ -0,0 +1,155 @@ +# frozen_string_literal: true + +require 'rails_helper' +require_relative 'shared_helpers' + +RSpec.describe External::VancouverCity::VancouverApiClient, 'request structure and parameters', type: :service do + include_context 'vancouver api client shared setup' + + let(:mock_adapter) { instance_double(External::VancouverCity::Adapters::FaradayAdapter) } + let(:test_client) { create_test_client_with_mock_adapter(mock_adapter) } + let(:mock_response) { create_successful_mock_response('{"results": []}') } + + before do + allow(mock_adapter).to receive(:get).and_return(mock_response) + end + + describe 'parameter edge cases' do + it 'handles special characters in parameters' do + params = { where: 'name = "O\'Reilly Park"', select: 'field with spaces' } + + test_client.get_dataset_records('test-dataset', **params) + + expect(mock_adapter).to have_received(:get) + .with("catalog/datasets/test-dataset/records", params) + end + + it 'handles large limit values' do + test_client.get_dataset_records('test-dataset', limit: 100) + + expect(mock_adapter).to have_received(:get) + .with("catalog/datasets/test-dataset/records", { limit: 100 }) + end + + it 'handles zero offset' do + test_client.get_dataset_records('test-dataset', offset: 0) + + expect(mock_adapter).to have_received(:get) + .with("catalog/datasets/test-dataset/records", { offset: 0 }) + end + end + + describe 'request structure and headers' do + it 'uses GET method for all requests' do + test_client.get_dataset_records('test-dataset') + test_client.get_dataset('test-dataset') + test_client.get_datasets + test_client.get_dataset_record('test-dataset', 'record-1') + + expect(mock_adapter).to have_received(:get).exactly(4).times + end + + it 'constructs proper paths for different endpoints' do + test_client.get_dataset_records('drinking-fountains') + expect(mock_adapter).to have_received(:get) + .with("catalog/datasets/drinking-fountains/records", {}) + + test_client.get_dataset('drinking-fountains') + expect(mock_adapter).to have_received(:get) + .with("catalog/datasets/drinking-fountains", {}) + + test_client.get_datasets + expect(mock_adapter).to have_received(:get) + .with("catalog/datasets", {}) + + test_client.get_dataset_record('drinking-fountains', 'DFPB0001') + expect(mock_adapter).to have_received(:get) + .with("catalog/datasets/drinking-fountains/records/DFPB0001", {}) + end + end + + describe 'JSON response parsing' do + context 'when response is successful but not JSON' do + let(:non_json_response) do + instance_double(Faraday::Response, + success?: true, + status: 200, + body: 'plain text response', + headers: { 'content-type' => 'text/plain' } + ) + end + + before do + allow(mock_adapter).to receive(:get).and_return(non_json_response) + end + + it 'returns response without parsing body' do + response = test_client.get_dataset_records('test-dataset') + + expect(response.success?).to be true + expect(response.body).to eq('plain text response') + end + end + + context 'when response has mixed content-type' do + let(:json_response_with_charset) { create_successful_mock_response('{"data": "test"}') } + + before do + allow(json_response_with_charset).to receive(:headers) + .and_return({ 'content-type' => 'application/json; charset=utf-8' }) + allow(mock_adapter).to receive(:get).and_return(json_response_with_charset) + end + + it 'still parses JSON correctly' do + response = test_client.get_dataset_records('test-dataset') + + expect(response.success?).to be true + end + end + end + + describe 'query parameter building' do + it 'maps options to parameter names correctly' do + options = { + select: 'name,location', + where: 'maintainer = "Parks"', + group_by: 'maintainer', + order_by: 'name asc', + limit: 50, + offset: 10, + refine: 'category:park', + exclude: 'status:inactive', + lang: 'en', + timezone: 'UTC', + include_links: true, + include_app_metas: false + } + + test_client.get_dataset_records('test-dataset', **options) + + expect(mock_adapter).to have_received(:get) + .with("catalog/datasets/test-dataset/records", options) + end + + it 'filters out nil values' do + options = { + select: 'name', + where: nil, + limit: 10, + offset: nil + } + + test_client.get_dataset_records('test-dataset', **options) + + expect(mock_adapter).to have_received(:get) + .with("catalog/datasets/test-dataset/records", { select: 'name', limit: 10 }) + end + + it 'handles empty options' do + test_client.get_dataset_records('test-dataset') + + expect(mock_adapter).to have_received(:get) + .with("catalog/datasets/test-dataset/records", {}) + end + end +end diff --git a/spec/services/external/vancouver_api/vancouver_api_client/shared_helpers.rb b/spec/services/external/vancouver_api/vancouver_api_client/shared_helpers.rb new file mode 100644 index 00000000..94744fab --- /dev/null +++ b/spec/services/external/vancouver_api/vancouver_api_client/shared_helpers.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +require 'rails_helper' + +RSpec.shared_context 'vancouver api client shared setup' do + let(:default_adapter) { External::VancouverCity::DEFAULT_ADAPTER } + let(:client) { described_class.new(adapter: default_adapter) } + let(:base_url) { 'https://opendata.vancouver.ca/api/explore/v2.1' } + + # Helper method to create a test client with a mock adapter + def create_test_client_with_mock_adapter(mock_adapter) + described_class.new(adapter: mock_adapter) + end + + # Helper to create a successful mock response + def create_successful_mock_response(body = '{"results": []}') + instance_double(Faraday::Response, + success?: true, + status: 200, + body: body, + headers: { 'content-type' => 'application/json' }, + env: double(body: nil) + ).tap do |response| + allow(response.env).to receive(:body=) + end + end + + # Helper to create an error mock response + def create_error_mock_response(status:, body:, content_type: 'text/html') + instance_double(Faraday::Response, + success?: false, + status: status, + body: body, + headers: { 'content-type' => content_type } + ) + end +end diff --git a/spec/services/external/vancouver_api/vancouver_api_error_spec.rb b/spec/services/external/vancouver_api/vancouver_api_error_spec.rb new file mode 100644 index 00000000..6cc866a9 --- /dev/null +++ b/spec/services/external/vancouver_api/vancouver_api_error_spec.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +require 'rails_helper' + +# Trigger autoloading +External::VancouverCity::VancouverApiClient if defined?(External::VancouverCity) + +# Test the custom error class +RSpec.describe External::VancouverCity::VancouverApiError, type: :service do + describe '#initialize' do + it 'sets message, status_code, and response_body' do + error = described_class.new('Test error', 404, '{"error": "Not found"}') + + expect(error.message).to eq('Test error') + expect(error.status_code).to eq(404) + expect(error.response_body).to eq('{"error": "Not found"}') + end + + it 'works with minimal parameters' do + error = described_class.new('Simple error') + + expect(error.message).to eq('Simple error') + expect(error.status_code).to be_nil + expect(error.response_body).to be_nil + end + + it 'inherits from StandardError' do + expect(described_class.new('test')).to be_a(StandardError) + end + end + + describe 'error attributes' do + let(:error) { described_class.new('Test message', 500, 'Error body') } + + it 'provides read access to status_code' do + expect(error.status_code).to eq(500) + end + + it 'provides read access to response_body' do + expect(error.response_body).to eq('Error body') + end + end +end diff --git a/spec/services/external/vancouver_city/facility_builder_spec.rb b/spec/services/external/vancouver_city/facility_builder_spec.rb new file mode 100644 index 00000000..ee89565c --- /dev/null +++ b/spec/services/external/vancouver_city/facility_builder_spec.rb @@ -0,0 +1,508 @@ +# frozen_string_literal: true + +require 'rails_helper' + +RSpec.describe External::VancouverCity::FacilityBuilder, type: :service do + let(:valid_api_key) { 'drinking-fountains' } + + let(:valid_record) do + { + 'mapid' => '12345', + 'name' => 'Test Fountain', + 'location' => 'Test Park', + 'geo_local_area' => 'Downtown', + 'phone' => '604-123-4567', + 'website' => 'https://vancouver.ca', + 'maintainer' => 'Parks Department', + 'in_operation' => 'Yes', + 'pet_friendly' => 'Yes', + 'geom' => { + 'geometry' => { + 'coordinates' => [-123.1207, 49.2827] + } + } + } + end + + let(:minimal_record) do + { + 'name' => 'Minimal Fountain', + 'geo_point_2d' => { + 'lat' => 49.2827, + 'lon' => -123.1207 + } + } + end + + describe '#initialize' do + it 'initializes with valid parameters' do + builder = described_class.new(record: valid_record, api_key: valid_api_key) + + expect(builder.record).to eq(valid_record) + expect(builder.api_key).to eq(valid_api_key) + end + end + + describe '#validate' do + context 'with valid parameters' do + let(:builder) { described_class.new(record: valid_record, api_key: valid_api_key) } + + it 'returns empty errors array' do + expect(builder.validate).to be_blank + end + + it 'is valid' do + expect(builder).to be_valid + end + end + + context 'with nil record' do + let(:builder) { described_class.new(record: nil, api_key: valid_api_key) } + + it 'returns validation errors' do + errors = builder.validate + expect(errors).to include('Record is required') + end + end + + context 'with non-hash record' do + let(:builder) { described_class.new(record: 'invalid', api_key: valid_api_key) } + + it 'returns validation errors' do + errors = builder.validate + expect(errors).to include('Record must be a Hash') + end + end + end + + describe '#call' do + let(:service) { create(:water_fountain_service) } + + before do + service # Ensure service exists + end + + context 'with valid parameters and complete record' do + let(:builder) { described_class.new(record: valid_record, api_key: valid_api_key) } + + it 'returns successful result' do + result = builder.call + + expect(result).to be_success + expect(result.errors).to be_blank + expect(result.data[:facility]).to be_a(Facility) + end + + it 'builds facility with correct attributes' do + result = builder.call + facility = result.data[:facility] + + expect(facility.external_id).to eq('12345') + expect(facility.name).to eq('Test Fountain') + expect(facility.address).to eq('Test Park, Downtown') + expect(facility.phone).to eq('604-123-4567') + expect(facility.website).to eq('https://vancouver.ca') + expect(facility.lat).to eq(49.2827) + expect(facility.long).to eq(-123.1207) + expect(facility.verified).to be true + end + + it 'builds notes from multiple fields' do + result = builder.call + facility = result.data[:facility] + + expect(facility.notes).to include('Maintained by: Parks Department') + expect(facility.notes).to include('Operation: Yes') + expect(facility.notes).to include('Pet friendly: Yes') + end + + it 'associates correct service' do + result = builder.call + facility = result.data[:facility] + + expect(facility.facility_services.size).to eq(1) + expect(facility.facility_services.first.service).to eq(service) + end + + it 'creates facility welcomes for all customers' do + result = builder.call + facility = result.data[:facility] + + expect(facility.facility_welcomes).not_to be_blank + # Test that welcomes are created (exact count depends on FacilityWelcome.all_customers) + end + + it 'creates schedules for all weekdays' do + result = builder.call + facility = result.data[:facility] + + expect(facility.schedules.size).to eq(7) # All weekdays + facility.schedules.each do |schedule| + expect(schedule.closed_all_day).to be false + expect(schedule.open_all_day).to be true + end + end + + describe 'schedule business logic' do + it 'creates exactly one schedule for each day of the week' do + result = builder.call + facility = result.data[:facility] + + # Test that we have all 7 days + expect(facility.schedules.size).to eq(7) + + # Test that each day is covered exactly once + week_days = facility.schedules.map(&:week_day) + expect(week_days.sort).to eq(FacilitySchedule.week_days.keys.sort) + end + + it 'sets all schedules to open_all_day = true and closed_all_day = false' do + result = builder.call + facility = result.data[:facility] + + facility.schedules.each do |schedule| + expect(schedule.open_all_day).to be(true), "Expected #{schedule.week_day} to be open_all_day" + expect(schedule.closed_all_day).to be(false), "Expected #{schedule.week_day} not to be closed_all_day" + end + end + + it 'creates schedules without time slots (consistent with open_all_day)' do + result = builder.call + facility = result.data[:facility] + + facility.schedules.each do |schedule| + expect(schedule.time_slots).to be_blank, "Expected #{schedule.week_day} to have no time slots when open_all_day" + end + end + + it 'creates valid schedule objects that pass model validations' do + result = builder.call + facility = result.data[:facility] + + facility.schedules.each do |schedule| + expect(schedule).to be_valid, "Expected #{schedule.week_day} schedule to be valid: #{schedule.errors.full_messages}" + end + end + + it 'sets schedule availability to :open for all days' do + result = builder.call + facility = result.data[:facility] + + facility.schedules.each do |schedule| + expect(schedule.availability).to eq(:open), "Expected #{schedule.week_day} availability to be :open" + end + end + + context 'when no fields are provided for schedules' do + it 'still creates open_all_day schedules for all weekdays' do + # Test with minimal record that has no schedule-related fields + minimal_builder = described_class.new(record: minimal_record, api_key: valid_api_key) + result = minimal_builder.call + facility = result.data[:facility] + + expect(facility.schedules.size).to eq(7) + facility.schedules.each do |schedule| + expect(schedule.open_all_day).to be true + expect(schedule.closed_all_day).to be false + end + end + end + + context 'business requirement verification' do + it 'ensures imported facilities are always accessible 24/7' do + result = builder.call + facility = result.data[:facility] + + # Verify that the facility is accessible any day of the week, any time + FacilitySchedule.week_days.each_key do |day| + schedule = facility.schedules.find { |s| s.week_day == day.to_s } + expect(schedule).to be_present, "Missing schedule for #{day}" + expect(schedule.open_all_day).to be(true), "Facility should be accessible 24/7 on #{day}" + expect(schedule.closed_all_day).to be(false), "Facility should not be closed on #{day}" + end + end + end + end + end + + context 'with minimal record' do + let(:builder) { described_class.new(record: minimal_record, api_key: valid_api_key) } + + it 'returns successful result' do + result = builder.call + + expect(result).to be_success + expect(result.data[:facility]).to be_a(Facility) + end + + it 'builds facility with minimal data' do + result = builder.call + facility = result.data[:facility] + + expect(facility.name).to eq('Minimal Fountain') + expect(facility.lat).to eq(49.2827) + expect(facility.long).to eq(-123.1207) + expect(facility.address).to be_nil + expect(facility.phone).to be_nil + expect(facility.website).to be_nil + end + end + + context 'with geo_point_2d coordinates' do + let(:record_with_geo_point) do + { + 'name' => 'Geo Point Fountain', + 'geo_point_2d' => { + 'lat' => 49.2827, + 'lon' => -123.1207 + } + } + end + let(:builder) { described_class.new(record: record_with_geo_point, api_key: valid_api_key) } + + it 'extracts coordinates from geo_point_2d' do + result = builder.call + facility = result.data[:facility] + + expect(facility.lat).to eq(49.2827) + expect(facility.long).to eq(-123.1207) + end + end + + context 'with geometry coordinates' do + let(:record_with_geometry) do + { + 'name' => 'Geometry Fountain', + 'geom' => { + 'geometry' => { + 'coordinates' => [-123.1207, 49.2827] # GeoJSON format: [longitude, latitude] + } + } + } + end + let(:builder) { described_class.new(record: record_with_geometry, api_key: valid_api_key) } + + it 'extracts coordinates from geometry in correct order' do + result = builder.call + facility = result.data[:facility] + + expect(facility.lat).to eq(49.2827) # Latitude from coordinates[1] + expect(facility.long).to eq(-123.1207) # Longitude from coordinates[0] + end + end + + context 'with special characters in name' do + let(:record_with_special_chars) do + { + 'name' => "Test\\nFountain\nWith\n\nSpecial Chars", + 'geo_point_2d' => { 'lat' => 49.2827, 'lon' => -123.1207 } + } + end + let(:builder) { described_class.new(record: record_with_special_chars, api_key: valid_api_key) } + + it 'cleans name by removing special characters and extra whitespace' do + result = builder.call + facility = result.data[:facility] + + expect(facility.name).to eq('Test Fountain With Special Chars') + end + end + + context 'with phone field variations' do + let(:record_with_phone_number) do + { + 'name' => 'Phone Test', + 'phone_number' => '604-555-1234', + 'geo_point_2d' => { 'lat' => 49.2827, 'lon' => -123.1207 } + } + end + let(:record_with_contact_phone) do + { + 'name' => 'Contact Phone Test', + 'contact_phone' => '604-555-5678', + 'geo_point_2d' => { 'lat' => 49.2827, 'lon' => -123.1207 } + } + end + + it 'extracts phone from phone_number field' do + builder = described_class.new(record: record_with_phone_number, api_key: valid_api_key) + result = builder.call + facility = result.data[:facility] + + expect(facility.phone).to eq('604-555-1234') + end + + it 'extracts phone from contact_phone field' do + builder = described_class.new(record: record_with_contact_phone, api_key: valid_api_key) + result = builder.call + facility = result.data[:facility] + + expect(facility.phone).to eq('604-555-5678') + end + end + + context 'with website field variations' do + let(:record_with_url) do + { + 'name' => 'URL Test', + 'url' => 'https://example.com', + 'geo_point_2d' => { 'lat' => 49.2827, 'lon' => -123.1207 } + } + end + + it 'extracts website from url field' do + builder = described_class.new(record: record_with_url, api_key: valid_api_key) + result = builder.call + facility = result.data[:facility] + + expect(facility.website).to eq('https://example.com') + end + end + + context 'with no coordinates' do + let(:record_without_coords) do + { + 'name' => 'No Coords Fountain' + } + end + let(:builder) { described_class.new(record: record_without_coords, api_key: valid_api_key) } + + it 'builds facility with nil coordinates' do + result = builder.call + facility = result.data[:facility] + + expect(result).not_to be_success + expect(facility).to be_nil + end + end + + context 'when service does not exist' do + let(:non_existent_api_key) { 'non-existent-service' } + let(:builder) { described_class.new(record: valid_record, api_key: non_existent_api_key) } + + before do + # Stub the API validation to pass + allow(External::ApiHelper).to receive(:supported_api?).with(non_existent_api_key).and_return(true) + end + + it 'builds facility without service association' do + result = builder.call + facility = result.data[:facility] + + expect(facility.facility_services).to be_blank + end + end + + context 'with invalid parameters' do + let(:builder) { described_class.new(record: nil, api_key: valid_api_key) } + + it 'returns error result without building facility' do + result = builder.call + + expect(result).to be_failed + expect(result.data).to be_blank + expect(result.errors).to include('Record is required') + end + end + + context 'when record has invalid data types that cause exceptions' do + context 'with non-string name field' do + let(:record_with_invalid_name) do + { + 'name' => 12345, # Integer instead of String + 'geo_point_2d' => { 'lat' => 49.2827, 'lon' => -123.1207 } + } + end + let(:builder) { described_class.new(record: record_with_invalid_name, api_key: valid_api_key) } + + it 'returns error result with exception message' do + result = builder.call + + expect(result).to be_failed + expect(result.data).to be_blank + expect(result.errors).to include(a_string_matching(/Failed to build facility from record:/)) + end + + it 'logs the error and record data' do + expect(Rails.logger).to receive(:warn).with(a_string_matching(/Failed to build facility from record:/)) + expect(Rails.logger).to receive(:warn).with("Record data: #{record_with_invalid_name.inspect}") + + builder.call + end + end + + context 'with invalid geometry coordinates' do + let(:record_with_invalid_geometry) do + { + 'name' => 'Test Fountain', + 'geom' => { + 'geometry' => { + 'coordinates' => 'invalid_string' # String instead of Array + } + } + } + end + let(:builder) { described_class.new(record: record_with_invalid_geometry, api_key: valid_api_key) } + + it 'returns error result with exception message' do + result = builder.call + + expect(result).to be_failed + expect(result.data).to be_blank + expect(result.errors).to include(a_string_matching(/Geometry should be/)) + end + end + + context 'with invalid geo_point_2d field' do + let(:record_with_invalid_geo_point) do + { + 'name' => 'Test Fountain', + 'geo_point_2d' => 'invalid_string' # String instead of Hash + } + end + let(:builder) { described_class.new(record: record_with_invalid_geo_point, api_key: valid_api_key) } + + it 'returns error result with exception message' do + result = builder.call + + expect(result).to be_failed + expect(result.data).to be_blank + expect(result.errors).to include(a_string_matching(/Geometry should be/)) + end + end + end + + context 'when built facility is invalid' do + let(:invalid_record) do + { + 'name' => '', # Empty name might make facility invalid + 'geo_point_2d' => { 'lat' => 49.2827, 'lon' => -123.1207 } + } + end + let(:builder) { described_class.new(record: invalid_record, api_key: valid_api_key) } + + it 'returns error result with validation messages' do + result = builder.call + + expect(result).to be_failed + expect(result.data).to be_blank + expect(result.errors.first).to match(/Facility .* is invalid:/) + end + end + end + + describe '.call class method' do + let(:service) { create(:water_fountain_service) } + + before do + service # Ensure service exists + end + + it 'works as a class method' do + result = described_class.call(record: valid_record, api_key: valid_api_key) + + expect(result).to be_success + expect(result.data[:facility]).to be_a(Facility) + end + end +end diff --git a/spec/services/external/vancouver_city/facility_schedule_builder_spec.rb b/spec/services/external/vancouver_city/facility_schedule_builder_spec.rb new file mode 100644 index 00000000..32cb365c --- /dev/null +++ b/spec/services/external/vancouver_city/facility_schedule_builder_spec.rb @@ -0,0 +1,133 @@ +# frozen_string_literal: true + +require 'rails_helper' + +RSpec.describe External::VancouverCity::FacilityScheduleBuilder, type: :service do + let(:facility) { build(:facility) } + let(:fields) { { 'name' => 'Test Facility' } } + + describe '#initialize' do + it 'initializes with valid parameters' do + builder = described_class.new(facility: facility, fields: fields) + + expect(builder.facility).to eq(facility) + expect(builder.fields).to eq(fields) + end + end + + describe '#validate' do + context 'with valid parameters' do + let(:builder) { described_class.new(facility: facility, fields: fields) } + + it 'returns empty errors array' do + expect(builder.validate).to be_empty + end + + it 'is valid' do + expect(builder).to be_valid + end + end + + context 'with nil facility' do + let(:builder) { described_class.new(facility: nil, fields: fields) } + + it 'returns validation errors' do + errors = builder.validate + expect(errors).to include('Facility is required') + end + + it 'is invalid' do + expect(builder).to be_invalid + end + end + + context 'with non-facility object' do + let(:builder) { described_class.new(facility: 'invalid', fields: fields) } + + it 'returns validation errors' do + errors = builder.validate + expect(errors).to include('Facility must be a Facility object') + end + end + + context 'with nil fields' do + let(:builder) { described_class.new(facility: facility, fields: nil) } + + it 'returns validation errors' do + errors = builder.validate + expect(errors).to include('Fields are required') + end + end + + context 'with non-hash fields' do + let(:builder) { described_class.new(facility: facility, fields: 'invalid') } + + it 'returns validation errors' do + errors = builder.validate + expect(errors).to include('Fields must be a Hash') + end + end + end + + describe '#call' do + context 'with valid parameters' do + let(:builder) { described_class.new(facility: facility, fields: fields) } + + it 'returns successful result' do + result = builder.call + + expect(result).to be_success + expect(result.errors).to be_empty + expect(result.data[:schedules_count]).to eq(7) + end + + it 'creates schedules for all weekdays' do + builder.call + + expect(facility.schedules.size).to eq(7) + facility.schedules.each do |schedule| + expect(schedule.closed_all_day).to be false + expect(schedule.open_all_day).to be true + end + end + + it 'creates exactly one schedule for each day of the week' do + builder.call + + # Test that each day is covered exactly once + week_days = facility.schedules.map(&:week_day) + expect(week_days.sort).to eq(FacilitySchedule.week_days.keys.sort) + end + + it 'creates valid schedule objects' do + builder.call + + facility.schedules.each do |schedule| + expect(schedule).to be_valid, "Expected #{schedule.week_day} schedule to be valid: #{schedule.errors.full_messages}" + end + end + end + + context 'with invalid parameters' do + let(:builder) { described_class.new(facility: nil, fields: nil) } + + it 'returns error result without building schedules' do + result = builder.call + + expect(result).to be_failed + expect(result.data).to be_nil + expect(result.errors).to include('Facility is required') + expect(result.errors).to include('Fields are required') + end + end + end + + describe '.call class method' do + it 'works as a class method' do + result = described_class.call(facility: facility, fields: fields) + + expect(result).to be_success + expect(result.data[:schedules_count]).to eq(7) + end + end +end diff --git a/spec/services/external/vancouver_city/facility_service_builder_spec.rb b/spec/services/external/vancouver_city/facility_service_builder_spec.rb new file mode 100644 index 00000000..9270f27c --- /dev/null +++ b/spec/services/external/vancouver_city/facility_service_builder_spec.rb @@ -0,0 +1,142 @@ +# frozen_string_literal: true + +require 'rails_helper' + +RSpec.describe External::VancouverCity::FacilityServiceBuilder, type: :service do + let(:facility) { build(:facility) } + let(:fields) { { 'name' => 'Test Facility' } } + let(:api_key) { 'drinking-fountains' } + + describe '#initialize' do + it 'initializes with valid parameters' do + builder = described_class.new(facility: facility, fields: fields, api_key: api_key) + + expect(builder.facility).to eq(facility) + expect(builder.fields).to eq(fields) + expect(builder.api_key).to eq(api_key) + end + end + + describe '#validate' do + context 'with valid parameters' do + let(:builder) { described_class.new(facility: facility, fields: fields, api_key: api_key) } + + it 'returns empty errors array' do + expect(builder.validate).to be_empty + end + + it 'is valid' do + expect(builder).to be_valid + end + end + + context 'with nil facility' do + let(:builder) { described_class.new(facility: nil, fields: fields, api_key: api_key) } + + it 'returns validation errors' do + errors = builder.validate + expect(errors).to include('Facility is required') + end + end + + context 'with non-facility object' do + let(:builder) { described_class.new(facility: 'invalid', fields: fields, api_key: api_key) } + + it 'returns validation errors' do + errors = builder.validate + expect(errors).to include('Facility must be a Facility object') + end + end + + context 'with nil fields' do + let(:builder) { described_class.new(facility: facility, fields: nil, api_key: api_key) } + + it 'returns validation errors' do + errors = builder.validate + expect(errors).to include('Fields are required') + end + end + + context 'with non-hash fields' do + let(:builder) { described_class.new(facility: facility, fields: 'invalid', api_key: api_key) } + + it 'returns validation errors' do + errors = builder.validate + expect(errors).to include('Fields must be a Hash') + end + end + + context 'with nil api_key' do + let(:builder) { described_class.new(facility: facility, fields: fields, api_key: nil) } + + it 'returns validation errors' do + errors = builder.validate + expect(errors).to include('API key is required') + end + end + + context 'with empty api_key' do + let(:builder) { described_class.new(facility: facility, fields: fields, api_key: '') } + + it 'returns validation errors' do + errors = builder.validate + expect(errors).to include('API key is required') + end + end + end + + describe '#call' do + context 'with valid parameters and existing service' do + let(:service) { create(:water_fountain_service) } + let(:builder) { described_class.new(facility: facility, fields: fields, api_key: api_key) } + + before do + service # Ensure service exists + end + + it 'returns successful result' do + result = builder.call + + expect(result).to be_success + expect(result.errors).to be_empty + expect(result.data[:services_count]).to eq(1) + end + + it 'associates correct service with facility' do + builder.call + + expect(facility.facility_services.size).to eq(1) + expect(facility.facility_services.first.service).to eq(service) + end + end + + context 'with invalid parameters' do + let(:builder) { described_class.new(facility: nil, fields: nil, api_key: nil) } + + it 'returns error result without building services' do + result = builder.call + + expect(result).to be_failed + expect(result.data).to be_nil + expect(result.errors).to include('Facility is required') + expect(result.errors).to include('Fields are required') + expect(result.errors).to include('API key is required') + end + end + end + + describe '.call class method' do + let(:service) { create(:water_fountain_service) } + + before do + service # Ensure service exists + end + + it 'works as a class method' do + result = described_class.call(facility: facility, fields: fields, api_key: api_key) + + expect(result).to be_success + expect(result.data[:services_count]).to eq(1) + end + end +end diff --git a/spec/services/external/vancouver_city/facility_syncer/create_operation_spec.rb b/spec/services/external/vancouver_city/facility_syncer/create_operation_spec.rb new file mode 100644 index 00000000..5447da4e --- /dev/null +++ b/spec/services/external/vancouver_city/facility_syncer/create_operation_spec.rb @@ -0,0 +1,305 @@ +# frozen_string_literal: true + +require 'rails_helper' + +RSpec.describe External::VancouverCity::FacilitySyncer, 'create operation', type: :service do + let(:api_key) { 'drinking-fountains' } + let(:service) { create(:water_fountain_service) } + + before { service } # Ensure service exists + + describe 'create operation (:create)' do + context 'when built facility is valid' do + let(:valid_record) do + { + 'mapid' => 'CREATE123', + 'name' => 'New Valid Fountain', + 'location' => 'Valid Park', + 'geo_local_area' => 'Downtown', + 'phone' => '604-123-4567', + 'website' => 'https://vancouver.ca', + 'geo_point_2d' => { 'lat' => 49.2827, 'lon' => -123.1207 } + } + end + + it 'saves the facility successfully' do + expect do + syncer = described_class.new(record: valid_record, api_key: api_key) + syncer.call + end.to change(Facility, :count).by(1) + end + + it 'returns success result with operation: :create' do + syncer = described_class.new(record: valid_record, api_key: api_key) + result = syncer.call + + expect(result).to be_success + expect(result.data.operation).to eq(:create) + expect(result.errors).to be_empty + end + + it 'sets result_facility to built_facility' do + syncer = described_class.new(record: valid_record, api_key: api_key) + result = syncer.call + + facility = result.data.facility + expect(facility).to be_persisted + expect(facility.name).to eq('New Valid Fountain') + expect(facility.external_id).to eq('CREATE123') + expect(facility.verified).to be true + end + + it 'creates facility with all expected attributes' do + syncer = described_class.new(record: valid_record, api_key: api_key) + result = syncer.call + + facility = result.data.facility + expect(facility.name).to eq('New Valid Fountain') + expect(facility.address).to eq('Valid Park, Downtown') + expect(facility.phone).to eq('604-123-4567') + expect(facility.website).to eq('https://vancouver.ca') + expect(facility.lat).to eq(49.2827) + expect(facility.long).to eq(-123.1207) + expect(facility.verified).to be true + expect(facility.external_id).to eq('CREATE123') + end + + it 'creates facility services' do + syncer = described_class.new(record: valid_record, api_key: api_key) + result = syncer.call + + facility = result.data.facility + expect(facility.facility_services.count).to eq(1) + expect(facility.services).to include(service) + end + + it 'logs creation message with external_id' do + expect(Rails.logger).to receive(:info).with("Creating new facility with external_id 'CREATE123'") + + syncer = described_class.new(record: valid_record, api_key: api_key) + syncer.call + end + end + + context 'when FacilityBuilder fails due to invalid data' do + let(:invalid_record) do + { + 'mapid' => 'INVALID123', + 'name' => '', # Empty name causes FacilityBuilder to fail + 'geo_point_2d' => { 'lat' => 49.2827, 'lon' => -123.1207 } + } + end + + it 'does not save facility' do + expect do + syncer = described_class.new(record: invalid_record, api_key: api_key) + syncer.call + end.not_to change(Facility, :count) + end + + it 'adds validation errors to errors array' do + syncer = described_class.new(record: invalid_record, api_key: api_key) + result = syncer.call + + expect(result).to be_failed + expect(result.errors).to include(a_string_matching(/can't be blank/i)) + end + + it 'sets result_facility to nil' do + syncer = described_class.new(record: invalid_record, api_key: api_key) + result = syncer.call + + expect(result.data.facility).to be_nil + end + + it 'returns early with operation: nil when FacilityBuilder fails' do + syncer = described_class.new(record: invalid_record, api_key: api_key) + result = syncer.call + + expect(result.data.operation).to be_nil # FacilityBuilder fails before operation is determined + expect(result).to be_failed + end + end + + context 'when save! raises other StandardError' do + let(:valid_record) do + { + 'mapid' => 'ERROR123', + 'name' => 'Error Test Fountain', + 'geo_point_2d' => { 'lat' => 49.2827, 'lon' => -123.1207 } + } + end + + before do + # Simulate a database connection error or similar + allow_any_instance_of(Facility).to receive(:save!).and_raise(StandardError.new('Database connection lost')) + end + + it 'catches exception and adds generic error message' do + syncer = described_class.new(record: valid_record, api_key: api_key) + result = syncer.call + + expect(result).to be_failed + expect(result.errors).to include(a_string_matching(/Unexpected error during facility sync:/)) + end + + it 'includes original error message' do + syncer = described_class.new(record: valid_record, api_key: api_key) + result = syncer.call + + expect(result.errors.first).to include('Database connection lost') + end + + it 'does not save facility on failure' do + expect do + syncer = described_class.new(record: valid_record, api_key: api_key) + syncer.call + end.not_to change(Facility, :count) + end + + it 'does not create any related records on failure' do + expect do + syncer = described_class.new(record: valid_record, api_key: api_key) + syncer.call + end.not_to change(FacilityService, :count) + end + end + + context 'when save! raises ActiveRecord::RecordInvalid' do + let(:invalid_save_record) do + { + 'mapid' => 'INVALID_SAVE123', + 'name' => 'Valid Name', + 'geo_point_2d' => { 'lat' => 49.2827, 'lon' => -123.1207 } + } + end + + before do + # Simulate a validation error during save + allow_any_instance_of(Facility).to receive(:save!).and_raise( + ActiveRecord::RecordInvalid.new(build(:facility)) + ) + end + + it 'catches RecordInvalid and adds error message' do + syncer = described_class.new(record: invalid_save_record, api_key: api_key) + result = syncer.call + + expect(result).to be_failed + expect(result.errors).to include(a_string_matching(/Failed to save facility:/)) + end + + it 'does not create facility record on validation failure' do + expect do + syncer = described_class.new(record: invalid_save_record, api_key: api_key) + syncer.call + end.not_to change(Facility, :count) + end + + it 'does not create any related records on validation failure' do + expect do + syncer = described_class.new(record: invalid_save_record, api_key: api_key) + syncer.call + end.not_to change(FacilityService, :count) + end + end + + context 'when service creation fails' do + let(:service_fail_record) do + { + 'mapid' => 'SERVICE_FAIL123', + 'name' => 'Service Fail Test', + 'geo_point_2d' => { 'lat' => 49.2827, 'lon' => -123.1207 } + } + end + + before do + # For create operations, service associations are built in memory by FacilityBuilder + # and saved together with the facility. To simulate failure, we need to make + # the facility save fail due to a constraint on the associations. + allow_any_instance_of(Facility).to receive(:save!).and_raise( + ActiveRecord::RecordInvalid.new(build(:facility, name: 'Service validation failed')) + ) + end + + it 'rolls back facility creation when facility save fails' do + expect do + syncer = described_class.new(record: service_fail_record, api_key: api_key) + syncer.call + end.not_to change(Facility, :count) + end + + it 'does not create any service records when transaction fails' do + expect do + syncer = described_class.new(record: service_fail_record, api_key: api_key) + syncer.call + end.not_to change(FacilityService, :count) + end + + it 'does not create any schedule records when transaction fails' do + expect do + syncer = described_class.new(record: service_fail_record, api_key: api_key) + syncer.call + end.not_to change(FacilitySchedule, :count) + end + + it 'returns failed result with proper error message' do + syncer = described_class.new(record: service_fail_record, api_key: api_key) + result = syncer.call + + expect(result).to be_failed + expect(result.errors).to include(a_string_matching(/Failed to save facility:/)) + end + end + + context 'database record creation on success' do + let(:success_record) do + { + 'mapid' => 'SUCCESS123', + 'name' => 'Success Test Fountain', + 'location' => 'Success Park', + 'geo_local_area' => 'Downtown', + 'geo_point_2d' => { 'lat' => 49.2827, 'lon' => -123.1207 } + } + end + + it 'creates facility with all related records atomically' do + syncer = described_class.new(record: success_record, api_key: api_key) + + expect { syncer.call }.to change { Facility.count }.by(1) + .and change { FacilityService.count }.by(1) + .and change { FacilitySchedule.count }.by(7) # 7 days of the week + .and change { FacilityWelcome.count }.by_at_least(1) + end + + it 'creates facility with correct attributes and relationships' do + syncer = described_class.new(record: success_record, api_key: api_key) + result = syncer.call + + facility = result.data.facility + expect(facility).to be_persisted + expect(facility.external_id).to eq('SUCCESS123') + expect(facility.name).to eq('Success Test Fountain') + expect(facility.verified).to be true + + # Verify related records are created + expect(facility.facility_services.count).to eq(1) + expect(facility.facility_services.first.service).to eq(service) + expect(facility.schedules.count).to eq(7) + expect(facility.facility_welcomes.count).to be > 0 + end + + it 'ensures all database records are properly linked' do + syncer = described_class.new(record: success_record, api_key: api_key) + result = syncer.call + + facility = result.data.facility + + # Verify foreign key relationships + expect(facility.facility_services.all? { |fs| fs.facility_id == facility.id }).to be true + expect(facility.schedules.all? { |s| s.facility_id == facility.id }).to be true + expect(facility.facility_welcomes.all? { |fw| fw.facility_id == facility.id }).to be true + end + end + end +end diff --git a/spec/services/external/vancouver_city/facility_syncer/error_handling_spec.rb b/spec/services/external/vancouver_city/facility_syncer/error_handling_spec.rb new file mode 100644 index 00000000..6850ff42 --- /dev/null +++ b/spec/services/external/vancouver_city/facility_syncer/error_handling_spec.rb @@ -0,0 +1,170 @@ +# frozen_string_literal: true + +require 'rails_helper' + +RSpec.describe External::VancouverCity::FacilitySyncer, 'error handling', type: :service do + let(:api_key) { 'drinking-fountains' } + let(:service) { create(:water_fountain_service) } + + before { service } + + describe 'transaction rollback scenarios' do + context 'when ActiveRecord::RecordInvalid occurs during external_update' do + let!(:existing_facility) do + create(:facility, + external_id: 'FAIL_UPDATE123', + name: 'Test Facility', + address: 'Test Address') + end + + let(:update_record) do + { + 'mapid' => 'FAIL_UPDATE123', + 'name' => 'Updated Name', + 'location' => 'Updated Location', + 'geo_local_area' => 'Updated Area', + 'geo_point_2d' => { 'lat' => 49.2827, 'lon' => -123.1207 } + } + end + + before do + # Stub update! to raise RecordInvalid to simulate validation failure + allow_any_instance_of(Facility).to receive(:update!).and_raise( + ActiveRecord::RecordInvalid.new(existing_facility) + ) + end + + it 'rolls back transaction and reports error' do + original_name = existing_facility.name + syncer = described_class.new(record: update_record, api_key: api_key) + result = syncer.call + + existing_facility.reload + expect(existing_facility.name).to eq(original_name) # No change due to rollback + expect(result).to be_failed + expect(result.errors).to include(a_string_matching(/Failed to save facility/)) + expect(result.data.operation).to eq(:external_update) + expect(result.data.facility).to be_nil + end + end + + context 'when StandardError occurs during service synchronization' do + let!(:existing_facility) do + create(:facility, + external_id: 'SERVICE_ERROR123', + name: 'Test Facility') + end + + let(:update_record) do + { + 'mapid' => 'SERVICE_ERROR123', + 'name' => 'Updated Name', + 'location' => 'Updated Location', + 'geo_local_area' => 'Updated Area', + 'geo_point_2d' => { 'lat' => 49.2827, 'lon' => -123.1207 } + } + end + + before do + # Stub facility_services.create! to raise StandardError + allow_any_instance_of(ActiveRecord::Associations::CollectionProxy) + .to receive(:create!).and_raise(StandardError.new('Database connection lost')) + end + + it 'rolls back transaction and reports error' do + original_service_count = existing_facility.facility_services.count + syncer = described_class.new(record: update_record, api_key: api_key) + result = syncer.call + + existing_facility.reload + expect(existing_facility.facility_services.count).to eq(original_service_count) + expect(result).to be_failed + expect(result.errors).to include(a_string_matching(/Unexpected error during facility sync.*Database connection lost/)) + expect(result.data.operation).to eq(:external_update) + expect(result.data.facility).to be_nil + end + end + end + + describe 'logging behavior during errors' do + let(:valid_record) do + { + 'mapid' => 'LOG_TEST123', + 'name' => 'Test Fountain', + 'location' => 'Test Park', + 'geo_local_area' => 'Downtown', + 'geo_point_2d' => { 'lat' => 49.2827, 'lon' => -123.1207 } + } + end + + before do + # Stub save! to raise an error to test logging + allow_any_instance_of(Facility).to receive(:save!).and_raise( + ActiveRecord::RecordInvalid.new(build(:facility)) + ) + end + + it 'logs errors appropriately' do + syncer = described_class.new(record: valid_record, api_key: api_key) + + expect(Rails.logger).to receive(:info).with( + a_string_matching(/Creating new facility with external_id 'LOG_TEST123'/) + ) + + syncer.call + end + end + + describe 'error message formatting' do + context 'when FacilityBuilder fails due to validation errors' do + let(:invalid_facility_record) do + { + 'mapid' => 'INVALID123', + 'name' => '', # Invalid name causes FacilityBuilder to fail + 'location' => 'Test Location', + 'geo_local_area' => 'Downtown', + 'geo_point_2d' => { 'lat' => 49.2827, 'lon' => -123.1207 } + } + end + + it 'includes detailed validation errors from FacilityBuilder' do + syncer = described_class.new(record: invalid_facility_record, api_key: api_key) + result = syncer.call + + expect(result).to be_failed + expect(result.errors.first).to match(/Name can't be blank/) + expect(result.data.operation).to be_nil # No operation determined when FacilityBuilder fails + expect(result.data.facility).to be_nil + end + end + + context 'when ActiveRecord::RecordInvalid provides detailed message' do + let(:valid_record) do + { + 'mapid' => 'DETAILED_ERROR123', + 'name' => 'Test Facility', + 'location' => 'Test Location', + 'geo_local_area' => 'Downtown', + 'geo_point_2d' => { 'lat' => 49.2827, 'lon' => -123.1207 } + } + end + + before do + facility = build(:facility) + facility.errors.add(:base, 'Custom validation error') + + allow_any_instance_of(Facility).to receive(:save!).and_raise( + ActiveRecord::RecordInvalid.new(facility) + ) + end + + it 'includes the detailed ActiveRecord error message' do + syncer = described_class.new(record: valid_record, api_key: api_key) + result = syncer.call + + expect(result).to be_failed + expect(result.errors).to include(a_string_matching(/Failed to save facility.*Custom validation error/)) + end + end + end +end diff --git a/spec/services/external/vancouver_city/facility_syncer/external_update_operation_spec.rb b/spec/services/external/vancouver_city/facility_syncer/external_update_operation_spec.rb new file mode 100644 index 00000000..9bb77d62 --- /dev/null +++ b/spec/services/external/vancouver_city/facility_syncer/external_update_operation_spec.rb @@ -0,0 +1,381 @@ +# frozen_string_literal: true + +require 'rails_helper' + +RSpec.describe External::VancouverCity::FacilitySyncer, 'external update operation', type: :service do + let(:api_key) { 'drinking-fountains' } + let(:service) { create(:water_fountain_service) } + let(:other_service) { create(:service, key: 'public-washrooms') } + + before do + service + other_service + end + + describe 'external_update operation (:external_update)' do + context 'when update succeeds' do + let!(:existing_external_facility) do + create(:facility, + external_id: 'EXT_UPDATE123', + name: 'Old Name', + address: 'Old Address', + lat: 49.0000, + long: -123.0000, + verified: false) + end + + let(:update_record) do + { + 'mapid' => 'EXT_UPDATE123', + 'name' => 'Updated Fountain Name', + 'location' => 'Updated Park', + 'geo_local_area' => 'Updated Area', + 'phone' => '604-999-8888', + 'geo_point_2d' => { 'lat' => 49.9999, 'lon' => -123.9999 } + } + end + + it 'updates facility attributes' do + syncer = described_class.new(record: update_record, api_key: api_key) + result = syncer.call + + facility = result.data.facility + expect(facility.name).to eq('Updated Fountain Name') + expect(facility.address).to eq('Updated Park, Updated Area') + expect(facility.lat).to eq(49.9999) + expect(facility.long).to eq(-123.9999) + expect(facility.verified).to be true + end + + it 'adds missing services' do + expect(existing_external_facility.services).not_to include(service) + + syncer = described_class.new(record: update_record, api_key: api_key) + result = syncer.call + + facility = result.data.facility + expect(facility.services).to include(service) + end + + it 'returns existing facility in result' do + syncer = described_class.new(record: update_record, api_key: api_key) + result = syncer.call + + expect(result.data.facility.id).to eq(existing_external_facility.id) + expect(result.data.operation).to eq(:external_update) + end + + it 'logs update message with external_id' do + expect(Rails.logger).to receive(:info).with("Facility with external_id 'EXT_UPDATE123' already exists, updating services") + + syncer = described_class.new(record: update_record, api_key: api_key) + syncer.call + end + + it 'returns success result' do + syncer = described_class.new(record: update_record, api_key: api_key) + result = syncer.call + + expect(result).to be_success + expect(result.errors).to be_empty + end + + it 'does not create new facility' do + expect do + syncer = described_class.new(record: update_record, api_key: api_key) + syncer.call + end.not_to change(Facility, :count) + end + end + + context 'when facility already has the service' do + let!(:existing_external_facility) do + facility = create(:facility, + external_id: 'EXT_HAS_SERVICE123', + name: 'Fountain with Service') + facility.facility_services.create!(service: service) + facility + end + + let(:update_record) do + { + 'mapid' => 'EXT_HAS_SERVICE123', + 'name' => 'Updated Name', + 'geo_point_2d' => { 'lat' => 49.2827, 'lon' => -123.1207 } + } + end + + it 'does not duplicate existing services' do + initial_service_count = existing_external_facility.facility_services.count + + syncer = described_class.new(record: update_record, api_key: api_key) + result = syncer.call + + facility = result.data.facility + expect(facility.facility_services.count).to eq(initial_service_count) + end + + it 'still updates facility attributes' do + syncer = described_class.new(record: update_record, api_key: api_key) + result = syncer.call + + facility = result.data.facility + expect(facility.name).to eq('Updated Name') + end + end + + context 'when update! raises ActiveRecord::RecordInvalid during attribute update' do + let!(:existing_external_facility) do + create(:facility, + external_id: 'EXT_INVALID123', + name: 'Test Facility') + end + + let(:update_record) do + { + 'mapid' => 'EXT_INVALID123', + 'name' => 'Updated Name', + 'geo_point_2d' => { 'lat' => 49.2827, 'lon' => -123.1207 } + } + end + + before do + # Simulate a validation error during update + allow_any_instance_of(Facility).to receive(:update!).and_raise( + ActiveRecord::RecordInvalid.new(existing_external_facility) + ) + end + + it 'catches exception during attribute update' do + syncer = described_class.new(record: update_record, api_key: api_key) + result = syncer.call + + expect(result).to be_failed + expect(result.errors).to include(a_string_matching(/Failed to save facility:/)) + end + end + + context 'when create! raises ActiveRecord::RecordInvalid during service creation' do + let!(:existing_external_facility) do + create(:facility, + external_id: 'EXT_SERVICE_ERROR123', + name: 'Test Facility') + end + + let(:update_record) do + { + 'mapid' => 'EXT_SERVICE_ERROR123', + 'name' => 'Updated Name', + 'geo_point_2d' => { 'lat' => 49.2827, 'lon' => -123.1207 } + } + end + + before do + # Simulate a constraint violation when creating facility service + allow_any_instance_of(ActiveRecord::Associations::CollectionProxy) + .to receive(:create!).and_raise( + ActiveRecord::RecordInvalid.new(FacilityService.new) + ) + end + + it 'catches exception during service creation' do + syncer = described_class.new(record: update_record, api_key: api_key) + result = syncer.call + + expect(result).to be_failed + expect(result.errors).to include(a_string_matching(/Failed to save facility:/)) + end + end + + context 'when update raises other StandardError' do + let!(:existing_external_facility) do + create(:facility, + external_id: 'EXT_STD_ERROR123', + name: 'Test Facility') + end + + let(:update_record) do + { + 'mapid' => 'EXT_STD_ERROR123', + 'name' => 'Updated Name', + 'geo_point_2d' => { 'lat' => 49.2827, 'lon' => -123.1207 } + } + end + + before do + # Force service creation to fail during add_missing_services + allow_any_instance_of(ActiveRecord::Associations::CollectionProxy) + .to receive(:create!).and_raise(StandardError.new('Service creation failed')) + end + + it 'catches and handles generic errors' do + syncer = described_class.new(record: update_record, api_key: api_key) + result = syncer.call + + expect(result).to be_failed + expect(result.errors).to include(a_string_matching(/Unexpected error during facility sync:/)) + expect(result.errors.first).to include('Service creation failed') + end + + it 'does not update facility attributes on error' do + original_name = existing_external_facility.name + original_address = existing_external_facility.address + + syncer = described_class.new(record: update_record, api_key: api_key) + syncer.call + + existing_external_facility.reload + expect(existing_external_facility.name).to eq(original_name) + expect(existing_external_facility.address).to eq(original_address) + end + + it 'does not create any new service records on error' do + expect do + syncer = described_class.new(record: update_record, api_key: api_key) + syncer.call + end.not_to change(FacilityService, :count) + end + end + + context 'database record updates on success' do + let!(:external_facility_with_data) do + facility = create(:facility, + external_id: 'DB_UPDATE123', + name: 'Original Name', + address: 'Original Address', + lat: 49.0000, + long: -123.0000, + verified: false) + + # Add existing service from different API + facility.facility_services.create!(service: other_service) + facility + end + + let(:comprehensive_update_record) do + { + 'mapid' => 'DB_UPDATE123', + 'name' => 'Completely Updated Name', + 'location' => 'New Location', + 'geo_local_area' => 'New Area', + 'phone' => '604-555-1234', + 'website' => 'https://updated.example.com', + 'geo_point_2d' => { 'lat' => 49.5555, 'lon' => -123.5555 } + } + end + + it 'updates all facility attributes correctly' do + syncer = described_class.new(record: comprehensive_update_record, api_key: api_key) + result = syncer.call + + facility = result.data.facility + # Only these attributes are updated in external_update operations + expect(facility.name).to eq('Completely Updated Name') + expect(facility.address).to eq('New Location, New Area') + expect(facility.lat).to eq(49.5555) + expect(facility.long).to eq(-123.5555) + expect(facility.verified).to be true + expect(facility.external_id).to eq('DB_UPDATE123') # Should remain unchanged + + # These attributes are NOT updated in external_update operations + expect(facility.phone).to eq('123') # Original value from factory + expect(facility.website).to eq('www.facility.test') # Original value from factory + end + + it 'adds new service without removing existing ones' do + initial_service_count = external_facility_with_data.facility_services.count + + syncer = described_class.new(record: comprehensive_update_record, api_key: api_key) + result = syncer.call + + facility = result.data.facility + expect(facility.facility_services.count).to eq(initial_service_count + 1) + expect(facility.services).to include(service) # New service added + expect(facility.services).to include(other_service) # Existing service preserved + end + + it 'maintains referential integrity during updates' do + syncer = described_class.new(record: comprehensive_update_record, api_key: api_key) + result = syncer.call + + facility = result.data.facility + + # Verify all related records still reference the correct facility + expect(facility.facility_services.all? { |fs| fs.facility_id == facility.id }).to be true + expect(facility.schedules.all? { |s| s.facility_id == facility.id }).to be true + expect(facility.facility_welcomes.all? { |fw| fw.facility_id == facility.id }).to be true + end + + it 'does not create duplicate services for same API key' do + # First update + syncer1 = described_class.new(record: comprehensive_update_record, api_key: api_key) + syncer1.call + + initial_count = external_facility_with_data.reload.facility_services.count + + # Second update with same API key + syncer2 = described_class.new(record: comprehensive_update_record, api_key: api_key) + syncer2.call + + external_facility_with_data.reload + expect(external_facility_with_data.facility_services.count).to eq(initial_count) + end + end + + context 'transaction rollback on failure' do + let!(:rollback_facility) do + create(:facility, + external_id: 'ROLLBACK123', + name: 'Rollback Test', + address: 'Original Address', + verified: false) + end + + let(:rollback_record) do + { + 'mapid' => 'ROLLBACK123', + 'name' => 'Updated Name', + 'location' => 'Updated Location', + 'geo_point_2d' => { 'lat' => 49.2827, 'lon' => -123.1207 } + } + end + + before do + # Force failure after attribute update but before service creation + allow_any_instance_of(ActiveRecord::Associations::CollectionProxy) + .to receive(:create!).and_raise(StandardError.new('Service creation failed')) + end + + it 'rolls back attribute changes when service creation fails' do + original_name = rollback_facility.name + original_address = rollback_facility.address + original_verified = rollback_facility.verified + + syncer = described_class.new(record: rollback_record, api_key: api_key) + syncer.call + + rollback_facility.reload + expect(rollback_facility.name).to eq(original_name) + expect(rollback_facility.address).to eq(original_address) + expect(rollback_facility.verified).to eq(original_verified) + end + + it 'does not create any service records when transaction fails' do + expect do + syncer = described_class.new(record: rollback_record, api_key: api_key) + syncer.call + end.not_to change(FacilityService, :count) + end + + it 'maintains database consistency after rollback' do + original_service_count = rollback_facility.facility_services.count + + syncer = described_class.new(record: rollback_record, api_key: api_key) + syncer.call + + rollback_facility.reload + expect(rollback_facility.facility_services.count).to eq(original_service_count) + end + end + end +end diff --git a/spec/services/external/vancouver_city/facility_syncer/facility_builder_integration_spec.rb b/spec/services/external/vancouver_city/facility_syncer/facility_builder_integration_spec.rb new file mode 100644 index 00000000..4e9ff26e --- /dev/null +++ b/spec/services/external/vancouver_city/facility_syncer/facility_builder_integration_spec.rb @@ -0,0 +1,108 @@ +# frozen_string_literal: true + +require 'rails_helper' + +RSpec.describe External::VancouverCity::FacilitySyncer, 'facility builder integration', type: :service do + let(:api_key) { 'drinking-fountains' } + let(:service) { create(:water_fountain_service) } + + before { service } # Ensure service exists + + describe 'FacilityBuilder integration' do + context 'when FacilityBuilder succeeds with valid facility' do + let(:valid_record) do + { + 'mapid' => '12345', + 'name' => 'Test Fountain', + 'location' => 'Test Park', + 'geo_local_area' => 'Downtown', + 'geo_point_2d' => { 'lat' => 49.2827, 'lon' => -123.1207 } + } + end + + it 'proceeds with sync operations' do + syncer = described_class.new(record: valid_record, api_key: api_key) + result = syncer.call + + expect(result).to be_success + expect(result.data.operation).to eq(:create) + expect(result.data.facility).to be_present + end + + it 'facility is created and persisted' do + syncer = described_class.new(record: valid_record, api_key: api_key) + result = syncer.call + + expect(result.data.facility).to be_persisted + expect(result.data.facility.name).to eq('Test Fountain') + expect(result.data.facility.external_id).to eq('12345') + end + end + + context 'when FacilityBuilder fails due to invalid record' do + let(:invalid_record) do + { + # Missing required fields like name and coordinates + } + end + + it 'returns early with FacilityBuilder errors' do + syncer = described_class.new(record: invalid_record, api_key: api_key) + result = syncer.call + + expect(result).to be_failed + expect(result.errors).to be_present + end + + it 'returns ResultData with operation: nil, facility: nil' do + syncer = described_class.new(record: invalid_record, api_key: api_key) + result = syncer.call + + expect(result.data.operation).to be_nil + expect(result.data.facility).to be_nil + end + + it 'does not attempt database operations' do + expect(Facility).not_to receive(:where) + + syncer = described_class.new(record: invalid_record, api_key: api_key) + syncer.call + end + end + + context 'when FacilityBuilder fails due to invalid facility data' do + # This scenario occurs when FacilityBuilder receives data that would create + # an invalid facility, so it fails validation and returns errors + let(:record_with_invalid_facility_data) do + { + 'mapid' => '12345', + 'name' => '', # Empty name will make facility invalid + 'geo_point_2d' => { 'lat' => 49.2827, 'lon' => -123.1207 } + } + end + + it 'returns early with validation errors' do + syncer = described_class.new(record: record_with_invalid_facility_data, api_key: api_key) + result = syncer.call + + expect(result).to be_failed + expect(result.data.operation).to be_nil # No operation determined + expect(result.data.facility).to be_nil # No facility created + end + + it 'includes FacilityBuilder validation errors' do + syncer = described_class.new(record: record_with_invalid_facility_data, api_key: api_key) + result = syncer.call + + expect(result.errors).to include(a_string_matching(/can't be blank/i)) + end + + it 'does not attempt to save anything' do + expect do + syncer = described_class.new(record: record_with_invalid_facility_data, api_key: api_key) + syncer.call + end.not_to change(Facility, :count) + end + end + end +end diff --git a/spec/services/external/vancouver_city/facility_syncer/initialization_spec.rb b/spec/services/external/vancouver_city/facility_syncer/initialization_spec.rb new file mode 100644 index 00000000..74755ae5 --- /dev/null +++ b/spec/services/external/vancouver_city/facility_syncer/initialization_spec.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +require 'rails_helper' + +RSpec.describe External::VancouverCity::FacilitySyncer, '#initialize', type: :service do + describe '#initialize' do + let(:record) { { 'name' => 'Test Facility' } } + let(:api_key) { 'test-api-key' } + + it 'sets record and api_key' do + syncer = described_class.new(record: record, api_key: api_key) + + expect(syncer.record).to eq(record) + expect(syncer.api_key).to eq(api_key) + end + + it 'inherits from ApplicationService' do + syncer = described_class.new(record: record, api_key: api_key) + + expect(syncer).to be_a(ApplicationService) + end + + it 'responds to call method' do + syncer = described_class.new(record: record, api_key: api_key) + + expect(syncer).to respond_to(:call) + end + end +end diff --git a/spec/services/external/vancouver_city/facility_syncer/integration_scenarios_spec.rb b/spec/services/external/vancouver_city/facility_syncer/integration_scenarios_spec.rb new file mode 100644 index 00000000..ce95af69 --- /dev/null +++ b/spec/services/external/vancouver_city/facility_syncer/integration_scenarios_spec.rb @@ -0,0 +1,232 @@ +# frozen_string_literal: true + +require 'rails_helper' + +RSpec.describe External::VancouverCity::FacilitySyncer, 'integration scenarios', type: :service do + let(:api_key) { 'drinking-fountains' } + let(:service) { create(:water_fountain_service) } + let(:secondary_service) { create(:service, key: 'public-washrooms') } + + before do + service + secondary_service + end + + describe 'complex data integration' do + context 'facility with comprehensive data' do + let(:comprehensive_record) do + { + 'mapid' => 'COMPREHENSIVE123', + 'name' => 'Downtown Community Fountain', + 'location' => 'Central Plaza', + 'geo_local_area' => 'Downtown Vancouver', + 'geo_point_2d' => { 'lat' => 49.2827, 'lon' => -123.1207 }, + 'phone' => '604-123-4567', + 'website' => 'https://vancouver.ca/fountains', + 'maintainer' => 'City of Vancouver', + 'in_operation' => 'Yes', + 'pet_friendly' => 'True' + } + end + + it 'creates facility with all available attributes' do + syncer = described_class.new(record: comprehensive_record, api_key: api_key) + result = syncer.call + + expect(result).to be_success + facility = result.data.facility + + expect(facility.external_id).to eq('COMPREHENSIVE123') + expect(facility.name).to eq('Downtown Community Fountain') + expect(facility.address).to eq('Central Plaza, Downtown Vancouver') + expect(facility.lat).to eq(49.2827) + expect(facility.long).to eq(-123.1207) + expect(facility.phone).to eq('604-123-4567') + expect(facility.website).to eq('https://vancouver.ca/fountains') + expect(facility.verified).to be true + expect(facility.external?).to be true + end + + it 'creates associated services, schedules, and welcomes' do + syncer = described_class.new(record: comprehensive_record, api_key: api_key) + result = syncer.call + + facility = result.data.facility + + # Services + expect(facility.facility_services.count).to eq(1) + expect(facility.services.first.key).to eq('water_fountain') + + # Schedules - should have open-all-day for all weekdays + expect(facility.schedules.count).to eq(7) + facility.schedules.each do |schedule| + expect(schedule.open_all_day).to be true + expect(schedule.closed_all_day).to be false + end + + # Welcomes - should welcome all customer types + expect(facility.facility_welcomes.count).to be > 0 + end + end + + context 'facility with minimal valid data' do + let(:minimal_record) do + { + 'mapid' => 'MINIMAL123', + 'name' => 'Basic Fountain', + 'geo_point_2d' => { 'lat' => 49.0, 'lon' => -123.0 } + } + end + + it 'creates facility with defaults for missing optional fields' do + syncer = described_class.new(record: minimal_record, api_key: api_key) + result = syncer.call + + expect(result).to be_success + facility = result.data.facility + + expect(facility.external_id).to eq('MINIMAL123') + expect(facility.name).to eq('Basic Fountain') + expect(facility.lat).to eq(49.0) + expect(facility.long).to eq(-123.0) + expect(facility.verified).to be true + expect(facility.external?).to be true + end + end + end + + describe 'edge case scenarios' do + context 'facility with special characters in name' do + let(:special_chars_record) do + { + 'mapid' => 'SPECIAL123', + 'name' => "O'Brien's Water Fountain & Rest Area", + 'location' => 'Québec Street', + 'geo_local_area' => 'Mount Pleasant', + 'geo_point_2d' => { 'lat' => 49.2627, 'lon' => -123.1007 } + } + end + + it 'handles special characters correctly' do + syncer = described_class.new(record: special_chars_record, api_key: api_key) + result = syncer.call + + expect(result).to be_success + facility = result.data.facility + + expect(facility.name).to eq("O'Brien's Water Fountain & Rest Area") + expect(facility.address).to eq('Québec Street, Mount Pleasant') + end + end + + context 'facility at edge coordinates' do + let(:edge_coords_record) do + { + 'mapid' => 'EDGE123', + 'name' => 'Edge Case Fountain', + 'location' => 'Boundary Road', + 'geo_local_area' => 'Boundary', + 'geo_point_2d' => { 'lat' => 90.0, 'lon' => -180.0 } # Edge coordinates + } + end + + it 'handles edge coordinate values' do + syncer = described_class.new(record: edge_coords_record, api_key: api_key) + result = syncer.call + + expect(result).to be_success + facility = result.data.facility + + expect(facility.lat).to eq(90.0) + expect(facility.long).to eq(-180.0) + end + end + end + + describe 'concurrent operation simulation' do + context 'when the same external_id is processed simultaneously' do + let(:concurrent_record1) do + { + 'mapid' => 'CONCURRENT123', + 'name' => 'First Version Fountain', + 'location' => 'First Location', + 'geo_local_area' => 'Downtown', + 'geo_point_2d' => { 'lat' => 49.2827, 'lon' => -123.1207 } + } + end + + let(:concurrent_record2) do + { + 'mapid' => 'CONCURRENT123', + 'name' => 'Second Version Fountain', + 'location' => 'Second Location', + 'geo_local_area' => 'Westside', + 'geo_point_2d' => { 'lat' => 49.2727, 'lon' => -123.1107 } + } + end + + it 'handles duplicate external_id creation gracefully' do + # First sync + syncer1 = described_class.new(record: concurrent_record1, api_key: api_key) + result1 = syncer1.call + + expect(result1).to be_success + expect(result1.data.operation).to eq(:create) + + # Second sync with same external_id but different data + syncer2 = described_class.new(record: concurrent_record2, api_key: api_key) + result2 = syncer2.call + + expect(result2).to be_success + expect(result2.data.operation).to eq(:external_update) + + # Verify final state + facility = Facility.find_by(external_id: 'CONCURRENT123') + expect(facility.name).to eq('Second Version Fountain') + expect(facility.address).to eq('Second Location, Westside') + end + end + end + + describe 'data consistency verification' do + let(:consistency_record) do + { + 'mapid' => 'CONSISTENCY123', + 'name' => 'Consistency Test Fountain', + 'location' => 'Test Park', + 'geo_local_area' => 'Test Area', + 'geo_point_2d' => { 'lat' => 49.2827, 'lon' => -123.1207 } + } + end + + it 'ensures data integrity across all related models' do + syncer = described_class.new(record: consistency_record, api_key: api_key) + result = syncer.call + + expect(result).to be_success + facility = result.data.facility + + # Verify facility + expect(facility).to be_persisted + expect(facility.external_id).to eq('CONSISTENCY123') + + # Verify services + expect(facility.facility_services.count).to eq(1) + expect(facility.facility_services.first.service.key).to eq('water_fountain') + + # Verify schedules + expect(facility.schedules.count).to eq(7) + facility.schedules.each do |schedule| + expect(schedule.facility_id).to eq(facility.id) + expect(schedule).to be_persisted + end + + # Verify welcomes + expect(facility.facility_welcomes.count).to be > 0 + facility.facility_welcomes.each do |welcome| + expect(welcome.facility_id).to eq(facility.id) + expect(welcome).to be_persisted + end + end + end +end diff --git a/spec/services/external/vancouver_city/facility_syncer/internal_update_operation_spec.rb b/spec/services/external/vancouver_city/facility_syncer/internal_update_operation_spec.rb new file mode 100644 index 00000000..5c95b8a6 --- /dev/null +++ b/spec/services/external/vancouver_city/facility_syncer/internal_update_operation_spec.rb @@ -0,0 +1,437 @@ +# frozen_string_literal: true + +require 'rails_helper' + +RSpec.describe External::VancouverCity::FacilitySyncer, 'internal update operation', type: :service do + let(:api_key) { 'drinking-fountains' } + let(:service) { create(:water_fountain_service) } + let(:other_service) { create(:service, key: 'public-washrooms') } + + before do + service + other_service + end + + describe 'internal_update operation (:internal_update)' do + context 'when update succeeds' do + let!(:existing_internal_facility) do + create(:facility, + external_id: nil, + name: 'Internal Fountain', + address: 'Original Address', + lat: 49.1111, + long: -123.1111, + verified: false) + end + + let(:update_record) do + { + 'mapid' => 'NEW_EXT_ID123', + 'name' => 'Internal Fountain', # Matches by name + 'location' => 'Different Location', + 'geo_local_area' => 'Different Area', + 'geo_point_2d' => { 'lat' => 49.9999, 'lon' => -123.9999 } + } + end + + it 'adds missing services only' do + expect(existing_internal_facility.services).not_to include(service) + + syncer = described_class.new(record: update_record, api_key: api_key) + result = syncer.call + + facility = result.data.facility + expect(facility.services).to include(service) + end + + it 'does not update facility attributes' do + original_name = existing_internal_facility.name + original_address = existing_internal_facility.address + original_lat = existing_internal_facility.lat + original_long = existing_internal_facility.long + original_verified = existing_internal_facility.verified + + syncer = described_class.new(record: update_record, api_key: api_key) + result = syncer.call + + facility = result.data.facility + expect(facility.name).to eq(original_name) + expect(facility.address).to eq(original_address) + expect(facility.lat).to eq(original_lat) + expect(facility.long).to eq(original_long) + expect(facility.verified).to eq(original_verified) + end + + it 'returns existing facility in result' do + syncer = described_class.new(record: update_record, api_key: api_key) + result = syncer.call + + expect(result.data.facility.id).to eq(existing_internal_facility.id) + expect(result.data.operation).to eq(:internal_update) + end + + it 'logs warning message with facility name' do + expect(Rails.logger).to receive(:warn).with("Facility with name 'Internal Fountain' already exists internally, adding services") + + syncer = described_class.new(record: update_record, api_key: api_key) + syncer.call + end + + it 'returns success result' do + syncer = described_class.new(record: update_record, api_key: api_key) + result = syncer.call + + expect(result).to be_success + expect(result.errors).to be_empty + end + + it 'does not create new facility' do + expect do + syncer = described_class.new(record: update_record, api_key: api_key) + syncer.call + end.not_to change(Facility, :count) + end + end + + context 'when facility already has the service' do + let!(:existing_internal_facility) do + facility = create(:facility, + external_id: nil, + name: 'Fountain with Service', + verified: false) + facility.facility_services.create!(service: service) + facility + end + + let(:update_record) do + { + 'mapid' => 'SOME_ID123', + 'name' => 'Fountain with Service', + 'geo_point_2d' => { 'lat' => 49.2827, 'lon' => -123.1207 } + } + end + + it 'does not duplicate existing services' do + initial_service_count = existing_internal_facility.facility_services.count + + syncer = described_class.new(record: update_record, api_key: api_key) + result = syncer.call + + facility = result.data.facility + expect(facility.facility_services.count).to eq(initial_service_count) + end + + it 'still succeeds even with no new services to add' do + syncer = described_class.new(record: update_record, api_key: api_key) + result = syncer.call + + expect(result).to be_success + expect(result.data.operation).to eq(:internal_update) + end + end + + context 'when service creation raises ActiveRecord::RecordInvalid' do + let!(:existing_internal_facility) do + create(:facility, + external_id: nil, + name: 'Service Error Fountain', + verified: false) + end + + let(:update_record) do + { + 'mapid' => 'ERROR_ID123', + 'name' => 'Service Error Fountain', + 'geo_point_2d' => { 'lat' => 49.2827, 'lon' => -123.1207 } + } + end + + before do + # Simulate a constraint violation when creating facility service + allow_any_instance_of(ActiveRecord::Associations::CollectionProxy) + .to receive(:create!).and_raise( + ActiveRecord::RecordInvalid.new(FacilityService.new) + ) + end + + it 'catches exception and adds error message' do + syncer = described_class.new(record: update_record, api_key: api_key) + result = syncer.call + + expect(result).to be_failed + expect(result.errors).to include(a_string_matching(/Failed to save facility:/)) + end + end + + context 'when update raises other StandardError' do + let!(:existing_internal_facility) do + create(:facility, + external_id: nil, + name: 'Generic Error Fountain', + verified: false) + end + + let(:update_record) do + { + 'mapid' => 'GENERIC_ERROR123', + 'name' => 'Generic Error Fountain', + 'geo_point_2d' => { 'lat' => 49.2827, 'lon' => -123.1207 } + } + end + + before do + # Simulate a database connection error during service creation + allow_any_instance_of(ActiveRecord::Associations::CollectionProxy) + .to receive(:create!).and_raise( + StandardError.new('Database connection failed') + ) + end + + it 'catches and handles generic errors' do + syncer = described_class.new(record: update_record, api_key: api_key) + result = syncer.call + + expect(result).to be_failed + expect(result.errors).to include(a_string_matching(/Unexpected error during facility sync:/)) + expect(result.errors.first).to include('Database connection failed') + end + end + + context 'when record would create new facility but matches internal by name' do + let!(:existing_internal_facility) do + create(:facility, + external_id: nil, + name: 'Exact Name Match', + verified: false) + end + + let(:new_record_matching_name) do + { + 'mapid' => 'COMPLETELY_NEW_ID', + 'name' => 'Exact Name Match', # Same name but would have different external_id + 'location' => 'New Location', + 'geo_point_2d' => { 'lat' => 49.2827, 'lon' => -123.1207 } + } + end + + it 'treats as internal update rather than create' do + syncer = described_class.new(record: new_record_matching_name, api_key: api_key) + result = syncer.call + + expect(result.data.operation).to eq(:internal_update) + expect(result.data.facility.id).to eq(existing_internal_facility.id) + end + + it 'does not change facility external_id' do + syncer = described_class.new(record: new_record_matching_name, api_key: api_key) + result = syncer.call + + facility = result.data.facility + expect(facility.external_id).to be_nil # Should remain nil + end + end + + context 'database record updates on success' do + let!(:internal_facility_with_services) do + facility = create(:facility, + external_id: nil, + name: 'Internal Service Test', + address: 'Original Internal Address', + verified: false) + + # Add existing service from different API + facility.facility_services.create!(service: other_service) + facility + end + + let(:internal_service_update_record) do + { + 'mapid' => 'NEW_EXTERNAL_ID456', + 'name' => 'Internal Service Test', # Matches by name + 'location' => 'Different Location', # Should NOT update + 'geo_point_2d' => { 'lat' => 49.9999, 'lon' => -123.9999 } # Should NOT update + } + end + + it 'adds new service without modifying facility attributes' do + original_name = internal_facility_with_services.name + original_address = internal_facility_with_services.address + original_lat = internal_facility_with_services.lat + original_long = internal_facility_with_services.long + original_verified = internal_facility_with_services.verified + original_external_id = internal_facility_with_services.external_id + + syncer = described_class.new(record: internal_service_update_record, api_key: api_key) + result = syncer.call + + facility = result.data.facility + + # Verify attributes remain unchanged + expect(facility.name).to eq(original_name) + expect(facility.address).to eq(original_address) + expect(facility.lat).to eq(original_lat) + expect(facility.long).to eq(original_long) + expect(facility.verified).to eq(original_verified) + expect(facility.external_id).to eq(original_external_id) + end + + it 'adds new service while preserving existing ones' do + initial_service_count = internal_facility_with_services.facility_services.count + + syncer = described_class.new(record: internal_service_update_record, api_key: api_key) + result = syncer.call + + facility = result.data.facility + expect(facility.facility_services.count).to eq(initial_service_count + 1) + expect(facility.services).to include(service) # New service added + expect(facility.services).to include(other_service) # Existing service preserved + end + + it 'maintains referential integrity when adding services' do + syncer = described_class.new(record: internal_service_update_record, api_key: api_key) + result = syncer.call + + facility = result.data.facility + + # Verify all services belong to the correct facility + expect(facility.facility_services.all? { |fs| fs.facility_id == facility.id }).to be true + + # Verify the new service was added correctly + new_service_record = facility.facility_services.find_by(service: service) + expect(new_service_record).to be_present + expect(new_service_record.facility_id).to eq(facility.id) + end + + it 'does not create duplicate services for same API key' do + # First update + syncer1 = described_class.new(record: internal_service_update_record, api_key: api_key) + syncer1.call + + initial_count = internal_facility_with_services.reload.facility_services.count + + # Second update with same API key + syncer2 = described_class.new(record: internal_service_update_record, api_key: api_key) + syncer2.call + + internal_facility_with_services.reload + expect(internal_facility_with_services.facility_services.count).to eq(initial_count) + end + end + + context 'transaction rollback on failure' do + let!(:rollback_internal_facility) do + facility = create(:facility, + external_id: nil, + name: 'Rollback Internal Test', + verified: false) + + # Add existing service + facility.facility_services.create!(service: other_service) + facility + end + + let(:rollback_internal_record) do + { + 'mapid' => 'ROLLBACK_INTERNAL123', + 'name' => 'Rollback Internal Test', + 'geo_point_2d' => { 'lat' => 49.2827, 'lon' => -123.1207 } + } + end + + before do + # Force service creation to fail + allow_any_instance_of(ActiveRecord::Associations::CollectionProxy) + .to receive(:create!).and_raise(StandardError.new('Service creation failed')) + end + + it 'does not create any service records when transaction fails' do + original_service_count = rollback_internal_facility.facility_services.count + + expect do + syncer = described_class.new(record: rollback_internal_record, api_key: api_key) + syncer.call + end.not_to change(FacilityService, :count) + + rollback_internal_facility.reload + expect(rollback_internal_facility.facility_services.count).to eq(original_service_count) + end + + it 'maintains existing facility state when service addition fails' do + original_attributes = rollback_internal_facility.attributes + original_service_ids = rollback_internal_facility.facility_services.pluck(:service_id) + + syncer = described_class.new(record: rollback_internal_record, api_key: api_key) + result = syncer.call + + rollback_internal_facility.reload + + # Verify facility attributes unchanged + # Compare all attributes, allowing updated_at and created_at to be within a small delta + expect(rollback_internal_facility.attributes.except('updated_at', 'created_at')).to eq(original_attributes.except('updated_at', 'created_at')) + expect(rollback_internal_facility.updated_at).to be_within(2.seconds).of(original_attributes['updated_at']) + expect(rollback_internal_facility.created_at).to be_within(2.seconds).of(original_attributes['created_at']) + + # Verify existing services unchanged + expect(rollback_internal_facility.facility_services.pluck(:service_id)).to match_array(original_service_ids) + + expect(result).to be_failed + end + + it 'does not affect other facilities when one fails' do + other_facility = create(:facility, external_id: nil, name: 'Other Facility') + + expect do + syncer = described_class.new(record: rollback_internal_record, api_key: api_key) + syncer.call + end.not_to change { other_facility.reload.facility_services.count } + end + end + + context 'validation error handling' do + let!(:validation_internal_facility) do + create(:facility, + external_id: nil, + name: 'Validation Test Facility', + verified: false) + end + + let(:validation_record) do + { + 'mapid' => 'VALIDATION123', + 'name' => 'Validation Test Facility', + 'geo_point_2d' => { 'lat' => 49.2827, 'lon' => -123.1207 } + } + end + + before do + # Simulate validation error during service creation + allow_any_instance_of(ActiveRecord::Associations::CollectionProxy) + .to receive(:create!).and_raise( + ActiveRecord::RecordInvalid.new(FacilityService.new) + ) + end + + it 'does not modify facility when service validation fails' do + original_service_count = validation_internal_facility.facility_services.count + original_updated_at = validation_internal_facility.updated_at + + syncer = described_class.new(record: validation_record, api_key: api_key) + syncer.call + + validation_internal_facility.reload + expect(validation_internal_facility.facility_services.count).to eq(original_service_count) + expect(validation_internal_facility.updated_at).to be_within(2.seconds).of(original_updated_at) + end + + it 'returns proper error information for validation failures' do + syncer = described_class.new(record: validation_record, api_key: api_key) + result = syncer.call + + expect(result).to be_failed + expect(result.errors).to include(a_string_matching(/Failed to save facility:/)) + expect(result.data.operation).to eq(:internal_update) + expect(result.data.facility).to be_nil # Should be nil when operation fails + end + end + end +end diff --git a/spec/services/external/vancouver_city/facility_syncer/operation_detection_spec.rb b/spec/services/external/vancouver_city/facility_syncer/operation_detection_spec.rb new file mode 100644 index 00000000..a94c4035 --- /dev/null +++ b/spec/services/external/vancouver_city/facility_syncer/operation_detection_spec.rb @@ -0,0 +1,160 @@ +# frozen_string_literal: true + +require 'rails_helper' + +RSpec.describe External::VancouverCity::FacilitySyncer, 'operation detection', type: :service do + let(:api_key) { 'drinking-fountains' } + let(:service) { create(:water_fountain_service) } + + before { service } # Ensure service exists + + describe 'operation detection' do + context 'when no existing facility found' do + let(:new_facility_record) do + { + 'mapid' => 'NEW123', + 'name' => 'Brand New Fountain', + 'location' => 'New Park', + 'geo_point_2d' => { 'lat' => 49.2827, 'lon' => -123.1207 } + } + end + + it 'sets operation to :create' do + syncer = described_class.new(record: new_facility_record, api_key: api_key) + result = syncer.call + + expect(result.data.operation).to eq(:create) + end + + it 'creates a new facility' do + expect do + syncer = described_class.new(record: new_facility_record, api_key: api_key) + syncer.call + end.to change(Facility, :count).by(1) + end + end + + context 'when existing facility has external_id' do + let!(:existing_external_facility) do + create(:facility, + :with_verified, + external_id: 'EXT123', + name: 'External Fountain') + end + + let(:update_record) do + { + 'mapid' => 'EXT123', + 'name' => 'Updated External Fountain', + 'location' => 'Updated Park', + 'geo_point_2d' => { 'lat' => 49.2827, 'lon' => -123.1207 } + } + end + + it 'sets operation to :external_update' do + syncer = described_class.new(record: update_record, api_key: api_key) + result = syncer.call + + expect(result.data.operation).to eq(:external_update) + end + + it 'returns the existing facility' do + syncer = described_class.new(record: update_record, api_key: api_key) + result = syncer.call + + expect(result.data.facility.id).to eq(existing_external_facility.id) + end + + it 'does not create a new facility' do + expect do + syncer = described_class.new(record: update_record, api_key: api_key) + syncer.call + end.not_to change(Facility, :count) + end + end + + context 'when existing facility found by name only' do + let!(:existing_internal_facility) do + create(:facility, + external_id: nil, + name: 'Internal Fountain', + verified: false) + end + + let(:name_match_record) do + { + 'mapid' => 'NEW456', + 'name' => 'Internal Fountain', # Matches existing facility name + 'location' => 'Same Park', + 'geo_point_2d' => { 'lat' => 49.2827, 'lon' => -123.1207 } + } + end + + it 'sets operation to :internal_update' do + syncer = described_class.new(record: name_match_record, api_key: api_key) + result = syncer.call + + expect(result.data.operation).to eq(:internal_update) + end + + it 'returns the existing facility' do + syncer = described_class.new(record: name_match_record, api_key: api_key) + result = syncer.call + + expect(result.data.facility.id).to eq(existing_internal_facility.id) + end + + it 'does not create a new facility' do + expect do + syncer = described_class.new(record: name_match_record, api_key: api_key) + syncer.call + end.not_to change(Facility, :count) + end + end + + context 'with complex matching scenarios' do + let!(:facility_with_external_id) do + create(:facility, + :with_verified, + external_id: 'EXT789', + name: 'Shared Name Fountain') + end + + let!(:facility_with_same_name) do + create(:facility, + external_id: nil, + name: 'Shared Name Fountain', + verified: false) + end + + it 'prioritizes external_id match over name match' do + record = { + 'mapid' => 'EXT789', + 'name' => 'Shared Name Fountain', + 'geo_point_2d' => { 'lat' => 49.2827, 'lon' => -123.1207 } + } + + syncer = described_class.new(record: record, api_key: api_key) + result = syncer.call + + expect(result.data.operation).to eq(:external_update) + expect(result.data.facility.id).to eq(facility_with_external_id.id) + end + + it 'handles facilities with same name but different external_id' do + record = { + 'mapid' => 'DIFFERENT123', + 'name' => 'Shared Name Fountain', + 'geo_point_2d' => { 'lat' => 49.2827, 'lon' => -123.1207 } + } + + syncer = described_class.new(record: record, api_key: api_key) + result = syncer.call + + # Should match by name since external_id is different + expect(result.data.operation).to eq(:internal_update) + expect(result.data.facility.id).to eq(facility_with_same_name.id) + end + end + end +end diff --git a/spec/services/external/vancouver_city/facility_syncer/result_structure_spec.rb b/spec/services/external/vancouver_city/facility_syncer/result_structure_spec.rb new file mode 100644 index 00000000..b293191d --- /dev/null +++ b/spec/services/external/vancouver_city/facility_syncer/result_structure_spec.rb @@ -0,0 +1,255 @@ +# frozen_string_literal: true + +require 'rails_helper' + +RSpec.describe External::VancouverCity::FacilitySyncer, 'result structure', type: :service do + let(:api_key) { 'drinking-fountains' } + let(:service) { create(:water_fountain_service) } + + before { service } + + describe 'ResultData structure' do + let(:valid_record) do + { + 'mapid' => 'RESULT123', + 'name' => 'Test Fountain', + 'location' => 'Test Park', + 'geo_local_area' => 'Downtown', + 'geo_point_2d' => { 'lat' => 49.2827, 'lon' => -123.1207 } + } + end + + it 'returns ResultData with operation and facility' do + syncer = described_class.new(record: valid_record, api_key: api_key) + result = syncer.call + + expect(result.data).to be_a(External::VancouverCity::FacilitySyncer::ResultData) + expect(result.data).to respond_to(:operation) + expect(result.data).to respond_to(:facility) + end + + it 'delegates present? and blank? to facility' do + syncer = described_class.new(record: valid_record, api_key: api_key) + result = syncer.call + + # When facility is present + expect(result.data.present?).to be true + expect(result.data.blank?).to be false + end + + context 'when FacilityBuilder fails' do + let(:invalid_record) do + { + 'mapid' => 'INVALID123', + 'name' => '', # Empty name causes FacilityBuilder to fail + 'location' => 'Test Location', + 'geo_local_area' => 'Downtown', + 'geo_point_2d' => { 'lat' => 49.2827, 'lon' => -123.1207 } + } + end + + it 'ResultData reflects early failure state' do + syncer = described_class.new(record: invalid_record, api_key: api_key) + result = syncer.call + + expect(result.data.operation).to be_nil # No operation determined when FacilityBuilder fails + expect(result.data.facility).to be_nil + expect(result.data.blank?).to be true + expect(result.data.present?).to be false + end + end + + context 'when FacilityBuilder fails' do + let(:malformed_record) do + { + 'mapid' => nil, + 'location' => 'Test Location' + } + end + + it 'ResultData shows nil operation and facility' do + syncer = described_class.new(record: malformed_record, api_key: api_key) + result = syncer.call + + expect(result.data.operation).to be_nil + expect(result.data.facility).to be_nil + expect(result.data.blank?).to be true + expect(result.data.present?).to be false + end + end + end + + describe 'Result object compliance with ApplicationService::Result' do + let(:valid_record) do + { + 'mapid' => 'COMPLIANCE123', + 'name' => 'Test Fountain', + 'location' => 'Test Park', + 'geo_local_area' => 'Downtown', + 'geo_point_2d' => { 'lat' => 49.2827, 'lon' => -123.1207 } + } + end + + it 'returns ApplicationService::Result object' do + syncer = described_class.new(record: valid_record, api_key: api_key) + result = syncer.call + + expect(result).to be_a(ApplicationService::Result) + expect(result).to respond_to(:data) + expect(result).to respond_to(:errors) + expect(result).to respond_to(:success?) + expect(result).to respond_to(:failed?) + end + + context 'when operation succeeds' do + it 'has success? true and failed? false' do + syncer = described_class.new(record: valid_record, api_key: api_key) + result = syncer.call + + expect(result.success?).to be true + expect(result.failed?).to be false + expect(result.errors).to be_blank + end + end + + context 'when operation fails' do + let(:invalid_record) do + { + 'mapid' => 'FAIL123', + 'name' => '', + 'location' => 'Test Location', + 'geo_local_area' => 'Downtown', + 'geo_point_2d' => { 'lat' => 49.2827, 'lon' => -123.1207 } + } + end + + it 'has success? false and failed? true' do + syncer = described_class.new(record: invalid_record, api_key: api_key) + result = syncer.call + + expect(result.success?).to be false + expect(result.failed?).to be true + expect(result.errors).to be_present + end + end + end + + describe 'operation type consistency' do + context 'for create operations' do + let(:create_record) do + { + 'mapid' => 'CREATE_OP123', + 'name' => 'New Fountain', + 'location' => 'New Park', + 'geo_local_area' => 'Downtown', + 'geo_point_2d' => { 'lat' => 49.2827, 'lon' => -123.1207 } + } + end + + it 'consistently reports :create operation' do + syncer = described_class.new(record: create_record, api_key: api_key) + result = syncer.call + + expect(result.data.operation).to eq(:create) + end + end + + context 'for external_update operations' do + let!(:existing_external_facility) do + create(:facility, + external_id: 'EXT_OP123', + name: 'Old Name') + end + + let(:update_record) do + { + 'mapid' => 'EXT_OP123', + 'name' => 'Updated Name', + 'location' => 'Updated Location', + 'geo_local_area' => 'Downtown', + 'geo_point_2d' => { 'lat' => 49.2827, 'lon' => -123.1207 } + } + end + + it 'consistently reports :external_update operation' do + syncer = described_class.new(record: update_record, api_key: api_key) + result = syncer.call + + expect(result.data.operation).to eq(:external_update) + end + end + + context 'for internal_update operations' do + let!(:existing_internal_facility) do + create(:facility, + external_id: nil, + name: 'Internal Facility') + end + + let(:update_record) do + { + 'mapid' => 'INT_OP123', + 'name' => 'Internal Facility', # Same name triggers internal_update + 'location' => 'Updated Location', + 'geo_local_area' => 'Downtown', + 'geo_point_2d' => { 'lat' => 49.2827, 'lon' => -123.1207 } + } + end + + it 'consistently reports :internal_update operation' do + syncer = described_class.new(record: update_record, api_key: api_key) + result = syncer.call + + expect(result.data.operation).to eq(:internal_update) + end + end + end + + describe 'facility reference consistency' do + let(:valid_record) do + { + 'mapid' => 'REF123', + 'name' => 'Reference Test Fountain', + 'location' => 'Test Park', + 'geo_local_area' => 'Downtown', + 'geo_point_2d' => { 'lat' => 49.2827, 'lon' => -123.1207 } + } + end + + it 'result facility matches database record' do + syncer = described_class.new(record: valid_record, api_key: api_key) + result = syncer.call + + db_facility = Facility.find(result.data.facility.id) + expect(result.data.facility).to eq(db_facility) + expect(result.data.facility.external_id).to eq('REF123') + expect(result.data.facility.name).to eq('Reference Test Fountain') + end + + context 'with update operations' do + let!(:existing_facility) do + create(:facility, + external_id: 'UPDATE_REF123', + name: 'Original Name') + end + + let(:update_record) do + { + 'mapid' => 'UPDATE_REF123', + 'name' => 'Updated Reference Name', + 'location' => 'Updated Location', + 'geo_local_area' => 'Downtown', + 'geo_point_2d' => { 'lat' => 49.2827, 'lon' => -123.1207 } + } + end + + it 'result facility is the same instance as existing facility' do + syncer = described_class.new(record: update_record, api_key: api_key) + result = syncer.call + + expect(result.data.facility.id).to eq(existing_facility.id) + expect(result.data.facility).to be_a(Facility) + end + end + end +end diff --git a/spec/services/external/vancouver_city/facility_syncer/service_synchronization_spec.rb b/spec/services/external/vancouver_city/facility_syncer/service_synchronization_spec.rb new file mode 100644 index 00000000..eb71cebd --- /dev/null +++ b/spec/services/external/vancouver_city/facility_syncer/service_synchronization_spec.rb @@ -0,0 +1,127 @@ +# frozen_string_literal: true + +require 'rails_helper' + +RSpec.describe External::VancouverCity::FacilitySyncer, 'service synchronization', type: :service do + let(:api_key) { 'drinking-fountains' } + let(:service) { create(:water_fountain_service) } + let(:other_service) { create(:service, key: 'public-washrooms') } + + before do + service + other_service + end + + describe 'service synchronization logic' do + context 'when built facility has new services' do + let!(:existing_facility) do + facility = create(:facility, external_id: 'SYNC_TEST123') + facility.facility_services.create!(service: other_service) + facility + end + + let(:record_with_new_service) do + { + 'mapid' => 'SYNC_TEST123', + 'name' => 'Service Sync Test', + 'geo_point_2d' => { 'lat' => 49.2827, 'lon' => -123.1207 } + } + end + + it 'adds only new services that do not exist on facility' do + # Facility starts with other_service, should get service added + expect(existing_facility.services).to include(other_service) + expect(existing_facility.services).not_to include(service) + + syncer = described_class.new(record: record_with_new_service, api_key: api_key) + result = syncer.call + + facility = result.data.facility + expect(facility.services).to include(other_service) # Keeps existing + expect(facility.services).to include(service) # Adds new one + end + + it 'increases facility services count' do + initial_count = existing_facility.facility_services.count + + syncer = described_class.new(record: record_with_new_service, api_key: api_key) + result = syncer.call + + facility = result.data.facility + expect(facility.facility_services.count).to eq(initial_count + 1) + end + end + + context 'when built facility has existing services' do + let!(:existing_facility) do + facility = create(:facility, external_id: 'EXISTING_SERVICES123') + facility.facility_services.create!(service: service) + facility.facility_services.create!(service: other_service) + facility + end + + let(:record_with_existing_services) do + { + 'mapid' => 'EXISTING_SERVICES123', + 'name' => 'Existing Services Test', + 'geo_point_2d' => { 'lat' => 49.2827, 'lon' => -123.1207 } + } + end + + it 'does not duplicate existing services' do + initial_count = existing_facility.facility_services.count + + syncer = described_class.new(record: record_with_existing_services, api_key: api_key) + result = syncer.call + + facility = result.data.facility + expect(facility.facility_services.count).to eq(initial_count) + end + + it 'maintains all existing services' do + syncer = described_class.new(record: record_with_existing_services, api_key: api_key) + result = syncer.call + + facility = result.data.facility + expect(facility.services).to include(service) + expect(facility.services).to include(other_service) + end + end + + + + context 'when built facility has duplicate services in builder' do + # This tests the .uniq call in add_missing_services + let!(:existing_facility) do + create(:facility, external_id: 'DUPLICATE_TEST123') + end + + let(:record) do + { + 'mapid' => 'DUPLICATE_TEST123', + 'name' => 'Duplicate Test', + 'geo_point_2d' => { 'lat' => 49.2827, 'lon' => -123.1207 } + } + end + + before do + # Mock the built facility to have duplicate services + # This would happen if FacilityBuilder creates duplicate associations + allow_any_instance_of(External::VancouverCity::FacilitySyncer) + .to receive(:add_missing_services).and_call_original + end + + it 'handles duplicate services gracefully' do + syncer = described_class.new(record: record, api_key: api_key) + result = syncer.call + + # Should succeed without errors + expect(result).to be_success + facility = result.data.facility + expect(facility.services).to include(service) + end + end + + + end +end diff --git a/spec/services/external/vancouver_city/facility_welcome_builder_spec.rb b/spec/services/external/vancouver_city/facility_welcome_builder_spec.rb new file mode 100644 index 00000000..5df6ca18 --- /dev/null +++ b/spec/services/external/vancouver_city/facility_welcome_builder_spec.rb @@ -0,0 +1,122 @@ +# frozen_string_literal: true + +require 'rails_helper' + +RSpec.describe External::VancouverCity::FacilityWelcomeBuilder, type: :service do + let(:facility) { build(:facility) } + let(:fields) { { 'name' => 'Test Facility' } } + + describe '#initialize' do + it 'initializes with valid parameters' do + builder = described_class.new(facility: facility, fields: fields) + + expect(builder.facility).to eq(facility) + expect(builder.fields).to eq(fields) + end + end + + describe '#validate' do + context 'with valid parameters' do + let(:builder) { described_class.new(facility: facility, fields: fields) } + + it 'returns empty errors array' do + expect(builder.validate).to be_empty + end + + it 'is valid' do + expect(builder).to be_valid + end + end + + context 'with nil facility' do + let(:builder) { described_class.new(facility: nil, fields: fields) } + + it 'returns validation errors' do + errors = builder.validate + expect(errors).to include('Facility is required') + end + + it 'is invalid' do + expect(builder).to be_invalid + end + end + + context 'with non-facility object' do + let(:builder) { described_class.new(facility: 'invalid', fields: fields) } + + it 'returns validation errors' do + errors = builder.validate + expect(errors).to include('Facility must be a Facility object') + end + end + + context 'with nil fields' do + let(:builder) { described_class.new(facility: facility, fields: nil) } + + it 'returns validation errors' do + errors = builder.validate + expect(errors).to include('Fields are required') + end + end + + context 'with non-hash fields' do + let(:builder) { described_class.new(facility: facility, fields: 'invalid') } + + it 'returns validation errors' do + errors = builder.validate + expect(errors).to include('Fields must be a Hash') + end + end + end + + describe '#call' do + context 'with valid parameters' do + let(:builder) { described_class.new(facility: facility, fields: fields) } + + it 'returns successful result' do + result = builder.call + + expect(result).to be_success + expect(result.errors).to be_empty + expect(result.data[:welcomes_count]).to be > 0 + end + + it 'creates facility welcomes for all customer types' do + builder.call + + expect(facility.facility_welcomes).not_to be_empty + # Test that welcomes are created (exact count depends on FacilityWelcome.all_customers) + end + + it 'creates valid welcome objects' do + builder.call + + facility.facility_welcomes.each do |welcome| + expect(welcome).to be_valid, "Expected welcome to be valid: #{welcome.errors.full_messages}" + end + end + end + + context 'with invalid parameters' do + let(:builder) { described_class.new(facility: nil, fields: nil) } + + it 'returns error result without building welcomes' do + result = builder.call + + expect(result).to be_failed + expect(result.data).to be_nil + expect(result.errors).to include('Facility is required') + expect(result.errors).to include('Fields are required') + end + end + end + + describe '.call class method' do + it 'works as a class method' do + result = described_class.call(facility: facility, fields: fields) + + expect(result).to be_success + expect(result.data[:welcomes_count]).to be > 0 + end + end +end