diff --git a/all/modules/datasources/MultiTileDataSource.i b/all/modules/datasources/MultiTileDataSource.i new file mode 100644 index 000000000..72965927b --- /dev/null +++ b/all/modules/datasources/MultiTileDataSource.i @@ -0,0 +1,35 @@ +#ifndef _LOCALPACKAGEMANAGERTILEDATASOURCE_I +#define _LOCALPACKAGEMANAGERTILEDATASOURCE_I + +%module(directors="1") MultiTileDataSource + +!proxy_imports(carto::MultiTileDataSource, core.MapTile, core.MapBounds, core.StringMap, datasources.TileDataSource, datasources.components.TileData) +%{ +#include "datasources/MultiTileDataSource.h" +#include "components/Exceptions.h" +#include +%} + +%include +%include +%include + +%import "core/MapTile.i" +%import "core/StringMap.i" +%import "datasources/TileDataSource.i" +#ifdef _CARTO_OFFLINE_SUPPORT +%import "datasources/MBTilesTileDataSource.i" +#endif +%import "datasources/components/TileData.i" + +!polymorphic_shared_ptr(carto::MultiTileDataSource, datasources.MultiTileDataSource) + +%std_exceptions(carto::MultiTileDataSource::MultiTileDataSource) +%std_exceptions(carto::LocalVectorDataSource::add) +%std_exceptions(carto::LocalVectorDataSource::remove) + +%feature("director") carto::MultiTileDataSource; + +%include "datasources/MultiTileDataSource.h" + +#endif diff --git a/all/native/datasources/MBTilesTileDataSource.cpp b/all/native/datasources/MBTilesTileDataSource.cpp index 17c057dd5..f87decc79 100644 --- a/all/native/datasources/MBTilesTileDataSource.cpp +++ b/all/native/datasources/MBTilesTileDataSource.cpp @@ -286,6 +286,22 @@ namespace carto { return true; } + std::string MBTilesTileDataSource::getMetaData(const std::string &key) const { + // As a first step, try to use metadata + std::string result; + try { + sqlite3pp::query query(*_database, "SELECT value FROM metadata WHERE name=:name"); + query.bind(":name", key.c_str()); + for (auto it = query.begin(); it != query.end(); it++) { + result = (*it).get(0); + } + query.finish(); + } + catch (const std::exception& ex) { + Log::Errorf("MBTilesTileDataSource::getMetaData: Exception while reading %s metadata: %s", key, ex.what()); + } + return result; + } } #endif diff --git a/all/native/datasources/MBTilesTileDataSource.h b/all/native/datasources/MBTilesTileDataSource.h index c5c3beadf..ed2aef605 100644 --- a/all/native/datasources/MBTilesTileDataSource.h +++ b/all/native/datasources/MBTilesTileDataSource.h @@ -81,6 +81,13 @@ namespace carto { * @return Map containing meta data information (parameter names mapped to parameter values). */ std::map getMetaData() const; + + /** + * Query a metadata value + * Possible parameters can be found in MBTiles specification. + * @return a string representation of the metadata value + */ + std::string MBTilesTileDataSource::getMetaData(const std::string &key) const; virtual int getMinZoom() const; diff --git a/all/native/datasources/MultiTileDataSource.cpp b/all/native/datasources/MultiTileDataSource.cpp new file mode 100644 index 000000000..bfbd455a6 --- /dev/null +++ b/all/native/datasources/MultiTileDataSource.cpp @@ -0,0 +1,251 @@ +#include "MultiTileDataSource.h" +#include "core/MapTile.h" +#include "components/Exceptions.h" +#include "utils/GeneralUtils.h" +#include "utils/Log.h" +#include "utils/Const.h" +#include "packagemanager/PackageTileMask.h" + +#ifdef _CARTO_OFFLINE_SUPPORT +#include "datasources/MBTilesTileDataSource.h" +#endif + +#include + +#include + +namespace carto { + + MultiTileDataSource::MultiTileDataSource(int maxOpenedPackages) : TileDataSource(0, Const::MAX_SUPPORTED_ZOOM_LEVEL), + _dataSources(), + _cachedOpenDataSources(), + _maxOpenedPackages(maxOpenedPackages), + _mutex() + { + _dataSourceListener = std::make_shared(*this); + } + MultiTileDataSource::MultiTileDataSource() : TileDataSource(0, Const::MAX_SUPPORTED_ZOOM_LEVEL), + _dataSources(), + _cachedOpenDataSources(), + _maxOpenedPackages(4), + _mutex() + { + _dataSourceListener = std::make_shared(*this); + } + + MultiTileDataSource::~MultiTileDataSource() { + _cachedOpenDataSources.clear(); + for (auto it = _dataSources.begin(); it != _dataSources.end(); it++) + { + if (auto dataSource = std::dynamic_pointer_cast(it->second)) + { + dataSource->unregisterOnChangeListener(_dataSourceListener); + } + } + _dataSourceListener.reset(); + } + + bool compare_datasource (std::pair, std::shared_ptr> dataSource1, std::pair, std::shared_ptr> dataSource2) { + return std::dynamic_pointer_cast(dataSource1.second) == std::dynamic_pointer_cast(dataSource2.second); + }; + + int MultiTileDataSource::getMinZoom() const { + int minZoom = Const::MAX_SUPPORTED_ZOOM_LEVEL; + for (auto it = _dataSources.begin(); it != _dataSources.end(); it++) + { + minZoom = std::min(minZoom, it->second->getMinZoom()); + } + return minZoom; + } + + int MultiTileDataSource::getMaxZoom() const { + int maxZoom = 0; + for (auto it = _dataSources.begin(); it != _dataSources.end(); it++) + { + maxZoom = std::max(maxZoom, it->second->getMaxZoom()); + } + return maxZoom; + } + + MapBounds MultiTileDataSource::getDataExtent() const { + MapBounds bounds; + for (auto it = _dataSources.begin(); it != _dataSources.end(); it++) + { + bounds.expandToContain( it->second->getDataExtent()); + } + return bounds; + } + + std::shared_ptr MultiTileDataSource::loadTile(const MapTile &mapTile) + { + Log::Infof("MultiTileDataSource::loadTile: Loading %s", mapTile.toString().c_str()); + try + { + MapTile mapTileFlipped = mapTile.getFlipped(); + + std::shared_ptr tileData; + std::lock_guard lock(_mutex); + bool tileOk = false; + const int zoom = mapTile.getZoom(); + // Fast path: try already open packages + for (auto it = _cachedOpenDataSources.begin(); it != _cachedOpenDataSources.end(); it++) + { + auto dataSource = it->second; + if (zoom < dataSource->getMinZoom() || zoom > dataSource->getMaxZoom()) { + continue; + } + + std::shared_ptr tileMask = it->first; + if (tileMask && tileMask->getTileStatus(mapTileFlipped) == PackageTileStatus::PACKAGE_TILE_STATUS_MISSING) { + continue; + } + + tileData = dataSource->loadTile(mapTile); + tileOk = tileData && tileData->getData(); + if (tileOk) + { + std::rotate(_cachedOpenDataSources.begin(), it, it + 1); + break; + } + } + if (!tileOk && _cachedOpenDataSources.size() != _dataSources.size()) + { + // Slow path: try other packages + for (auto it = _dataSources.begin(); it != _dataSources.end(); it++) + { + if (auto dataSource = std::dynamic_pointer_cast(it->second)) + { + auto it2 = std::find_if(_cachedOpenDataSources.begin(), _cachedOpenDataSources.end(), [&it](const std::pair, std::shared_ptr> pair) + { return pair.second == it->second; }); + if (it2 != _cachedOpenDataSources.end() && it2->second == it->second) { + continue; + } + if (zoom < dataSource->getMinZoom() || zoom > dataSource->getMaxZoom()) { + continue; + } + std::shared_ptr tileMask = it->first; + if (tileMask && tileMask->getTileStatus(mapTileFlipped) == PackageTileStatus::PACKAGE_TILE_STATUS_MISSING) { + continue; + } + + tileData = dataSource->loadTile(mapTile); + tileOk = tileData && tileData->getData(); + if (tileOk) { + _cachedOpenDataSources.insert(_cachedOpenDataSources.begin(), std::make_pair(tileMask, dataSource)); + if (_cachedOpenDataSources.size() > _maxOpenedPackages) + { + _cachedOpenDataSources.pop_back(); + } + break; + } + } + } + } + + if (!tileOk) + { + + if (mapTile.getZoom() > getMinZoom()) + { + Log::Infof("MultiTileDataSource::loadTile: Tile data doesn't exist in the database, redirecting to parent"); + if (!tileData){ + tileData = std::make_shared(std::shared_ptr()); + } + tileData->setReplaceWithParent(true); + } + else + { + Log::Infof("MultiTileDataSource::loadTile: Tile data doesn't exist in the database"); + return std::shared_ptr(); + } + } + return tileData; + } + catch (const std::exception &ex) + { + Log::Errorf("PackageManagerTileDataSource::loadTile: Exception: %s", ex.what()); + } + return std::shared_ptr(); + } + + void MultiTileDataSource::onPackagesChanged(ChangeType changeType) + { + { + std::lock_guard lock(_mutex); + _cachedOpenDataSources.clear(); + } + notifyTilesChanged(changeType == PACKAGES_DELETED); // we need to remove tiles only if packages were deleted + } + + void MultiTileDataSource::add(const std::shared_ptr &dataSource) + { + add(dataSource, std::string()); + } + + void MultiTileDataSource::add(const std::shared_ptr &dataSource, const std::string& tileMaskArg) + { + { + std::lock_guard lock(_mutex); + auto it = std::find_if(_dataSources.begin(), _dataSources.end(), [&dataSource](const std::pair, std::shared_ptr> pair) + { return pair.second == dataSource; }); + if (it != _dataSources.end()) { + return; + } + std::shared_ptr tileMask; + std::string tileMaskStr = tileMaskArg; + if (tileMaskStr.empty()) { + if (auto mbtilesDatasource = std::dynamic_pointer_cast(dataSource)) { + tileMaskStr = mbtilesDatasource->getMetaData("tilemask"); + } + } + if (!tileMaskStr.empty()) + { + std::vector parts = GeneralUtils::Split(tileMaskStr, ':'); + if (!parts.empty()) + { + int zoomLevel; + if (parts.size() > 1) + { + zoomLevel = boost::lexical_cast(parts[1]); + } + else + { + zoomLevel = dataSource->getMaxZoom(); + } + tileMask = std::make_shared(parts[0], zoomLevel); + } + } + dataSource->registerOnChangeListener(_dataSourceListener); + _dataSources.emplace_back(tileMask, dataSource); + } + onPackagesChanged(PACKAGES_ADDED); + } + + + bool MultiTileDataSource::remove(const std::shared_ptr &dataSource) + { + { + std::lock_guard lock(_mutex); + auto it = std::remove_if(_dataSources.begin(), _dataSources.end(), [&dataSource]( + const std::pair, std::shared_ptr> pair) { + return pair.second == dataSource; + }); + if (it == _dataSources.end()) { + return false; + } + dataSource->unregisterOnChangeListener(_dataSourceListener); + _dataSources.erase(it); + } + onPackagesChanged(PACKAGES_DELETED); + return true; + } + + MultiTileDataSource::DataSourceListener::DataSourceListener(MultiTileDataSource& combinedDataSource) : + _combinedDataSource(combinedDataSource) + { + } + + void MultiTileDataSource::DataSourceListener::onTilesChanged(bool removeTiles) { + _combinedDataSource.notifyTilesChanged(removeTiles); + } +} diff --git a/all/native/datasources/MultiTileDataSource.h b/all/native/datasources/MultiTileDataSource.h new file mode 100644 index 000000000..a778d918f --- /dev/null +++ b/all/native/datasources/MultiTileDataSource.h @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2016 CartoDB. All rights reserved. + * Copying and using this code is allowed only according + * to license terms, as given in https://cartodb.com/terms/ + */ + + +#ifndef _CARTO_LOCALPACKAGEMANAGERTILEDATASOURCE_H_ +#define _CARTO_LOCALPACKAGEMANAGERTILEDATASOURCE_H_ + +#include "datasources/TileDataSource.h" +#include "packagemanager/PackageTileMask.h" + +#include +#include +#include +#include + +namespace carto { + + /** + * A tile data source that handles multiple data sources. + */ + class MultiTileDataSource : public TileDataSource { + public: + /** + * Constructs a PackageManagerTileDataSource object. + * @param packageManager The package manager that is used to retrieve requested tiles. + */ + explicit MultiTileDataSource(); + MultiTileDataSource(int maxOpenedPackages); + virtual ~MultiTileDataSource(); + + virtual int getMinZoom() const; + virtual int getMaxZoom() const; + + virtual MapBounds getDataExtent() const; + + virtual std::shared_ptr loadTile(const MapTile& mapTile); + + + /** + * Adds a new data source to the data source stack. The new data source will be the last (and topmost) data source. + * @param datasource The data source to be added. + */ + void add(const std::shared_ptr& datasource); + + /** + * Adds a new data source to the data source stack. The new data source will be the last (and topmost) data source. + * @param datasource The data source to be added. + */ + void add(const std::shared_ptr& datasource, const std::string& tileMask); + + /** + * Removes a data source from the sources stack. + * @param datasource The data source to be removed. + * @return True if the data source was removed. False otherwise ( data source was not found). + */ + bool remove(const std::shared_ptr& datasource); + + protected: + enum ChangeType { + PACKAGES_ADDED, + PACKAGES_DELETED + }; + + class DataSourceListener : public TileDataSource::OnChangeListener { + public: + explicit DataSourceListener(MultiTileDataSource& combinedDataSource); + + virtual void onTilesChanged(bool removeTiles); + + private: + MultiTileDataSource& _combinedDataSource; + }; + + mutable std::optional _maxOpenedPackages; + + mutable std::vector, std::shared_ptr > > _dataSources; + mutable std::vector, std::shared_ptr > > _cachedOpenDataSources; + + mutable std::mutex _mutex; + + void onPackagesChanged(ChangeType changeType); + + private: + std::shared_ptr _dataSourceListener; + }; + +} + +#endif