Importer simple

atlas/core/config/config.hpp

@@ -328,7 +328,7 @@ SETTINGS_D(
 SETTINGS_D( threads, import_threads, int, 2 )
 SETTINGS_D( threads, image_import_threads, int, 4 )
 SETTINGS_D( threads, image_loader_threads, int, 2 )
-SETTINGS_D( threads, import_pre_loader_threads, int, 4 )
+SETTINGS_D( threads, import_pre_loader_threads, int, 8 )
 
 SETTINGS_D( experimental, local_match, bool, false )
 SETTINGS_D( experimental, loading_preview, bool, false )

atlas/core/database/Binder.cpp

@@ -8,10 +8,34 @@
 
 Binder::Binder( const std::string_view sql )
 {
+	ZoneScoped;
+	const char* unused { nullptr };
 	const auto prepare_ret {
-		sqlite3_prepare_v2( &Database::ref(), sql.data(), static_cast< int >( sql.size() + 1 ), &stmt, nullptr )
+		sqlite3_prepare_v2( &Database::ref(), sql.data(), static_cast< int >( sql.size() + 1 ), &stmt, &unused )
 	};
 
+	if ( unused != nullptr && strlen( unused ) > 0 )
+	{
+		//Check if the string is just empty (\n or \t)
+		const std::string_view leftovers { unused };
+		auto itter { leftovers.begin() };
+		while ( itter != leftovers.end() )
+		{
+			if ( *itter == '\n' || *itter == '\t' )
+			{
+				++itter;
+				continue;
+			}
+			else
+				throw DatabaseException(
+					format_ns::format( "Query had unused portions of the input. Unused: \"{}\"", unused ) );
+		}
+	}
+
+	if ( stmt == nullptr )
+		throw DatabaseException( format_ns::
+		                             format( "Failed to prepare stmt, {}", sqlite3_errmsg( &Database::ref() ) ) );
+
 	if ( prepare_ret != SQLITE_OK )
 	{
 		throw DatabaseException( format_ns::format(
@@ -21,14 +45,25 @@ Binder::Binder( const std::string_view sql )
 	max_param_count = sqlite3_bind_parameter_count( stmt );
 }
 
-Binder::~Binder() noexcept( false )
+Binder::~Binder()
 {
-	if ( !ran ) [[unlikely]]
+	ZoneScoped;
+	try
 	{
-		atlas::logging::debug( "Binder falloff. Running query" );
-		std::optional< std::tuple<> > tpl;
-		executeQuery( tpl );
-	}
+		if ( !ran )
+		{
+			std::optional< std::tuple<> > tpl;
+			executeQuery( tpl );
+		}
 
-	sqlite3_finalize( stmt );
-}
+		sqlite3_finalize( stmt );
+	}
+	catch ( std::exception& e )
+	{
+		atlas::logging::critical( "Binder's dtor has thrown!, {}", e.what() );
+	}
+	catch ( ... )
+	{
+		atlas::logging::critical( "Binder's dtor has thrown!, ..." );
+	}
+}

atlas/core/database/Binder.hpp

@@ -6,6 +6,8 @@
 #ifndef ATLASGAMEMANAGER_BINDER_HPP
 #define ATLASGAMEMANAGER_BINDER_HPP
 
+#include <tracy/Tracy.hpp>
+
 #include <sqlite3.h>
 #include <string>
 
@@ -87,6 +89,7 @@ class Binder
 		return *this;
 	}
 
+	// Feed into value directly
 	template < typename T >
 		requires( (!is_optional< T >) && (!is_tuple< T >))
 	void operator>>( T& t )
@@ -95,9 +98,13 @@ class Binder
 
 		executeQuery( tpl );
 
-		if ( tpl.has_value() ) t = std::move( std::get< 0, T >( tpl.value() ) );
+		if ( tpl.has_value() )
+			t = std::move( std::get< 0, T >( tpl.value() ) );
+		else
+			throw DatabaseRowMismatch( format_ns::format( "No rows returned for query \"{}\"", sqlite3_sql( stmt ) ) );
 	}
 
+	// Feed output into optional
 	template < typename T >
 		requires( !is_optional< T > && (!is_tuple< T >))
 	void operator>>( std::optional< T >& t )
@@ -112,6 +119,7 @@ class Binder
 			t = std::nullopt;
 	}
 
+	// Call function using output
 	template < typename Function >
 		requires( (!is_optional< Function >) && (!is_tuple< Function >))
 	void operator>>( Function&& func )
@@ -129,42 +137,48 @@ class Binder
 		}
 	}
 
+	// Feed output into tuple
 	template < typename... Ts >
 		requires( !( is_optional< Ts > && ... ) ) && ( !( is_tuple< Ts > && ... ) )
 	void operator>>( std::tuple< Ts... >& tpl )
 	{
 		ran = true;
 
 		std::optional< std::tuple< Ts... > > opt_tpl { std::nullopt };
-		executeQuery( opt_tpl );
+		executeQuery< Ts... >( opt_tpl );
 
 		if ( opt_tpl.has_value() )
-		{
 			tpl = std::move( opt_tpl.value() );
-		}
+		else
+			throw DatabaseRowMismatch( "No rows returned for query" );
+
 		return;
 	}
 
   private:
 
 	template < typename... Ts >
 		requires( !( is_optional< Ts > || ... ) && !( is_tuple< Ts > || ... ) )
-	void executeQuery( std::optional< std::tuple< Ts... > >& tpl_opt )
+	void executeQuery( [[maybe_unused]] std::optional< std::tuple< Ts... > >& tpl_opt )
 	{
+		ZoneScoped;
 		if ( param_counter != max_param_count )
 			throw AtlasException( format_ns::format(
 				"Not enough parameters given for query! Given {}, Expected {}. param_counter != max_param_count = {} != {} for query \"{}\"",
 				param_counter,
 				max_param_count,
 				param_counter,
 				max_param_count,
-				std::string( sqlite3_sql( stmt ) ) ) );
+				std::string_view( sqlite3_sql(
+					stmt ) ) ) ); // String view is safe here since the string is owned by sqlite3 and not freed until the statement is finalized
 
 		ran = true;
 
 		if ( stmt == nullptr ) throw DatabaseException( "stmt was nullptr" );
 
+#ifdef LOG_SQL_QUERIES
 		atlas::logging::debug( "Executing query {}", sqlite3_expanded_sql( stmt ) );
+#endif
 
 		const auto step_ret { sqlite3_step( stmt ) };
 
@@ -182,14 +196,18 @@ class Binder
 					}
 					else
 					{
-						tpl_opt = std::nullopt;
-						return;
+						throw AtlasException( format_ns::format(
+							"No rows were expected but rows were returned for query: \"{}\". Is this intentional?",
+							sqlite3_expanded_sql( stmt ) ) );
 					}
 				}
 			case SQLITE_DONE:
 				{
+#ifdef LOG_SQL_QUERIES
 					atlas::logging::debug( "Finished query {}", sqlite3_expanded_sql( stmt ) );
-					tpl_opt = std::nullopt;
+#endif
+					//Help hint to the compiler that it shouldn't keep an empty tuple around
+					if constexpr ( sizeof...( Ts ) > 0 ) tpl_opt = std::nullopt;
 					return;
 
 					default:
@@ -211,7 +229,7 @@ class Binder
 
   public:
 
-	~Binder() noexcept( false );
+	~Binder();
 };
 
 #endif //ATLASGAMEMANAGER_BINDER_HPP

atlas/core/database/Transaction.hpp

@@ -35,6 +35,7 @@ namespace atlas::database
 
 		inline Binder operator<<( std::string_view sql )
 		{
+			ZoneScopedN( "TransactionBase::operator<<" );
 			if constexpr ( is_commitable )
 				sqlite3_exec( &Database::ref(), "BEGIN TRANSACTION;", nullptr, nullptr, nullptr );
 
@@ -44,6 +45,7 @@ namespace atlas::database
 		template < std::uint64_t size >
 		inline Binder operator<<( const char ( &raw_str )[ size - 1 ] )
 		{
+			ZoneScopedN( "TransactionBase::operator<<" );
 			const std::string_view str_view { std::string_view( raw_str, size - 1 ) };
 			return *this << str_view;
 		}

atlas/core/database/extractors.hpp

@@ -12,6 +12,22 @@
 template < std::uint64_t, typename T >
 void extract( sqlite3_stmt*, T& ) = delete;
 
+/*
+template < std::uint64_t index, typename T >
+	requires std::same_as< std::string, T >
+void extract( [[maybe_unused]] sqlite3_stmt* stmt, [[maybe_unused]] std::string& t ) noexcept
+{
+	static_assert( false, "You should use std::string_view instead of std::string to reduce the need to memove/copy" );
+}
+
+template < std::uint64_t index, typename T >
+	requires( !std::move_constructible< T > )
+void extract( [[maybe_unused]] sqlite3_stmt* stmt, [[maybe_unused]] std::string& t ) noexcept
+{
+	static_assert( false, "T is not move constructable" );
+}
+*/
+
 template < std::uint64_t index, typename T >
 	requires std::is_integral_v< T >
 void extract( sqlite3_stmt* stmt, T& t ) noexcept
@@ -34,8 +50,8 @@ void extract( sqlite3_stmt* stmt, T& t ) noexcept
 }
 
 template < std::uint64_t index, typename T >
-	requires std::is_same_v< T, std::u8string >
-void extract( sqlite3_stmt* stmt, std::u8string& t ) noexcept
+	requires std::is_same_v< T, std::u8string_view >
+void extract( sqlite3_stmt* stmt, std::u8string_view& t ) noexcept
 {
 	const unsigned char* const txt { sqlite3_column_text( stmt, index ) };
 
@@ -47,16 +63,20 @@ void extract( sqlite3_stmt* stmt, std::u8string& t ) noexcept
 	else
 	{
 		const auto len { strlen( reinterpret_cast< const char* const >( txt ) ) };
-		t = { reinterpret_cast< const char8_t* >( txt ), len };
+		t = std::u8string_view( reinterpret_cast< const char8_t* const >( txt ), len );
 		return;
 	}
 }
 
 template < std::uint64_t index, typename T >
-	requires std::is_same_v< T, QString >
-void extract( sqlite3_stmt* stmt, QString& t ) noexcept
+	requires std::is_same_v< T, std::string_view >
+void extract( sqlite3_stmt* stmt, std::string_view& t ) noexcept
 {
+#ifdef __linux__
 	const unsigned char* const txt { sqlite3_column_text( stmt, index ) };
+#else
+	const unsigned char* const txt { reinterpret_cast< const unsigned char* >( sqlite3_column_text16( stmt, index ) ) };
+#endif
 
 	if ( txt == nullptr )
 	{
@@ -65,7 +85,8 @@ void extract( sqlite3_stmt* stmt, QString& t ) noexcept
 	}
 	else
 	{
-		t = QString::fromUtf8( txt );
+		const auto len { strlen( reinterpret_cast< const char* const >( txt ) ) };
+		t = std::string_view( reinterpret_cast< const char* const >( txt ), len );
 		return;
 	}
 }
@@ -74,18 +95,27 @@ template < std::uint64_t index, typename T >
 	requires std::is_same_v< T, std::filesystem::path >
 void extract( sqlite3_stmt* stmt, std::filesystem::path& t ) noexcept
 {
-	std::u8string str;
-	extract< index, std::u8string >( stmt, str );
-	t = { std::move( str ) };
+	std::string_view path_str;
+	extract< index, std::string_view >( stmt, path_str );
+	t = path_str;
 }
 
 template < std::uint64_t index, typename T >
-	requires std::is_same_v< T, std::string >
-void extract( sqlite3_stmt* stmt, std::string& t ) noexcept
+	requires std::is_same_v< T, QString >
+void extract( sqlite3_stmt* stmt, QString& t ) noexcept
 {
-	QString str;
-	extract< index, QString >( stmt, str );
-	t = { str.toStdString() };
+	const unsigned char* const txt { sqlite3_column_text( stmt, index ) };
+
+	if ( txt == nullptr )
+	{
+		t = {};
+		return;
+	}
+	else
+	{
+		t = QString::fromUtf8( txt );
+		return;
+	}
 }
 
 template < std::uint64_t index, typename T >

atlas/core/database/migrations/migration-run.cpp

@@ -52,8 +52,10 @@ namespace atlas::database::migrations
 		}
 
 		int current_migration { -1 };
+		std::optional< int > last_migration;
 		RapidTransaction() << "SELECT migration_id FROM migrations ORDER BY migration_id DESC limit 1"
-			>> current_migration;
+			>> last_migration;
+		if ( last_migration.has_value() ) current_migration = last_migration.value();
 
 		try
 		{

atlas/core/database/record/GameData.cpp

@@ -44,13 +44,13 @@ namespace atlas::records
 
 		RapidTransaction() << "SELECT count(*) FROM previews WHERE record_id = ?" << m_game_id >> m_preview_count;
 
-		AtlasID atlas_id { INVALID_ATLAS_ID };
+		std::optional< AtlasID > atlas_id;
 		RapidTransaction() << "SELECT atlas_id FROM atlas_mappings WHERE record_id = ? " << m_game_id >> atlas_id;
-		if ( atlas_id != INVALID_ATLAS_ID ) atlas_data = { atlas_id };
+		if ( atlas_id.has_value() ) atlas_data = { atlas_id.value() };
 
-		F95ID f95_id { INVALID_F95_ID };
+		std::optional< F95ID > f95_id;
 		RapidTransaction() << "SELECT f95_id FROM f95_zone_mappings WHERE record_id = ?" << m_game_id >> f95_id;
-		if ( f95_id != INVALID_F95_ID ) f95_data = { f95_id };
+		if ( f95_id.has_value() ) f95_data = { f95_id.value() };
 
 		RapidTransaction() << "SELECT version FROM versions WHERE record_id = ?" << m_game_id >>
 			[ & ]( const QString version ) { m_versions.emplace_back( Version( this->m_game_id, version ) ); };
@@ -67,14 +67,14 @@ namespace atlas::records
 	{
 		ZoneScoped;
 		RapidTransaction transaction;
-		RecordID record_id { INVALID_RECORD_ID };
+		std::optional< RecordID > record_id;
 		transaction << "SELECT record_id FROM games WHERE title = ? AND creator = ? AND engine = ?" << title_in
 					<< creator_in << engine_in
 			>> record_id;
 
-		if ( record_id != INVALID_RECORD_ID )
+		if ( record_id.has_value() )
 		{
-			Game game { record_id };
+			Game game { record_id.value() };
 			throw RecordAlreadyExists( game );
 		}
 
@@ -95,14 +95,17 @@ namespace atlas::records
 	RecordID recordID( const QString& title, const QString& creator, const QString& engine )
 	{
 		ZoneScoped;
-		RecordID record_id { INVALID_RECORD_ID };
+		std::optional< RecordID > record_id;
 
 		RapidTransaction transaction;
 		transaction << "SELECT record_id FROM games WHERE title = ? AND creator = ? AND engine = ?" << title << creator
 					<< engine
 			>> record_id;
 
-		return record_id;
+		if ( record_id.has_value() )
+			return record_id.value();
+		else
+			return INVALID_RECORD_ID;
 	}
 
 	//! Helper function. Returns if a title,creator,engine combo can be found.