database: Add basic database validation and error description

README.md

@@ -19,6 +19,7 @@ Usage: the_dude_to_human.exe [options] <filename>
 -o, --out                                  Save json database file
 -c, --credentials                          Save credentials in plain text
 -m, --mikrotik user:password@address:port  Connect to the specified mikrotik device
+-i, --integrity                            Validate database health
 -h, --help                                 Display this help and exit
 -v, --version                              Print tool version
 

src/common/string_util.cpp

@@ -5,6 +5,7 @@
 #include <algorithm>
 #include <cctype>
 #include <codecvt>
+#include <iomanip>
 #include <locale>
 #include <sstream>
 #include <vector>
@@ -43,6 +44,16 @@ std::string StringFromBuffer(std::span<const char> data) {
     return std::string(data.begin(), std::find(data.begin(), data.end(), '\0'));
 }
 
+std::string HexStringFromBuffer(std::span<const u8> data) {
+    std::stringstream ss;
+    ss << std::hex << std::uppercase;
+
+    for (uint8_t byte : data) {
+        ss << std::setw(2) << std::setfill('0') << static_cast<int>(byte);
+    }
+    return ss.str();
+}
+
 // Turns "  hej " into "hej". Also handles tabs.
 std::string StripSpaces(const std::string& str) {
     const std::size_t s = str.find_first_not_of(" \t\r\n");

src/common/string_util.h

@@ -21,6 +21,7 @@ namespace Common {
 
 [[nodiscard]] std::string StringFromBuffer(std::span<const u8> data);
 [[nodiscard]] std::string StringFromBuffer(std::span<const char> data);
+[[nodiscard]] std::string HexStringFromBuffer(std::span<const u8> data);
 
 [[nodiscard]] std::string StripSpaces(const std::string& s);
 [[nodiscard]] std::string StripQuotes(const std::string& s);

src/the_dude_to_human/TheDudeToHuman.cpp

@@ -35,6 +35,7 @@ static void PrintHelp(const char* argv0) {
            "-c, --credentials                          Save credentials in plain text\n"
            "-m, --mikrotik user:password@address:port  Connect to the specified mikrotik device\n"
            //"-d, --database=user:password@address:port  Connect to the specified database\n"
+           "-i, --integrity                            Validate database health\n"
            "-h, --help                                 Display this help and exit\n"
            "-v, --version                              Print tool version\n";
     // clang-format on
@@ -119,12 +120,15 @@ int main(int argc, char** argv) {
     std::string database_address{};
     [[maybe_unused]] u16 database_port = {3306};
 
+    bool check_integrity{};
+
     static struct option long_options[] = {
         // clang-format off
         {"file", required_argument, 0, 'f'},
         {"out", required_argument, 0, 'o'},
         {"credentials", no_argument, 0, 'c'},
         {"mikrotik", required_argument, 0, 'm'},
+        {"integrity", no_argument, 0, 'i'},
         //{"database", optional_argument, 0, 'd'},
         {"help", no_argument, 0, 'h'},
         {"version", no_argument, 0, 'v'},
@@ -133,7 +137,7 @@ int main(int argc, char** argv) {
     };
 
     while (optind < argc) {
-        int arg = getopt_long(argc, argv, "f:o:cm:hv", long_options, &option_index);
+        int arg = getopt_long(argc, argv, "f:o:cm:ihv", long_options, &option_index);
         if (arg != -1) {
             switch (static_cast<char>(arg)) {
             case 'f': {
@@ -151,6 +155,9 @@ int main(int argc, char** argv) {
             case 'c':
                 has_credentials = true;
                 break;
+            case 'i':
+                check_integrity = true;
+                break;
             case 'h':
                 PrintHelp(argv[0]);
                 return 0;
@@ -253,8 +260,13 @@ int main(int argc, char** argv) {
     if (has_filepath) {
         std::cout << "Reading database " << filepath << "\n";
         Database::DudeDatabase db{filepath};
-        db.ListMapData();
-        db.ListDeviceData();
+
+        if (check_integrity) {
+            db.CheckIntegrity();
+        } else {
+            db.ListMapData();
+            db.ListDeviceData();
+        }
 
         if (has_out_filepath) {
             std::cout << "Saving database " << out_filepath << "\n";

src/the_dude_to_human/database/dude_database.cpp

@@ -8,6 +8,7 @@
 #include "the_dude_to_human/database/dude_database.h"
 #include "the_dude_to_human/database/dude_field_parser.h"
 #include "the_dude_to_human/database/dude_json.h"
+#include "the_dude_to_human/database/dude_validator.h"
 
 namespace Database {
 DudeDatabase::DudeDatabase(const std::string& db_file) : db{db_file} {
@@ -51,6 +52,10 @@ int DudeDatabase::SaveDatabase(const std::string& db_file, bool has_credentials)
     return SerializeDatabaseJson(this, db_file, has_credentials);
 }
 
+int DudeDatabase::CheckIntegrity() {
+    return ValidateDatabase(this);
+}
+
 std::vector<DataFormat> DudeDatabase::ListUsedDataFormats() const {
     std::vector<DataFormat> data_formats{};
     Sqlite::SqlData sql_data{};
@@ -66,7 +71,7 @@ std::vector<DataFormat> DudeDatabase::ListUsedDataFormats() const {
                 continue;
             }
 
-            printf("New Format 0x%02x in row %d \n", data_format, id);
+            // printf("New Format 0x%02x in row %d \n", data_format, id);
             data_formats.push_back(format);
         }
     }
@@ -109,8 +114,14 @@ std::vector<T> DudeDatabase::GetObjectData(DataFormat format,
 
         const T obj_data = (this->*RawToObjData)(parser);
 
-        if (id != (u32)obj_data.object_id.value) {
-            printf("Corrupted Entry %d\n", id);
+        if (!parser.IsDataValid()) {
+            printf("Corrupted Entry id %d: %s\n\t'%s'\n", id, parser.GetErrorMessage().c_str(),
+                   Common::HexStringFromBuffer(blob).c_str());
+            continue;
+        } else if (id != (u32)obj_data.object_id.value) {
+            printf("Invalid Entry expected %d, found %d\n", id, obj_data.object_id.value);
+            parser.PrintFieldInfo();
+            printf("\n");
             continue;
         }
 

src/the_dude_to_human/database/dude_database.h

@@ -28,6 +28,7 @@ class DudeDatabase {
     int GetOutages(Sqlite::SqlData& data) const;
 
     int SaveDatabase(const std::string& db_file, bool has_credentials);
+    int CheckIntegrity();
 
     // Usefull to find new unsuported types
     std::vector<DataFormat> ListUsedDataFormats() const;

src/the_dude_to_human/database/dude_field_parser.cpp

@@ -11,43 +11,44 @@ DudeFieldParser::DudeFieldParser(std::span<const u8> data) : raw_data{data} {
 }
 
 void DudeFieldParser::Reset() {
-    is_data_valid = true;
+    status = ParserResult::Success;
+    error_count = 0;
     offset = 0;
 
     if (ReadData(&magic, sizeof(u16)) != ParserResult::Success) {
-        is_data_valid = false;
+        status = ParserResult::InvalidMagic;
         return;
     }
 
     if (ReadField(data_format, FieldId::DataFormat) != ParserResult::Success) {
-        is_data_valid = false;
+        status = ParserResult::InvalidHeader;
         return;
     }
 
     if (data_format.entries == 0) {
-        is_data_valid = false;
+        status = ParserResult::InvalidHeader;
         return;
     }
 }
 
 u16 DudeFieldParser::GetMagic() const {
-    if (!is_data_valid) {
+    if (!IsDataValid()) {
         return {};
     }
 
     return magic;
 }
 
 IntArrayField DudeFieldParser::GetFormat() const {
-    if (!is_data_valid) {
+    if (!IsDataValid()) {
         return {};
     }
 
     return data_format;
 }
 
 DataFormat DudeFieldParser::GetMainFormat() const {
-    if (!is_data_valid) {
+    if (!IsDataValid()) {
         return {};
     }
 
@@ -69,7 +70,7 @@ ParserResult DudeFieldParser::SkipField() {
     ParserResult result = GetFieldInfo(info);
 
     if (result != ParserResult::Success) {
-        return result;
+        return ReturnWithError(result);
     }
 
     BoolField bool_field;
@@ -103,7 +104,7 @@ ParserResult DudeFieldParser::SkipField() {
     case FieldType::StringArray:
         return ReadField(string_array_field, FieldId::None);
     default:
-        return ParserResult::InvalidFieldType;
+        return ReturnWithError(ParserResult::InvalidFieldType);
     }
 }
 
@@ -112,13 +113,13 @@ ParserResult DudeFieldParser::ReadData(void* data, std::size_t size) {
         return ParserResult::Success;
     }
     if (data == nullptr) {
-        return ParserResult::InvalidFieldArguments;
+        return ReturnWithError(ParserResult::InvalidFieldArguments);
     }
-    if (!is_data_valid) {
+    if (!IsDataValid()) {
         return ParserResult::Corrupted;
     }
     if (raw_data.size() < size + offset) {
-        return ParserResult::EndOfFile;
+        return ReturnWithError(ParserResult::EndOfFile);
     }
 
     std::memcpy(data, raw_data.data() + offset, size);
@@ -142,11 +143,11 @@ ParserResult DudeFieldParser::ValidataFieldInfo(const FieldInfo& field_info, Fie
     case FieldType::StringArray:
         break;
     default:
-        return ParserResult::InvalidFieldType;
+        return ReturnWithError(ParserResult::InvalidFieldType);
     }
 
     // Allow all id if none is specified
-    if (field_info.id.Value() == FieldId::None) {
+    if (id == FieldId::None) {
         return ParserResult::Success;
     }
 
@@ -455,4 +456,131 @@ void DudeFieldParser::RestoreOffset() {
     offset = previous_offset;
 }
 
+ParserResult DudeFieldParser::ReturnWithError(ParserResult result) {
+    if (result == ParserResult::Success) {
+        return ParserResult::Success;
+    }
+
+    error_count++;
+    status = result;
+    return result;
+}
+
+bool DudeFieldParser::IsDataValid() const {
+    return status == ParserResult::Success;
+}
+
+ParserResult DudeFieldParser::GetStatus() const {
+    return status;
+}
+
+std::string DudeFieldParser::GetErrorMessage() const {
+    return GetErrorMessage(status);
+}
+
+std::string DudeFieldParser::GetErrorMessage(ParserResult result) {
+    switch (result) {
+    case ParserResult::Success:
+        return "OK";
+    case ParserResult::Corrupted:
+        return "The field can't be fully parsed";
+    case ParserResult::FieldTypeMismatch:
+        return "Requested field type mismatch";
+    case ParserResult::FieldIdMismatch:
+        return "Requested field id mismatch";
+    case ParserResult::InvalidFieldType:
+        return "Unsupported field type";
+    case ParserResult::InvalidFieldArguments:
+        return "Arguments given are invalid";
+    case ParserResult::InvalidMagic:
+        return "Magic bytes can't be read";
+    case ParserResult::InvalidHeader:
+        return "Header is invalid";
+    case ParserResult::EndOfFile:
+        return "Reached end of file while parsing data";
+    default:
+        return "Unexpected error";
+    }
+}
+
+void DudeFieldParser::PrintFieldInfo() {
+    printf("Field data: ");
+    Reset();
+    if (!IsDataValid()) {
+        printf(" Error: %s\n", GetErrorMessage().c_str());
+        return;
+    }
+    printf(" Format 0x%x\n", static_cast<u32>(GetMainFormat()));
+
+    while (offset < raw_data.size()) {
+        FieldInfo info{};
+        ParserResult result = GetFieldInfo(info);
+
+        if (result != ParserResult::Success) {
+            printf("\tError: %s\n", GetErrorMessage(result).c_str());
+            return;
+        }
+
+        std::string field_json = "";
+        BoolField bool_field;
+        IntField int_field;
+        LongField long_field;
+        LongLongField long_long_field;
+        TextField text_field;
+        LongArrayField long_array_field;
+        IntArrayField int_array_field;
+        StringArrayField string_array_field;
+
+        RestoreOffset();
+        switch (info.type) {
+        case FieldType::BoolFalse:
+        case FieldType::BoolTrue:
+            ReadField(bool_field, info.id);
+            field_json = bool_field.SerializeJson();
+            break;
+        case FieldType::Int:
+        case FieldType::Byte:
+            ReadField(int_field, info.id);
+            field_json = int_field.SerializeJson();
+            break;
+        case FieldType::Long:
+            ReadField(long_field, info.id);
+            field_json = long_field.SerializeJson();
+            break;
+        case FieldType::LongLong:
+            ReadField(long_long_field, info.id);
+            field_json = long_long_field.SerializeJson();
+            break;
+        case FieldType::LongString:
+        case FieldType::ShortString:
+            ReadField(text_field, info.id);
+            field_json = text_field.SerializeJson();
+            break;
+        case FieldType::LongArray:
+            ReadField(long_array_field, info.id);
+            field_json = long_array_field.SerializeJson();
+            break;
+        case FieldType::IntArray:
+            ReadField(int_array_field, info.id);
+            field_json = int_array_field.SerializeJson();
+            break;
+        case FieldType::StringArray:
+            ReadField(string_array_field, info.id);
+            field_json = string_array_field.SerializeJson();
+            break;
+        default:
+            printf("\tError: %s\n", GetErrorMessage(ParserResult::InvalidFieldType).c_str());
+            return;
+        }
+
+        if (!IsDataValid()) {
+            printf("\tError: %s\n", GetErrorMessage().c_str());
+            return;
+        }
+
+        printf("\tCategory 0x%x, 0x%x: %s\n", static_cast<u32>(info.id.Value()),
+               static_cast<u32>(info.type.Value()), field_json.c_str());
+    }
+}
+
 } // namespace Database