diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..1a25f40 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,79 @@ +cmake_minimum_required(VERSION 3.2) +project(terminator CXX) + +find_package(PkgConfig) +pkg_search_module(KYOTOCABINET REQUIRED kyotocabinet) + +if (NOT CMAKE_BUILD_TYPE) + message(STATUS "Setting build type to 'Debug' as none was specified.") + set(CMAKE_BUILD_TYPE Debug CACHE STRING "Choose the type of build." FORCE) +endif() +string(TOUPPER "${CMAKE_BUILD_TYPE}" CMAKE_BUILD_TYPE_UPPER) + +set ( CMAKE_CXX_FLAGS -std=c++17 ) +set ( CMAKE_CXX_FLAGS "-Wl,--as-needed -Wl,--allow-shlib-undefined") +set ( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC") + +set ( TERMINATOR_INCLUDE ${CMAKE_CURRENT_SOURCE_DIR}/include) +set ( TERMINATOR_SRC ${CMAKE_CURRENT_SOURCE_DIR}/src) +set ( TERMINATOR_BUILD_DIR "${CMAKE_BINARY_DIR}/build" CACHE STRING "Set build dir") + +set ( TERMINATOR_BUILD_LIB_DIR ${TERMINATOR_BUILD_DIR}/lib) +set ( TERMINATOR_BUILD_INCLUDE_DIR ${TERMINATOR_BUILD_DIR}/include) +set ( LIBRARY_OUTPUT_PATH ${TERMINATOR_BUILD_LIB_DIR} CACHE PATH "Build directory" FORCE) + +include_directories(${TERMINATOR_INCLUDE}) + +set(SOURCE_LIBTERMINATOR + ${TERMINATOR_SRC}/terminator.cc + ${TERMINATOR_SRC}/terminator_classifier_base.cc + ${TERMINATOR_SRC}/terminator_classifier_bwinnow.cc + ${TERMINATOR_SRC}/terminator_classifier_hit.cc + ${TERMINATOR_SRC}/terminator_classifier_lr.cc + ${TERMINATOR_SRC}/terminator_classifier_nb.cc + ${TERMINATOR_SRC}/terminator_classifier_nsnb.cc + ${TERMINATOR_SRC}/terminator_classifier_owv.cc + ${TERMINATOR_SRC}/terminator_classifier_pa.cc + ${TERMINATOR_SRC}/terminator_classifier_pam.cc + ${TERMINATOR_SRC}/terminator_classifier_winnow.cc +) + + + +add_library( + ${PROJECT_NAME}_obj + OBJECT + ${SOURCE_LIBTERMINATOR} +) + +add_library( + ${PROJECT_NAME}_static + STATIC + $ +) + +add_library( + ${PROJECT_NAME} + SHARED + $ +) + +target_link_libraries(${PROJECT_NAME} ${KYOTOCABINET_LIBRARIES}) +target_link_libraries(${PROJECT_NAME}_static PUBLIC ${KYOTOCABINET_LIBRARIES}) + +install(TARGETS ${PROJECT_NAME} ${PROJECT_NAME}_static DESTINATION lib) +install(FILES ${TERMINATOR_INCLUDE}/terminator_classifier_base.h + ${TERMINATOR_INCLUDE}/terminator_classifier_bwinnow.h + ${TERMINATOR_INCLUDE}/terminator_classifier_hit.h + ${TERMINATOR_INCLUDE}/terminator_classifier_lr.h + ${TERMINATOR_INCLUDE}/terminator_classifier_nb.h + ${TERMINATOR_INCLUDE}/terminator_classifier_nsnb.h + ${TERMINATOR_INCLUDE}/terminator_classifier_owv.h + ${TERMINATOR_INCLUDE}/terminator_classifier_pa.h + ${TERMINATOR_INCLUDE}/terminator_classifier_pam.h + ${TERMINATOR_INCLUDE}/terminator_classifier_winnow.h + ${TERMINATOR_INCLUDE}/terminator_common.h + ${TERMINATOR_INCLUDE}/terminator.h + + DESTINATION include) + diff --git a/include/terminator.h b/include/terminator.h new file mode 100644 index 0000000..22b9aef --- /dev/null +++ b/include/terminator.h @@ -0,0 +1,50 @@ +// +// terminator.h +// terminator +// +// @author freiz +// @email freizsu@gmail.com + + +#ifndef terminator_terminator_h +#define terminator_terminator_h + +#include +#include +#include +#include +#include +#include + +#include "terminator_classifier_owv.h" + +#define MAX_READ_LEN (3000) +#define NGRAM (4) + +class Terminator { + private: + static const node DEFALULT_NODE_VALUE; + static const std::string IDENTIFIER_CLASSIFIER_WEIGHTS; + static const std::string IDENTIFIER_TOTAL_SPAM; + static const std::string IDENTIFIER_TOTAL_HAM; + + std::string db_path_; + size_t mem_cache_; + kyotocabinet::HashDB db_; + TerminatorClassifierBase* classifier_; + ptr_node cache_node_; + double classifier_weights_[CLASSIFIER_NUMBER]; + + bool InitDB(std::string db_path, size_t mem_cache); + void PrepareMetaData(); + void SaveWeights(std::map& weights); + void Vectorization(std::string email_content, std::map& weights); + public: + Terminator(std::string db_path, size_t mem_cache); + ~Terminator(); + double Predict(std::string email_content); + void Train(std::string email_content, bool is_spam); +}; + + +#endif diff --git a/include/terminator_classifier_base.h b/include/terminator_classifier_base.h new file mode 100644 index 0000000..4fb592c --- /dev/null +++ b/include/terminator_classifier_base.h @@ -0,0 +1,20 @@ +// +// terminator_classifier_base.h +// terminator +// + +#ifndef terminator_terminator_classifier_base_h +#define terminator_terminator_classifier_base_h + +#include "terminator_common.h" + +class TerminatorClassifierBase { + public: + static unsigned long long TotalSpam; + static unsigned long long TotalHam; + static const double CLASSIFIER_THRESHOLD; + virtual void Train(std::map& weights, bool is_spam) = 0; + virtual double Predict(std::map& weights) = 0; + virtual ~TerminatorClassifierBase() = default; +}; +#endif diff --git a/include/terminator_classifier_bwinnow.h b/include/terminator_classifier_bwinnow.h new file mode 100644 index 0000000..2579d64 --- /dev/null +++ b/include/terminator_classifier_bwinnow.h @@ -0,0 +1,34 @@ +// +// terminator_classifier_bwinnow.h +// terminator +// + +#ifndef terminator_terminator_classifier_bwinnow_h +#define terminator_terminator_classifier_bwinnow_h + +#include "terminator_classifier_base.h" + +class TerminatorClassifierBWinnow: public TerminatorClassifierBase { + private: + static const double DEFAULT_BWINNOW_ALPHA; + static const double DEFAULT_BWINNOW_BETA; + static const double DEFAULT_BWINNOW_SHIFT; + static const double DEFAULT_BWINNOW_THRESHOLD; + static const double DEFAULT_BWINNOW_THICKNESS; + static const double DEFAULT_BWINNOW_MAX_ITERATIONS; + + double bwinnow_alpha_; + double bwinnow_beta_; + double bwinnow_shift_; + double bwinnow_threshold_; + double bwinnow_thickness_; + double bwinnow_max_iterations_; + + public: + TerminatorClassifierBWinnow(); + virtual double Predict(std::map& weights); + virtual void Train(std::map& weights, bool is_spam); +}; + + +#endif diff --git a/include/terminator_classifier_hit.h b/include/terminator_classifier_hit.h new file mode 100644 index 0000000..fee621f --- /dev/null +++ b/include/terminator_classifier_hit.h @@ -0,0 +1,36 @@ +// +// terminator_classifier_hit.h +// terminator +// + +#ifndef terminator_terminator_classifier_hit_h +#define terminator_terminator_classifier_hit_h + +#include "terminator_classifier_base.h" + +class TerminatorClassifierHIT: public TerminatorClassifierBase { + + private: + + static const double DEFAULT_HIT_RATE; + static const double DEFAULT_HIT_SHIFT; + static const double DEFAULT_HIT_THICKNESS; + static const double DEFAULT_HIT_SMOOTH; + static const int DEFAULT_HIT_MAX_ITERATIONS; + + double hit_rate_; + double hit_shift_; + double hit_thickness_; + double hit_smooth_; + int hit_max_iterations_; + + public: + + TerminatorClassifierHIT(); + virtual void Train(std::map& weights, bool is_spam); + virtual double Predict(std::map& weights); + +}; + + +#endif diff --git a/include/terminator_classifier_lr.h b/include/terminator_classifier_lr.h new file mode 100644 index 0000000..e9ef7c9 --- /dev/null +++ b/include/terminator_classifier_lr.h @@ -0,0 +1,31 @@ +// +// terminator_classifier_lr.h +// terminator +// + +#ifndef terminator_terminator_classifier_lr_h +#define terminator_terminator_classifier_lr_h + +#include "terminator_classifier_base.h" + +class TerminatorClassifierLR: public TerminatorClassifierBase { + private: + static const double DEFAULT_LOGISTIC_LEARNING_RATE; + static const double DEFAULT_LOGISTIC_SHIFT; + static const double DEFAULT_LOGISTIC_THICKNESS; + static const double DEFAULT_LOGISTIC_MAX_ITERATIONS; + + double logistic_learning_rate_; + double logistic_shift_; + double logistic_thickness_; + double logistic_max_iterations_; + + public: + TerminatorClassifierLR(); + virtual void Train(std::map& weights, bool is_spam); + virtual double Predict(std::map& weights); + +}; + + +#endif diff --git a/include/terminator_classifier_nb.h b/include/terminator_classifier_nb.h new file mode 100644 index 0000000..33daf16 --- /dev/null +++ b/include/terminator_classifier_nb.h @@ -0,0 +1,35 @@ +// +// terminator_classifier_nb.h +// terminator +// + +#ifndef terminator_terminator_classifier_nb_h +#define terminator_terminator_classifier_nb_h + +#include "terminator_classifier_base.h" + +class TerminatorClassifierNB: public TerminatorClassifierBase { + private: + static const double DEFAULT_NB_SHIFT; + static const double DEFAULT_NB_SMOOTH; + static const double DEFAULT_NB_THICKNESS; + static const int DEFAULT_NB_INCREASING; + static const int DEFAULT_NB_MAX_ITERATIONS; + + double nb_shift_; + double nb_smooth_; + double nb_thickness_; + int nb_increasing_; + int nb_max_iterations_; + + void TrainCell(std::map& weights, bool is_spam); + + public: + TerminatorClassifierNB(); + virtual double Predict(std::map& weights); + virtual void Train(std::map& weights, bool is_spam); + +}; + + +#endif diff --git a/include/terminator_classifier_nsnb.h b/include/terminator_classifier_nsnb.h new file mode 100644 index 0000000..35e084d --- /dev/null +++ b/include/terminator_classifier_nsnb.h @@ -0,0 +1,34 @@ +// +// terminator_classifier_nsnb.h +// terminator +// + +#ifndef terminator_terminator_classifier_nsnb_h +#define terminator_terminator_classifier_nsnb_h + +#include "terminator_classifier_base.h" + +class TermiantorClassifierNSNB: public TerminatorClassifierBase { + private: + static const double DEFAULT_NSNB_SHIFT; + static const double DEFAULT_NSNB_SMOOTH; + static const double DEFAULT_NSNB_THICKNESS; + static const double DEFAULT_NSNB_LEARNING_RATE; + static const double DEFAULT_NSNB_MAX_ITERATIONS; + + double nsnb_shift_; + double nsnb_smooth_; + double nsnb_thickness_; + double nsnb_learning_rate_; + double nsnb_max_iterations_; + + void TrainCell(std::map& weights, bool is_spam); + + public: + TermiantorClassifierNSNB(); + virtual double Predict(std::map& weights); + virtual void Train(std::map& weights, bool is_spam); +}; + + +#endif diff --git a/include/terminator_classifier_owv.h b/include/terminator_classifier_owv.h new file mode 100644 index 0000000..c379007 --- /dev/null +++ b/include/terminator_classifier_owv.h @@ -0,0 +1,42 @@ +// +// terminator_classifier_owv.h +// terminator +// + +#ifndef terminator_terminator_classifier_owv_h +#define terminator_terminator_classifier_owv_h + +#include "terminator_classifier_base.h" +#include "terminator_classifier_bwinnow.h" +#include "terminator_classifier_lr.h" +#include "terminator_classifier_nb.h" +#include "terminator_classifier_nsnb.h" +#include "terminator_classifier_winnow.h" +#include "terminator_classifier_pa.h" +#include "terminator_classifier_pam.h" +#include "terminator_classifier_hit.h" + +#define CLASSIFIER_NUMBER (8) + +class TerminatorClassifierOWV: public TerminatorClassifierBase { + + private: + + static const double DEFAULT_OWV_STEP; + static const unsigned DEFAULT_CLASSIFIER_NUMBER; + static const double DEFAULT_OWV_SPAM_TRADEOFF; + + double owv_step_; + double owv_spam_tradeoff; + + TerminatorClassifierBase* classifiers_[CLASSIFIER_NUMBER]; + double* weights_classifier_; + + public: + TerminatorClassifierOWV(double* weights_classifier); + ~TerminatorClassifierOWV(); + virtual void Train(std::map& weights, bool is_spam); + virtual double Predict(std::map& weights); +}; + +#endif diff --git a/include/terminator_classifier_pa.h b/include/terminator_classifier_pa.h new file mode 100644 index 0000000..96457a1 --- /dev/null +++ b/include/terminator_classifier_pa.h @@ -0,0 +1,25 @@ +// +// terminator_classifier_pa.h +// terminator +// + +#ifndef terminator_terminator_classifier_pa_h +#define terminator_terminator_classifier_pa_h + +#include "terminator_classifier_base.h" + +class TerminatorClassifierPA: public TerminatorClassifierBase { + private: + + static const double DEFAULT_PA_SHIFT; + + double pa_shift_; + + public: + + TerminatorClassifierPA(); + virtual double Predict(std::map& weights); + virtual void Train(std::map& weights, bool is_spam); +}; + +#endif diff --git a/include/terminator_classifier_pam.h b/include/terminator_classifier_pam.h new file mode 100644 index 0000000..7675c42 --- /dev/null +++ b/include/terminator_classifier_pam.h @@ -0,0 +1,30 @@ +// +// terminator_classifier_pam.h +// terminator +// + +#ifndef terminator_terminator_classifier_pam_h +#define terminator_terminator_classifier_pam_h + +#include "terminator_classifier_base.h" + +class TerminatorClassifierPAM: public TerminatorClassifierBase { + private: + + static const double DEFAULT_PAM_SHIFT; + static const double DEFAULT_PAM_LAMBDA; + static const int DEFAULT_PAM_MAX_ITERATIONS; + + double pam_shift_; + double pam_lambda_; + double pam_max_iterations_; + + public: + + TerminatorClassifierPAM(); + virtual double Predict(std::map& weights); + virtual void Train(std::map& weights, bool is_spam); + +}; + +#endif diff --git a/include/terminator_classifier_winnow.h b/include/terminator_classifier_winnow.h new file mode 100644 index 0000000..518dd4f --- /dev/null +++ b/include/terminator_classifier_winnow.h @@ -0,0 +1,37 @@ +// +// terminator_classifier_winnow.h +// terminator +// + +#ifndef terminator_terminator_classifier_winnow_h +#define terminator_terminator_classifier_winnow_h + +#include "terminator_classifier_base.h" + +class TerminatorClassifierWinnow: public TerminatorClassifierBase { + private: + + static const double DEFAULT_WINNOW_THRESHOLD; + static const double DEFAULT_WINNOW_SHIFT; + static const double DEFAULT_WINNOW_THICKNESS; + static const double DEFAULT_WINNOW_ALPHA; + static const double DEFAULT_WINNOW_BETA; + static const double DEFAULT_WINNOW_MAX_ITERATIONS; + + double winnow_threshold_; + double winnow_shift_; + double winnow_thickness_; + double winnow_alpha_; + double winnow_beta_; + double winnow_max_iterations_; + + public: + + TerminatorClassifierWinnow(); + virtual double Predict(std::map& weights); + virtual void Train(std::map& weights, bool is_spam); + +}; + + +#endif diff --git a/include/terminator_common.h b/include/terminator_common.h new file mode 100644 index 0000000..af035ec --- /dev/null +++ b/include/terminator_common.h @@ -0,0 +1,45 @@ +// +// terminator_common.h +// terminator +// + +#ifndef terminator_terminator_common_h +#define terminator_terminator_common_h + +#include +#include +#include + +/** + * base model of terminator + * store information of each feature + * */ +struct Node { + float logist; + float bwinnow_upper; + float bwinnow_lower; + int nsnb_spam; + int nsnb_ham; + double nsnb_confidence; + float pam; + float pa; + float winnow; + int hit_spam; + int hit_ham; + float hit; + int nb_spam; + int nb_ham; +}; + +typedef struct Node node; +typedef struct Node * ptr_node; + +inline double logist(double x) { + return 1.0 / (1.0 + exp(-x)); +} + +inline double invlogist(double x) { + return log(x / (1 - x)); +} + +#endif