From 5894b804c721fc73cc66209367bc2a1e8711da3a Mon Sep 17 00:00:00 2001 From: Yannick Payot Date: Thu, 7 Sep 2023 19:38:01 +0200 Subject: [PATCH 1/2] feat: Give dedicated cachedb dir for volume mount In order to extract the DB dump from the image we want this folder to be standalone especially in the CI --- bin/runmigration | 46 +++++++++++++++++++++--------------------- bin/runtests | 52 +++++++++++++++++++++++------------------------- 2 files changed, 48 insertions(+), 50 deletions(-) diff --git a/bin/runmigration b/bin/runmigration index ca1bc2ad..ecd1beba 100755 --- a/bin/runmigration +++ b/bin/runmigration @@ -35,7 +35,7 @@ set -e wait_postgres.sh -CACHE_DIR=/.cachedb +CACHE_DIR=/tmp/cachedb echo $CACHE_DIR @@ -45,38 +45,38 @@ CACHED_DUMP="$CACHE_DIR/odoo_sample_$VERSION.dmp" if [ "$LOAD_DB_CACHE" != "false" ]; then - # If we want to run the migration steps on top of a previous dump - # useful when odoo/VERSION was edited - if [ -n "$MIG_LOAD_VERSION_CEIL" ]; then - echo "New version - Searching for previous version dump 🔭" - if [ -d "$CACHE_DIR" ]; then - # Filter dumps of higher releases - export MAX_DUMP="$CACHE_DIR/odoo_sample_${MIG_LOAD_VERSION_CEIL}.dmp" - CACHED_DUMP=$(ls -v $CACHE_DIR/odoo_sample_*.dmp | awk '$0 < ENVIRON["MAX_DUMP"]' | tail -n1) - else - echo "No cached migration sample dump found" - fi + # If we want to run the migration steps on top of a previous dump + # useful when odoo/VERSION was edited + if [ -n "$MIG_LOAD_VERSION_CEIL" ]; then + echo "New version - Searching for previous version dump 🔭" + if [ -d "$CACHE_DIR" ]; then + # Filter dumps of higher releases + export MAX_DUMP="$CACHE_DIR/odoo_sample_${MIG_LOAD_VERSION_CEIL}.dmp" + CACHED_DUMP=$(ls -v $CACHE_DIR/odoo_sample_*.dmp | awk '$0 < ENVIRON["MAX_DUMP"]' | tail -n1) + else + echo "No cached migration sample dump found" fi + fi else - echo "Dump cache load disabled." + echo "Dump cache load disabled." fi if [ "$LOAD_DB_CACHE" != "false" -a -f "$CACHED_DUMP" ]; then - echo "🐘 🐘 Database dump ${CACHED_DUMP} found 🐘 🐘" - echo "Restore Database dump from cache 📦⮕ 🐘" - createdb -O $DB_USER $DB_NAME - psql -q -o /dev/null -f "$CACHED_DUMP" - echo "Do migration on top of restored dump" + echo "🐘 🐘 Database dump ${CACHED_DUMP} found 🐘 🐘" + echo "Restore Database dump from cache 📦⮕ 🐘" + createdb -O $DB_USER $DB_NAME + psql -q -o /dev/null -f "$CACHED_DUMP" + echo "Do migration on top of restored dump" else - echo "Do migration from scratch 🐢 🐢 🐢" + echo "Do migration from scratch 🐢 🐢 🐢" fi gosu odoo migrate # Create a dump if none exist for the current VERSION if [ "$CREATE_DB_CACHE" == "true" -a ! -f "$CACHED_DUMP" ]; then - echo "Save DB to cache $CACHED_DUMP 🐘⮕ 📦" - mkdir -p "$CACHE_DIR" - pg_dump -Fp -O -f "$CACHED_DUMP" - ls -l $CACHED_DUMP + echo "Save DB to cache $CACHED_DUMP 🐘⮕ 📦" + mkdir -p "$CACHE_DIR" + pg_dump -Fp -O -f "$CACHED_DUMP" + ls -l $CACHED_DUMP fi diff --git a/bin/runtests b/bin/runtests index 6fec8c8e..553af984 100755 --- a/bin/runtests +++ b/bin/runtests @@ -31,17 +31,16 @@ set -e wait_postgres.sh LOCAL_SRC_DIR=/odoo/local-src case "$ODOO_VERSION" in - "7.0") ODOO_BIN_PATH=/odoo/src/openerp-server ;; - "8.0"|"9.0") ODOO_BIN_PATH=/odoo/src/odoo.py ;; - *) ODOO_BIN_PATH=/odoo/src/odoo-bin ;; +"7.0") ODOO_BIN_PATH=/odoo/src/openerp-server ;; +"8.0" | "9.0") ODOO_BIN_PATH=/odoo/src/odoo.py ;; +*) ODOO_BIN_PATH=/odoo/src/odoo-bin ;; esac -CACHE_DIR=/.cachedb +CACHE_DIR=/tmp/cachedb -if [ -z $1 ] -then - LOCAL_ADDONS=$(find ${LOCAL_SRC_DIR}/* -maxdepth 0 -type d -and -not -name server_environment_files -printf "%f\n"\ - | awk -vORS=, '{print $1}'\ - | sed 's/,$/\n/') +if [ -z $1 ]; then + LOCAL_ADDONS=$(find ${LOCAL_SRC_DIR}/* -maxdepth 0 -type d -and -not -name server_environment_files -printf "%f\n" | + awk -vORS=, '{print $1}' | + sed 's/,$/\n/') else LOCAL_ADDONS=$1 fi @@ -50,34 +49,33 @@ DEPS_ADDONS=$(list_dependencies.py "$LOCAL_ADDONS") DB_NAME_TEST=${DB_NAME}_test - echo "Create database" createdb -O $DB_USER ${DB_NAME_TEST} if [[ ! -z "$SUBS_MD5" ]]; then - CACHED_DUMP="$CACHE_DIR/odoo_test_$SUBS_MD5.dmp" + CACHED_DUMP="$CACHE_DIR/odoo_test_$SUBS_MD5.dmp" fi echo "Submodule addons MD5 is: $SUBS_MD5" if [ "$LOAD_DB_CACHE" != "false" -a -f "$CACHED_DUMP" ]; then - echo "🐘 🐘 Database dump ${CACHED_DUMP} found 🐘 🐘" - echo "Restore Database dump from cache matching MD5 📦⮕ 🐘" - psql -q -o /dev/null -d $DB_NAME_TEST -f "$CACHED_DUMP" - psql -d $DB_NAME_TEST -P pager=off -c "SELECT name as installed_module FROM ir_module_module WHERE state = 'installed' ORDER BY name" + echo "🐘 🐘 Database dump ${CACHED_DUMP} found 🐘 🐘" + echo "Restore Database dump from cache matching MD5 📦⮕ 🐘" + psql -q -o /dev/null -d $DB_NAME_TEST -f "$CACHED_DUMP" + psql -d $DB_NAME_TEST -P pager=off -c "SELECT name as installed_module FROM ir_module_module WHERE state = 'installed' ORDER BY name" else - if [ "$LOAD_DB_CACHE" == "false" ]; then - echo "Dump cache load disabled." - else - echo "No cached dump found matching MD5 🐢 🐢 🐢" - fi - echo "🔨🔨 Install official/OCA modules 🔨🔨" - gosu odoo odoo --stop-after-init --workers=0 --database $DB_NAME_TEST --log-level=warn --without-demo="" --db-filter=$DB_NAME_TEST -i ${DEPS_ADDONS} - if [ "$CREATE_DB_CACHE" == "true" -a ! -z "$CACHED_DUMP" ]; then - echo "Generate dump $CACHED_DUMP into cache 🐘⮕ 📦" - mkdir -p "$CACHE_DIR" - pg_dump -Fp -d $DB_NAME_TEST -O -f "$CACHED_DUMP" - fi + if [ "$LOAD_DB_CACHE" == "false" ]; then + echo "Dump cache load disabled." + else + echo "No cached dump found matching MD5 🐢 🐢 🐢" + fi + echo "🔨🔨 Install official/OCA modules 🔨🔨" + gosu odoo odoo --stop-after-init --workers=0 --database $DB_NAME_TEST --log-level=warn --without-demo="" --db-filter=$DB_NAME_TEST -i ${DEPS_ADDONS} + if [ "$CREATE_DB_CACHE" == "true" -a ! -z "$CACHED_DUMP" ]; then + echo "Generate dump $CACHED_DUMP into cache 🐘⮕ 📦" + mkdir -p "$CACHE_DIR" + pg_dump -Fp -d $DB_NAME_TEST -O -f "$CACHED_DUMP" + fi fi echo "🔧🔧 Install local-src modules 🔧🔧" gosu odoo odoo --stop-after-init --workers=0 --database $DB_NAME_TEST --log-level=warn --without-demo="" --db-filter=$DB_NAME_TEST -i ${LOCAL_ADDONS} From d965672986402231483358790001a50b10d7b1c8 Mon Sep 17 00:00:00 2001 From: Yannick Payot Date: Thu, 7 Sep 2023 20:55:03 +0200 Subject: [PATCH 2/2] refactor: cached dumps in CI helpers --- bin/db.sh | 33 ++++++++++++ bin/runmigration | 41 +++++++-------- bin/runtests | 52 +++++++++---------- .../{__openerp__.py => __manifest__.py} | 0 test.sh | 3 -- 5 files changed, 79 insertions(+), 50 deletions(-) create mode 100755 bin/db.sh rename example/odoo/local-src/dummy_test/{__openerp__.py => __manifest__.py} (100%) diff --git a/bin/db.sh b/bin/db.sh new file mode 100755 index 00000000..e31cc45f --- /dev/null +++ b/bin/db.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +db_restore() { + DUMP=$1 + DB_NAME=$2 + + echo "Removing database '$DB_NAME'" + dropdb $DB_NAME --if-exists + echo "Create database '$DB_NAME'" + createdb -O $DB_USER $DB_NAME + + if [ -f "$DUMP" ]; then + echo "🐘 🐘 Database dump $DUMP found 🐘 🐘" + echo "Restore Database dump from $DUMP 📦⮕ 🐘" + psql -q -o /dev/null -d $DB_NAME -f "$DUMP" + psql -d $DB_NAME -P pager=off -c "SELECT name as installed_module FROM ir_module_module WHERE state = 'installed' ORDER BY name" + return 1 + else + echo "No dump found matching" + return 0 + fi +} + +db_save() { + DB_NAME=$1 + DUMP=$2 + if [ ! -z "$DUMP" ]; then + echo "Dumping $DB_NAME into $DUMP 🐘⮕ 📦" + mkdir -p $(dirname $DUMP) + pg_dump -Fp -d $DB_NAME -O -f "$DUMP" + ls $DUMP + fi +} diff --git a/bin/runmigration b/bin/runmigration index ecd1beba..38df7dfc 100755 --- a/bin/runmigration +++ b/bin/runmigration @@ -11,8 +11,6 @@ # # And finally make a database dump if none exists for current VERSION. # -# TODO: store cache on S3 to store one DB per version -# # Environment variables: # # CREATE_DB_CACHE: @@ -34,14 +32,16 @@ # steps on top of it. set -e +source db.sh wait_postgres.sh -CACHE_DIR=/tmp/cachedb -echo $CACHE_DIR +CREATE_DB_CACHE=${CREATE_DB_CACHE:=false} +LOAD_DB_CACHE=${LOAD_DB_CACHE:=true} -VERSION=$(cat /odoo/VERSION) +CACHE_DIR=${CACHE_DIR:=/tmp/cachedb} +DB_NAME=${DB_NAME:=odoo_mig} -CACHED_DUMP="$CACHE_DIR/odoo_sample_$VERSION.dmp" +VERSION=$(cat /odoo/VERSION) if [ "$LOAD_DB_CACHE" != "false" ]; then @@ -52,31 +52,30 @@ if [ "$LOAD_DB_CACHE" != "false" ]; then if [ -d "$CACHE_DIR" ]; then # Filter dumps of higher releases export MAX_DUMP="$CACHE_DIR/odoo_sample_${MIG_LOAD_VERSION_CEIL}.dmp" - CACHED_DUMP=$(ls -v $CACHE_DIR/odoo_sample_*.dmp | awk '$0 < ENVIRON["MAX_DUMP"]' | tail -n1) - else + CACHED_DUMP=$(ls -v $CACHE_DIR/odoo_sample_*.dmp || true | awk '$0 < ENVIRON["MAX_DUMP"]' | tail -n1) + fi + if [ -n "$CACHED_DUMP" ]; then echo "No cached migration sample dump found" fi fi else - echo "Dump cache load disabled." + echo "DB cache load disabled." fi -if [ "$LOAD_DB_CACHE" != "false" -a -f "$CACHED_DUMP" ]; then - echo "🐘 🐘 Database dump ${CACHED_DUMP} found 🐘 🐘" - echo "Restore Database dump from cache 📦⮕ 🐘" - createdb -O $DB_USER $DB_NAME - psql -q -o /dev/null -f "$CACHED_DUMP" - echo "Do migration on top of restored dump" -else - echo "Do migration from scratch 🐢 🐢 🐢" +if [ "$LOAD_DB_CACHE" != "false" ]; then + db_restore "$CACHED_DUMP" "$DB_NAME" + if [ $? = 1 ]; then + echo "Do migration on top of restored dump" + else + echo "Do migration from scratch 🐢 🐢 🐢" + fi fi gosu odoo migrate +DUMP="$CACHE_DIR/odoo_sample_$VERSION.dmp" # Create a dump if none exist for the current VERSION -if [ "$CREATE_DB_CACHE" == "true" -a ! -f "$CACHED_DUMP" ]; then - echo "Save DB to cache $CACHED_DUMP 🐘⮕ 📦" +if [ "$CREATE_DB_CACHE" == "true" -a ! -f $DUMP ]; then mkdir -p "$CACHE_DIR" - pg_dump -Fp -O -f "$CACHED_DUMP" - ls -l $CACHED_DUMP + db_save $DB_NAME $DUMP fi diff --git a/bin/runtests b/bin/runtests index 553af984..923621de 100755 --- a/bin/runtests +++ b/bin/runtests @@ -27,7 +27,7 @@ set -e # TODO: if we are not in TRAVIS, make a template then run tests on a copy - +source db.sh wait_postgres.sh LOCAL_SRC_DIR=/odoo/local-src case "$ODOO_VERSION" in @@ -35,7 +35,6 @@ case "$ODOO_VERSION" in "8.0" | "9.0") ODOO_BIN_PATH=/odoo/src/odoo.py ;; *) ODOO_BIN_PATH=/odoo/src/odoo-bin ;; esac -CACHE_DIR=/tmp/cachedb if [ -z $1 ]; then LOCAL_ADDONS=$(find ${LOCAL_SRC_DIR}/* -maxdepth 0 -type d -and -not -name server_environment_files -printf "%f\n" | @@ -45,43 +44,44 @@ else LOCAL_ADDONS=$1 fi -DEPS_ADDONS=$(list_dependencies.py "$LOCAL_ADDONS") +CREATE_DB_CACHE=${CREATE_DB_CACHE:=false} +LOAD_DB_CACHE=${LOAD_DB_CACHE:=true} -DB_NAME_TEST=${DB_NAME}_test +LOCAL_SRC_DIR=${LOCAL_SRC_DIR:=/odoo/local-src} +CACHE_DIR=${CACHE_DIR:=/tmp/cachedb} -echo "Create database" -createdb -O $DB_USER ${DB_NAME_TEST} +DEPS_ADDONS=$(list_dependencies.py "$LOCAL_ADDONS") + +DB_NAME=${DB_NAME:=odoo_test} if [[ ! -z "$SUBS_MD5" ]]; then - CACHED_DUMP="$CACHE_DIR/odoo_test_$SUBS_MD5.dmp" + DUMP="$CACHE_DIR/odoo_test_$SUBS_MD5.dmp" fi echo "Submodule addons MD5 is: $SUBS_MD5" -if [ "$LOAD_DB_CACHE" != "false" -a -f "$CACHED_DUMP" ]; then - echo "🐘 🐘 Database dump ${CACHED_DUMP} found 🐘 🐘" - echo "Restore Database dump from cache matching MD5 📦⮕ 🐘" - psql -q -o /dev/null -d $DB_NAME_TEST -f "$CACHED_DUMP" - psql -d $DB_NAME_TEST -P pager=off -c "SELECT name as installed_module FROM ir_module_module WHERE state = 'installed' ORDER BY name" -else - if [ "$LOAD_DB_CACHE" == "false" ]; then - echo "Dump cache load disabled." +DUMP_LOADED=0 +if [ "$LOAD_DB_CACHE" != "false" ]; then + db_restore "$DUMP" "$DB_NAME" + DUMP_LOADED=$? + if [ $DUMP_LOADED = 1 ]; then + echo "Use restored dump with official/OCA modules" else - echo "No cached dump found matching MD5 🐢 🐢 🐢" + echo "Reinstall all addons from scratch 🐢 🐢 🐢" fi +else + echo "DB cache load disabled." +fi + +if [ "$DUMP_LOADED" = 0 ]; then echo "🔨🔨 Install official/OCA modules 🔨🔨" - gosu odoo odoo --stop-after-init --workers=0 --database $DB_NAME_TEST --log-level=warn --without-demo="" --db-filter=$DB_NAME_TEST -i ${DEPS_ADDONS} - if [ "$CREATE_DB_CACHE" == "true" -a ! -z "$CACHED_DUMP" ]; then - echo "Generate dump $CACHED_DUMP into cache 🐘⮕ 📦" - mkdir -p "$CACHE_DIR" - pg_dump -Fp -d $DB_NAME_TEST -O -f "$CACHED_DUMP" + odoo --stop-after-init --workers=0 --database $DB_NAME --log-level=warn --without-demo="" --db-filter=$DB_NAME -i ${DEPS_ADDONS} + if [ "$CREATE_DB_CACHE" != "false" ]; then + db_save $DB_NAME $DUMP fi fi echo "🔧🔧 Install local-src modules 🔧🔧" -gosu odoo odoo --stop-after-init --workers=0 --database $DB_NAME_TEST --log-level=warn --without-demo="" --db-filter=$DB_NAME_TEST -i ${LOCAL_ADDONS} - -export COVERAGE_FILE=/home/odoo/.coverage -gosu odoo coverage run --source="${LOCAL_SRC_DIR}" "${ODOO_BIN_PATH}" --stop-after-init --workers=0 --database $DB_NAME_TEST --test-enable --log-level=test --log-handler=":INFO" --db-filter=$DB_NAME_TEST -u ${LOCAL_ADDONS} -dropdb ${DB_NAME_TEST} +coverage run --source="${LOCAL_SRC_DIR}" "${ODOO_BIN_PATH}" --stop-after-init --workers=0 --database $DB_NAME --test-enable --log-level=test --log-handler=":INFO" --without-demo="" --db-filter=$DB_NAME -i ${LOCAL_ADDONS} +dropdb ${DB_NAME} coverage report -m diff --git a/example/odoo/local-src/dummy_test/__openerp__.py b/example/odoo/local-src/dummy_test/__manifest__.py similarity index 100% rename from example/odoo/local-src/dummy_test/__openerp__.py rename to example/odoo/local-src/dummy_test/__manifest__.py diff --git a/test.sh b/test.sh index f95d4d6a..45cbec19 100755 --- a/test.sh +++ b/test.sh @@ -88,8 +88,6 @@ echo " post:" >>odoo/migration.yml echo " - anthem songs.install.demo::create_partners" >>odoo/migration.yml docoruncmd odoo dropdb odoodb - - echo '>>> * run unit tests with runtests' docoruntests -e LOAD_DB_CACHE="false" -e CREATE_DB_CACHE="false" @@ -99,4 +97,3 @@ docoruntests -e CREATE_DB_CACHE="true" -e SUBS_MD5=testcache echo '>>> * run unit tests with runtests and re-use a dump' docoruntests -e LOAD_DB_CACHE="true" -e SUBS_MD5=testcache docodown -