Merge pull request #231 from arangodb-helper/bug-fix/fix-upgrade-from-346

Special upgrade procedure for from 3.4.6.

Author: neunhoef

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## Changes from 0.14.4 to 0.14.5
 
+- Implement special upgrade procedure for upgrading from 3.4.6
 - Move to alpine 3.9 for Docker image
 
 ## Changes from 0.14.3 to 0.14.4

MAINTAINERS.md

@@ -4,7 +4,15 @@
 
 A subset of all tests are frequently run in Travis.
 
-To run the entire test set, run:
+To run the entire test set, set the following environment variables,
+
+  - `ARANGODB`: name of Docker image for ArangoDB to use
+  - `VERBOSE` (optional): set to `1` for more output
+  - `IP` (for Docker tests): set to a valid IP address on the local machine
+  - `TESTOPTIONS`` (optional): set to `-test.run=REGEXPTOMATCHTESTS` to
+    run only some tests
+
+Then run:
 
 ```bash
 make run-tests

client/client.go

@@ -29,6 +29,7 @@ import (
 	"io/ioutil"
 	"net/http"
 	"net/url"
+	"time"
 
 	driver "github.com/arangodb/go-driver"
 	"github.com/pkg/errors"
@@ -245,6 +246,7 @@ func (c *client) RemovePeer(ctx context.Context, id string, force bool) error {
 func (c *client) StartDatabaseUpgrade(ctx context.Context) error {
 	url := c.createURL("/database-auto-upgrade", nil)
 
+	c.client.Timeout = time.Minute * 5
 	req, err := http.NewRequest("POST", url, nil)
 	if err != nil {
 		return maskAny(err)

service/peer.go

@@ -126,6 +126,21 @@ func (p Peer) CreateCoordinatorAPI(clientBuilder ClientBuilder) (driver.Client,
 	return nil, maskAny(fmt.Errorf("Peer has no coordinator"))
 }
 
+// CreateAgentAPI creates a client for the agent of the peer
+func (p Peer) CreateAgentAPI(clientBuilder ClientBuilder) (driver.Client, error) {
+	if p.HasAgent() {
+		port := p.Port + p.PortOffset + ServerType(ServerTypeAgent).PortOffset()
+		scheme := NewURLSchemes(p.IsSecure).Browser
+		ep := fmt.Sprintf("%s://%s", scheme, net.JoinHostPort(p.Address, strconv.Itoa(port)))
+		c, err := clientBuilder([]string{ep}, ConnectionTypeDatabase)
+		if err != nil {
+			return nil, maskAny(err)
+		}
+		return c, nil
+	}
+	return nil, maskAny(fmt.Errorf("Peer has no agent"))
+}
+
 // PortRangeOverlaps returns true if the port range of this peer overlaps with a port
 // range starting at given port.
 func (p Peer) PortRangeOverlaps(otherPort int, clusterConfig ClusterConfig) bool {

service/upgrade_manager.go

@@ -32,7 +32,7 @@ import (
 
 	driver "github.com/arangodb/go-driver"
 	"github.com/arangodb/go-driver/agency"
-	"github.com/arangodb/go-upgrade-rules"
+	upgraderules "github.com/arangodb/go-upgrade-rules"
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
 	"github.com/ryanuber/columnize"
@@ -93,7 +93,7 @@ type UpgradeManagerContext interface {
 // NewUpgradeManager creates a new upgrade manager.
 func NewUpgradeManager(log zerolog.Logger, upgradeManagerContext UpgradeManagerContext) UpgradeManager {
 	return &upgradeManager{
-		log: log,
+		log:                   log,
 		upgradeManagerContext: upgradeManagerContext,
 	}
 }
@@ -252,10 +252,14 @@ func (m *upgradeManager) StartDatabaseUpgrade(ctx context.Context) error {
 	}
 
 	// Check if we can upgrade from running to binary versions
+	specialUpgradeFrom346 := false
 	for _, from := range runningDBVersions {
 		if err := upgraderules.CheckUpgradeRules(from, toVersion); err != nil {
 			return maskAny(errors.Wrap(err, "Found incompatible upgrade versions"))
 		}
+		if from.CompareTo("3.4.6") == 0 {
+			specialUpgradeFrom346 = true
+		}
 	}
 
 	// Fetch mode
@@ -300,18 +304,61 @@ func (m *upgradeManager) StartDatabaseUpgrade(ctx context.Context) error {
 		lock.Unlock(context.Background())
 	}()
 
+	m.log.Debug().Msg("Reading upgrade plan...")
 	// Check existing plan
 	plan, err := m.readUpgradePlan(ctx)
 	if err != nil && !agency.IsKeyNotFound(err) {
 		// Failed to read upgrade plan
+		m.log.Error().Msg("Failed to read upgrade plan")
 		return errors.Wrap(err, "Failed to read upgrade plan")
 	}
 
+	m.log.Debug().Msg("Checking if plan is ready...")
 	// Check plan status
 	if !plan.IsReady() {
+		m.log.Debug().Msg("Current upgrade plan has not finished yet.")
 		return maskAny(client.NewBadRequestError("Current upgrade plan has not finished yet"))
 	}
 
+	// Special measure for upgrades from 3.4.6:
+	if specialUpgradeFrom346 {
+		// Write 1000 dummy values into agency to advance the log:
+		for i := 0; i < 1000; i++ {
+			err := api.WriteKey(nil, []string{"/arangodb-helper/dummy"}, 17, 0)
+			if err != nil {
+				m.log.Error().Msg("Could not append log entries to agency.")
+				return maskAny(err)
+			}
+		}
+		m.log.Debug().Msg("Have written 1000 log entries into agency.")
+
+		// wait for the compaction to be created
+		time.Sleep(3 * time.Second)
+
+		// Repair each agent's persistent snapshots:
+		for _, p := range config.AllPeers {
+			if p.HasAgent() {
+				cli, err := p.CreateAgentAPI(m.upgradeManagerContext.CreateClient)
+				if err != nil {
+					m.log.Error().Msgf("Could not create client for agent of peer %s", p.ID)
+					return maskAny(err)
+				}
+				db, err := cli.Database(nil, "_system")
+				if err != nil {
+					m.log.Error().Msgf("Could not find _system database for agent of peer %s", p.ID)
+					return maskAny(err)
+				}
+				_, err = db.Query(nil, "FOR x IN compact LET old = x.readDB LET new = (FOR i IN 0..LENGTH(old)-1 RETURN i == 1 ? {} : old[i]) UPDATE x._key WITH {readDB: new} IN compact", nil)
+				if err != nil {
+					m.log.Error().Msgf("Could not repair agent log compaction for agent of peer %s", p.ID)
+				}
+				m.log.Debug().Msgf("Finished repair of log compaction for agent of peer %s", p.ID)
+			}
+		}
+
+		m.log.Info().Msg("Applied special update procedure for 3.4.6")
+	}
+
 	// Create upgrade plan
 	m.log.Debug().Msg("Creating upgrade plan")
 	plan = UpgradePlan{
@@ -387,8 +434,10 @@ func (m *upgradeManager) StartDatabaseUpgrade(ctx context.Context) error {
 	m.log.Debug().Msg("Writing upgrade plan")
 	overwrite := true
 	if _, err := m.writeUpgradePlan(ctx, plan, overwrite); driver.IsPreconditionFailed(err) {
+		m.log.Error().Msg("Failed to write upgrade plan because it was outdated or removed.")
 		return errors.Wrap(err, "Failed to write upgrade plan because is was outdated or removed")
 	} else if err != nil {
+		m.log.Error().Msgf("Failed to write upgrade plan %v.", err)
 		return errors.Wrap(err, "Failed to write upgrade plan")
 	}