From 664b5be699fc4ceac4aebe47e3437ad45151aaaa Mon Sep 17 00:00:00 2001
From: Dmitry Shulyak <yashulyak@gmail.com>
Date: Fri, 10 Mar 2017 15:52:36 +0200
Subject: [PATCH 01/20] Refactor appcontroller to work as long running
 controller

Instead of running ac-run inside of the appcontroller pod user will
have to create documented config map with several fields (concurrency, labels selector).

appcontroller process will listen to changes for that config map and run deployment graph
processing, on deletion of configmap we will terminate deployment workflow.

It will allow us to provide guarantees that workflow will be finished even if appcontroller
pod will crash.
---
 cmd/deploy.go               |  82 ++----------------------
 pkg/scheduler/controller.go | 121 ++++++++++++++++++++++++++++++++++++
 2 files changed, 125 insertions(+), 78 deletions(-)
 create mode 100644 pkg/scheduler/controller.go

diff --git a/cmd/deploy.go b/cmd/deploy.go
index 2d72bb8..fab90c7 100644
--- a/cmd/deploy.go
+++ b/cmd/deploy.go
@@ -15,14 +15,10 @@
 package cmd
 
 import (
-	"fmt"
 	"log"
 	"os"
-	"strconv"
-	"strings"
 
 	"github.com/spf13/cobra"
-	"k8s.io/client-go/pkg/labels"
 
 	"github.com/Mirantis/k8s-AppController/pkg/client"
 	"github.com/Mirantis/k8s-AppController/pkg/scheduler"
@@ -31,18 +27,6 @@ import (
 func deploy(cmd *cobra.Command, args []string) {
 	var err error
 
-	concurrency, err := cmd.Flags().GetInt("concurrency")
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	labelSelector, err := getLabelSelector(cmd)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	log.Println("Using concurrency:", concurrency)
-
 	var url string
 	if len(args) > 0 {
 		url = args[0]
@@ -55,75 +39,17 @@ func deploy(cmd *cobra.Command, args []string) {
 	if err != nil {
 		log.Fatal(err)
 	}
-
-	sel, err := labels.Parse(labelSelector)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	log.Println("Using label selector:", labelSelector)
-
-	depGraph, err := scheduler.BuildDependencyGraph(c, sel)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	log.Println("Checking for circular dependencies.")
-	cycles := scheduler.DetectCycles(depGraph)
-	if len(cycles) > 0 {
-		message := "Cycles detected, terminating:\n"
-		for _, cycle := range cycles {
-			keys := make([]string, 0, len(cycle))
-			for _, vertex := range cycle {
-				keys = append(keys, vertex.Key())
-			}
-			message = fmt.Sprintf("%sCycle: %s\n", message, strings.Join(keys, ", "))
-		}
-
-		log.Fatal(message)
-	} else {
-		log.Println("No cycles detected.")
-	}
+	controller := scheduler.NewDeploymentController(c)
 	stopChan := make(chan struct{})
-	scheduler.Create(depGraph, concurrency, stopChan)
-
-	log.Println("Done")
-
-}
-
-func getLabelSelector(cmd *cobra.Command) (string, error) {
-	labelSelector, err := cmd.Flags().GetString("label")
-	if labelSelector == "" {
-		labelSelector = os.Getenv("KUBERNETES_AC_LABEL_SELECTOR")
-	}
-	return labelSelector, err
+	controller.Run(stopChan)
 }
 
 // InitRunCommand returns cobra command for performing AppController graph deployment
 func InitRunCommand() (*cobra.Command, error) {
-	run := &cobra.Command{
+	return &cobra.Command{
 		Use:   "run",
 		Short: "Start deployment of AppController graph",
 		Long:  "Start deployment of AppController graph",
 		Run:   deploy,
-	}
-
-	var labelSelector string
-	run.Flags().StringVarP(&labelSelector, "label", "l", "", "Label selector. Overrides KUBERNETES_AC_LABEL_SELECTOR env variable in AppController pod.")
-
-	concurrencyString := os.Getenv("KUBERNETES_AC_CONCURRENCY")
-
-	var err error
-	var concurrencyDefault int
-	if len(concurrencyString) > 0 {
-		concurrencyDefault, err = strconv.Atoi(concurrencyString)
-		if err != nil {
-			log.Printf("KUBERNETES_AC_CONCURRENCY is set to '%s' but it does not look like an integer: %v",
-				concurrencyString, err)
-			concurrencyDefault = 0
-		}
-	}
-	var concurrency int
-	run.Flags().IntVarP(&concurrency, "concurrency", "c", concurrencyDefault, "concurrency")
-	return run, err
+	}, nil
 }
diff --git a/pkg/scheduler/controller.go b/pkg/scheduler/controller.go
new file mode 100644
index 0000000..de6c3ba
--- /dev/null
+++ b/pkg/scheduler/controller.go
@@ -0,0 +1,121 @@
+// Copyright 2017 Mirantis
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package scheduler
+
+import (
+	"fmt"
+	"log"
+	"strconv"
+	"strings"
+
+	"github.com/Mirantis/k8s-AppController/pkg/client"
+
+	"k8s.io/client-go/pkg/api"
+	"k8s.io/client-go/pkg/api/v1"
+	"k8s.io/client-go/pkg/fields"
+	"k8s.io/client-go/pkg/labels"
+	"k8s.io/client-go/pkg/runtime"
+	"k8s.io/client-go/pkg/watch"
+	"k8s.io/client-go/tools/cache"
+)
+
+const (
+	controlName    = "AppcontrollerDeployment"
+	concurrencyKey = "concurrency"
+	selector       = "selector"
+)
+
+func singleObjectOpts() v1.ListOptions {
+	return v1.ListOptions{
+		FieldSelector: fields.OneTermEqualSelector("metadata.name", controlName).String(),
+	}
+}
+
+func NewDeploymentController(appc client.Interface) cache.ControllerInterface {
+	opts := singleObjectOpts()
+	var stopChan chan struct{}
+	_, cfgController := cache.NewInformer(
+		&cache.ListWatch{
+			ListFunc: func(options api.ListOptions) (runtime.Object, error) {
+				return appc.ConfigMaps().List(opts)
+			},
+			WatchFunc: func(options api.ListOptions) (watch.Interface, error) {
+				return appc.ConfigMaps().Watch(opts)
+			},
+		},
+		&api.ConfigMap{},
+		0,
+		cache.ResourceEventHandlerFuncs{
+			AddFunc: func(obj interface{}) {
+				cfg := obj.(*api.ConfigMap)
+				stopChan = make(chan struct{})
+				var concurrency int
+				concurrencyVal, ok := cfg.Data[concurrencyKey]
+				if !ok {
+					concurrency = 1
+				} else {
+					var err error
+					concurrency, err = strconv.Atoi(concurrencyVal)
+					if err != nil {
+						fmt.Println(err)
+					}
+					return
+				}
+				labelSelector, ok := cfg.Data[selector]
+				if !ok {
+					labelSelector = ""
+				}
+				depGraph := initializeDependencyGraph(appc, labelSelector)
+				fmt.Println("Running deployment with labels ", labelSelector)
+				go Create(depGraph, concurrency, stopChan)
+			},
+			DeleteFunc: func(_ interface{}) {
+				close(stopChan)
+			},
+		},
+	)
+	return cfgController
+}
+
+func initializeDependencyGraph(c client.Interface, labelSelector string) DependencyGraph {
+	sel, err := labels.Parse(labelSelector)
+	if err != nil {
+		log.Println(err)
+	}
+	log.Println("Using label selector:", labelSelector)
+
+	depGraph, err := BuildDependencyGraph(c, sel)
+	if err != nil {
+		log.Println(err)
+	}
+
+	log.Println("Checking for circular dependencies.")
+	cycles := DetectCycles(depGraph)
+	if len(cycles) > 0 {
+		message := "Cycles detected, terminating:\n"
+		for _, cycle := range cycles {
+			keys := make([]string, 0, len(cycle))
+			for _, vertex := range cycle {
+				keys = append(keys, vertex.Key())
+			}
+			message = fmt.Sprintf("%sCycle: %s\n", message, strings.Join(keys, ", "))
+		}
+
+		log.Println(message)
+	} else {
+		log.Println("No cycles detected.")
+	}
+	return depGraph
+}

From 45073b758085db8ca28a16b83e05a577afd7f8ce Mon Sep 17 00:00:00 2001
From: Dmitry Shulyak <yashulyak@gmail.com>
Date: Mon, 13 Mar 2017 12:56:34 +0200
Subject: [PATCH 02/20] Add unit test to cover controller behaviour

---
 pkg/scheduler/controller.go      | 43 +++++++++++++++----------
 pkg/scheduler/controller_test.go | 54 ++++++++++++++++++++++++++++++++
 2 files changed, 80 insertions(+), 17 deletions(-)
 create mode 100644 pkg/scheduler/controller_test.go

diff --git a/pkg/scheduler/controller.go b/pkg/scheduler/controller.go
index de6c3ba..b385b61 100644
--- a/pkg/scheduler/controller.go
+++ b/pkg/scheduler/controller.go
@@ -43,45 +43,41 @@ func singleObjectOpts() v1.ListOptions {
 	}
 }
 
-func NewDeploymentController(appc client.Interface) cache.ControllerInterface {
-	opts := singleObjectOpts()
+type CreateFunc func(depGraph DependencyGraph, concurrency int, stopChan <-chan struct{})
+
+func newDeploymentController(appc client.Interface, source cache.ListerWatcher, create CreateFunc) cache.ControllerInterface {
 	var stopChan chan struct{}
 	_, cfgController := cache.NewInformer(
-		&cache.ListWatch{
-			ListFunc: func(options api.ListOptions) (runtime.Object, error) {
-				return appc.ConfigMaps().List(opts)
-			},
-			WatchFunc: func(options api.ListOptions) (watch.Interface, error) {
-				return appc.ConfigMaps().Watch(opts)
-			},
-		},
-		&api.ConfigMap{},
+		source,
+		&v1.ConfigMap{},
 		0,
 		cache.ResourceEventHandlerFuncs{
 			AddFunc: func(obj interface{}) {
-				cfg := obj.(*api.ConfigMap)
+				log.Println("Received required configmap. Starting deployment workflow.")
+				cfg := obj.(*v1.ConfigMap)
 				stopChan = make(chan struct{})
 				var concurrency int
 				concurrencyVal, ok := cfg.Data[concurrencyKey]
 				if !ok {
-					concurrency = 1
+					concurrency = 0
 				} else {
 					var err error
 					concurrency, err = strconv.Atoi(concurrencyVal)
 					if err != nil {
-						fmt.Println(err)
+						log.Println(err)
+						return
 					}
-					return
 				}
 				labelSelector, ok := cfg.Data[selector]
 				if !ok {
 					labelSelector = ""
 				}
 				depGraph := initializeDependencyGraph(appc, labelSelector)
-				fmt.Println("Running deployment with labels ", labelSelector)
-				go Create(depGraph, concurrency, stopChan)
+				log.Println("Running deployment with labels ", labelSelector)
+				go create(depGraph, concurrency, stopChan)
 			},
 			DeleteFunc: func(_ interface{}) {
+				log.Println("Stopping deployment workflow")
 				close(stopChan)
 			},
 		},
@@ -89,6 +85,19 @@ func NewDeploymentController(appc client.Interface) cache.ControllerInterface {
 	return cfgController
 }
 
+func NewDeploymentController(appc client.Interface) cache.ControllerInterface {
+	opts := singleObjectOpts()
+	source := &cache.ListWatch{
+		ListFunc: func(options api.ListOptions) (runtime.Object, error) {
+			return appc.ConfigMaps().List(opts)
+		},
+		WatchFunc: func(options api.ListOptions) (watch.Interface, error) {
+			return appc.ConfigMaps().Watch(opts)
+		},
+	}
+	return newDeploymentController(appc, source, Create)
+}
+
 func initializeDependencyGraph(c client.Interface, labelSelector string) DependencyGraph {
 	sel, err := labels.Parse(labelSelector)
 	if err != nil {
diff --git a/pkg/scheduler/controller_test.go b/pkg/scheduler/controller_test.go
new file mode 100644
index 0000000..2a676d2
--- /dev/null
+++ b/pkg/scheduler/controller_test.go
@@ -0,0 +1,54 @@
+// Copyright 2017 Mirantis
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package scheduler
+
+import (
+	"testing"
+
+	"k8s.io/client-go/pkg/api/v1"
+	fcache "k8s.io/client-go/tools/cache/testing"
+
+	"github.com/Mirantis/k8s-AppController/pkg/mocks"
+)
+
+func TestDeploymentWorkflow(t *testing.T) {
+	c := mocks.NewClient()
+	source := fcache.NewFakeControllerSource()
+
+	waitCh := make(chan struct{})
+	var stopCh <-chan struct{}
+	f := func(_ DependencyGraph, _ int, stopChan <-chan struct{}) {
+		t.Log("Closing wait chanell")
+		waitCh <- struct{}{}
+		stopCh = stopChan
+	}
+
+	controller := newDeploymentController(c, source, f)
+	controllerCh := make(chan struct{})
+	defer close(controllerCh)
+	go controller.Run(controllerCh)
+
+	cfg := &v1.ConfigMap{
+		ObjectMeta: v1.ObjectMeta{Name: controlName},
+		Data: map[string]string{
+			concurrencyKey: "0",
+			selector:       "",
+		},
+	}
+	source.Add(cfg)
+	<-waitCh
+	source.Delete(cfg)
+	<-stopCh
+}

From b88b6d32f7cf4d71ec942fbc4c9fe0bf528b4cb7 Mon Sep 17 00:00:00 2001
From: Dmitry Shulyak <yashulyak@gmail.com>
Date: Mon, 13 Mar 2017 13:03:26 +0200
Subject: [PATCH 03/20] Change run and prepare logic in e2e tests

---
 e2e/utils/appcmanager.go | 37 +++++++++++++++----------------------
 1 file changed, 15 insertions(+), 22 deletions(-)

diff --git a/e2e/utils/appcmanager.go b/e2e/utils/appcmanager.go
index 1192aa7..1600945 100644
--- a/e2e/utils/appcmanager.go
+++ b/e2e/utils/appcmanager.go
@@ -15,7 +15,6 @@
 package utils
 
 import (
-	"os/exec"
 	"time"
 
 	. "github.com/onsi/ginkgo"
@@ -27,6 +26,12 @@ import (
 	"k8s.io/client-go/pkg/api/v1"
 )
 
+const (
+	controlName    = "AppcontrollerDeployment"
+	concurrencyKey = "concurrency"
+	selector       = "selector"
+)
+
 type AppControllerManager struct {
 	Client    client.Interface
 	Clientset *kubernetes.Clientset
@@ -37,27 +42,15 @@ type AppControllerManager struct {
 }
 
 func (a *AppControllerManager) Run() {
-	cmd := exec.Command(
-		"kubectl",
-		"--namespace",
-		a.Namespace.Name,
-		"exec",
-		"k8s-appcontroller",
-		"--",
-		"ac-run",
-		"-l",
-		"ns:"+a.Namespace.Name,
-	)
-	out, err := cmd.Output()
-	if err != nil {
-		switch err.(type) {
-		case *exec.ExitError:
-			exErr := err.(*exec.ExitError)
-			Fail(string(out) + string(exErr.Stderr))
-		default:
-			Expect(err).NotTo(HaveOccurred())
-		}
+	cfg := &v1.ConfigMap{
+		ObjectMeta: v1.ObjectMeta{Name: controlName},
+		Data: map[string]string{
+			concurrencyKey: "0",
+			selector:       "",
+		},
 	}
+	_, err := a.Client.ConfigMaps().Create(cfg)
+	Expect(err).NotTo(HaveOccurred())
 }
 
 func (a *AppControllerManager) Prepare() {
@@ -74,7 +67,7 @@ func (a *AppControllerManager) Prepare() {
 				{
 					Name:            "kubeac",
 					Image:           "mirantis/k8s-appcontroller",
-					Command:         []string{"/usr/bin/run_runit"},
+					Command:         []string{"kubeac", "run"},
 					ImagePullPolicy: v1.PullNever,
 					Env: []v1.EnvVar{
 						{

From 82ee0fbf6da9297d956bd1af8cdf4c546d6db336 Mon Sep 17 00:00:00 2001
From: Jedrzej Nowak <pigmej@gmail.com>
Date: Mon, 13 Mar 2017 12:23:00 +0100
Subject: [PATCH 04/20] Updated roadmap in readme

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 4cfd34b..a27fb16 100644
--- a/README.md
+++ b/README.md
@@ -140,7 +140,7 @@ Here is the brief list of our mid term Roadmap:
 * Joining Kubernetes Incubator
 * Cooperation with [Helm](https://github.com/kubernetes/helm) project
 * [Failure handling](https://github.com/Mirantis/k8s-AppController/blob/master/docs/research/failure-handling.md)
-* Implementation of [AppController Mysql Multi Slave research](https://github.com/Mirantis/k8s-AppController/blob/master/docs/research/lcm.md)
-* Real life examples
+* [Flows](https://github.com/Mirantis/k8s-AppController/pull/212) implementation
 * HA for AppController Pod
+* Real life examples
 * Documentation improvements

From 2dc2a0190e7787ff9fa04f2a94e97c58593ef638 Mon Sep 17 00:00:00 2001
From: Dmitry Shulyak <yashulyak@gmail.com>
Date: Mon, 13 Mar 2017 16:58:04 +0200
Subject: [PATCH 05/20] Fix misc errors

---
 cmd/get-status.go           | 8 ++++++++
 e2e/utils/appcmanager.go    | 2 +-
 pkg/scheduler/controller.go | 2 +-
 3 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/cmd/get-status.go b/cmd/get-status.go
index c66ad5c..fe2fc1f 100644
--- a/cmd/get-status.go
+++ b/cmd/get-status.go
@@ -27,6 +27,14 @@ import (
 	"k8s.io/client-go/pkg/labels"
 )
 
+func getLabelSelector(cmd *cobra.Command) (string, error) {
+	labelSelector, err := cmd.Flags().GetString("label")
+	if labelSelector == "" {
+		labelSelector = os.Getenv("KUBERNETES_AC_LABEL_SELECTOR")
+	}
+	return labelSelector, err
+}
+
 // GetStatus is a command that prints the deployment status
 func getStatus(cmd *cobra.Command, args []string) {
 	var err error
diff --git a/e2e/utils/appcmanager.go b/e2e/utils/appcmanager.go
index 1600945..ba1b6ac 100644
--- a/e2e/utils/appcmanager.go
+++ b/e2e/utils/appcmanager.go
@@ -27,7 +27,7 @@ import (
 )
 
 const (
-	controlName    = "AppcontrollerDeployment"
+	controlName    = "appcontrollerdeployment"
 	concurrencyKey = "concurrency"
 	selector       = "selector"
 )
diff --git a/pkg/scheduler/controller.go b/pkg/scheduler/controller.go
index b385b61..3631631 100644
--- a/pkg/scheduler/controller.go
+++ b/pkg/scheduler/controller.go
@@ -32,7 +32,7 @@ import (
 )
 
 const (
-	controlName    = "AppcontrollerDeployment"
+	controlName    = "appcontrollerdeployment"
 	concurrencyKey = "concurrency"
 	selector       = "selector"
 )

From 4fd659126765ceed5578a9146e3b90a84fc2503b Mon Sep 17 00:00:00 2001
From: Jedrzej Nowak <pigmej@gmail.com>
Date: Mon, 13 Mar 2017 20:49:14 +0100
Subject: [PATCH 06/20] Bump k8s on CI to 1.5.4 and 1.4.9

---
 .travis.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 0243e67..d79fb62 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -6,14 +6,14 @@ go:
   - 1.6
   - 1.7
 env:
-  - WORKING=$HOME/kube1.5 K8S_TAG=v1.5.2 PUBLISH=1
-  - WORKING=$HOME/kube1.4 K8S_TAG=v1.4.8
+  - WORKING=$HOME/kube1.5 K8S_TAG=v1.5.4 PUBLISH=1
+  - WORKING=$HOME/kube1.4 K8S_TAG=v1.4.9
 matrix:
   exclude:
     - go: 1.6
-      env: WORKING=$HOME/kube1.5 K8S_TAG=v1.5.2 PUBLISH=1
+      env: WORKING=$HOME/kube1.5 K8S_TAG=v1.5.4 PUBLISH=1
     - go: 1.7
-      env: WORKING=$HOME/kube1.4 K8S_TAG=v1.4.8
+      env: WORKING=$HOME/kube1.4 K8S_TAG=v1.4.9
 cache:
   directories:
     - $HOME/kube1.5

From 63626e4b0324d8ae97864aa0b804cd294ff63491 Mon Sep 17 00:00:00 2001
From: Dmitry Shulyak <yashulyak@gmail.com>
Date: Tue, 14 Mar 2017 11:39:05 +0200
Subject: [PATCH 07/20] Verify that deployment will succeed even if workflow
 was terminated

---
 e2e/basic_test.go        | 17 +++++++++++++++++
 e2e/utils/appcmanager.go | 20 ++++++++++++++++----
 2 files changed, 33 insertions(+), 4 deletions(-)

diff --git a/e2e/basic_test.go b/e2e/basic_test.go
index 6f33e76..3785358 100644
--- a/e2e/basic_test.go
+++ b/e2e/basic_test.go
@@ -21,6 +21,7 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 	"k8s.io/client-go/pkg/api"
+	"k8s.io/client-go/pkg/api/errors"
 	"k8s.io/client-go/pkg/api/v1"
 	"k8s.io/client-go/pkg/runtime"
 	"k8s.io/client-go/pkg/util/intstr"
@@ -99,6 +100,22 @@ var _ = Describe("Basic Suite", func() {
 		testutils.WaitForPod(framework.Clientset, framework.Namespace.Name, pod2.Name, v1.PodRunning)
 	})
 
+	It("Deployment should finish even if appcontroller pod was terminated", func() {
+		framework.DeletePod()
+		By("Creating resource definition with single pod")
+		pod1 := PodPause("pod1")
+		framework.WrapAndCreate(pod1)
+		framework.Run()
+		By("Verify that pod is consistently not found")
+		Consistently(func() bool {
+			_, err := framework.Client.Pods().Get(pod1.Name)
+			return errors.IsNotFound(err)
+		}, 5*time.Second, 1*time.Second).Should(BeTrue(), "Pod was unexpectadly created")
+		By("Recreate appcontroller pod and verify that pod was successfully created")
+		framework.Prepare()
+		testutils.WaitForPod(framework.Clientset, framework.Namespace.Name, pod1.Name, v1.PodRunning)
+	})
+
 	Describe("Failure handling - subgraph", func() {
 		var parentPod *v1.Pod
 		var childPod *v1.Pod
diff --git a/e2e/utils/appcmanager.go b/e2e/utils/appcmanager.go
index ba1b6ac..8928b29 100644
--- a/e2e/utils/appcmanager.go
+++ b/e2e/utils/appcmanager.go
@@ -23,13 +23,15 @@ import (
 	"github.com/Mirantis/k8s-AppController/pkg/client"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/pkg/api"
+	"k8s.io/client-go/pkg/api/errors"
 	"k8s.io/client-go/pkg/api/v1"
 )
 
 const (
-	controlName    = "appcontrollerdeployment"
-	concurrencyKey = "concurrency"
-	selector       = "selector"
+	controlName      = "appcontrollerdeployment"
+	concurrencyKey   = "concurrency"
+	selector         = "selector"
+	appcontrollerPod = "k8s-appcontroller"
 )
 
 type AppControllerManager struct {
@@ -53,10 +55,20 @@ func (a *AppControllerManager) Run() {
 	Expect(err).NotTo(HaveOccurred())
 }
 
+func (a *AppControllerManager) DeletePod() {
+	By("Removing pod " + appcontrollerPod)
+	err := a.Client.Pods().Delete(appcontrollerPod, nil)
+	Expect(err).NotTo(HaveOccurred())
+	Eventually(func() bool {
+		_, err := a.Client.Pods().Get(appcontrollerPod)
+		return errors.IsNotFound(err)
+	}, 20*time.Second, 1*time.Second).Should(BeTrue(), "Appcontroller pod wasn't removed in time")
+}
+
 func (a *AppControllerManager) Prepare() {
 	appControllerObj := &v1.Pod{
 		ObjectMeta: v1.ObjectMeta{
-			Name: "k8s-appcontroller",
+			Name: appcontrollerPod,
 			Annotations: map[string]string{
 				"pod.alpha.kubernetes.io/init-containers": `[{"name": "kubeac-bootstrap", "image": "mirantis/k8s-appcontroller", "imagePullPolicy": "Never", "command": ["kubeac", "bootstrap", "/opt/kubeac/manifests"]}]`,
 			},

From e0a0f003fdf6ab5e557216972c8b744fddfa80e0 Mon Sep 17 00:00:00 2001
From: Dmitry Shulyak <yashulyak@gmail.com>
Date: Tue, 14 Mar 2017 12:03:22 +0200
Subject: [PATCH 08/20] Create cfg.yaml in examples instead of ac-run

---
 examples/extended/create.sh          | 2 +-
 examples/extended/delete.sh          | 1 +
 examples/on-error-subgraph/delete.sh | 1 +
 examples/services/create.sh          | 3 +--
 examples/simple/create.sh            | 3 +--
 examples/timeout/create.sh           | 2 +-
 examples/timeout/delete.sh           | 1 +
 manifests/appcontroller.yaml         | 2 +-
 manifests/cfg.yaml                   | 7 +++++++
 9 files changed, 15 insertions(+), 7 deletions(-)
 create mode 100644 manifests/cfg.yaml

diff --git a/examples/extended/create.sh b/examples/extended/create.sh
index ce8e062..d9f865f 100755
--- a/examples/extended/create.sh
+++ b/examples/extended/create.sh
@@ -40,5 +40,5 @@ cat serviceaccount.yaml | $KUBECTL_NAME exec -i k8s-appcontroller kubeac wrap |
 
 cat deployment.yaml | $KUBECTL_NAME exec -i k8s-appcontroller kubeac wrap | $KUBECTL_NAME create -f -
 
-$KUBECTL_NAME exec k8s-appcontroller ac-run
+$KUBECTL_NAME create -f ../../manifests/cfg.yaml
 $KUBECTL_NAME logs -f k8s-appcontroller
diff --git a/examples/extended/delete.sh b/examples/extended/delete.sh
index d05c740..a848e39 100755
--- a/examples/extended/delete.sh
+++ b/examples/extended/delete.sh
@@ -40,3 +40,4 @@ cat deployment.yaml | $KUBECTL_NAME exec -i k8s-appcontroller kubeac wrap | $KUB
 cat pvc.yaml | $KUBECTL_NAME exec -i k8s-appcontroller kubeac wrap | $KUBECTL_NAME delete -f -
 
 $KUBECTL_NAME delete -f ../../manifests/appcontroller.yaml
+$KUBECTL_NAME delete -f ../../manifests/cfg.yaml
diff --git a/examples/on-error-subgraph/delete.sh b/examples/on-error-subgraph/delete.sh
index 68f61e8..684c00d 100755
--- a/examples/on-error-subgraph/delete.sh
+++ b/examples/on-error-subgraph/delete.sh
@@ -8,3 +8,4 @@ cat pod.yaml | $KUBECTL_NAME exec -i k8s-appcontroller kubeac wrap pod1 | $KUBEC
 cat pod2.yaml | $KUBECTL_NAME exec -i k8s-appcontroller kubeac wrap pod2 | $KUBECTL_NAME delete -f -
 
 $KUBECTL_NAME delete -f ../../appcontroller.yaml
+$KUBECTL_NAME delete -f ../../cfg.yaml
diff --git a/examples/services/create.sh b/examples/services/create.sh
index 272d1f9..fd85455 100755
--- a/examples/services/create.sh
+++ b/examples/services/create.sh
@@ -17,6 +17,5 @@ cat pod2.yaml | $KUBECTL_NAME exec -i k8s-appcontroller kubeac wrap | $KUBECTL_N
 cat pod3.yaml | $KUBECTL_NAME exec -i k8s-appcontroller kubeac wrap | $KUBECTL_NAME create -f -
 cat pod4.yaml | $KUBECTL_NAME exec -i k8s-appcontroller kubeac wrap | $KUBECTL_NAME create -f -
 
-$KUBECTL_NAME exec k8s-appcontroller ac-run
-
+$KUBECTL_NAME create -f ../../manifests/appcontroller.yaml
 $KUBECTL_NAME logs -f k8s-appcontroller
diff --git a/examples/simple/create.sh b/examples/simple/create.sh
index 83272f8..e6f9a90 100755
--- a/examples/simple/create.sh
+++ b/examples/simple/create.sh
@@ -30,5 +30,4 @@ echo "cat pod3.yaml | $KUBECTL_NAME exec -i k8s-appcontroller kubeac wrap pod3 |
 cat pod3.yaml | $KUBECTL_NAME exec -i k8s-appcontroller kubeac wrap | $KUBECTL_NAME create -f -
 
 echo "Here we are running appcontroller binary itself. As the log will say, it retrieves dependencies and resource definitions from the k8s cluster and creates underlying objects accordingly."
-echo "$KUBECTL_NAME exec k8s-appcontroller ac-run"
-$KUBECTL_NAME exec k8s-appcontroller ac-run
+$KUBECTL_NAME create -f ../../manifests/cfg.yaml
diff --git a/examples/timeout/create.sh b/examples/timeout/create.sh
index d6d98e0..840d22d 100755
--- a/examples/timeout/create.sh
+++ b/examples/timeout/create.sh
@@ -11,5 +11,5 @@ cat pod.yaml | $KUBECTL_NAME create -f -
 cat pod2.yaml | $KUBECTL_NAME exec -i k8s-appcontroller kubeac wrap | $KUBECTL_NAME create -f -
 cat timedout-pod.yaml | $KUBECTL_NAME create -f -
 
-$KUBECTL_NAME exec k8s-appcontroller ac-run
+$KUBECTL_NAME create -f ../../manifests/cfg.yaml
 $KUBECTL_NAME logs -f k8s-appcontroller
diff --git a/examples/timeout/delete.sh b/examples/timeout/delete.sh
index d773187..560da73 100755
--- a/examples/timeout/delete.sh
+++ b/examples/timeout/delete.sh
@@ -9,3 +9,4 @@ cat pod2.yaml | $KUBECTL_NAME exec -i k8s-appcontroller kubeac wrap | $KUBECTL_N
 cat timedout-pod.yaml | $KUBECTL_NAME delete -f -
 
 $KUBECTL_NAME delete -f ../../manifests/appcontroller.yaml
+$KUBECTL_NAME delete -f ../../manifests/cfg.yaml
diff --git a/manifests/appcontroller.yaml b/manifests/appcontroller.yaml
index 8e05417..de5fb94 100644
--- a/manifests/appcontroller.yaml
+++ b/manifests/appcontroller.yaml
@@ -10,7 +10,7 @@ spec:
   - name: kubeac
     image: mirantis/k8s-appcontroller
     imagePullPolicy: IfNotPresent
-    command:  ["/usr/bin/run_runit"]
+    command:  ["kubeac", "run"]
     env:
     - name: KUBERNETES_AC_LABEL_SELECTOR
       value: ""
diff --git a/manifests/cfg.yaml b/manifests/cfg.yaml
new file mode 100644
index 0000000..bf1b942
--- /dev/null
+++ b/manifests/cfg.yaml
@@ -0,0 +1,7 @@
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: appcontrollerdeployment
+data:
+  concurrency: 3
+  selector: ""

From a656d590f0f9e9c210b8bbc2110a4f8ecb292b38 Mon Sep 17 00:00:00 2001
From: Dmitry Shulyak <yashulyak@gmail.com>
Date: Tue, 14 Mar 2017 14:41:16 +0200
Subject: [PATCH 09/20] Ignore part of the label flag test

---
 cmd/deploy_test.go | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/cmd/deploy_test.go b/cmd/deploy_test.go
index c3ce320..1403db1 100644
--- a/cmd/deploy_test.go
+++ b/cmd/deploy_test.go
@@ -46,13 +46,10 @@ func TestLabelFlag(t *testing.T) {
 	cmd, _ := InitRunCommand()
 
 	val := "TEST_KEY=TEST_VALUE"
-	val2 := "TEST_OTHER_KEY=TEST_OTHER_VALUE"
 	os.Setenv("KUBERNETES_AC_LABEL_SELECTOR", val)
-	cmd.Flags().Parse([]string{"-l", val2})
-
 	label, _ := getLabelSelector(cmd)
 
-	if label != val2 {
-		t.Errorf("label selector should be equal to `%s`, is `%s` instead", val2, label)
+	if label != val {
+		t.Errorf("label selector should be equal to `%s`, is `%s` instead", val, label)
 	}
 }

From c8b7afe30ec9e52727ed1ce41291be64b049b1d3 Mon Sep 17 00:00:00 2001
From: Dmitry Shulyak <yashulyak@gmail.com>
Date: Thu, 16 Mar 2017 10:59:34 +0200
Subject: [PATCH 10/20] Change README to reflect current change

---
 README.md                 | 14 ++++++++++++--
 examples/simple/create.sh |  1 -
 2 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index 4cfd34b..d04a355 100644
--- a/README.md
+++ b/README.md
@@ -89,9 +89,19 @@ Load it to k8s:
 
 `kubectl create -f dependencies_file.yaml`
 
-Start appcontroller process:
+Create configmap to start appcontroller deployment workflow:
 
-`kubectl exec k8s-appcontroller ac-run`
+```
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: appcontrollerdeployment
+data:
+  # limit concurrency for appcontroller
+  concurrency: 3
+  # specify label selector
+  selector: ""
+```
 
 You can stop appcontroller process by:
 
diff --git a/examples/simple/create.sh b/examples/simple/create.sh
index e6f9a90..03eb9c3 100755
--- a/examples/simple/create.sh
+++ b/examples/simple/create.sh
@@ -29,5 +29,4 @@ cat pod2.yaml | $KUBECTL_NAME exec -i k8s-appcontroller kubeac wrap | $KUBECTL_N
 echo "cat pod3.yaml | $KUBECTL_NAME exec -i k8s-appcontroller kubeac wrap pod3 | $KUBECTL_NAME create -f -"
 cat pod3.yaml | $KUBECTL_NAME exec -i k8s-appcontroller kubeac wrap | $KUBECTL_NAME create -f -
 
-echo "Here we are running appcontroller binary itself. As the log will say, it retrieves dependencies and resource definitions from the k8s cluster and creates underlying objects accordingly."
 $KUBECTL_NAME create -f ../../manifests/cfg.yaml

From a049044d01a692f50d35d6d87b5edc9ba243af80 Mon Sep 17 00:00:00 2001
From: Dmitry Shulyak <yashulyak@gmail.com>
Date: Tue, 21 Mar 2017 16:53:07 +0200
Subject: [PATCH 11/20] Change patterns to work with kubeadm-dind

---
 scripts/import.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/import.sh b/scripts/import.sh
index 91eab56..1aa24f7 100755
--- a/scripts/import.sh
+++ b/scripts/import.sh
@@ -9,8 +9,8 @@ IMAGE_REPO=${IMAGE_REPO:-mirantis/k8s-appcontroller}
 IMAGE_TAG=${IMAGE_TAG:-latest}
 NUM_NODES=${NUM_NODES:-2}
 TMP_IMAGE_PATH=${TMP_IMAGE_PATH:-/tmp/image.tar}
-MASTER_NAME=${MASTER_NAME=}
-SLAVE_PATTERN=${SLAVE_PATTERN:-"dind_node_"}
+MASTER_NAME=${MASTER_NAME="kube-master"}
+SLAVE_PATTERN=${SLAVE_PATTERN:-"kube-node-"}
 
 
 function import-image {

From c4cf34fbef7a8992a51eb46b734284012fc57ed9 Mon Sep 17 00:00:00 2001
From: Dmitry Shulyak <yashulyak@gmail.com>
Date: Tue, 21 Mar 2017 16:53:25 +0200
Subject: [PATCH 12/20] Add cluster role binding that will add serviceaccount
 to admin group

This is mainly required for tests and not supposed to be used on
production
---
 e2e/utils/appcmanager.go |  2 ++
 e2e/utils/utils.go       | 33 +++++++++++++++++++++++++++++++++
 manifests/auth.yaml      | 11 +++++++++++
 3 files changed, 46 insertions(+)
 create mode 100644 manifests/auth.yaml

diff --git a/e2e/utils/appcmanager.go b/e2e/utils/appcmanager.go
index 8928b29..91a74fe 100644
--- a/e2e/utils/appcmanager.go
+++ b/e2e/utils/appcmanager.go
@@ -109,6 +109,7 @@ func (a *AppControllerManager) Prepare() {
 func (a *AppControllerManager) BeforeEach() {
 	var err error
 	a.Clientset, err = KubeClient()
+	AddServiceAccountToAdmins(a.Clientset)
 	Expect(err).NotTo(HaveOccurred())
 	By("Creating namespace and initializing test framework")
 	namespaceObj := &v1.Namespace{
@@ -148,6 +149,7 @@ func (a *AppControllerManager) AfterEach() {
 		err := a.Client.Dependencies().Delete(dep.Name, nil)
 		Expect(err).NotTo(HaveOccurred())
 	}
+	RemoveServiceAccountFromAdmins(a.Clientset)
 }
 
 func NewAppControllerManager() *AppControllerManager {
diff --git a/e2e/utils/utils.go b/e2e/utils/utils.go
index 66b9165..39757ae 100644
--- a/e2e/utils/utils.go
+++ b/e2e/utils/utils.go
@@ -31,6 +31,11 @@ import (
 	"strings"
 
 	"github.com/Mirantis/k8s-AppController/pkg/client"
+	"k8s.io/client-go/pkg/apis/rbac/v1alpha1"
+)
+
+const (
+	rbacServiceAccountAdmin = "system:serviceaccount-admin"
 )
 
 type TContext struct {
@@ -45,6 +50,34 @@ func SkipIf14() {
 	}
 }
 
+// AddServiceAccountToAdmins will add system:serviceaccounts to cluster-admin ClusterRole
+func AddServiceAccountToAdmins(c kubernetes.Interface) {
+	if strings.Contains(TestContext.Version, "1.6") {
+		return
+	}
+	roleBinding := &v1alpha1.ClusterRoleBinding{
+		ObjectMeta: v1.ObjectMeta{
+			Name: rbacServiceAccountAdmin,
+		},
+		Subjects: []v1alpha1.Subject{{
+			Kind: "Group",
+			Name: "system:serviceaccounts",
+		}},
+		RoleRef: v1alpha1.RoleRef{
+			Kind:     "ClusterRole",
+			Name:     "cluster-admin",
+			APIGroup: "rbac.authorization.k8s.io",
+		},
+	}
+	_, err := c.Rbac().ClusterRoleBindings().Create(roleBinding)
+	Expect(err).NotTo(HaveOccurred(), "Wasnt able to create role binding for serviceaccounts")
+}
+
+func RemoveServiceAccountFromAdmins(c kubernetes.Interface) {
+	err := c.Rbac().ClusterRoleBindings().Delete(rbacServiceAccountAdmin, nil)
+	Expect(err).NotTo(HaveOccurred(), "Failed to remove serviceaccount from admin group")
+}
+
 var TestContext = TContext{}
 
 var url string
diff --git a/manifests/auth.yaml b/manifests/auth.yaml
new file mode 100644
index 0000000..973df2d
--- /dev/null
+++ b/manifests/auth.yaml
@@ -0,0 +1,11 @@
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1alpha1
+metadata:
+  name: system:serviceaccounts
+subjects:
+- kind: Group
+  name: system:serviceaccounts
+roleRef:
+  kind: ClusterRole
+  name: cluster-admin
+  apiGroup: rbac.authorization.k8s.io
\ No newline at end of file

From 53a2cac27ce83875788b1bad3eeee6a003b27f27 Mon Sep 17 00:00:00 2001
From: Dmitry Shulyak <yashulyak@gmail.com>
Date: Tue, 21 Mar 2017 17:04:23 +0200
Subject: [PATCH 13/20] Refactor CI scripts to use kubeadm-dind

---
 .travis.yml             | 15 +++++++--------
 Makefile                | 18 ++++++------------
 e2e/utils/utils.go      |  9 +++++++--
 scripts/checkout_k8s.sh | 28 ----------------------------
 scripts/dind_down.sh    |  7 -------
 scripts/prepare_dind.sh | 24 ------------------------
 6 files changed, 20 insertions(+), 81 deletions(-)
 delete mode 100755 scripts/checkout_k8s.sh
 delete mode 100755 scripts/dind_down.sh
 delete mode 100755 scripts/prepare_dind.sh

diff --git a/.travis.yml b/.travis.yml
index d79fb62..ce76284 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -6,18 +6,17 @@ go:
   - 1.6
   - 1.7
 env:
-  - WORKING=$HOME/kube1.5 K8S_TAG=v1.5.4 PUBLISH=1
-  - WORKING=$HOME/kube1.4 K8S_TAG=v1.4.9
+  - K8S_VERSION=v1.6 PUBLISH=1
+  - K8S_VERSION=v1.5
+  - K8S_VERSION=v1.4
 matrix:
   exclude:
     - go: 1.6
-      env: WORKING=$HOME/kube1.5 K8S_TAG=v1.5.4 PUBLISH=1
+      env: K8S_VERSION=v1.6 PUBLISH=1
+    - go: 1.6
+      env: K8S_VERSION=v1.5
     - go: 1.7
-      env: WORKING=$HOME/kube1.4 K8S_TAG=v1.4.9
-cache:
-  directories:
-    - $HOME/kube1.5
-    - $HOME/kube1.4
+      env: K8S_VERSION=v1.4
 script:
   - go get github.com/Masterminds/glide
   - make test
diff --git a/Makefile b/Makefile
index ed48d48..3ba3c25 100644
--- a/Makefile
+++ b/Makefile
@@ -1,9 +1,7 @@
 
 IMAGE_REPO ?= mirantis/k8s-appcontroller
-WORKING ?= ~/testappcontroller
-K8S_SOURCE_LOCATION = .k8s-source
 K8S_CLUSTER_MARKER = .k8s-cluster
-K8S_TAG ?= v1.5.2
+K8S_VERSION ?= v1.5
 
 .PHONY: docker
 docker: kubeac Makefile
@@ -24,12 +22,12 @@ docker-publish:
 
 .PHONY: img-in-dind
 img-in-dind: docker $(K8S_CLUSTER_MARKER)
-	IMAGE_REPO=$(IMAGE_REPO) bash scripts/import.sh
+	IMAGE_REPO=$(IMAGE_REPO) ./scripts/import.sh
 
 .PHONY: e2e
 e2e: $(K8S_CLUSTER_MARKER) img-in-dind
 	go test -c -o e2e.test ./e2e/
-	PATH=$(PATH):$(WORKING)/kubernetes/_output/bin/ ./e2e.test --cluster-url=http://0.0.0.0:8888 --k8s-version=$(K8S_TAG)
+	./e2e.test --cluster-url=http://0.0.0.0:8080 --k8s-version=$(K8S_VERSION)
 
 .PHONY: clean-all
 clean-all: clean clean-k8s
@@ -42,14 +40,10 @@ clean:
 
 .PHONY: clean-k8s
 clean-k8s:
-	WORKING=$(WORKING) scripts/dind_down.sh
-	<$(K8S_SOURCE_LOCATION) xargs rm -rf
-	-rm $(K8S_SOURCE_LOCATION)
+	 ./kubeadm-dind-cluster/fixed/dind-cluster-$(K8S_VERSION).sh {down,clean}
 	-rm $(K8S_CLUSTER_MARKER)
 
-$(K8S_SOURCE_LOCATION):
-	WORKING=$(WORKING) scripts/checkout_k8s.sh > $(K8S_SOURCE_LOCATION)
-
 $(K8S_CLUSTER_MARKER): $(K8S_SOURCE_LOCATION)
-	WORKING=$(WORKING) ./scripts/prepare_dind.sh
+	git clone https://github.com/Mirantis/kubeadm-dind-cluster.git 
+	./kubeadm-dind-cluster/fixed/dind-cluster-$(K8S_VERSION).sh up
 	touch $(K8S_CLUSTER_MARKER)
diff --git a/e2e/utils/utils.go b/e2e/utils/utils.go
index 39757ae..c05faf0 100644
--- a/e2e/utils/utils.go
+++ b/e2e/utils/utils.go
@@ -55,6 +55,7 @@ func AddServiceAccountToAdmins(c kubernetes.Interface) {
 	if strings.Contains(TestContext.Version, "1.6") {
 		return
 	}
+	By("Adding service account group to cluster-admin role")
 	roleBinding := &v1alpha1.ClusterRoleBinding{
 		ObjectMeta: v1.ObjectMeta{
 			Name: rbacServiceAccountAdmin,
@@ -70,12 +71,16 @@ func AddServiceAccountToAdmins(c kubernetes.Interface) {
 		},
 	}
 	_, err := c.Rbac().ClusterRoleBindings().Create(roleBinding)
-	Expect(err).NotTo(HaveOccurred(), "Wasnt able to create role binding for serviceaccounts")
+	Expect(err).NotTo(HaveOccurred(), "Failed to create role binding for serviceaccounts")
 }
 
 func RemoveServiceAccountFromAdmins(c kubernetes.Interface) {
+	if strings.Contains(TestContext.Version, "1.6") {
+		return
+	}
+	By("Remowing service account group from cluster-admin role")
 	err := c.Rbac().ClusterRoleBindings().Delete(rbacServiceAccountAdmin, nil)
-	Expect(err).NotTo(HaveOccurred(), "Failed to remove serviceaccount from admin group")
+	Expect(err).NotTo(HaveOccurred(), "Failed to remove serviceaccount from cluster-admin role")
 }
 
 var TestContext = TContext{}
diff --git a/scripts/checkout_k8s.sh b/scripts/checkout_k8s.sh
deleted file mode 100755
index 93c168a..0000000
--- a/scripts/checkout_k8s.sh
+++ /dev/null
@@ -1,28 +0,0 @@
-#!/bin/bash
-
-set -o errexit
-set -o nounset
-set -o pipefail
-set -o xtrace
-
-WORKING=${WORKING:-`mktemp -d`}
-K8S_TAG=${K8S_TAG:-v1.5.2}
-DIND_COMPATIBLE_COMMIT=${DIND_COMPATIBLE_COMMIT:-897ad95a8e0e1fe674ff81533d4198a3cecee41e}
-mkdir -p $WORKING
-
-pushd $WORKING &> /dev/null
-if [ ! -d kubernetes ]; then
-  git clone --branch $K8S_TAG --depth 1 --single-branch https://github.com/kubernetes/kubernetes.git
-fi
-pushd kubernetes &> /dev/null
-if [ ! -d dind ]; then 
-  git clone --single-branch https://github.com/sttts/kubernetes-dind-cluster.git dind
-fi
-go get -u github.com/jteeuwen/go-bindata/go-bindata
-pushd dind &> /dev/null
-git checkout $DIND_COMPATIBLE_COMMIT
-popd &> /dev/null
-popd &> /dev/null
-popd &> /dev/null
-
-echo $WORKING
diff --git a/scripts/dind_down.sh b/scripts/dind_down.sh
deleted file mode 100755
index 0eb5d30..0000000
--- a/scripts/dind_down.sh
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/bin/bash
-
-set -o errexit
-
-pushd $WORKING/kubernetes/
-./dind/dind-down-cluster.sh
-popd
diff --git a/scripts/prepare_dind.sh b/scripts/prepare_dind.sh
deleted file mode 100755
index 6a52481..0000000
--- a/scripts/prepare_dind.sh
+++ /dev/null
@@ -1,24 +0,0 @@
-#!/bin/bash
-
-set -o errexit
-set -o nounset
-set -o pipefail
-set -o xtrace
-
-function ensure-build {
-  local name=$1
-  if [ ! -f _output/bin/$name ]; then
-    make WHAT="cmd/$name"
-  fi
-}
-
-function prepare-dind {
-  pushd $WORKING/kubernetes
-  ensure-build hyperkube
-  ensure-build kubectl
-  ./dind/dind-down-cluster.sh
-  ./dind/dind-up-cluster.sh
-  popd
-}
-
-prepare-dind

From 26e604b6fba72e352e405ce39c4c151877a3290b Mon Sep 17 00:00:00 2001
From: Dmitry Shulyak <yashulyak@gmail.com>
Date: Wed, 22 Mar 2017 10:10:45 +0200
Subject: [PATCH 14/20] Change method of comparison for kubernetes version

---
 Makefile           |  7 ++++---
 e2e/utils/utils.go | 27 +++++++++++++++++++++------
 2 files changed, 25 insertions(+), 9 deletions(-)

diff --git a/Makefile b/Makefile
index 3ba3c25..9468b25 100644
--- a/Makefile
+++ b/Makefile
@@ -40,10 +40,11 @@ clean:
 
 .PHONY: clean-k8s
 clean-k8s:
-	 ./kubeadm-dind-cluster/fixed/dind-cluster-$(K8S_VERSION).sh {down,clean}
+	./kubeadm-dind-cluster/fixed/dind-cluster-$(K8S_VERSION).sh down
+	./kubeadm-dind-cluster/fixed/dind-cluster-$(K8S_VERSION).sh clean
 	-rm $(K8S_CLUSTER_MARKER)
 
-$(K8S_CLUSTER_MARKER): $(K8S_SOURCE_LOCATION)
-	git clone https://github.com/Mirantis/kubeadm-dind-cluster.git 
+$(K8S_CLUSTER_MARKER):
+	if [ ! -d "kubeadm-dind-cluster" ]; then git clone https://github.com/Mirantis/kubeadm-dind-cluster.git; fi
 	./kubeadm-dind-cluster/fixed/dind-cluster-$(K8S_VERSION).sh up
 	touch $(K8S_CLUSTER_MARKER)
diff --git a/e2e/utils/utils.go b/e2e/utils/utils.go
index c05faf0..31162ed 100644
--- a/e2e/utils/utils.go
+++ b/e2e/utils/utils.go
@@ -18,6 +18,8 @@ import (
 	"flag"
 	"fmt"
 	"io"
+	"strconv"
+	"strings"
 
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/pkg/api/errors"
@@ -28,8 +30,6 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
-	"strings"
-
 	"github.com/Mirantis/k8s-AppController/pkg/client"
 	"k8s.io/client-go/pkg/apis/rbac/v1alpha1"
 )
@@ -44,15 +44,14 @@ type TContext struct {
 }
 
 func SkipIf14() {
-	// TODO semver compare
 	if strings.Contains(TestContext.Version, "1.4") {
-		Skip("This test is disabled on kubernetes of version 1.4")
+		Skip("This test is disabled on kubernetes of versions 1.4")
 	}
 }
 
 // AddServiceAccountToAdmins will add system:serviceaccounts to cluster-admin ClusterRole
 func AddServiceAccountToAdmins(c kubernetes.Interface) {
-	if strings.Contains(TestContext.Version, "1.6") {
+	if IsVersionOlderThan16() {
 		return
 	}
 	By("Adding service account group to cluster-admin role")
@@ -75,7 +74,7 @@ func AddServiceAccountToAdmins(c kubernetes.Interface) {
 }
 
 func RemoveServiceAccountFromAdmins(c kubernetes.Interface) {
-	if strings.Contains(TestContext.Version, "1.6") {
+	if IsVersionOlderThan16() {
 		return
 	}
 	By("Remowing service account group from cluster-admin role")
@@ -93,6 +92,22 @@ func init() {
 	flag.StringVar(&url, "cluster-url", "http://127.0.0.1:8080", "apiserver address to use with restclient")
 }
 
+func SanitizedVersion() float64 {
+	var dirtyVersion string
+	if strings.HasPrefix(TestContext.Version, "v") {
+		dirtyVersion = TestContext.Version[1:]
+	} else {
+		dirtyVersion = TestContext.Version
+	}
+	version, err := strconv.ParseFloat(dirtyVersion, 64)
+	Expect(err).NotTo(HaveOccurred())
+	return version
+}
+
+func IsVersionOlderThan16() bool {
+	return SanitizedVersion() < 1.6
+}
+
 func Logf(format string, a ...interface{}) {
 	fmt.Fprintf(GinkgoWriter, format, a...)
 }

From 11bbd1ccce54e64e60e96683016771b479d19d22 Mon Sep 17 00:00:00 2001
From: Dmitry Shulyak <yashulyak@gmail.com>
Date: Wed, 22 Mar 2017 11:42:15 +0200
Subject: [PATCH 15/20] Address review comments

---
 Makefile            | 1 -
 manifests/auth.yaml | 2 +-
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index 9468b25..cc29a86 100644
--- a/Makefile
+++ b/Makefile
@@ -40,7 +40,6 @@ clean:
 
 .PHONY: clean-k8s
 clean-k8s:
-	./kubeadm-dind-cluster/fixed/dind-cluster-$(K8S_VERSION).sh down
 	./kubeadm-dind-cluster/fixed/dind-cluster-$(K8S_VERSION).sh clean
 	-rm $(K8S_CLUSTER_MARKER)
 
diff --git a/manifests/auth.yaml b/manifests/auth.yaml
index 973df2d..d8b3edb 100644
--- a/manifests/auth.yaml
+++ b/manifests/auth.yaml
@@ -8,4 +8,4 @@ subjects:
 roleRef:
   kind: ClusterRole
   name: cluster-admin
-  apiGroup: rbac.authorization.k8s.io
\ No newline at end of file
+  apiGroup: rbac.authorization.k8s.io

From 6ecfb1ba7c304ff650c0321dcd5a96ad28c59607 Mon Sep 17 00:00:00 2001
From: Dmitry Shulyak <yashulyak@gmail.com>
Date: Wed, 22 Mar 2017 15:03:29 +0200
Subject: [PATCH 16/20] Make job with kube 1.5 authoritative

---
 .travis.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index ce76284..b015598 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -6,15 +6,15 @@ go:
   - 1.6
   - 1.7
 env:
-  - K8S_VERSION=v1.6 PUBLISH=1
-  - K8S_VERSION=v1.5
+  - K8S_VERSION=v1.6
+  - K8S_VERSION=v1.5 PUBLISH=1
   - K8S_VERSION=v1.4
 matrix:
   exclude:
     - go: 1.6
-      env: K8S_VERSION=v1.6 PUBLISH=1
+      env: K8S_VERSION=v1.6
     - go: 1.6
-      env: K8S_VERSION=v1.5
+      env: K8S_VERSION=v1.5 PUBLISH=1
     - go: 1.7
       env: K8S_VERSION=v1.4
 script:

From 0fb60a2b9644a4cb1d70128c2dfb0049f5d47d05 Mon Sep 17 00:00:00 2001
From: Dmitry Shulyak <yashulyak@gmail.com>
Date: Thu, 23 Mar 2017 10:37:09 +0200
Subject: [PATCH 17/20] Remove all runit related scripts

---
 Dockerfile    | 4 ----
 ac-run.sh     | 3 ---
 ac-stop.sh    | 3 ---
 ac_service.sh | 7 -------
 run_runit.sh  | 4 ----
 5 files changed, 21 deletions(-)
 delete mode 100755 ac-run.sh
 delete mode 100755 ac-stop.sh
 delete mode 100755 ac_service.sh
 delete mode 100755 run_runit.sh

diff --git a/Dockerfile b/Dockerfile
index c393523..af9762f 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -7,8 +7,4 @@ RUN apk --no-cache add runit@community &&\
     touch /etc/sv/ac/down
 
 ADD manifests /opt/kubeac/manifests
-ADD ac_service.sh /etc/sv/ac/run
-ADD run_runit.sh /usr/bin/run_runit
-ADD ac-run.sh /usr/bin/ac-run
-ADD ac-stop.sh /usr/bin/ac-stop
 ADD kubeac /usr/bin/kubeac
diff --git a/ac-run.sh b/ac-run.sh
deleted file mode 100755
index dd4fbbd..0000000
--- a/ac-run.sh
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/bin/sh
-
-exec printf o >/etc/sv/ac/supervise/control
diff --git a/ac-stop.sh b/ac-stop.sh
deleted file mode 100755
index e0af1d7..0000000
--- a/ac-stop.sh
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/bin/sh
-
-exec printf d >/etc/sv/ac/supervise/control
diff --git a/ac_service.sh b/ac_service.sh
deleted file mode 100755
index ba24bd1..0000000
--- a/ac_service.sh
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/bin/sh
-
-exec 2>&1
-
-source /tmp/envvars
-
-exec /usr/bin/kubeac run
diff --git a/run_runit.sh b/run_runit.sh
deleted file mode 100755
index 91961f4..0000000
--- a/run_runit.sh
+++ /dev/null
@@ -1,4 +0,0 @@
-#!/bin/sh
-
-export > /tmp/envvars
-exec runsv /etc/sv/ac

From 98cb3e45775a04ed0d1536a7122816d3c14f1063 Mon Sep 17 00:00:00 2001
From: Alexander Arzhanov <aarzhanov@mirantis.com>
Date: Thu, 23 Mar 2017 19:46:41 +0300
Subject: [PATCH 18/20] Fixed docker publish

- Add copyright notices

- Currently, CI pushing images only with latest tag:
  https://travis-ci.org/Mirantis/k8s-AppController/jobs/203410745#L1998-L2001
  No one paid attention to this, because the "after_success" step failure
  doesn't fail build:
  https://github.com/travis-ci/travis-ci/issues/758
  And at the moment a good fix is to use this as the last entry of
  "script" step instead of "after_success" step.
---
 .travis.yml               |  3 +-
 scripts/docker_publish.sh | 84 +++++++++++++++++++++++++++++----------
 scripts/import.sh         | 44 ++++++++++++--------
 3 files changed, 91 insertions(+), 40 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index b015598..769a141 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,7 +1,7 @@
 sudo: required
 language: go
 services:
-    - docker
+  - docker
 go:
   - 1.6
   - 1.7
@@ -21,5 +21,4 @@ script:
   - go get github.com/Masterminds/glide
   - make test
   - make e2e
-after_success:
   - make docker-publish
diff --git a/scripts/docker_publish.sh b/scripts/docker_publish.sh
index 4f8b95d..5b77a27 100755
--- a/scripts/docker_publish.sh
+++ b/scripts/docker_publish.sh
@@ -1,33 +1,73 @@
 #!/bin/bash
+# Copyright 2017 Mirantis
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 
+set -o xtrace
+set -o pipefail
 set -o errexit
+set -o nounset
 
+
+TRAVIS_PULL_REQUEST_BRANCH=${TRAVIS_PULL_REQUEST_BRANCH:-}
+TRAVIS_TEST_RESULT=${TRAVIS_TEST_RESULT:-}
+TRAVIS_BRANCH=${TRAVIS_BRANCH:-}
 IMAGE_REPO=${IMAGE_REPO:-mirantis/k8s-appcontroller}
+TRAVIS_TAG=${TRAVIS_TAG:-}
 PUBLISH=${PUBLISH:-}
 
+
 function push-to-docker {
-    if [ -z "$PUBLISH" ]; then
-        echo "Publish is disabled"
-        exit 0
-    fi
-    if [ "$TRAVIS_PULL_REQUEST_BRANCH" != "" ]; then
-         echo "Processing PR $TRAVIS_PULL_REQUEST_BRANCH"
-         exit 0
-    fi
-    docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD"
-    if [ ! -z "$TRAVIS_TAG" ]; then
-        echo "Pushing with tag - $TRAVIS_TAG"
-        docker push $IMAGE_REPO:$TRAVIS_TAG
-        exit 0
-    fi
-    if [ $TRAVIS_BRANCH == "master" ]; then
-        echo "Pushing with tag - latest"
-        docker push $IMAGE_REPO:latest
-        exit 0
-    fi
-    echo "Pushing with tag - $TRAVIS_BRANCH"
-    docker push $IMAGE_REPO:$TRAVIS_BRANCH
+  if [ -z "${PUBLISH}" ]; then
+    echo "Publish is disabled."
+    exit 0
+  fi
+
+  if [ -z "${TRAVIS_TEST_RESULT}" ]; then
+    echo "TRAVIS_TEST_RESULT is not set!"
+    exit 1
+  else
+      if [ "${TRAVIS_TEST_RESULT}" -ne 0 ]; then
+        echo "Some of the previous steps ended with an errors! The build is broken!"
+        exit 1
+      fi
+  fi
+
+  if [ "${TRAVIS_PULL_REQUEST_BRANCH}" != "" ]; then
+    echo "Processing PR ${TRAVIS_PULL_REQUEST_BRANCH}"
+    exit 0
+  else
+      set +o xtrace
+      docker login -u="${DOCKER_USERNAME}" -p="${DOCKER_PASSWORD}"
+      set -o xtrace
+  fi
+
+  if [ ! -z "${TRAVIS_TAG}" ]; then
+    echo "Pushing with tag - ${TRAVIS_TAG}"
+    docker tag "${IMAGE_REPO}" "${IMAGE_REPO}":"${TRAVIS_TAG}"
+    docker push "${IMAGE_REPO}":"${TRAVIS_TAG}"
+    exit
+  fi
+
+  if [ "${TRAVIS_BRANCH}" == "master" ]; then
+    echo "Pushing with tag - latest"
+    docker push "${IMAGE_REPO}":latest
+    exit
+  fi
+
+  echo "Pushing with tag - ${TRAVIS_BRANCH}"
+  docker tag "${IMAGE_REPO}" "${IMAGE_REPO}":"${TRAVIS_BRANCH}"
+  docker push "${IMAGE_REPO}":"${TRAVIS_BRANCH}"
 }
 
 push-to-docker
-
diff --git a/scripts/import.sh b/scripts/import.sh
index 1aa24f7..8033b2c 100755
--- a/scripts/import.sh
+++ b/scripts/import.sh
@@ -1,9 +1,23 @@
 #!/bin/bash
+# Copyright 2017 Mirantis
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 
+set -o xtrace
+set -o pipefail
 set -o errexit
 set -o nounset
-set -o pipefail
-set -o xtrace
+
 
 IMAGE_REPO=${IMAGE_REPO:-mirantis/k8s-appcontroller}
 IMAGE_TAG=${IMAGE_TAG:-latest}
@@ -14,20 +28,18 @@ SLAVE_PATTERN=${SLAVE_PATTERN:-"kube-node-"}
 
 
 function import-image {
-    docker save ${IMAGE_REPO}:${IMAGE_TAG} -o "${TMP_IMAGE_PATH}"
-
-    if [ -n "$MASTER_NAME" ]; then
-        docker cp "${TMP_IMAGE_PATH}" kube-master:/image.tar
-        docker exec -ti "${MASTER_NAME}" docker load -i /image.tar
-    fi
-
-    for i in `seq 1 "${NUM_NODES}"`;
-    do
-        docker cp "${TMP_IMAGE_PATH}" "${SLAVE_PATTERN}$i":/image.tar
-        docker exec -ti "${SLAVE_PATTERN}$i" docker load -i /image.tar
-    done
-    set +o xtrace
-    echo "Finished copying docker image to dind nodes"
+  docker save -o "${TMP_IMAGE_PATH}" "${IMAGE_REPO}":"${IMAGE_TAG}"
+
+  if [ -n "${MASTER_NAME}" ]; then
+    docker cp "${TMP_IMAGE_PATH}" "${MASTER_NAME}":/image.tar
+    docker exec -ti "${MASTER_NAME}" docker load -i /image.tar
+  fi
+
+  for node in $(seq 1 "${NUM_NODES}"); do
+    docker cp "${TMP_IMAGE_PATH}" "${SLAVE_PATTERN}""${node}":/image.tar
+    docker exec -ti "${SLAVE_PATTERN}""${node}" docker load -i /image.tar
+  done
+  echo "Finished copying docker image to dind nodes"
 }
 
 import-image

From 12baf08362ab953d7c498ac6b749e25f57032c7e Mon Sep 17 00:00:00 2001
From: Alexander Arzhanov <aarzhanov@mirantis.com>
Date: Fri, 24 Mar 2017 12:58:48 +0300
Subject: [PATCH 19/20] Fixed syntax

- Fixed syntax
---
 scripts/docker_publish.sh | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/scripts/docker_publish.sh b/scripts/docker_publish.sh
index 5b77a27..cc835b1 100755
--- a/scripts/docker_publish.sh
+++ b/scripts/docker_publish.sh
@@ -37,22 +37,22 @@ function push-to-docker {
     echo "TRAVIS_TEST_RESULT is not set!"
     exit 1
   else
-      if [ "${TRAVIS_TEST_RESULT}" -ne 0 ]; then
-        echo "Some of the previous steps ended with an errors! The build is broken!"
-        exit 1
-      fi
+    if [ "${TRAVIS_TEST_RESULT}" -ne 0 ]; then
+      echo "Some of the previous steps ended with an errors! The build is broken!"
+      exit 1
+    fi
   fi
 
-  if [ "${TRAVIS_PULL_REQUEST_BRANCH}" != "" ]; then
+  if [ -n "${TRAVIS_PULL_REQUEST_BRANCH}" ]; then
     echo "Processing PR ${TRAVIS_PULL_REQUEST_BRANCH}"
     exit 0
   else
-      set +o xtrace
-      docker login -u="${DOCKER_USERNAME}" -p="${DOCKER_PASSWORD}"
-      set -o xtrace
+    set +o xtrace
+    docker login -u="${DOCKER_USERNAME}" -p="${DOCKER_PASSWORD}"
+    set -o xtrace
   fi
 
-  if [ ! -z "${TRAVIS_TAG}" ]; then
+  if [ -n "${TRAVIS_TAG}" ]; then
     echo "Pushing with tag - ${TRAVIS_TAG}"
     docker tag "${IMAGE_REPO}" "${IMAGE_REPO}":"${TRAVIS_TAG}"
     docker push "${IMAGE_REPO}":"${TRAVIS_TAG}"

From 034b6f441a00e9a92e0ead5716e2c422903db175 Mon Sep 17 00:00:00 2001
From: Jedrzej Nowak <pigmej@gmail.com>
Date: Mon, 8 May 2017 10:40:52 +0200
Subject: [PATCH 20/20] Testing k8s 1.4 on travis with go 1.7

---
 .travis.yml | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 769a141..237b732 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -6,16 +6,13 @@ go:
   - 1.6
   - 1.7
 env:
-  - K8S_VERSION=v1.6
   - K8S_VERSION=v1.5 PUBLISH=1
   - K8S_VERSION=v1.4
 matrix:
   exclude:
-    - go: 1.6
-      env: K8S_VERSION=v1.6
     - go: 1.6
       env: K8S_VERSION=v1.5 PUBLISH=1
-    - go: 1.7
+    - go: 1.6
       env: K8S_VERSION=v1.4
 script:
   - go get github.com/Masterminds/glide