diff --git a/.travis.yml b/.travis.yml index 0243e67..237b732 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,26 +1,21 @@ sudo: required language: go services: - - docker + - docker go: - 1.6 - 1.7 env: - - WORKING=$HOME/kube1.5 K8S_TAG=v1.5.2 PUBLISH=1 - - WORKING=$HOME/kube1.4 K8S_TAG=v1.4.8 + - K8S_VERSION=v1.5 PUBLISH=1 + - K8S_VERSION=v1.4 matrix: exclude: - go: 1.6 - env: WORKING=$HOME/kube1.5 K8S_TAG=v1.5.2 PUBLISH=1 - - go: 1.7 - env: WORKING=$HOME/kube1.4 K8S_TAG=v1.4.8 -cache: - directories: - - $HOME/kube1.5 - - $HOME/kube1.4 + env: K8S_VERSION=v1.5 PUBLISH=1 + - go: 1.6 + env: K8S_VERSION=v1.4 script: - go get github.com/Masterminds/glide - make test - make e2e -after_success: - make docker-publish diff --git a/Dockerfile b/Dockerfile index c393523..af9762f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,8 +7,4 @@ RUN apk --no-cache add runit@community &&\ touch /etc/sv/ac/down ADD manifests /opt/kubeac/manifests -ADD ac_service.sh /etc/sv/ac/run -ADD run_runit.sh /usr/bin/run_runit -ADD ac-run.sh /usr/bin/ac-run -ADD ac-stop.sh /usr/bin/ac-stop ADD kubeac /usr/bin/kubeac diff --git a/Makefile b/Makefile index ed48d48..cc29a86 100644 --- a/Makefile +++ b/Makefile @@ -1,9 +1,7 @@ IMAGE_REPO ?= mirantis/k8s-appcontroller -WORKING ?= ~/testappcontroller -K8S_SOURCE_LOCATION = .k8s-source K8S_CLUSTER_MARKER = .k8s-cluster -K8S_TAG ?= v1.5.2 +K8S_VERSION ?= v1.5 .PHONY: docker docker: kubeac Makefile @@ -24,12 +22,12 @@ docker-publish: .PHONY: img-in-dind img-in-dind: docker $(K8S_CLUSTER_MARKER) - IMAGE_REPO=$(IMAGE_REPO) bash scripts/import.sh + IMAGE_REPO=$(IMAGE_REPO) ./scripts/import.sh .PHONY: e2e e2e: $(K8S_CLUSTER_MARKER) img-in-dind go test -c -o e2e.test ./e2e/ - PATH=$(PATH):$(WORKING)/kubernetes/_output/bin/ ./e2e.test --cluster-url=http://0.0.0.0:8888 --k8s-version=$(K8S_TAG) + ./e2e.test --cluster-url=http://0.0.0.0:8080 --k8s-version=$(K8S_VERSION) .PHONY: clean-all clean-all: clean clean-k8s @@ -42,14 +40,10 @@ clean: .PHONY: clean-k8s clean-k8s: - WORKING=$(WORKING) scripts/dind_down.sh - <$(K8S_SOURCE_LOCATION) xargs rm -rf - -rm $(K8S_SOURCE_LOCATION) + ./kubeadm-dind-cluster/fixed/dind-cluster-$(K8S_VERSION).sh clean -rm $(K8S_CLUSTER_MARKER) -$(K8S_SOURCE_LOCATION): - WORKING=$(WORKING) scripts/checkout_k8s.sh > $(K8S_SOURCE_LOCATION) - -$(K8S_CLUSTER_MARKER): $(K8S_SOURCE_LOCATION) - WORKING=$(WORKING) ./scripts/prepare_dind.sh +$(K8S_CLUSTER_MARKER): + if [ ! -d "kubeadm-dind-cluster" ]; then git clone https://github.com/Mirantis/kubeadm-dind-cluster.git; fi + ./kubeadm-dind-cluster/fixed/dind-cluster-$(K8S_VERSION).sh up touch $(K8S_CLUSTER_MARKER) diff --git a/README.md b/README.md index 4cfd34b..130de0b 100644 --- a/README.md +++ b/README.md @@ -89,9 +89,19 @@ Load it to k8s: `kubectl create -f dependencies_file.yaml` -Start appcontroller process: +Create configmap to start appcontroller deployment workflow: -`kubectl exec k8s-appcontroller ac-run` +``` +kind: ConfigMap +apiVersion: v1 +metadata: + name: appcontrollerdeployment +data: + # limit concurrency for appcontroller + concurrency: 3 + # specify label selector + selector: "" +``` You can stop appcontroller process by: @@ -140,7 +150,7 @@ Here is the brief list of our mid term Roadmap: * Joining Kubernetes Incubator * Cooperation with [Helm](https://github.com/kubernetes/helm) project * [Failure handling](https://github.com/Mirantis/k8s-AppController/blob/master/docs/research/failure-handling.md) -* Implementation of [AppController Mysql Multi Slave research](https://github.com/Mirantis/k8s-AppController/blob/master/docs/research/lcm.md) -* Real life examples +* [Flows](https://github.com/Mirantis/k8s-AppController/pull/212) implementation * HA for AppController Pod +* Real life examples * Documentation improvements diff --git a/ac-run.sh b/ac-run.sh deleted file mode 100755 index dd4fbbd..0000000 --- a/ac-run.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -exec printf o >/etc/sv/ac/supervise/control diff --git a/ac-stop.sh b/ac-stop.sh deleted file mode 100755 index e0af1d7..0000000 --- a/ac-stop.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -exec printf d >/etc/sv/ac/supervise/control diff --git a/ac_service.sh b/ac_service.sh deleted file mode 100755 index ba24bd1..0000000 --- a/ac_service.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/sh - -exec 2>&1 - -source /tmp/envvars - -exec /usr/bin/kubeac run diff --git a/cmd/deploy.go b/cmd/deploy.go index 2d72bb8..fab90c7 100644 --- a/cmd/deploy.go +++ b/cmd/deploy.go @@ -15,14 +15,10 @@ package cmd import ( - "fmt" "log" "os" - "strconv" - "strings" "github.com/spf13/cobra" - "k8s.io/client-go/pkg/labels" "github.com/Mirantis/k8s-AppController/pkg/client" "github.com/Mirantis/k8s-AppController/pkg/scheduler" @@ -31,18 +27,6 @@ import ( func deploy(cmd *cobra.Command, args []string) { var err error - concurrency, err := cmd.Flags().GetInt("concurrency") - if err != nil { - log.Fatal(err) - } - - labelSelector, err := getLabelSelector(cmd) - if err != nil { - log.Fatal(err) - } - - log.Println("Using concurrency:", concurrency) - var url string if len(args) > 0 { url = args[0] @@ -55,75 +39,17 @@ func deploy(cmd *cobra.Command, args []string) { if err != nil { log.Fatal(err) } - - sel, err := labels.Parse(labelSelector) - if err != nil { - log.Fatal(err) - } - - log.Println("Using label selector:", labelSelector) - - depGraph, err := scheduler.BuildDependencyGraph(c, sel) - if err != nil { - log.Fatal(err) - } - - log.Println("Checking for circular dependencies.") - cycles := scheduler.DetectCycles(depGraph) - if len(cycles) > 0 { - message := "Cycles detected, terminating:\n" - for _, cycle := range cycles { - keys := make([]string, 0, len(cycle)) - for _, vertex := range cycle { - keys = append(keys, vertex.Key()) - } - message = fmt.Sprintf("%sCycle: %s\n", message, strings.Join(keys, ", ")) - } - - log.Fatal(message) - } else { - log.Println("No cycles detected.") - } + controller := scheduler.NewDeploymentController(c) stopChan := make(chan struct{}) - scheduler.Create(depGraph, concurrency, stopChan) - - log.Println("Done") - -} - -func getLabelSelector(cmd *cobra.Command) (string, error) { - labelSelector, err := cmd.Flags().GetString("label") - if labelSelector == "" { - labelSelector = os.Getenv("KUBERNETES_AC_LABEL_SELECTOR") - } - return labelSelector, err + controller.Run(stopChan) } // InitRunCommand returns cobra command for performing AppController graph deployment func InitRunCommand() (*cobra.Command, error) { - run := &cobra.Command{ + return &cobra.Command{ Use: "run", Short: "Start deployment of AppController graph", Long: "Start deployment of AppController graph", Run: deploy, - } - - var labelSelector string - run.Flags().StringVarP(&labelSelector, "label", "l", "", "Label selector. Overrides KUBERNETES_AC_LABEL_SELECTOR env variable in AppController pod.") - - concurrencyString := os.Getenv("KUBERNETES_AC_CONCURRENCY") - - var err error - var concurrencyDefault int - if len(concurrencyString) > 0 { - concurrencyDefault, err = strconv.Atoi(concurrencyString) - if err != nil { - log.Printf("KUBERNETES_AC_CONCURRENCY is set to '%s' but it does not look like an integer: %v", - concurrencyString, err) - concurrencyDefault = 0 - } - } - var concurrency int - run.Flags().IntVarP(&concurrency, "concurrency", "c", concurrencyDefault, "concurrency") - return run, err + }, nil } diff --git a/cmd/deploy_test.go b/cmd/deploy_test.go index c3ce320..1403db1 100644 --- a/cmd/deploy_test.go +++ b/cmd/deploy_test.go @@ -46,13 +46,10 @@ func TestLabelFlag(t *testing.T) { cmd, _ := InitRunCommand() val := "TEST_KEY=TEST_VALUE" - val2 := "TEST_OTHER_KEY=TEST_OTHER_VALUE" os.Setenv("KUBERNETES_AC_LABEL_SELECTOR", val) - cmd.Flags().Parse([]string{"-l", val2}) - label, _ := getLabelSelector(cmd) - if label != val2 { - t.Errorf("label selector should be equal to `%s`, is `%s` instead", val2, label) + if label != val { + t.Errorf("label selector should be equal to `%s`, is `%s` instead", val, label) } } diff --git a/cmd/get-status.go b/cmd/get-status.go index c66ad5c..fe2fc1f 100644 --- a/cmd/get-status.go +++ b/cmd/get-status.go @@ -27,6 +27,14 @@ import ( "k8s.io/client-go/pkg/labels" ) +func getLabelSelector(cmd *cobra.Command) (string, error) { + labelSelector, err := cmd.Flags().GetString("label") + if labelSelector == "" { + labelSelector = os.Getenv("KUBERNETES_AC_LABEL_SELECTOR") + } + return labelSelector, err +} + // GetStatus is a command that prints the deployment status func getStatus(cmd *cobra.Command, args []string) { var err error diff --git a/e2e/basic_test.go b/e2e/basic_test.go index 6f33e76..3785358 100644 --- a/e2e/basic_test.go +++ b/e2e/basic_test.go @@ -21,6 +21,7 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "k8s.io/client-go/pkg/api" + "k8s.io/client-go/pkg/api/errors" "k8s.io/client-go/pkg/api/v1" "k8s.io/client-go/pkg/runtime" "k8s.io/client-go/pkg/util/intstr" @@ -99,6 +100,22 @@ var _ = Describe("Basic Suite", func() { testutils.WaitForPod(framework.Clientset, framework.Namespace.Name, pod2.Name, v1.PodRunning) }) + It("Deployment should finish even if appcontroller pod was terminated", func() { + framework.DeletePod() + By("Creating resource definition with single pod") + pod1 := PodPause("pod1") + framework.WrapAndCreate(pod1) + framework.Run() + By("Verify that pod is consistently not found") + Consistently(func() bool { + _, err := framework.Client.Pods().Get(pod1.Name) + return errors.IsNotFound(err) + }, 5*time.Second, 1*time.Second).Should(BeTrue(), "Pod was unexpectadly created") + By("Recreate appcontroller pod and verify that pod was successfully created") + framework.Prepare() + testutils.WaitForPod(framework.Clientset, framework.Namespace.Name, pod1.Name, v1.PodRunning) + }) + Describe("Failure handling - subgraph", func() { var parentPod *v1.Pod var childPod *v1.Pod diff --git a/e2e/utils/appcmanager.go b/e2e/utils/appcmanager.go index 1192aa7..91a74fe 100644 --- a/e2e/utils/appcmanager.go +++ b/e2e/utils/appcmanager.go @@ -15,7 +15,6 @@ package utils import ( - "os/exec" "time" . "github.com/onsi/ginkgo" @@ -24,9 +23,17 @@ import ( "github.com/Mirantis/k8s-AppController/pkg/client" "k8s.io/client-go/kubernetes" "k8s.io/client-go/pkg/api" + "k8s.io/client-go/pkg/api/errors" "k8s.io/client-go/pkg/api/v1" ) +const ( + controlName = "appcontrollerdeployment" + concurrencyKey = "concurrency" + selector = "selector" + appcontrollerPod = "k8s-appcontroller" +) + type AppControllerManager struct { Client client.Interface Clientset *kubernetes.Clientset @@ -37,33 +44,31 @@ type AppControllerManager struct { } func (a *AppControllerManager) Run() { - cmd := exec.Command( - "kubectl", - "--namespace", - a.Namespace.Name, - "exec", - "k8s-appcontroller", - "--", - "ac-run", - "-l", - "ns:"+a.Namespace.Name, - ) - out, err := cmd.Output() - if err != nil { - switch err.(type) { - case *exec.ExitError: - exErr := err.(*exec.ExitError) - Fail(string(out) + string(exErr.Stderr)) - default: - Expect(err).NotTo(HaveOccurred()) - } + cfg := &v1.ConfigMap{ + ObjectMeta: v1.ObjectMeta{Name: controlName}, + Data: map[string]string{ + concurrencyKey: "0", + selector: "", + }, } + _, err := a.Client.ConfigMaps().Create(cfg) + Expect(err).NotTo(HaveOccurred()) +} + +func (a *AppControllerManager) DeletePod() { + By("Removing pod " + appcontrollerPod) + err := a.Client.Pods().Delete(appcontrollerPod, nil) + Expect(err).NotTo(HaveOccurred()) + Eventually(func() bool { + _, err := a.Client.Pods().Get(appcontrollerPod) + return errors.IsNotFound(err) + }, 20*time.Second, 1*time.Second).Should(BeTrue(), "Appcontroller pod wasn't removed in time") } func (a *AppControllerManager) Prepare() { appControllerObj := &v1.Pod{ ObjectMeta: v1.ObjectMeta{ - Name: "k8s-appcontroller", + Name: appcontrollerPod, Annotations: map[string]string{ "pod.alpha.kubernetes.io/init-containers": `[{"name": "kubeac-bootstrap", "image": "mirantis/k8s-appcontroller", "imagePullPolicy": "Never", "command": ["kubeac", "bootstrap", "/opt/kubeac/manifests"]}]`, }, @@ -74,7 +79,7 @@ func (a *AppControllerManager) Prepare() { { Name: "kubeac", Image: "mirantis/k8s-appcontroller", - Command: []string{"/usr/bin/run_runit"}, + Command: []string{"kubeac", "run"}, ImagePullPolicy: v1.PullNever, Env: []v1.EnvVar{ { @@ -104,6 +109,7 @@ func (a *AppControllerManager) Prepare() { func (a *AppControllerManager) BeforeEach() { var err error a.Clientset, err = KubeClient() + AddServiceAccountToAdmins(a.Clientset) Expect(err).NotTo(HaveOccurred()) By("Creating namespace and initializing test framework") namespaceObj := &v1.Namespace{ @@ -143,6 +149,7 @@ func (a *AppControllerManager) AfterEach() { err := a.Client.Dependencies().Delete(dep.Name, nil) Expect(err).NotTo(HaveOccurred()) } + RemoveServiceAccountFromAdmins(a.Clientset) } func NewAppControllerManager() *AppControllerManager { diff --git a/e2e/utils/utils.go b/e2e/utils/utils.go index 66b9165..31162ed 100644 --- a/e2e/utils/utils.go +++ b/e2e/utils/utils.go @@ -18,6 +18,8 @@ import ( "flag" "fmt" "io" + "strconv" + "strings" "k8s.io/client-go/kubernetes" "k8s.io/client-go/pkg/api/errors" @@ -28,9 +30,12 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "strings" - "github.com/Mirantis/k8s-AppController/pkg/client" + "k8s.io/client-go/pkg/apis/rbac/v1alpha1" +) + +const ( + rbacServiceAccountAdmin = "system:serviceaccount-admin" ) type TContext struct { @@ -39,10 +44,42 @@ type TContext struct { } func SkipIf14() { - // TODO semver compare if strings.Contains(TestContext.Version, "1.4") { - Skip("This test is disabled on kubernetes of version 1.4") + Skip("This test is disabled on kubernetes of versions 1.4") + } +} + +// AddServiceAccountToAdmins will add system:serviceaccounts to cluster-admin ClusterRole +func AddServiceAccountToAdmins(c kubernetes.Interface) { + if IsVersionOlderThan16() { + return + } + By("Adding service account group to cluster-admin role") + roleBinding := &v1alpha1.ClusterRoleBinding{ + ObjectMeta: v1.ObjectMeta{ + Name: rbacServiceAccountAdmin, + }, + Subjects: []v1alpha1.Subject{{ + Kind: "Group", + Name: "system:serviceaccounts", + }}, + RoleRef: v1alpha1.RoleRef{ + Kind: "ClusterRole", + Name: "cluster-admin", + APIGroup: "rbac.authorization.k8s.io", + }, } + _, err := c.Rbac().ClusterRoleBindings().Create(roleBinding) + Expect(err).NotTo(HaveOccurred(), "Failed to create role binding for serviceaccounts") +} + +func RemoveServiceAccountFromAdmins(c kubernetes.Interface) { + if IsVersionOlderThan16() { + return + } + By("Remowing service account group from cluster-admin role") + err := c.Rbac().ClusterRoleBindings().Delete(rbacServiceAccountAdmin, nil) + Expect(err).NotTo(HaveOccurred(), "Failed to remove serviceaccount from cluster-admin role") } var TestContext = TContext{} @@ -55,6 +92,22 @@ func init() { flag.StringVar(&url, "cluster-url", "http://127.0.0.1:8080", "apiserver address to use with restclient") } +func SanitizedVersion() float64 { + var dirtyVersion string + if strings.HasPrefix(TestContext.Version, "v") { + dirtyVersion = TestContext.Version[1:] + } else { + dirtyVersion = TestContext.Version + } + version, err := strconv.ParseFloat(dirtyVersion, 64) + Expect(err).NotTo(HaveOccurred()) + return version +} + +func IsVersionOlderThan16() bool { + return SanitizedVersion() < 1.6 +} + func Logf(format string, a ...interface{}) { fmt.Fprintf(GinkgoWriter, format, a...) } diff --git a/examples/extended/create.sh b/examples/extended/create.sh index ce8e062..d9f865f 100755 --- a/examples/extended/create.sh +++ b/examples/extended/create.sh @@ -40,5 +40,5 @@ cat serviceaccount.yaml | $KUBECTL_NAME exec -i k8s-appcontroller kubeac wrap | cat deployment.yaml | $KUBECTL_NAME exec -i k8s-appcontroller kubeac wrap | $KUBECTL_NAME create -f - -$KUBECTL_NAME exec k8s-appcontroller ac-run +$KUBECTL_NAME create -f ../../manifests/cfg.yaml $KUBECTL_NAME logs -f k8s-appcontroller diff --git a/examples/extended/delete.sh b/examples/extended/delete.sh index d05c740..a848e39 100755 --- a/examples/extended/delete.sh +++ b/examples/extended/delete.sh @@ -40,3 +40,4 @@ cat deployment.yaml | $KUBECTL_NAME exec -i k8s-appcontroller kubeac wrap | $KUB cat pvc.yaml | $KUBECTL_NAME exec -i k8s-appcontroller kubeac wrap | $KUBECTL_NAME delete -f - $KUBECTL_NAME delete -f ../../manifests/appcontroller.yaml +$KUBECTL_NAME delete -f ../../manifests/cfg.yaml diff --git a/examples/on-error-subgraph/delete.sh b/examples/on-error-subgraph/delete.sh index 68f61e8..684c00d 100755 --- a/examples/on-error-subgraph/delete.sh +++ b/examples/on-error-subgraph/delete.sh @@ -8,3 +8,4 @@ cat pod.yaml | $KUBECTL_NAME exec -i k8s-appcontroller kubeac wrap pod1 | $KUBEC cat pod2.yaml | $KUBECTL_NAME exec -i k8s-appcontroller kubeac wrap pod2 | $KUBECTL_NAME delete -f - $KUBECTL_NAME delete -f ../../appcontroller.yaml +$KUBECTL_NAME delete -f ../../cfg.yaml diff --git a/examples/services/create.sh b/examples/services/create.sh index 272d1f9..fd85455 100755 --- a/examples/services/create.sh +++ b/examples/services/create.sh @@ -17,6 +17,5 @@ cat pod2.yaml | $KUBECTL_NAME exec -i k8s-appcontroller kubeac wrap | $KUBECTL_N cat pod3.yaml | $KUBECTL_NAME exec -i k8s-appcontroller kubeac wrap | $KUBECTL_NAME create -f - cat pod4.yaml | $KUBECTL_NAME exec -i k8s-appcontroller kubeac wrap | $KUBECTL_NAME create -f - -$KUBECTL_NAME exec k8s-appcontroller ac-run - +$KUBECTL_NAME create -f ../../manifests/appcontroller.yaml $KUBECTL_NAME logs -f k8s-appcontroller diff --git a/examples/simple/create.sh b/examples/simple/create.sh index 83272f8..03eb9c3 100755 --- a/examples/simple/create.sh +++ b/examples/simple/create.sh @@ -29,6 +29,4 @@ cat pod2.yaml | $KUBECTL_NAME exec -i k8s-appcontroller kubeac wrap | $KUBECTL_N echo "cat pod3.yaml | $KUBECTL_NAME exec -i k8s-appcontroller kubeac wrap pod3 | $KUBECTL_NAME create -f -" cat pod3.yaml | $KUBECTL_NAME exec -i k8s-appcontroller kubeac wrap | $KUBECTL_NAME create -f - -echo "Here we are running appcontroller binary itself. As the log will say, it retrieves dependencies and resource definitions from the k8s cluster and creates underlying objects accordingly." -echo "$KUBECTL_NAME exec k8s-appcontroller ac-run" -$KUBECTL_NAME exec k8s-appcontroller ac-run +$KUBECTL_NAME create -f ../../manifests/cfg.yaml diff --git a/examples/timeout/create.sh b/examples/timeout/create.sh index d6d98e0..840d22d 100755 --- a/examples/timeout/create.sh +++ b/examples/timeout/create.sh @@ -11,5 +11,5 @@ cat pod.yaml | $KUBECTL_NAME create -f - cat pod2.yaml | $KUBECTL_NAME exec -i k8s-appcontroller kubeac wrap | $KUBECTL_NAME create -f - cat timedout-pod.yaml | $KUBECTL_NAME create -f - -$KUBECTL_NAME exec k8s-appcontroller ac-run +$KUBECTL_NAME create -f ../../manifests/cfg.yaml $KUBECTL_NAME logs -f k8s-appcontroller diff --git a/examples/timeout/delete.sh b/examples/timeout/delete.sh index d773187..560da73 100755 --- a/examples/timeout/delete.sh +++ b/examples/timeout/delete.sh @@ -9,3 +9,4 @@ cat pod2.yaml | $KUBECTL_NAME exec -i k8s-appcontroller kubeac wrap | $KUBECTL_N cat timedout-pod.yaml | $KUBECTL_NAME delete -f - $KUBECTL_NAME delete -f ../../manifests/appcontroller.yaml +$KUBECTL_NAME delete -f ../../manifests/cfg.yaml diff --git a/manifests/appcontroller.yaml b/manifests/appcontroller.yaml index 8e05417..de5fb94 100644 --- a/manifests/appcontroller.yaml +++ b/manifests/appcontroller.yaml @@ -10,7 +10,7 @@ spec: - name: kubeac image: mirantis/k8s-appcontroller imagePullPolicy: IfNotPresent - command: ["/usr/bin/run_runit"] + command: ["kubeac", "run"] env: - name: KUBERNETES_AC_LABEL_SELECTOR value: "" diff --git a/manifests/auth.yaml b/manifests/auth.yaml new file mode 100644 index 0000000..d8b3edb --- /dev/null +++ b/manifests/auth.yaml @@ -0,0 +1,11 @@ +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1alpha1 +metadata: + name: system:serviceaccounts +subjects: +- kind: Group + name: system:serviceaccounts +roleRef: + kind: ClusterRole + name: cluster-admin + apiGroup: rbac.authorization.k8s.io diff --git a/manifests/cfg.yaml b/manifests/cfg.yaml new file mode 100644 index 0000000..bf1b942 --- /dev/null +++ b/manifests/cfg.yaml @@ -0,0 +1,7 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: appcontrollerdeployment +data: + concurrency: 3 + selector: "" diff --git a/pkg/scheduler/controller.go b/pkg/scheduler/controller.go new file mode 100644 index 0000000..3631631 --- /dev/null +++ b/pkg/scheduler/controller.go @@ -0,0 +1,130 @@ +// Copyright 2017 Mirantis +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scheduler + +import ( + "fmt" + "log" + "strconv" + "strings" + + "github.com/Mirantis/k8s-AppController/pkg/client" + + "k8s.io/client-go/pkg/api" + "k8s.io/client-go/pkg/api/v1" + "k8s.io/client-go/pkg/fields" + "k8s.io/client-go/pkg/labels" + "k8s.io/client-go/pkg/runtime" + "k8s.io/client-go/pkg/watch" + "k8s.io/client-go/tools/cache" +) + +const ( + controlName = "appcontrollerdeployment" + concurrencyKey = "concurrency" + selector = "selector" +) + +func singleObjectOpts() v1.ListOptions { + return v1.ListOptions{ + FieldSelector: fields.OneTermEqualSelector("metadata.name", controlName).String(), + } +} + +type CreateFunc func(depGraph DependencyGraph, concurrency int, stopChan <-chan struct{}) + +func newDeploymentController(appc client.Interface, source cache.ListerWatcher, create CreateFunc) cache.ControllerInterface { + var stopChan chan struct{} + _, cfgController := cache.NewInformer( + source, + &v1.ConfigMap{}, + 0, + cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + log.Println("Received required configmap. Starting deployment workflow.") + cfg := obj.(*v1.ConfigMap) + stopChan = make(chan struct{}) + var concurrency int + concurrencyVal, ok := cfg.Data[concurrencyKey] + if !ok { + concurrency = 0 + } else { + var err error + concurrency, err = strconv.Atoi(concurrencyVal) + if err != nil { + log.Println(err) + return + } + } + labelSelector, ok := cfg.Data[selector] + if !ok { + labelSelector = "" + } + depGraph := initializeDependencyGraph(appc, labelSelector) + log.Println("Running deployment with labels ", labelSelector) + go create(depGraph, concurrency, stopChan) + }, + DeleteFunc: func(_ interface{}) { + log.Println("Stopping deployment workflow") + close(stopChan) + }, + }, + ) + return cfgController +} + +func NewDeploymentController(appc client.Interface) cache.ControllerInterface { + opts := singleObjectOpts() + source := &cache.ListWatch{ + ListFunc: func(options api.ListOptions) (runtime.Object, error) { + return appc.ConfigMaps().List(opts) + }, + WatchFunc: func(options api.ListOptions) (watch.Interface, error) { + return appc.ConfigMaps().Watch(opts) + }, + } + return newDeploymentController(appc, source, Create) +} + +func initializeDependencyGraph(c client.Interface, labelSelector string) DependencyGraph { + sel, err := labels.Parse(labelSelector) + if err != nil { + log.Println(err) + } + log.Println("Using label selector:", labelSelector) + + depGraph, err := BuildDependencyGraph(c, sel) + if err != nil { + log.Println(err) + } + + log.Println("Checking for circular dependencies.") + cycles := DetectCycles(depGraph) + if len(cycles) > 0 { + message := "Cycles detected, terminating:\n" + for _, cycle := range cycles { + keys := make([]string, 0, len(cycle)) + for _, vertex := range cycle { + keys = append(keys, vertex.Key()) + } + message = fmt.Sprintf("%sCycle: %s\n", message, strings.Join(keys, ", ")) + } + + log.Println(message) + } else { + log.Println("No cycles detected.") + } + return depGraph +} diff --git a/pkg/scheduler/controller_test.go b/pkg/scheduler/controller_test.go new file mode 100644 index 0000000..2a676d2 --- /dev/null +++ b/pkg/scheduler/controller_test.go @@ -0,0 +1,54 @@ +// Copyright 2017 Mirantis +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scheduler + +import ( + "testing" + + "k8s.io/client-go/pkg/api/v1" + fcache "k8s.io/client-go/tools/cache/testing" + + "github.com/Mirantis/k8s-AppController/pkg/mocks" +) + +func TestDeploymentWorkflow(t *testing.T) { + c := mocks.NewClient() + source := fcache.NewFakeControllerSource() + + waitCh := make(chan struct{}) + var stopCh <-chan struct{} + f := func(_ DependencyGraph, _ int, stopChan <-chan struct{}) { + t.Log("Closing wait chanell") + waitCh <- struct{}{} + stopCh = stopChan + } + + controller := newDeploymentController(c, source, f) + controllerCh := make(chan struct{}) + defer close(controllerCh) + go controller.Run(controllerCh) + + cfg := &v1.ConfigMap{ + ObjectMeta: v1.ObjectMeta{Name: controlName}, + Data: map[string]string{ + concurrencyKey: "0", + selector: "", + }, + } + source.Add(cfg) + <-waitCh + source.Delete(cfg) + <-stopCh +} diff --git a/run_runit.sh b/run_runit.sh deleted file mode 100755 index 91961f4..0000000 --- a/run_runit.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh - -export > /tmp/envvars -exec runsv /etc/sv/ac diff --git a/scripts/checkout_k8s.sh b/scripts/checkout_k8s.sh deleted file mode 100755 index 93c168a..0000000 --- a/scripts/checkout_k8s.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash - -set -o errexit -set -o nounset -set -o pipefail -set -o xtrace - -WORKING=${WORKING:-`mktemp -d`} -K8S_TAG=${K8S_TAG:-v1.5.2} -DIND_COMPATIBLE_COMMIT=${DIND_COMPATIBLE_COMMIT:-897ad95a8e0e1fe674ff81533d4198a3cecee41e} -mkdir -p $WORKING - -pushd $WORKING &> /dev/null -if [ ! -d kubernetes ]; then - git clone --branch $K8S_TAG --depth 1 --single-branch https://github.com/kubernetes/kubernetes.git -fi -pushd kubernetes &> /dev/null -if [ ! -d dind ]; then - git clone --single-branch https://github.com/sttts/kubernetes-dind-cluster.git dind -fi -go get -u github.com/jteeuwen/go-bindata/go-bindata -pushd dind &> /dev/null -git checkout $DIND_COMPATIBLE_COMMIT -popd &> /dev/null -popd &> /dev/null -popd &> /dev/null - -echo $WORKING diff --git a/scripts/dind_down.sh b/scripts/dind_down.sh deleted file mode 100755 index 0eb5d30..0000000 --- a/scripts/dind_down.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -set -o errexit - -pushd $WORKING/kubernetes/ -./dind/dind-down-cluster.sh -popd diff --git a/scripts/docker_publish.sh b/scripts/docker_publish.sh index 4f8b95d..cc835b1 100755 --- a/scripts/docker_publish.sh +++ b/scripts/docker_publish.sh @@ -1,33 +1,73 @@ #!/bin/bash +# Copyright 2017 Mirantis +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -o xtrace +set -o pipefail set -o errexit +set -o nounset + +TRAVIS_PULL_REQUEST_BRANCH=${TRAVIS_PULL_REQUEST_BRANCH:-} +TRAVIS_TEST_RESULT=${TRAVIS_TEST_RESULT:-} +TRAVIS_BRANCH=${TRAVIS_BRANCH:-} IMAGE_REPO=${IMAGE_REPO:-mirantis/k8s-appcontroller} +TRAVIS_TAG=${TRAVIS_TAG:-} PUBLISH=${PUBLISH:-} + function push-to-docker { - if [ -z "$PUBLISH" ]; then - echo "Publish is disabled" - exit 0 - fi - if [ "$TRAVIS_PULL_REQUEST_BRANCH" != "" ]; then - echo "Processing PR $TRAVIS_PULL_REQUEST_BRANCH" - exit 0 - fi - docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD" - if [ ! -z "$TRAVIS_TAG" ]; then - echo "Pushing with tag - $TRAVIS_TAG" - docker push $IMAGE_REPO:$TRAVIS_TAG - exit 0 - fi - if [ $TRAVIS_BRANCH == "master" ]; then - echo "Pushing with tag - latest" - docker push $IMAGE_REPO:latest - exit 0 + if [ -z "${PUBLISH}" ]; then + echo "Publish is disabled." + exit 0 + fi + + if [ -z "${TRAVIS_TEST_RESULT}" ]; then + echo "TRAVIS_TEST_RESULT is not set!" + exit 1 + else + if [ "${TRAVIS_TEST_RESULT}" -ne 0 ]; then + echo "Some of the previous steps ended with an errors! The build is broken!" + exit 1 fi - echo "Pushing with tag - $TRAVIS_BRANCH" - docker push $IMAGE_REPO:$TRAVIS_BRANCH + fi + + if [ -n "${TRAVIS_PULL_REQUEST_BRANCH}" ]; then + echo "Processing PR ${TRAVIS_PULL_REQUEST_BRANCH}" + exit 0 + else + set +o xtrace + docker login -u="${DOCKER_USERNAME}" -p="${DOCKER_PASSWORD}" + set -o xtrace + fi + + if [ -n "${TRAVIS_TAG}" ]; then + echo "Pushing with tag - ${TRAVIS_TAG}" + docker tag "${IMAGE_REPO}" "${IMAGE_REPO}":"${TRAVIS_TAG}" + docker push "${IMAGE_REPO}":"${TRAVIS_TAG}" + exit + fi + + if [ "${TRAVIS_BRANCH}" == "master" ]; then + echo "Pushing with tag - latest" + docker push "${IMAGE_REPO}":latest + exit + fi + + echo "Pushing with tag - ${TRAVIS_BRANCH}" + docker tag "${IMAGE_REPO}" "${IMAGE_REPO}":"${TRAVIS_BRANCH}" + docker push "${IMAGE_REPO}":"${TRAVIS_BRANCH}" } push-to-docker - diff --git a/scripts/import.sh b/scripts/import.sh index 91eab56..8033b2c 100755 --- a/scripts/import.sh +++ b/scripts/import.sh @@ -1,33 +1,45 @@ #!/bin/bash +# Copyright 2017 Mirantis +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -o xtrace +set -o pipefail set -o errexit set -o nounset -set -o pipefail -set -o xtrace + IMAGE_REPO=${IMAGE_REPO:-mirantis/k8s-appcontroller} IMAGE_TAG=${IMAGE_TAG:-latest} NUM_NODES=${NUM_NODES:-2} TMP_IMAGE_PATH=${TMP_IMAGE_PATH:-/tmp/image.tar} -MASTER_NAME=${MASTER_NAME=} -SLAVE_PATTERN=${SLAVE_PATTERN:-"dind_node_"} +MASTER_NAME=${MASTER_NAME="kube-master"} +SLAVE_PATTERN=${SLAVE_PATTERN:-"kube-node-"} function import-image { - docker save ${IMAGE_REPO}:${IMAGE_TAG} -o "${TMP_IMAGE_PATH}" - - if [ -n "$MASTER_NAME" ]; then - docker cp "${TMP_IMAGE_PATH}" kube-master:/image.tar - docker exec -ti "${MASTER_NAME}" docker load -i /image.tar - fi - - for i in `seq 1 "${NUM_NODES}"`; - do - docker cp "${TMP_IMAGE_PATH}" "${SLAVE_PATTERN}$i":/image.tar - docker exec -ti "${SLAVE_PATTERN}$i" docker load -i /image.tar - done - set +o xtrace - echo "Finished copying docker image to dind nodes" + docker save -o "${TMP_IMAGE_PATH}" "${IMAGE_REPO}":"${IMAGE_TAG}" + + if [ -n "${MASTER_NAME}" ]; then + docker cp "${TMP_IMAGE_PATH}" "${MASTER_NAME}":/image.tar + docker exec -ti "${MASTER_NAME}" docker load -i /image.tar + fi + + for node in $(seq 1 "${NUM_NODES}"); do + docker cp "${TMP_IMAGE_PATH}" "${SLAVE_PATTERN}""${node}":/image.tar + docker exec -ti "${SLAVE_PATTERN}""${node}" docker load -i /image.tar + done + echo "Finished copying docker image to dind nodes" } import-image diff --git a/scripts/prepare_dind.sh b/scripts/prepare_dind.sh deleted file mode 100755 index 6a52481..0000000 --- a/scripts/prepare_dind.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -set -o errexit -set -o nounset -set -o pipefail -set -o xtrace - -function ensure-build { - local name=$1 - if [ ! -f _output/bin/$name ]; then - make WHAT="cmd/$name" - fi -} - -function prepare-dind { - pushd $WORKING/kubernetes - ensure-build hyperkube - ensure-build kubectl - ./dind/dind-down-cluster.sh - ./dind/dind-up-cluster.sh - popd -} - -prepare-dind