diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..155fbfd --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +.idea/* +*.tar +*/file/tmp \ No newline at end of file diff --git a/ansible/.gitignore b/ansible/.gitignore index feeb43b..59bd828 100644 --- a/ansible/.gitignore +++ b/ansible/.gitignore @@ -1,2 +1 @@ **/files/tmp/ -hosts diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml index f665031..3c71ead 100644 --- a/ansible/group_vars/all.yml +++ b/ansible/group_vars/all.yml @@ -1,5 +1,9 @@ -keepalived_version: 2.0.18 -rabbitmq_version: 3.7.23 -local_cache_path: "~/.ansible/cache" -erlang_solution_version: 1.0 -haproxy_version: 2.0 \ No newline at end of file +appAgentVersion: v1.0.7 +arping_version: "20211215-1" +jq_version: 1.8.1 +local_cache_path: "file/tmp" +caddy_version: v2.9.1 +haproxy_version: 3.0.9 +keepalived_version: 2.3.3 +erlang_version: 26.2.5.9 +rabbitmq_version: 3.13.7 diff --git a/ansible/host_vars/kylin-amd64.yml b/ansible/host_vars/kylin-amd64.yml new file mode 100644 index 0000000..d41ed55 --- /dev/null +++ b/ansible/host_vars/kylin-amd64.yml @@ -0,0 +1,3 @@ +default_os: kylin +default_osv: 10sp3 +default_arch: amd64 \ No newline at end of file diff --git a/ansible/host_vars/kylin-arm64.yml b/ansible/host_vars/kylin-arm64.yml new file mode 100644 index 0000000..d4f264e --- /dev/null +++ b/ansible/host_vars/kylin-arm64.yml @@ -0,0 +1,3 @@ +default_os: kylin +default_osv: 10sp3 +default_arch: arm64 \ No newline at end of file diff --git a/ansible/host_vars/ubuntu-amd64.yml b/ansible/host_vars/ubuntu-amd64.yml new file mode 100644 index 0000000..f1522b9 --- /dev/null +++ b/ansible/host_vars/ubuntu-amd64.yml @@ -0,0 +1,3 @@ +default_os: ubuntu +default_osv: "22.04" +default_arch: amd64 \ No newline at end of file diff --git a/ansible/hosts b/ansible/hosts new file mode 100644 index 0000000..39c647d --- /dev/null +++ b/ansible/hosts @@ -0,0 +1,3 @@ +ubuntu-amd64 ansible_host=yourip +kylin-arm64 ansible_host=youip ansible_port=30023 +kylin-amd64 ansible_host=youip ansible_port=30024 \ No newline at end of file diff --git a/ansible/make.yml b/ansible/make.yml index ab13b2e..4cb4233 100644 --- a/ansible/make.yml +++ b/ansible/make.yml @@ -8,18 +8,23 @@ - include_role: name: "{{ service_name }}" loop: - - disable-apt-jobs-1.0.0 - - disable-motd-1.0.0 - - app-agent-1.0.1 - - appctl-1.0.9 - - arping-1.0.0 - - rabbitmq-server - - haproxy - - keepalived + - tar + - ncncat + - jq + - os-update + - disable-package-manager-timers + - disable-motd + - app-agent + - appctl + - arping + - caddy - node-all - node-client - - node-proxy + - node-proxy-keepalived - node-rabbitmq - - caddy-1.0.6 + - users_groups + - crashkernel + - pwquality + - rsyslog loop_control: loop_var: service_name diff --git a/ansible/requirements.yml b/ansible/requirements.yml deleted file mode 100644 index de56304..0000000 --- a/ansible/requirements.yml +++ /dev/null @@ -1,8 +0,0 @@ -- src: https://qingcloudappcenter.github.io/ansible-roles/disable-apt-jobs-1.0.0.tar.gz -- src: https://qingcloudappcenter.github.io/ansible-roles/disable-motd-1.0.0.tar.gz -- src: https://qingcloudappcenter.github.io/ansible-roles/app-agent-1.0.1.tar.gz -- src: https://qingcloudappcenter.github.io/ansible-roles/appctl-1.0.9.tar.gz -- src: https://qingcloudappcenter.github.io/ansible-roles/arping-1.0.0.tar.gz -- src: https://qingcloudappcenter.github.io/ansible-roles/confd-files-1.0.2.tar.gz -- src: https://qingcloudappcenter.github.io/ansible-roles/install-1.0.5.tar.gz -- src: https://qingcloudappcenter.github.io/ansible-roles/caddy-1.0.6.tar.gz \ No newline at end of file diff --git a/ansible/roles/app-agent/meta/main.yml b/ansible/roles/app-agent/meta/main.yml new file mode 100644 index 0000000..0a4c119 --- /dev/null +++ b/ansible/roles/app-agent/meta/main.yml @@ -0,0 +1,13 @@ +galaxy_info: + role_name: app-agent + role_version: 1.0.1 + author: Hongliang Wang + description: installs app agent + + license: Apache + + min_ansible_version: 2.4 + + galaxy_tags: [] + +dependencies: [] diff --git a/ansible/roles/app-agent/tasks/main.yml b/ansible/roles/app-agent/tasks/main.yml new file mode 100644 index 0000000..753cf40 --- /dev/null +++ b/ansible/roles/app-agent/tasks/main.yml @@ -0,0 +1,78 @@ +--- +- name: set up variables + set_fact: + os: "{{ os | d(default_os) }}" + appAgentVersion: "{{ appAgentVersion | d('v1.0.7') }}" + arch: "{{ arch | d(default_arch) }}" + tmpPath: "/tmp/app-agent" + +# will set var downloaded +- include_role: + name: downloader + vars: + opts: + pkg_name: app-agent + pkg_version: "{{ appAgentVersion }}" + pkg_feature: "linux-{{ arch }}" + pkg_fileType: ".tgz" + pkg_url: "https://github.com/QingCloudAppcenter/AppcenterAgent/releases/download/{{ appAgentVersion }}/app-agent-linux-{{ arch }}.tar.gz" + +- name: prepare dir + file: + path: "{{ tmpPath }}" + state: directory + +- name: decompress file + unarchive: + src: "{{ downloaded }}" + dest: "{{ tmpPath }}" + creates: "{{ tmpPath }}/bin" + extra_opts: + - --strip-components=1 + +- name: hack for kylin + lineinfile: + path: "{{ tmpPath }}/install.sh" + regexp: "'fedora'" + line: "elif echo \"$os\" | grep '[fedora|kylin]' > /dev/null; then" + +- name: install app-agent + shell: + cmd: ./install.sh + chdir: "{{ tmpPath }}" + +- name: adjust logrotate + replace: + path: /etc/logrotate.d/app-agent + regexp: '^(\s+size).*' + replace: '\1 2M' + +- name: fix logrotate + lineinfile: + path: /etc/logrotate.d/app-agent + line: ' copytruncate' + insertbefore: '^}' + validate: 'logrotate -d %s' + +- name: set up variables + set_fact: + confd_path: "/etc/init.d/confd" + +- name: fix {{ confd_path }} + block: + - name: fix - PIDFILE + replace: + path: "{{ confd_path }}" + regexp: '^PIDFILE.*' + replace: 'PIDFILE=/run/$PROG.pid' + - name: fix - start + replace: + path: "{{ confd_path }}" + regexp: 'touch.*' + replace: 'touch $LOCKFILE; $(echo `pidofproc $PROG_BIN` > $PIDFILE)' + - name: fix - stop + replace: + path: "{{ confd_path }}" + regexp: 'rm -f \$LOCKFILE.*' + replace: 'rm -f $LOCKFILE && rm -f $PIDFILE' + when: os == 'kylin' \ No newline at end of file diff --git a/ansible/roles/appctl/files/etc/logrotate.d/appctl b/ansible/roles/appctl/files/etc/logrotate.d/appctl new file mode 100644 index 0000000..37d2466 --- /dev/null +++ b/ansible/roles/appctl/files/etc/logrotate.d/appctl @@ -0,0 +1,11 @@ +/data/log/appctl/appctl.log { + weekly + maxsize 2M + rotate 5 + missingok + notifempty + copytruncate + postrotate + /usr/bin/systemctl kill -s HUP rsyslog.service >/dev/null 2>&1 || true + endscript +} diff --git a/ansible/roles/appctl/files/etc/rsyslog.d/49-appctl.conf b/ansible/roles/appctl/files/etc/rsyslog.d/49-appctl.conf new file mode 100644 index 0000000..c2243d0 --- /dev/null +++ b/ansible/roles/appctl/files/etc/rsyslog.d/49-appctl.conf @@ -0,0 +1,11 @@ +if $programname startswith 'appctl' then { + action( + type="omfile" + file="/data/log/appctl/appctl.log" + FileOwner="syslog" + FileGroup="svc" + FileCreateMode="0640" + DirCreateMode="0755" + ) + stop +} diff --git a/ansible/roles/appctl/files/opt/app/bin/ctl.sh b/ansible/roles/appctl/files/opt/app/bin/ctl.sh new file mode 100755 index 0000000..de041c8 --- /dev/null +++ b/ansible/roles/appctl/files/opt/app/bin/ctl.sh @@ -0,0 +1,266 @@ +#!/usr/bin/env bash + +# Default hook functions named starting with _, e.g. _init(), _start(), etc. +# Specific roles can override the default hooks like: +# start() { +# _start +# ... +# } +# +# Specific hooks will be executed if exist, otherwise the default ones. + +# Error codes +EC_CHECK_INACTIVE=200 +EC_CHECK_PORT_ERR=201 +EC_CHECK_PROTO_ERR=202 +EC_ENV_ERR=203 +EC_CHECK_HTTP_REQ_ERR=204 +EC_CHECK_HTTP_CODE_ERR=205 +EC_CHECK_TCP_ERR=206 + +command=$1 +args="${@:2}" + +log() { + if [ "$1" == "--debug" ]; then + [ "$APPCTL_ENV" == "dev" ] || return 0 + shift + fi + logger -S 5000 -t appctl --id=$$ -- "[cmd=$command args='$args'] $@" +} + +retry() { + local tried=0 + local maxAttempts=$1 + local interval=$2 + local stopCode=$3 + local cmd="${@:4}" + local retCode=0 + while [ $tried -lt $maxAttempts ]; do + $cmd && return 0 || { + retCode=$? + if [ "$retCode" = "$stopCode" ]; then + log "'$cmd' returned with stop code $stopCode. Stopping ..." + return $retCode + fi + } + sleep $interval + tried=$((tried+1)) + done + + log "'$cmd' still returned errors after $tried attempts. Stopping ..." + return $retCode +} + +rotate() { + local maxFilesCount=5 + for path in $@; do + for i in $(seq 1 $maxFilesCount | tac); do + if [ -f "${path}.$i" ]; then mv ${path}.$i ${path}.$(($i+1)); fi + done + if [ -f "$path" ]; then cp $path ${path}.1; fi + done +} + +execute() { + local cmd=$1; log --debug "Executing command ..." + [ "$(type -t $cmd)" = "function" ] || cmd=_$cmd + $cmd ${@:2} +} + +applyEnvFiles() { + local envFile; for envFile in $(find /opt/app/bin/envs -name "*.env"); do . $envFile; done +} + +applyRoleScripts() { + local scriptFile=/opt/app/bin/node/$NODE_CTL.sh + if [ -f "$scriptFile" ]; then . $scriptFile; fi +} + +checkEnv() { + test -n "$1" +} + +checkMounts() { + test -n "${MY_HYPER_TYPE}" || { + log "ERROR: MY_HYPER_TYPE variable is required to be set. " + return 1 + } + test -n "${DATA_MOUNTS+x}" || { + log "ERROR: DATA_MOUNTS variable is required to be set. " + return 1 + } + case $MY_HYPER_TYPE in + kvm) + local dataDir; for dataDir in $DATA_MOUNTS; do + grep -qs " $dataDir " /proc/mounts || { + log "ERROR: Failed to mount disk . " + return 1 + } + done + ;; + lxc) + local dataDir; for dataDir in $DATA_MOUNTS; do + dataDir=$(echo $dataDir|tr -s [:space:]) + if [ -d $dataDir ]; then + : + else + log "ERROR: $dataDir is not found in this container . " + return 1 + fi + done + ;; + *) + log "ERROR: unrecognized hyper type: $MY_HYPER_TYPE. " + return 1 + ;; + esac +} + +getServices() { + if [ "$1" = "-a" ]; then + echo $SERVICES + else + echo $SERVICES | xargs -n1 | awk -F/ '$2=="true"' | xargs + fi +} + +isSvcEnabled() { + local svc="${1%%/*}" + [ "$(echo $(getServices -a) | xargs -n1 | awk -F/ '$1=="'$svc'" {print $2}')" = "true" ] +} + +checkActive() { + systemctl is-active -q $1 +} + +checkEndpoint() { + local proto=${1%:*} host=${2-$MY_IP} port=${1#*:} + if [ "$proto" = "tcp" ]; then + if ! nc -z -w5 $host $port; then + log "ERROR: TCP timeout - failed to check $host:$port" + return $EC_CHECK_TCP_ERR + fi + elif [ "$proto" = "http" ]; then + local code + code="$(curl -s -m5 -o /dev/null -w "%{http_code}" $host:$port)" || { + log "ERROR: HTTP $code - failed to check http://$host:$port ($?)." + return $EC_CHECK_HTTP_REQ_ERR + } + [[ "$code" =~ ^(200|302|401|403|404)$ ]] || { + log "ERROR: unexpected HTTP code $code." + return $EC_CHECK_HTTP_CODE_ERR + } + else + return $EC_CHECK_PROTO_ERR + fi +} + +isNodeInitialized() { + test -f $APPCTL_NODE_FILE +} + +initSvc() { + systemctl unmask -q ${1%%/*} +} + +_checkSvc() { + checkActive ${1%%/*} || { + # log "Service '$1' is inactive." + return $EC_CHECK_INACTIVE + } + local endpoints=$(echo $1 | awk -F/ '{print $3}') + local endpoint; for endpoint in ${endpoints//,/ }; do + checkEndpoint $endpoint || { + # log "Endpoint '$endpoint' is unreachable." + return $EC_CHECK_PORT_ERR + } + done +} + +startSvc() { + systemctl enable ${1%%/*} + systemctl start ${1%%/*} +} + +stopSvc() { + systemctl stop ${1%%/*} +} + +restartSvc() { + stopSvc $1 + startSvc $1 +} + +### app management + +_preCheck() { + checkEnv "$MY_IP" +} + +_initNode() { + checkMounts + rm -rf /data/lost+found + install -d -o syslog -g svc /data/log/appctl/ + local svc; for svc in $(getServices -a); do initSvc $svc; done + touch $APPCTL_NODE_FILE +} + +_revive() { + log "INFO: Application is asked to revive . " + local svc; for svc in $(getServices); do + execute checkSvc $svc || restartSvc $svc || log "ERROR: failed to restart '$svc' ($?)." + done + log "INFO: Application revived successfully . " +} + +_check() { + local svc; for svc in $(getServices); do + execute checkSvc $svc || (log "ERROR: $svc failed the health check . " && return 1) + done +} + +_start() { + isNodeInitialized || { + execute initNode + systemctl restart rsyslog # output to log files under /data + } + local svc; for svc in $(getServices); do + startSvc $svc || (log "ERROR: service $svc failed to start . " && return 1) + done +} + +_stop() { + local svc; for svc in $(getServices -a | xargs -n1 | tac); do + stopSvc $svc + done +} + +_restart() { + log "INFO: Application is asked to restart . " + execute stop + execute start + log "INFO: Application restarted successfully . " +} + +_reload() { + if ! isNodeInitialized; then return 0; fi # only reload after initialized + local svcs="${@:-$(getServices -a)}" + local svc; for svc in $(echo $svcs | xargs -n1 | tac); do stopSvc $svc; done + local svc; for svc in $svcs; do + if isSvcEnabled $svc; then + log "INFO: $svc is asked to reload by appctl . " + startSvc $svc + log "INFO: $svc reloaded successfully . " + fi + done +} + +applyEnvFiles +applyRoleScripts + +[ "$APPCTL_ENV" == "dev" ] && set -x +set -eo pipefail + +execute preCheck +execute $command $args diff --git a/ansible/roles/appctl/files/opt/app/bin/envs/appctl.env b/ansible/roles/appctl/files/opt/app/bin/envs/appctl.env new file mode 100644 index 0000000..3e167db --- /dev/null +++ b/ansible/roles/appctl/files/opt/app/bin/envs/appctl.env @@ -0,0 +1 @@ +APPCTL_ENV=prod diff --git a/ansible/roles/appctl/files/opt/app/bin/envs/readme.md b/ansible/roles/appctl/files/opt/app/bin/envs/readme.md new file mode 100644 index 0000000..0bae9b9 --- /dev/null +++ b/ansible/roles/appctl/files/opt/app/bin/envs/readme.md @@ -0,0 +1,7 @@ +This directory holds env files, e.g. + +svc-rabbitmq-server.env: + +```env +SERVICES="$SERVICES rabbitmq-server/true/tcp:5672,http:15672" +``` \ No newline at end of file diff --git a/ansible/roles/appctl/files/opt/app/bin/node/readme.md b/ansible/roles/appctl/files/opt/app/bin/node/readme.md new file mode 100644 index 0000000..fae06e7 --- /dev/null +++ b/ansible/roles/appctl/files/opt/app/bin/node/readme.md @@ -0,0 +1 @@ +Place node-specific scripts under this directory. \ No newline at end of file diff --git a/ansible/roles/appctl/files/opt/app/bin/tmpl/readme.md b/ansible/roles/appctl/files/opt/app/bin/tmpl/readme.md new file mode 100644 index 0000000..9835102 --- /dev/null +++ b/ansible/roles/appctl/files/opt/app/bin/tmpl/readme.md @@ -0,0 +1 @@ +This directory holds bundled confd template files. diff --git a/ansible/roles/appctl/meta/main.yml b/ansible/roles/appctl/meta/main.yml new file mode 100644 index 0000000..a995235 --- /dev/null +++ b/ansible/roles/appctl/meta/main.yml @@ -0,0 +1,13 @@ +galaxy_info: + role_name: appctl + role_version: 1.0.9 + author: Hongliang Wang + description: installs appctl + + license: Apache + + min_ansible_version: 2.4 + + galaxy_tags: [] + +dependencies: [] diff --git a/ansible/roles/appctl/tasks/main.yml b/ansible/roles/appctl/tasks/main.yml new file mode 100644 index 0000000..8851e37 --- /dev/null +++ b/ansible/roles/appctl/tasks/main.yml @@ -0,0 +1,46 @@ +--- +- name: copy logging config files + copy: + src: files/{{ path }}/ + dest: /{{ path }} + owner: root + group: root + mode: u=rw,go=r + loop: + - etc/logrotate.d + - etc/rsyslog.d + loop_control: + loop_var: path + +- name: create svc group + group: + name: svc + state: present + +- name: copy app files + copy: + src: files/opt/app/ + dest: /opt/app + owner: root + group: svc + mode: preserve + directory_mode: u=rwx,g=rx,o= + +- name: Change file permissions + file: + path: /opt/app/bin/ctl.sh + mode: '0755' + +- name: create symbolic link + file: + src: /opt/app/bin/ctl.sh + dest: /usr/bin/appctl + state: link + +- name: set up dev env + copy: + dest: /opt/app/bin/envs/appdev.env + content: | + APPCTL_ENV=dev + + when: target_env == 'dev' diff --git a/ansible/roles/arping/tasks/main.yml b/ansible/roles/arping/tasks/main.yml new file mode 100644 index 0000000..c71d0b2 --- /dev/null +++ b/ansible/roles/arping/tasks/main.yml @@ -0,0 +1,45 @@ +--- +- name: set up variables + set_fact: + arping_version: "{{ arping_version | d('20211215-1') }}" + os: "{{ os | d(default_os) }}" + arch: "{{ arch | d(default_arch) }}" + +- name: set up url_prefix for ubuntu amd64 + set_fact: + url_prefix: "http://archive.ubuntu.com/ubuntu" + when: + - os == 'ubuntu' + - arch == 'amd64' + +- name: set up url_prefix for ubuntu arm64 + set_fact: + url_prefix: "http://ports.ubuntu.com" + when: + - os == 'ubuntu' + - arch == 'arm64' + +# will set var downloaded +- include_role: + name: downloader + vars: + opts: + pkg_name: arping + pkg_version: "{{ arping_version }}" + pkg_feature: "linux-{{ arch }}" + pkg_fileType: ".deb" + pkg_url: "{{ url_prefix }}/pool/main/i/iputils/iputils-arping_{{ arping_version }}_{{ arch }}.deb" + when: os == 'ubuntu' + +- name: copy file + copy: + src: "{{ downloaded }}" + dest: "/tmp" + when: os == 'ubuntu' + +- name: install + apt: + deb: "/tmp/{{ downloaded | basename }}" + policy_rc_d: 101 + state: present + when: os == 'ubuntu' \ No newline at end of file diff --git a/ansible/roles/caddy/files/lib/systemd/system/caddy.service b/ansible/roles/caddy/files/lib/systemd/system/caddy.service new file mode 100644 index 0000000..4e69946 --- /dev/null +++ b/ansible/roles/caddy/files/lib/systemd/system/caddy.service @@ -0,0 +1,36 @@ +[Unit] +Description=Caddy HTTP/2 web server +Documentation=https://caddyserver.com/docs +After=network-online.target +Wants=network-online.target +ConditionFileNotEmpty=/opt/app/conf/caddy/caddyfile + +[Service] +Restart=on-abnormal + +User=caddy +Group=caddy + +Environment=XDG_DATA_HOME=/data +; Always set "-root" to something safe in case it gets forgotten in the Caddyfile. +ExecStart=/opt/caddy/current/caddy run --config=/opt/app/conf/caddy/caddyfile +ExecReload=/bin/kill -USR1 $MAINPID + +KillMode=mixed +KillSignal=SIGQUIT +TimeoutStopSec=5s + +LimitNOFILE=1024 +LimitNPROC=512 + +PrivateTmp=true +PrivateDevices=false +ProtectHome=true +ProtectSystem=full + +CapabilityBoundingSet=CAP_NET_BIND_SERVICE +AmbientCapabilities=CAP_NET_BIND_SERVICE +NoNewPrivileges=true + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/caddy/files/opt/app/bin/envs/svc-caddy.env b/ansible/roles/caddy/files/opt/app/bin/envs/svc-caddy.env new file mode 100644 index 0000000..c98b9d4 --- /dev/null +++ b/ansible/roles/caddy/files/opt/app/bin/envs/svc-caddy.env @@ -0,0 +1 @@ +SERVICES="$SERVICES caddy/false/http:80" \ No newline at end of file diff --git a/ansible/roles/node-rabbitmq/files/opt/app/conf/rabbitmq-server/enabled-plugins b/ansible/roles/caddy/files/opt/app/conf/caddy/caddyfile similarity index 100% rename from ansible/roles/node-rabbitmq/files/opt/app/conf/rabbitmq-server/enabled-plugins rename to ansible/roles/caddy/files/opt/app/conf/caddy/caddyfile diff --git a/ansible/roles/caddy/files/opt/app/conf/caddy/templates/print-headers-template.txt b/ansible/roles/caddy/files/opt/app/conf/caddy/templates/print-headers-template.txt new file mode 100644 index 0000000..2b2f282 --- /dev/null +++ b/ansible/roles/caddy/files/opt/app/conf/caddy/templates/print-headers-template.txt @@ -0,0 +1,21 @@ +Remote-Host: {{.Req.Host}} +Remote-IP: {{.RemoteIP}} +Client-IP: {{.ClientIP}} +X-Forwarded-For: {{.Req.Header.Get "X-Forwarded-For"}} +X-Forwarded-Host: {{.Req.Header.Get "X-Forwarded-Host"}} +X-Forwarded-Port: {{.Req.Header.Get "X-Forwarded-Port"}} +X-Forwarded-Proto: {{.Req.Header.Get "X-Forwarded-Proto"}} + +{{ if eq (.Req.Header.Get "X-Forwarded-Proto") "https"}} +I'm https +{{end}} + +Forwarded: {{.Req.Header.Get "Forwarded"}} + +UA: {{.Req.Header.Get "User-Agent"}} + +All Headers: + +{{range $field, $val := .Req.Header}} + {{$field}}: {{$val}} +{{end}} \ No newline at end of file diff --git a/ansible/roles/caddy/files/opt/app/conf/caddy/templates/upload-resp-template.txt b/ansible/roles/caddy/files/opt/app/conf/caddy/templates/upload-resp-template.txt new file mode 100644 index 0000000..a69c5d3 --- /dev/null +++ b/ansible/roles/caddy/files/opt/app/conf/caddy/templates/upload-resp-template.txt @@ -0,0 +1,11 @@ + +http.request.uri.path: {{placeholder "http.request.uri.path"}} + +http.request.uuid {{placeholder "http.request.uuid" }} +http.request.host {{placeholder "http.request.host" }} + +http.upload.filename: {{placeholder "http.upload.filename"}} +http.upload.filesize: {{placeholder "http.upload.filesize"}} +http.upload.directory: {{placeholder "http.upload.directory"}} + +http.upload.uuiddir: {{placeholder "http.upload.uuiddir"}} diff --git a/ansible/roles/caddy/files/opt/app/conf/caddy/templates/upload-template.html b/ansible/roles/caddy/files/opt/app/conf/caddy/templates/upload-template.html new file mode 100644 index 0000000..1885c25 --- /dev/null +++ b/ansible/roles/caddy/files/opt/app/conf/caddy/templates/upload-template.html @@ -0,0 +1,23 @@ + + + + + + + Document + + +
+ Maxfilesize is: {{ placeholder "http.upload.max_filesize"}}   Bytes + + Maxfilesize is: {{ placeholder "http.vars.maxfilesize"}}   Bytes +
+ + +
+ + \ No newline at end of file diff --git a/ansible/roles/caddy/tasks/main.yml b/ansible/roles/caddy/tasks/main.yml new file mode 100644 index 0000000..f1636eb --- /dev/null +++ b/ansible/roles/caddy/tasks/main.yml @@ -0,0 +1,69 @@ +--- +- name: set up variables + set_fact: + caddy_version: "{{ caddy_version | d('v2.8.4') }}" + arch: "{{ arch | d(default_arch) }}" + installPath: "/opt/caddy" + +- name: prepare service group + group: + name: svc + state: present + +- name: prepare service user + user: + name: caddy + groups: svc + shell: /sbin/nologin + create_home: no + append: yes + comment: "Service User" + state: present + +- name: prepare binary directory + file: + path: "{{ installPath }}/{{ caddy_version }}" + state: directory + +- name: link binary dir + file: + src: "{{ installPath }}/{{ caddy_version }}" + dest: "{{ installPath }}/current" + state: link + +# will set var downloaded +- include_role: + name: downloader + vars: + opts: + pkg_name: caddy + pkg_version: "{{ caddy_version }}" + pkg_feature: "linux-{{ arch }}" + pkg_fileType: "" + pkg_url: https://github.com/djangoyi-yunify/caddy-compiler/releases/download/caddy-{{ caddy_version }}_linux/caddy-{{ arch }} + +- name: copy bin file + copy: + src: "{{ downloaded }}" + dest: "{{ installPath }}/{{ caddy_version }}/caddy" + mode: "0755" + +- name: transfer files + copy: + src: files/{{ file_path }} + dest: /{{ file_path }} + owner: root + group: root + mode: preserve + loop: + - lib/systemd/system/ + - opt/app/ + loop_control: + loop_var: file_path + +- name: mask caddy service + systemd: + name: caddy + enabled: no + masked: yes + state: stopped \ No newline at end of file diff --git a/ansible/roles/confd-files/files/etc/confd/templates/01.header.sh.tmpl b/ansible/roles/confd-files/files/etc/confd/templates/01.header.sh.tmpl new file mode 100644 index 0000000..5fa74a8 --- /dev/null +++ b/ansible/roles/confd-files/files/etc/confd/templates/01.header.sh.tmpl @@ -0,0 +1,40 @@ +#!/usr/bin/env bash + +set -eo pipefail + +myPath="$0" + +cleanUp() { + local rc=$? + [ "$rc" -eq 0 ] || echo "# Failed ($rc)! Please check confd logs." >> $myPath + return $rc +} + +trap cleanUp EXIT + +rotate() { + local path=$1 maxFilesCount=5 + for i in $(seq 1 $maxFilesCount | tac); do + if [ -f "${path}.$i" ]; then mv ${path}.$i ${path}.$(($i+1)); fi + done + if [ -f "$path" ]; then cp $path ${path}.1; fi +} + +flush() { + local targetFile=$1 + if [ -n "$targetFile" ]; then + rotate $targetFile + cat > $targetFile - + else + cat - + fi +} + +applyEnvs() { + local -r envFile=/opt/app/bin/envs/confd.env + if [ -f "$envFile" ]; then . $envFile; fi + local -r nodeEnvFile=/opt/app/bin/envs/nodectl.env + if [ -f "$nodeEnvFile" ]; then . $nodeEnvFile; fi +} + +applyEnvs diff --git a/ansible/roles/confd-files/meta/main.yml b/ansible/roles/confd-files/meta/main.yml new file mode 100644 index 0000000..8913e8e --- /dev/null +++ b/ansible/roles/confd-files/meta/main.yml @@ -0,0 +1,13 @@ +galaxy_info: + role_name: confd-files + role_version: 1.0.2 + author: Hongliang Wang + description: installs confd-files + + license: Apache + + min_ansible_version: 2.9 + + galaxy_tags: [] + +dependencies: [] diff --git a/ansible/roles/confd-files/tasks/main.yml b/ansible/roles/confd-files/tasks/main.yml new file mode 100644 index 0000000..c175cf4 --- /dev/null +++ b/ansible/roles/confd-files/tasks/main.yml @@ -0,0 +1,40 @@ +--- +- name: compile + shell: | + compileTmpls() { + local commonFiles="$(ls {{ role_path }}/files/etc/confd/templates/*)" + local srcFiles="$commonFiles ${@:2}" destFile=$1 + for tmpl in $srcFiles; do + (cat $tmpl; echo) >> $destFile + done + } + + set -euo pipefail + + parentRolePath="{{ ansible_parent_role_paths[0] }}" + srcRootDir="$parentRolePath/files/etc/confd/templates" + [ -d "$srcRootDir" ] + destDir=$parentRolePath/files/tmp/confd/templates + mkdir -p $destDir + rm -rf $destDir/* + for srcDir in $(ls -d $srcRootDir/*); do + compileTmpls $destDir/$(basename $srcDir).tmpl $(ls $srcDir/*) + done + args: + executable: /bin/bash + delegate_to: localhost + run_once: True + +- name: install + copy: + src: "{{ ansible_parent_role_paths[0] }}/files/{{ path.src }}/" + dest: /etc/confd/{{ path.dest }}/ + owner: root + group: root + loop: + - src: etc/confd/conf.d + dest: conf.d + - src: tmp/confd/templates + dest: templates + loop_control: + loop_var: path diff --git a/ansible/roles/crashkernel/tasks/main.yml b/ansible/roles/crashkernel/tasks/main.yml new file mode 100644 index 0000000..6e2d146 --- /dev/null +++ b/ansible/roles/crashkernel/tasks/main.yml @@ -0,0 +1,27 @@ +- name: set up variables + set_fact: + os: "{{ os | d(default_os) }}" + +- name: Remove crashkernel parameter from GRUB_CMDLINE_LINUX + replace: + path: /etc/default/grub + regexp: 'crashkernel=[^ \"]*' + replace: '' + when: os == 'kylin' + +- name: update GRUB for efi + command: + grub2-mkconfig -o /boot/efi/EFI/kylin/grub.cfg + when: os == 'kylin' + +- name: update GRUB for bios + command: + grub2-mkconfig -o /boot/grub2/grub.cfg + when: os == 'kylin' + +- name: disable kdump.service + systemd: + name: kdump.service + state: stopped + enabled: no + when: os == 'kylin' \ No newline at end of file diff --git a/ansible/roles/disable-apt-jobs/meta/main.yml b/ansible/roles/disable-apt-jobs/meta/main.yml new file mode 100644 index 0000000..b8c81f8 --- /dev/null +++ b/ansible/roles/disable-apt-jobs/meta/main.yml @@ -0,0 +1,13 @@ +galaxy_info: + role_name: disable-apt-jobs + role_version: 1.0.0 + author: Hongliang Wang + description: disable apt jobs + + license: Apache + + min_ansible_version: 2.4 + + galaxy_tags: [] + +dependencies: [] diff --git a/ansible/roles/disable-apt-jobs/tasks/main.yml b/ansible/roles/disable-apt-jobs/tasks/main.yml new file mode 100644 index 0000000..c461176 --- /dev/null +++ b/ansible/roles/disable-apt-jobs/tasks/main.yml @@ -0,0 +1,16 @@ +--- +- name: disable apt auto upgrade + systemd: + name: "{{ svc_name }}" + state: stopped + masked: yes + loop: + - apt-daily.timer + - apt-daily.service + - apt-daily-upgrade.timer + - apt-daily-upgrade.service + loop_control: + loop_var: svc_name + +- name: fix apt issues + shell: dpkg --configure -a diff --git a/ansible/roles/disable-motd/meta/main.yml b/ansible/roles/disable-motd/meta/main.yml new file mode 100644 index 0000000..9bdfd23 --- /dev/null +++ b/ansible/roles/disable-motd/meta/main.yml @@ -0,0 +1,13 @@ +galaxy_info: + role_name: disable-motd + role_version: 1.0.0 + author: Hongliang Wang + description: disable motd + + license: Apache + + min_ansible_version: 2.4 + + galaxy_tags: [] + +dependencies: [] diff --git a/ansible/roles/disable-motd/tasks/main.yml b/ansible/roles/disable-motd/tasks/main.yml new file mode 100644 index 0000000..929ffa7 --- /dev/null +++ b/ansible/roles/disable-motd/tasks/main.yml @@ -0,0 +1,33 @@ +--- +- name: set vars + set_fact: + os: "{{ os | d(default_os) }}" + +- name: ubuntu + file: + path: "{{ remote_usr.home }}/.hushlogin" + owner: "{{ remote_usr.name }}" + state: touch + loop: + - name: ubuntu + home: /home/ubuntu + - name: root + home: /root + loop_control: + loop_var: remote_usr + when: os == 'ubuntu' + +- name: kylin - cockpit + file: + src: /dev/null + dest: /etc/motd.d/cockpit + state: link + force: yes + when: os == 'kylin' + +- name: kylin - /etc/motd + copy: + content: "" + dest: /etc/motd + force: yes + when: os == 'kylin' \ No newline at end of file diff --git a/ansible/roles/disable-package-manager-timers/tasks/main.yml b/ansible/roles/disable-package-manager-timers/tasks/main.yml new file mode 100644 index 0000000..3d3660a --- /dev/null +++ b/ansible/roles/disable-package-manager-timers/tasks/main.yml @@ -0,0 +1,59 @@ +--- +- name: set vars + set_fact: + os: "{{ os | d(default_os) }}" + osv: "{{ osv | d(default_osv) }}" + +- name: ubuntu + systemd: + name: "{{ svc_name }}" + state: stopped + masked: yes + loop: + - apt-daily.timer + - apt-daily.service + - apt-daily-upgrade.timer + - apt-daily-upgrade.service + loop_control: + loop_var: svc_name + when: os == 'ubuntu' + +- name: ubuntu fix apt issues + shell: dpkg --configure -a + when: os == 'ubuntu' + +- name: kylin - 10sp2 + systemd: + name: "{{ svc_name }}" + state: stopped + masked: yes + loop: + - dnf-automatic-download.timer + - dnf-automatic-download.service + - dnf-automatic-install.timer + - dnf-automatic-install.service + - dnf-automatic-notifyonly.timer + - dnf-automatic-notifyonly.service + - dnf-automatic.timer + - dnf-automatic.service + - dnf-makecache.timer + - dnf-makecache.service + loop_control: + loop_var: svc_name + when: + - os == 'kylin' + - osv == '10sp2' + +- name: kylin - 10sp3 + systemd: + name: "{{ svc_name }}" + state: stopped + masked: yes + loop: + - dnf-makecache.timer + - dnf-makecache.service + loop_control: + loop_var: svc_name + when: + - os == 'kylin' + - osv == '10sp3' \ No newline at end of file diff --git a/ansible/roles/downloader/tasks/main.yml b/ansible/roles/downloader/tasks/main.yml new file mode 100644 index 0000000..a1927ee --- /dev/null +++ b/ansible/roles/downloader/tasks/main.yml @@ -0,0 +1,34 @@ +--- +- name: set up variables + set_fact: + local_cache_path: "{{ local_cache_path | d('file/tmp') }}" + pkg_name: "{{ opts.pkg_name }}" + pkg_version: "{{ opts.pkg_version }}" + pkg_feature: "{{ opts.pkg_feature }}" + pkg_fileType: "{{ opts.pkg_fileType }}" + pkg_url: "{{ opts.pkg_url }}" + +- name: prepare dir + file: + path: "{{ local_cache_path }}/{{ pkg_name }}" + state: directory + mode: '0755' + delegate_to: localhost + +- name: check if file exists + stat: + path: "{{ local_cache_path }}/{{ pkg_name }}/{{ pkg_name }}-{{ pkg_version }}-{{ pkg_feature }}{{ pkg_fileType }}" + register: cached_file + delegate_to: localhost + +- name: download file + get_url: + url: "{{ pkg_url }}" + dest: "{{ local_cache_path }}/{{ pkg_name }}/{{ pkg_name }}-{{ pkg_version }}-{{ pkg_feature }}{{ pkg_fileType }}" + delegate_to: localhost + when: + - cached_file.stat.exists == false + +- name: setup output + set_fact: + downloaded: "{{ local_cache_path }}/{{ pkg_name }}/{{ pkg_name }}-{{ pkg_version }}-{{ pkg_feature }}{{ pkg_fileType }}" diff --git a/ansible/roles/haproxy/tasks/main.yml b/ansible/roles/haproxy/tasks/main.yml deleted file mode 100644 index 47a1156..0000000 --- a/ansible/roles/haproxy/tasks/main.yml +++ /dev/null @@ -1,33 +0,0 @@ ---- -- name: install haproxy plugins - apt: - name: ['hatop','arping','software-properties-common'] - update_cache: yes - state: present - -- name: add-apt-repository - apt_repository: - repo: "ppa:vbernat/haproxy-{{ haproxy_version }}" - -- name: install haproxy - apt: - name: "haproxy={{ haproxy_version }}.*" - update_cache: yes - state: present - -- name: copy binaries - copy: - src: files/lib/ - dest: /lib - owner: root - group: root - mode: preserve - directory_mode: u=rwx,g=rx,o= - -- name: disable auto startup on boot - systemd: - name: haproxy - enabled: no - masked: yes - state: stopped - diff --git a/ansible/roles/install/meta/main.yml b/ansible/roles/install/meta/main.yml new file mode 100644 index 0000000..c461b3b --- /dev/null +++ b/ansible/roles/install/meta/main.yml @@ -0,0 +1,13 @@ +galaxy_info: + role_name: install + role_version: 1.0.5 + author: Hongliang Wang + description: download files with local cache and install + + license: Apache + + min_ansible_version: 2.9 + + galaxy_tags: [] + +dependencies: [] diff --git a/ansible/roles/install/tasks/main.yml b/ansible/roles/install/tasks/main.yml new file mode 100644 index 0000000..2b89787 --- /dev/null +++ b/ansible/roles/install/tasks/main.yml @@ -0,0 +1,114 @@ +--- +- name: set up default home var + set_fact: + default_dest_dir: "/opt/{{ opts.pkg_name }}/{{ opts.pkg_version }}/" + when: opts.pkg_version is defined + +- name: set up variables + set_fact: + dest_path: "{{ opts.dest_path | d(default_dest_dir) }}" + pkg_name: "{{ opts.pkg_name }}" + pkg_type: "{{ opts.pkg_type | d('') }}" + pkg_version: "{{ opts.pkg_version | d('') }}" + extracts: "{{ opts.extracts | d(false) }}" + extra_opts: "{{ opts.extra_opts | d(['--strip-components=1'] if opts.pkg_type == 'tgz' else []) }}" + creates: "{{ opts.creates | d('') }}" + parent_role_name: "{{ ansible_parent_role_names[0] | regex_replace('-[0-9.]+$', '') }}" + target_owner: "{{ opts.target_owner | d('root') }}" + target_group: "{{ opts.target_group | d('root') }}" + +- name: set other variables + set_fact: + local_path: "{{ local_cache_path }}/{{ parent_role_name }}/{{ opts.local_path + '/' if opts.local_path is defined else '' }}{{ pkg_name }}{{ '-' + pkg_version if pkg_version else '' }}{{ '.' + pkg_type if pkg_type else '' }}" + +- name: install tools + apt: + name: ['unzip'] + state: present + update_cache: no + when: + - extracts + - pkg_type == 'zip' + +- name: check if package file exists + stat: + path: "{{ local_path }}" + register: cached_file + run_once: True + delegate_to: localhost + +- name: prepare local dir + file: + dest: "{{ local_path | dirname }}" + state: directory + delegate_to: localhost + when: + - cached_file.stat.exists == False + +- name: download package - {{ pkg_name }} + get_url: + url: "{{ opts.pkg_url }}" + dest: "{{ local_path }}" + delegate_to: localhost + run_once: True + when: + - cached_file.stat.exists == False + +- name: Prepare directories - {{ pkg_name }} + file: + path: "{{ dest_path if extracts or dest_path[-1] == '/' else dest_path | dirname }}" + owner: "{{ target_owner }}" + group: "{{ target_group }}" + state: directory + +- name: extract package - {{ pkg_name }} + unarchive: + src: "{{ local_path }}" + dest: "{{ dest_path }}" + owner: "{{ target_owner }}" + group: "{{ target_group }}" + creates: "{{ dest_path | regex_replace('/*$', '') }}/{{ creates }}" + extra_opts: "{{ extra_opts }}" + when: + - extracts + +- name: set permissions - {{ pkg_name }} + file: + path: "{{ dest_path }}" + owner: "{{ target_owner }}" + group: "{{ target_group }}" + recurse: yes + when: + - extracts + +- name: transfer package - {{ pkg_name }}0 + copy: + src: "{{ local_path }}" + dest: "{{ dest_path }}" + mode: "{{ opts.file_mode | d('644') }}" + when: + - extracts == False + +- name: create symbolic link - {{ pkg_name }} + file: + src: "{{ pkg_version }}" + dest: "/opt/{{ pkg_name }}/current" + state: link + when: dest_path == default_dest_dir + +- name: add bin path to PATH + copy: + dest: /etc/profile.d/{{ pkg_name }}-path.sh + content: PATH={{ dest_path | regex_replace('/*$', '') }}/{{ opts.bin_path }}:$PATH + when: opts.bin_path is defined + +- name: create bin symbolic links - {{ pkg_name }} + file: + src: "{{ '/opt/' + pkg_name + '/current' if dest_path == default_dest_dir else dest_path }}{{ '/' + bin_link.src if bin_link.src else '' }}" + dest: "{{ bin_link.dest | d('/usr/bin/' + (bin_link.src | basename)) }}" + force: yes + state: link + loop: "{{ opts.bin_links }}" + loop_control: + loop_var: bin_link + when: opts.bin_links is defined diff --git a/ansible/roles/jq/tasks/main.yml b/ansible/roles/jq/tasks/main.yml new file mode 100644 index 0000000..8f3b68b --- /dev/null +++ b/ansible/roles/jq/tasks/main.yml @@ -0,0 +1,40 @@ +--- +- name: set up variables + set_fact: + jq_version: "{{ jq_version | d('1.7') }}" + arch: "{{ arch | d(default_arch) }}" + installPath: "/opt/jq" + +# will set var downloaded +- include_role: + name: downloader + vars: + opts: + pkg_name: jq + pkg_version: "{{ jq_version }}" + pkg_feature: "linux-{{ arch }}" + pkg_fileType: "" + pkg_url: https://github.com/jqlang/jq/releases/download/jq-{{ jq_version }}/jq-linux-{{ arch }} + +- name: prepare dir + file: + path: "{{ installPath }}/{{ jq_version }}" + state: directory + +- name: copy file + copy: + src: "{{ downloaded }}" + dest: "{{ installPath }}/{{ jq_version }}/jq" + mode: "0755" + +- name: link for current + file: + src: "{{ installPath }}/{{ jq_version }}" + dest: "{{ installPath }}/current" + state: link + +- name: link for jq + file: + src: "{{ installPath }}/current/jq" + dest: "/usr/bin/jq" + state: link \ No newline at end of file diff --git a/ansible/roles/keepalived/tasks/main.yml b/ansible/roles/keepalived/tasks/main.yml deleted file mode 100644 index 8035964..0000000 --- a/ansible/roles/keepalived/tasks/main.yml +++ /dev/null @@ -1,67 +0,0 @@ ---- -- name: Prepare dev env for keepalived - apt: - update_cache: yes - name: ['tree', 'curl', 'gcc', 'libssl-dev', 'libnl-3-dev', 'libnl-genl-3-dev', 'libsnmp-dev', 'make'] - state: present - delegate_to: localhost - -- name: Prepare directories for keepalived - file: - path: /etc/keepalived - owner: root - group: root - state: directory - -- name: download keepalived - include_role: - name: install-1.0.5 - vars: - opts: - pkg_name: keepalived - pkg_version: "{{ item }}" - pkg_type: tgz - pkg_url: "https://www.keepalived.org/software/keepalived-{{ item }}.tar.gz" - extracts: yes - creates: "configure" - bin_path: - with_items: "{{ keepalived_version }}" - -- name: build keepalived - vars: - build_path: "/root/.ansible/cache/keepalived/keepalived-{{ keepalived_version }}/build-dir" - shell: | - tar -zxvf keepalived-{{ keepalived_version }}.tgz - cd keepalived-{{ keepalived_version }}/ - ./configure --prefix={{ build_path }} - make - sudo make install - args: - chdir: "/root/.ansible/cache/keepalived" - creates: "{{ build_path }}/bin/keepalived" - delegate_to: localhost - -- name: install keepalived bin - copy: - src: "{{ item }}" - dest: /opt/keepalived/{{ keepalived_version }} - owner: root - group: root - mode: preserve - directory_mode: u=rwx,g=rx,o= - with_items: - - "/root/.ansible/cache/keepalived/keepalived-{{ keepalived_version }}/build-dir/" - - "files/lib/" - -- name: create keepalived service link - file: - src: /opt/keepalived/current/systemd/system/keepalived.service - dest: /lib/systemd/system/keepalived.service - state: link - -- name: mask services - systemd: - name: keepalived - enabled: no - masked: yes - state: stopped \ No newline at end of file diff --git a/ansible/roles/ncncat/tasks/main.yml b/ansible/roles/ncncat/tasks/main.yml new file mode 100644 index 0000000..d710bd7 --- /dev/null +++ b/ansible/roles/ncncat/tasks/main.yml @@ -0,0 +1,23 @@ +- name: set up variables + set_fact: + os: "{{ os | d(default_os) }}" + +- name: install ncat on ubuntu + apt: + name: "ncat" + state: present + when: os == 'ubuntu' + +- name: link nc to ncat + file: + src: /usr/bin/ncat + dest: /usr/bin/nc + state: link + force: yes + when: os == "ubuntu" + +- name: Install nc and ncat on kylin + dnf: + name: "nmap" + state: present + when: os == 'kylin' \ No newline at end of file diff --git a/ansible/roles/node-all/files/etc/confd/conf.d/nodectl.sh.toml b/ansible/roles/node-all/files/etc/confd/conf.d/0-nodectl.sh.toml similarity index 100% rename from ansible/roles/node-all/files/etc/confd/conf.d/nodectl.sh.toml rename to ansible/roles/node-all/files/etc/confd/conf.d/0-nodectl.sh.toml diff --git a/ansible/roles/node-all/files/etc/confd/conf.d/journal.sh.toml b/ansible/roles/node-all/files/etc/confd/conf.d/journal.sh.toml new file mode 100644 index 0000000..e7e4538 --- /dev/null +++ b/ansible/roles/node-all/files/etc/confd/conf.d/journal.sh.toml @@ -0,0 +1,8 @@ +[template] +src = "journal.sh.tmpl" +dest = "/opt/app/bin/tmpl/journal.sh" +mode = "0700" +keys = [ + "/", +] +reload_cmd = "/opt/app/bin/tmpl/journal.sh; systemctl restart rsyslog" diff --git a/ansible/roles/node-all/files/etc/confd/templates/appctl.sh/01.appctl.env.tmpl b/ansible/roles/node-all/files/etc/confd/templates/appctl.sh/01.appctl.env.tmpl index a078af2..4ef8b5f 100644 --- a/ansible/roles/node-all/files/etc/confd/templates/appctl.sh/01.appctl.env.tmpl +++ b/ansible/roles/node-all/files/etc/confd/templates/appctl.sh/01.appctl.env.tmpl @@ -1,5 +1,6 @@ - +mkdir -p /opt/app/conf/appctl flush /opt/app/bin/envs/appctl.env << APPCTL_ENV_EOF +APPCTL_NODE_FILE=/opt/app/conf/appctl/node.init MY_IP={{ getv "/host/ip" }} MY_ROLE={{ getv "/host/role" }} CLUSTER_ID={{ getv "/cluster/cluster_id" }} @@ -16,6 +17,13 @@ RAM_NODES="$(echo " {{ getv (printf "/hosts/ram/%s/sid" .) }}/{{ getv (printf "/hosts/ram/%s/instance_id" .) }}/{{ getv (printf "/hosts/ram/%s/ip" .) }} {{- end }} " | xargs -n1 | sort -V | xargs)" - +{{ if (exists "/links/etcd_service/cluster/cluster_id") and (ne (getv "/links/etcd_service/cluster/cluster_id")) }} +PEER_DISCOVERY_BACKEND_TYPE=etcd +{{ else }} +PEER_DISCOVERY_BACKEND_TYPE=classic_config +{{ end }} +PATH=/opt/rabbitmq/current/sbin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin +export RABBITMQ_HOME=/opt/rabbitmq/current +export HOME=/root APPCTL_ENV_EOF diff --git a/ansible/roles/node-all/files/etc/confd/templates/journal.sh/01.journald.conf.tmpl b/ansible/roles/node-all/files/etc/confd/templates/journal.sh/01.journald.conf.tmpl new file mode 100644 index 0000000..d2a66bb --- /dev/null +++ b/ansible/roles/node-all/files/etc/confd/templates/journal.sh/01.journald.conf.tmpl @@ -0,0 +1,16 @@ +{{- if not (getvs "/host/role" | filter "client") }} +mkdir -p ${DATA_MOUNTS}/log/journald/ +chmod 755 ${DATA_MOUNTS}/log/journald/ +chown syslog:svc ${DATA_MOUNTS}/log/journald/ +flush /etc/rsyslog.d/49-journald.conf << JOURNALD_CONFIG_EOF +module(load="imjournal" PersistStateInterval="100") #load imjournal module +module(load="mmjsonparse") #load mmjsonparse module for structured logs + +#template(name="CEETemplate" type="string" string="%TIMESTAMP% %HOSTNAME% %syslogtag% @cee: %$!all-json%\n" ) #template for messages + +#action(type="mmjsonparse") +#action(type="omfile" file="${DATA_MOUNTS}/log/journald/journald.log" template="CEETemplate") +action(type="omfile" file="${DATA_MOUNTS}/log/journald/journald.log" FileOwner="syslog" FileGroup="svc" FileCreateMode="0640" DirCreateMode="0755") + +JOURNALD_CONFIG_EOF +{{- end }} \ No newline at end of file diff --git a/ansible/roles/node-all/files/etc/confd/templates/journal.sh/02.logrotate.tmpl b/ansible/roles/node-all/files/etc/confd/templates/journal.sh/02.logrotate.tmpl new file mode 100644 index 0000000..e65257a --- /dev/null +++ b/ansible/roles/node-all/files/etc/confd/templates/journal.sh/02.logrotate.tmpl @@ -0,0 +1,16 @@ +{{- if not (getvs "/host/role" | filter "client") }} +flush /etc/logrotate.d/journald << LOGROTATE_EOF +${DATA_MOUNTS}/log/journald/journald.log { + daily + rotate 20 + missingok + notifempty + compress + delaycompress + copytruncate + postrotate + /usr/bin/systemctl kill -s HUP rsyslog.service >/dev/null 2>&1 || true + endscript +} +LOGROTATE_EOF +{{- end }} \ No newline at end of file diff --git a/ansible/roles/node-all/files/etc/confd/templates/nodectl.sh/01.nodectl.env.tmpl b/ansible/roles/node-all/files/etc/confd/templates/nodectl.sh/01.nodectl.env.tmpl index a7d523b..27d7076 100644 --- a/ansible/roles/node-all/files/etc/confd/templates/nodectl.sh/01.nodectl.env.tmpl +++ b/ansible/roles/node-all/files/etc/confd/templates/nodectl.sh/01.nodectl.env.tmpl @@ -1,23 +1,44 @@ +{{- if getvs "/host/role" | filter "client" }} +getSshServiceName() { + local res + res=$(grep '^ID=' /etc/os-release | cut -d= -f2- | tr -d '"') + if [ "$res" = "kylin" ]; then + echo "sshd" + return 0 + fi + echo "ssh" +} +tmpSshName=$(getSshServiceName) +{{- end }} flush /opt/app/bin/envs/nodectl.env << NODE_ENV_EOF {{- if getvs "/host/role" | filter "(disc|ram)" }} SERVICES="\$SERVICES -rabbitmq-server/true/tcp:5672,http:15672" +rabbitmq-server/true/tcp:5672" NODE_CTL="rabbitmq-node" DATA_MOUNTS="/data" MY_HYPER_TYPE={{ getv "/host/hypervisor" }} {{- else if getvs "/host/role" | filter "haproxy" }} SERVICES="\$SERVICES \$(echo " {{- $HPPORT := getv "/env/haproxy_web_port" "8100" }} -haproxy/true/tcp:5672,http:15672,tcp:61613,tcp:1883,http:{{ $HPPORT }} +haproxy/true/http:{{ $HPPORT }} keepalived/true/ " | xargs)" -DATA_MOUNTS="" +DATA_MOUNTS="/data" MY_HYPER_TYPE={{ getv "/host/hypervisor" }} NODE_CTL="proxy-node" {{- else if getvs "/host/role" | filter "client" }} -SERVICES="\$SERVICES ssh/true/tcp:22" +SERVICES="\$SERVICES $tmpSshName/{{ getv "/env/ssh_enabled" }}/tcp:22" NODE_CTL="client-node" DATA_MOUNTS="" MY_HYPER_TYPE={{ getv "/host/hypervisor" }} {{- end }} NODE_ENV_EOF + +{{- if getvs "/host/role" | filter "client" }} +# client node will change status of ssh service + {{- if eq (getv "/env/ssh_enabled") "true" }} +systemctl start $tmpSshName || : + {{- else }} +systemctl stop $tmpSshName || : + {{- end }} +{{- end }} diff --git a/ansible/roles/node-all/tasks/main.yml b/ansible/roles/node-all/tasks/main.yml index e1c67ea..56e2545 100644 --- a/ansible/roles/node-all/tasks/main.yml +++ b/ansible/roles/node-all/tasks/main.yml @@ -1,4 +1,69 @@ --- - name: install confd files include_role: - name: confd-files-1.0.2 \ No newline at end of file + name: confd-files + +- name: Set net.ipv4.tcp_keepalive_time to 30 + lineinfile: + path: /etc/sysctl.conf + line: 'net.ipv4.tcp_keepalive_time=30' + regexp: '^net.ipv4.tcp_keepalive_time=3' + state: present + +- name: Set net.ipv4.tcp_keepalive_intvl to 10 + lineinfile: + path: /etc/sysctl.conf + line: 'net.ipv4.tcp_keepalive_intvl=10' + regexp: '^net.ipv4.tcp_keepalive_intvl=' + state: present + +- name: Set net.ipv4.tcp_keepalive_probes to 4 + lineinfile: + path: /etc/sysctl.conf + line: 'net.ipv4.tcp_keepalive_probes=4' + regexp: '^net.ipv4.tcp_keepalive_probes=' + state: present + +- name: Set net.ipv4.ip_local_port_range to 102465023 + lineinfile: + path: /etc/sysctl.conf + line: 'net.ipv4.ip_local_port_range=102465023' + regexp: '^net.ipv4.ip_local_port_range=' + state: present + +- name: Set net.core.somaxconn to 65535 + lineinfile: + path: /etc/sysctl.conf + line: 'net.core.somaxconn=65535' + regexp: '^net.core.somaxconn=' + state: present + +- name: Set net.ipv4.icmp_ratelimit to 10 + lineinfile: + path: /etc/sysctl.conf + line: 'net.ipv4.icmp_ratelimit=10' + regexp: '^net.ipv4.icmp_ratelimit=' + state: present + +- name: Set net.ipv4.tcp_tw_reuse to 0 + lineinfile: + path: /etc/sysctl.conf + line: 'net.ipv4.tcp_tw_reuse=0' + regexp: '^net.ipv4.tcp_tw_reuse=' + state: present + +- name: Set net.ipv4.tcp_tw_recycle to 0 + lineinfile: + path: /etc/sysctl.conf + line: 'net.ipv4.tcp_tw_recycle=0' + regexp: '^net.ipv4.tcp_tw_recycle=' + state: present + +- name: update configuration in /etc/rsyslog.conf + ansible.builtin.blockinfile: + path: /etc/rsyslog.conf + block: | + $ModLoad imudp + $UDPServerAddress 0.0.0.0 + $UDPServerRun 514 + marker: "# {mark} ANSIBLE MANAGED BLOCK FOR UDP CONFIG" \ No newline at end of file diff --git a/ansible/roles/node-client/files/opt/app/bin/node/client-node.sh b/ansible/roles/node-client/files/opt/app/bin/node/client-node.sh index ee84cf2..eda64de 100644 --- a/ansible/roles/node-client/files/opt/app/bin/node/client-node.sh +++ b/ansible/roles/node-client/files/opt/app/bin/node/client-node.sh @@ -1,5 +1,5 @@ initNode() { _initNode - echo 'root:rabbitmq' | chpasswd - echo 'ubuntu:rabbitmq' | chpasswd + echo 'root:rabbitmq123' | chpasswd + echo -e "client\nclient\n" | adduser client > /dev/null 2>&1 && echo "client:rabbitmq123" | chpasswd } \ No newline at end of file diff --git a/ansible/roles/node-proxy/files/etc/confd/conf.d/haproxy.sh.toml b/ansible/roles/node-proxy-keepalived/files/etc/confd/conf.d/haproxy.sh.toml similarity index 100% rename from ansible/roles/node-proxy/files/etc/confd/conf.d/haproxy.sh.toml rename to ansible/roles/node-proxy-keepalived/files/etc/confd/conf.d/haproxy.sh.toml diff --git a/ansible/roles/node-proxy/files/etc/confd/conf.d/keepalived.sh.toml b/ansible/roles/node-proxy-keepalived/files/etc/confd/conf.d/keepalived.sh.toml similarity index 100% rename from ansible/roles/node-proxy/files/etc/confd/conf.d/keepalived.sh.toml rename to ansible/roles/node-proxy-keepalived/files/etc/confd/conf.d/keepalived.sh.toml diff --git a/ansible/roles/node-proxy-keepalived/files/etc/confd/templates/haproxy.sh/01.haproxy.cfg.tmpl b/ansible/roles/node-proxy-keepalived/files/etc/confd/templates/haproxy.sh/01.haproxy.cfg.tmpl new file mode 100644 index 0000000..63a72e8 --- /dev/null +++ b/ansible/roles/node-proxy-keepalived/files/etc/confd/templates/haproxy.sh/01.haproxy.cfg.tmpl @@ -0,0 +1,124 @@ +{{- $enabledPlugins := print (getv "/env/plugins_enabled") "," }} +flush /opt/app/conf/haproxy/haproxy.cfg << HAPROXY_CONF_EOF +#logging options +global + log /dev/log local0 info + maxconn 65535 + quiet + {{- $cores := getv "/host/cpu" }} + nbthread {{ $cores }} + +defaults + log global + mode tcp + option tcplog + option dontlognull + retries 3 + option redispatch + maxconn 65535 + timeout connect 5s + timeout client 120s + timeout server 120s + +#front-end IP for consumers and producters +listen rabbitmq_cluster + bind :5672 + mode tcp + #balance rdp-cookie + #balance leastconn + #balance source + #balance roundrobin + #simple polling + balance {{ getv "/env/haproxy_balance_policy" }} + #rabbitmq cluster node config + {{- range $dir := lsdir "/hosts/disc" }}{{ $ip := printf "/hosts/disc/%s/ip" $dir }} + server rabbit_{{ $dir }} {{ getv $ip }}:5672 check inter 2000 rise 3 fall 3 on-marked-down shutdown-sessions {{ end }} + {{- range $dir := lsdir "/hosts/ram" }}{{ $ip := printf "/hosts/ram/%s/ip" $dir }} + server rabbit_{{ $dir }} {{ getv $ip }}:5672 check inter 2000 rise 3 fall 3 on-marked-down shutdown-sessions {{ end }} + +{{- if or (contains $enabledPlugins "rabbitmq_stomp,") (contains $enabledPlugins "rabbitmq_web_stomp,") }} +#front-end IP for stomp +listen rabbitmq_cluster_stomp + bind :61613 + #TCP mode + mode tcp + balance {{ getv "/env/haproxy_balance_policy" }} + timeout client 3h + timeout server 3h + {{- range $dir := lsdir "/hosts/disc" }}{{ $ip := printf "/hosts/disc/%s/ip" $dir }} + server rabbit_{{ $dir }} {{ getv $ip }}:61613 check inter 5000 rise 2 fall 2 {{ end }} + {{- range $dir := lsdir "/hosts/ram" }}{{ $ip := printf "/hosts/ram/%s/ip" $dir }} + server rabbit_{{ $dir }} {{ getv $ip }}:61613 check inter 5000 rise 2 fall 2 {{ end }} +{{- end }} + +{{- if contains $enabledPlugins "rabbitmq_web_stomp," }} +#front-end IP for web_stomp +listen rabbitmq_cluster_web_stomp + bind :15674 + #TCP mode + mode tcp + balance {{ getv "/env/haproxy_balance_policy" }} + timeout client 3h + timeout server 3h + {{- range $dir := lsdir "/hosts/disc" }}{{ $ip := printf "/hosts/disc/%s/ip" $dir }} + server rabbit_{{ $dir }} {{ getv $ip }}:61613 check inter 5000 rise 2 fall 2 {{ end }} + {{- range $dir := lsdir "/hosts/ram" }}{{ $ip := printf "/hosts/ram/%s/ip" $dir }} + server rabbit_{{ $dir }} {{ getv $ip }}:61613 check inter 5000 rise 2 fall 2 {{ end }} +{{- end }} + +{{- if or (contains $enabledPlugins "rabbitmq_mqtt,") (contains $enabledPlugins "rabbitmq_web_mqtt,") }} +#front-end IP for mqtt +listen rabbitmq_cluster_mqtt + bind :1883 + #TCP mode + mode tcp + balance {{ getv "/env/haproxy_balance_policy" }} + timeout client 3h + timeout server 3h + {{- range $dir := lsdir "/hosts/disc" }}{{ $ip := printf "/hosts/disc/%s/ip" $dir }} + server rabbit_{{ $dir }} {{ getv $ip }}:1883 check inter 5000 rise 2 fall 2 {{ end }} + {{- range $dir := lsdir "/hosts/ram" }}{{ $ip := printf "/hosts/ram/%s/ip" $dir }} + server rabbit_{{ $dir }} {{ getv $ip }}:1883 check inter 5000 rise 2 fall 2 {{ end }} +{{- end }} + +{{- if contains $enabledPlugins "rabbitmq_web_mqtt," }} +#front-end IP for web_mqtt +listen rabbitmq_cluster_web_mqtt + bind :15675 + #TCP mode + mode tcp + balance {{ getv "/env/haproxy_balance_policy" }} + timeout client 3h + timeout server 3h + {{- range $dir := lsdir "/hosts/disc" }}{{ $ip := printf "/hosts/disc/%s/ip" $dir }} + server rabbit_{{ $dir }} {{ getv $ip }}:1883 check inter 5000 rise 2 fall 2 {{ end }} + {{- range $dir := lsdir "/hosts/ram" }}{{ $ip := printf "/hosts/ram/%s/ip" $dir }} + server rabbit_{{ $dir }} {{ getv $ip }}:1883 check inter 5000 rise 2 fall 2 {{ end }} +{{- end }} + +{{- if contains $enabledPlugins "rabbitmq_management," }} +#rabbitmq-management +# optional, for proxying management site +frontend front_rabbitmq_management + bind :15672 + default_backend backend_rabbitmq_management + +backend backend_rabbitmq_management + balance source + {{- range $dir := lsdir "/hosts/disc" }}{{ $ip := printf "/hosts/disc/%s/ip" $dir }} + server rabbit_{{ $dir }} {{ getv $ip }}:15672 check {{ end }} + {{- range $dir := lsdir "/hosts/ram" }}{{ $ip := printf "/hosts/ram/%s/ip" $dir }} + server rabbit_{{ $dir }} {{ getv $ip }}:15672 check {{ end }} +{{- end }} + +# haproxy web monitor infomation +listen statistics + bind :{{ getv "/env/haproxy_web_port" "8100" }} + mode http + stats enable + stats hide-version + stats refresh 10s + stats uri / + stats auth {{ getv "/env/haproxy_username" "haproxy" }}:{{ replace (getv "/env/haproxy_password" "haproxy") `$` `\$` -1 }} + +HAPROXY_CONF_EOF diff --git a/ansible/roles/node-proxy-keepalived/files/etc/confd/templates/haproxy.sh/02.haproxy.log.conf.tmpl b/ansible/roles/node-proxy-keepalived/files/etc/confd/templates/haproxy.sh/02.haproxy.log.conf.tmpl new file mode 100644 index 0000000..166d335 --- /dev/null +++ b/ansible/roles/node-proxy-keepalived/files/etc/confd/templates/haproxy.sh/02.haproxy.log.conf.tmpl @@ -0,0 +1,25 @@ +{{- if getvs "/host/role" | filter "haproxy" }} +mkdir -p ${DATA_MOUNTS}/log/haproxy/ +chmod 755 ${DATA_MOUNTS}/log/haproxy/ +chown syslog:svc ${DATA_MOUNTS}/log/haproxy/ + +flush /etc/rsyslog.d/49-haproxy.conf << HAPROXY_LOG_CONF_EOF + +# Create an additional socket in haproxy's chroot in order to allow logging via +# /dev/log to chroot'ed HAProxy processes +# \$AddUnixListenSocket /var/lib/haproxy/dev/log + +# Send HAProxy messages to a dedicated logfile +:programname, startswith, "haproxy" { + action( + type="omfile" + file="${DATA_MOUNTS}/log/haproxy/haproxy.log" + FileOwner="syslog" + FileGroup="svc" + FileCreateMode="0640" + DirCreateMode="0755" + ) + stop +} +HAPROXY_LOG_CONF_EOF +{{- end }} diff --git a/ansible/roles/node-proxy-keepalived/files/etc/confd/templates/haproxy.sh/03.logrotate.tmpl b/ansible/roles/node-proxy-keepalived/files/etc/confd/templates/haproxy.sh/03.logrotate.tmpl new file mode 100644 index 0000000..f09c473 --- /dev/null +++ b/ansible/roles/node-proxy-keepalived/files/etc/confd/templates/haproxy.sh/03.logrotate.tmpl @@ -0,0 +1,15 @@ + +flush /etc/logrotate.d/haproxy << LOGROTATE_EOF +${DATA_MOUNTS}/log/haproxy/haproxy.log { + daily + maxsize 2M + rotate 10 + missingok + notifempty + compress + delaycompress + postrotate + /usr/bin/systemctl kill -s HUP rsyslog.service >/dev/null 2>&1 || true + endscript +} +LOGROTATE_EOF \ No newline at end of file diff --git a/ansible/roles/node-proxy-keepalived/files/etc/confd/templates/keepalived.sh/01.keepalived.conf.tmpl b/ansible/roles/node-proxy-keepalived/files/etc/confd/templates/keepalived.sh/01.keepalived.conf.tmpl new file mode 100644 index 0000000..ed426f0 --- /dev/null +++ b/ansible/roles/node-proxy-keepalived/files/etc/confd/templates/keepalived.sh/01.keepalived.conf.tmpl @@ -0,0 +1,44 @@ +flush /etc/keepalived/keepalived.conf << KEEPALIVED_EOF +{{- $replicaIPs := split (getv "/cluster/endpoints/reserved_ips/vip/value") "." }} +{{- $myIp := getv "/host/ip" }} +global_defs { + enable_script_security + script_user root root +} + +vrrp_script check_haproxy { + script "/usr/bin/pkill -0 haproxy" + interval 2 + weight 10 +} + +vrrp_instance HAProxy_HA { + state BACKUP + interface eth0 + virtual_router_id {{ index $replicaIPs 3 }} + priority 100 + advert_int 2 + unicast_src_ip {{ $myIp }} + unicast_peer { + {{- range $dir := lsdir "/hosts/haproxy" }} + {{- $ip := getv (printf "/hosts/haproxy/%s/ip" $dir) }} + {{- if ne $ip $myIp }} + {{ $ip }} + {{- end }} + {{- end }} + } + authentication { + auth_type PASS + auth_pass {{ getv "/cluster/cluster_id" }} + } + + virtual_ipaddress { #set VIP + {{ getv "/cluster/endpoints/reserved_ips/vip/value" }}/32 + } + + track_script { + check_haproxy + } +} + +KEEPALIVED_EOF diff --git a/ansible/roles/node-proxy-keepalived/files/etc/confd/templates/keepalived.sh/02.keepalived.sysconfig.tmpl b/ansible/roles/node-proxy-keepalived/files/etc/confd/templates/keepalived.sh/02.keepalived.sysconfig.tmpl new file mode 100644 index 0000000..52f1848 --- /dev/null +++ b/ansible/roles/node-proxy-keepalived/files/etc/confd/templates/keepalived.sh/02.keepalived.sysconfig.tmpl @@ -0,0 +1,18 @@ +flush /opt/keepalived/current/etc/sysconfig/keepalived << 'KEEPALIVED_SYSCONFIG_EOF' + +# Options for keepalived. See `keepalived --help' output and keepalived(8) and +# keepalived.conf(5) man pages for a list of all options. Here are the most +# common ones : +# +# --vrrp -P Only run with VRRP subsystem. +# --check -C Only run with Health-checker subsystem. +# --dont-release-vrrp -V Dont remove VRRP VIPs & VROUTEs on daemon stop. +# --dont-release-ipvs -I Dont remove IPVS topology on daemon stop. +# --dump-conf -d Dump the configuration data. +# --log-detail -D Detailed log messages. +# --log-facility -S 0-7 Set local syslog facility (default=LOG_DAEMON) +# + +KEEPALIVED_OPTIONS="-D -d -S 0" + +KEEPALIVED_SYSCONFIG_EOF diff --git a/ansible/roles/node-proxy-keepalived/files/etc/confd/templates/keepalived.sh/03.keepalived.rsyslog.tmpl b/ansible/roles/node-proxy-keepalived/files/etc/confd/templates/keepalived.sh/03.keepalived.rsyslog.tmpl new file mode 100644 index 0000000..2d73844 --- /dev/null +++ b/ansible/roles/node-proxy-keepalived/files/etc/confd/templates/keepalived.sh/03.keepalived.rsyslog.tmpl @@ -0,0 +1,20 @@ +{{- if getvs "/host/role" | filter "haproxy" }} +mkdir -p ${DATA_MOUNTS}/log/keepalived/ +chmod 755 ${DATA_MOUNTS}/log/keepalived/ +chown syslog:svc ${DATA_MOUNTS}/log/keepalived/ +flush /etc/rsyslog.d/49-keepalived.conf << KEEPALIVED_LOG_EOF + +if \$programname startswith 'Keepalived' then { + action( + type="omfile" + file="${DATA_MOUNTS}/log/keepalived/keepalived.log" + FileOwner="syslog" + FileGroup="svc" + FileCreateMode="0640" + DirCreateMode="0755" + ) + stop +} + +KEEPALIVED_LOG_EOF +{{- end }} diff --git a/ansible/roles/node-proxy-keepalived/files/etc/confd/templates/keepalived.sh/04.logrotate.tmpl b/ansible/roles/node-proxy-keepalived/files/etc/confd/templates/keepalived.sh/04.logrotate.tmpl new file mode 100644 index 0000000..d5bcde7 --- /dev/null +++ b/ansible/roles/node-proxy-keepalived/files/etc/confd/templates/keepalived.sh/04.logrotate.tmpl @@ -0,0 +1,14 @@ +flush /etc/logrotate.d/keepalived << LOGROTATE_EOF +${DATA_MOUNTS}/log/keepalived/keepalived.log { + daily + maxsize 2M + rotate 10 + missingok + notifempty + compress + delaycompress + postrotate + /usr/bin/systemctl kill -s HUP rsyslog.service >/dev/null 2>&1 || true + endscript +} +LOGROTATE_EOF \ No newline at end of file diff --git a/ansible/roles/haproxy/files/lib/systemd/system/haproxy.service b/ansible/roles/node-proxy-keepalived/files/lib/systemd/system/haproxy.service similarity index 73% rename from ansible/roles/haproxy/files/lib/systemd/system/haproxy.service rename to ansible/roles/node-proxy-keepalived/files/lib/systemd/system/haproxy.service index f55e301..4ac0555 100644 --- a/ansible/roles/haproxy/files/lib/systemd/system/haproxy.service +++ b/ansible/roles/node-proxy-keepalived/files/lib/systemd/system/haproxy.service @@ -3,16 +3,15 @@ Description=HAProxy Load Balancer Documentation=man:haproxy(1) Documentation=file:/usr/share/doc/haproxy/configuration.txt.gz After=network.target rsyslog.service +ConditionFileNotEmpty=/opt/app/conf/haproxy/haproxy.cfg [Service] -User=root -Group=root -EnvironmentFile=-/etc/default/haproxy -EnvironmentFile=-/etc/sysconfig/haproxy -Environment="CONFIG=/opt/app/conf/haproxy/haproxy.cfg" "PIDFILE=/run/haproxy.pid" "EXTRAOPTS=-S /run/haproxy-master.sock" -ExecStartPre=/usr/sbin/haproxy -f $CONFIG -c -q $EXTRAOPTS -ExecStart=/usr/sbin/haproxy -Ws -f $CONFIG -p $PIDFILE $EXTRAOPTS -ExecReload=/usr/sbin/haproxy -f $CONFIG -c -q $EXTRAOPTS +User=haproxy +Group=svc +RuntimeDirectory=haproxy +Environment="CONFIG=/opt/app/conf/haproxy/haproxy.cfg" "PIDFILE=/var/run/haproxy/haproxy.pid" "BINDOPT=/var/run/haproxy/haproxy-master.sock" +ExecStartPre=/opt/haproxy/current/haproxy -f $CONFIG -c -q +ExecStart=/opt/haproxy/current/haproxy -Ws -f $CONFIG -p $PIDFILE -S $BINDOPT ExecReload=/bin/kill -USR2 $MAINPID KillMode=mixed Restart=always @@ -25,6 +24,9 @@ Type=notify # reduced performance. See systemd.service(5) and systemd.exec(5) for further # information. +LimitNOFILE=infinity +CapabilityBoundingSet=CAP_NET_BIND_SERVICE +AmbientCapabilities=CAP_NET_BIND_SERVICE # NoNewPrivileges=true # ProtectHome=true # If you want to use 'ProtectSystem=strict' you should whitelist the PIDFILE, diff --git a/ansible/roles/keepalived/files/lib/systemd/system/keepalived.service b/ansible/roles/node-proxy-keepalived/files/lib/systemd/system/keepalived.service similarity index 60% rename from ansible/roles/keepalived/files/lib/systemd/system/keepalived.service rename to ansible/roles/node-proxy-keepalived/files/lib/systemd/system/keepalived.service index 6fa3222..e25495d 100644 --- a/ansible/roles/keepalived/files/lib/systemd/system/keepalived.service +++ b/ansible/roles/node-proxy-keepalived/files/lib/systemd/system/keepalived.service @@ -2,13 +2,17 @@ Description=LVS and VRRP High Availability Monitor After=network-online.target syslog.target Wants=network-online.target +Documentation=man:keepalived(8) +Documentation=man:keepalived.conf(5) +Documentation=man:genhash(1) +Documentation=https://keepalived.org [Service] -Type=forking +Type=notify PIDFile=/run/keepalived.pid KillMode=process EnvironmentFile=-/opt/keepalived/current/etc/sysconfig/keepalived -ExecStart=/opt/keepalived/current/sbin/keepalived $KEEPALIVED_OPTIONS +ExecStart=/usr/local/sbin/keepalived --dont-fork $KEEPALIVED_OPTIONS ExecReload=/bin/kill -HUP $MAINPID [Install] diff --git a/ansible/roles/node-proxy-keepalived/files/opt/app/bin/node/proxy-node.sh b/ansible/roles/node-proxy-keepalived/files/opt/app/bin/node/proxy-node.sh new file mode 100644 index 0000000..d77f2b6 --- /dev/null +++ b/ansible/roles/node-proxy-keepalived/files/opt/app/bin/node/proxy-node.sh @@ -0,0 +1,43 @@ +checkSvc() { + checkActive ${1%%/*} || { + log "Service '$1' is inactive." + return $EC_CHECK_INACTIVE + } + local endpoints=$(echo $1 | awk -F/ '{print $3}') + local endpoint; for endpoint in ${endpoints//,/ }; do + checkEndpoint $endpoint || { + log "Endpoint '$endpoint' is unreachable." + return 0 + } + done +} + +stop() { + log "INFO: Application is asked to stop . " + _stop || (log "ERROR: services in Node ${MY_INSTANCE_ID} failed to stop . " && return 1) + log "INFO: Application stopped successfully . " +} + +start() { + log "INFO: Application is asked to start . " + _start || (log "ERROR: services in Node ${MY_INSTANCE_ID} failed to start . " && return 1) + log "INFO: Application started successfully . " +} + +initNode() { + log "INFO: Application is about to initialize . " + _initNode || ( log "ERROR: Application failed to initialize . " && return 1 ) + mkdir -p /data/haproxy/logs + chown -R haproxy.haproxy /data/haproxy + mkdir -p /data/keepalived/logs + chown -R root.root /data/keepalived + mkdir -p /data/caddy + chown -R caddy:svc /data/caddy + log "INFO: Application initialization completed . " +} + +reload() { + log "INFO: Application is asked to reload . " + _reload $@ + log "INFO: Application reloaded completely . " +} \ No newline at end of file diff --git a/ansible/roles/node-proxy-keepalived/tasks/main.yml b/ansible/roles/node-proxy-keepalived/tasks/main.yml new file mode 100644 index 0000000..fbf1ac7 --- /dev/null +++ b/ansible/roles/node-proxy-keepalived/tasks/main.yml @@ -0,0 +1,156 @@ +--- +- name: copy conf files + copy: + src: files/opt/app/ + dest: /opt/app/ + owner: root + group: svc + mode: preserve + directory_mode: u=rwx,g=rx,o= + +- name: install confd files + include_role: + name: confd-files + +# haproxy +- name: set vars + set_fact: + haproxy_version: "{{ haproxy_version | d('3.0.9') }}" + os: "{{ os | d(default_os) }}" + arch: "{{ arch | d(default_arch) }}" + osv: "{{ osv | d(default_osv) }}" + installPath: "/opt/haproxy" + +- name: group svc + group: + name: svc + state: present + +- name: prepare service user + user: + name: haproxy + groups: svc + shell: /sbin/nologin + create_home: no + append: yes + comment: "Service User" + state: present + +# will set var downloaded +- include_role: + name: downloader + vars: + opts: + pkg_name: haproxy + pkg_version: "{{ haproxy_version }}" + pkg_feature: "{{ os }}-{{ osv }}-{{ arch }}" + pkg_fileType: ".tgz" + pkg_url: "https://github.com/djangoyi-yunify/haproxy-compiler/releases/download/haproxy-{{ haproxy_version }}_{{ os }}-{{ osv }}/haproxy-{{ haproxy_version }}-{{ arch }}.tgz" + +- name: prepare dir + file: + path: "{{ installPath }}/{{ haproxy_version }}" + state: directory + +- name: link for current + file: + src: "{{ installPath }}/{{ haproxy_version }}" + dest: "{{ installPath }}/current" + state: link + +- name: decompress file + unarchive: + src: "{{ downloaded }}" + dest: "{{ installPath }}/current" + creates: "{{ installPath }}/current/bin/haproxy" + extra_opts: + - --strip-components=4 + - --wildcards + - output/usr/local/sbin/haproxy + +- name: creates directory + file: + path: /opt/app/conf/haproxy + state: directory + +# keepalived +- name: set vars + set_fact: + keepalived_version: "{{ keepalived_version | d('2.3.3') }}" + os: "{{ os | d(default_os) }}" + arch: "{{ arch | d(default_arch) }}" + osv: "{{ osv | d(default_osv) }}" + optPath: "/opt/keepalived" + +- name: group svc + group: + name: svc + state: present + +- name: prepare service user + user: + name: keepalived + groups: svc + shell: /sbin/nologin + create_home: no + append: yes + comment: "Service User" + state: present + +# will set var downloaded +- include_role: + name: downloader + vars: + opts: + pkg_name: keepalived + pkg_version: "{{ keepalived_version }}" + pkg_feature: "{{ os }}-{{ osv }}-{{ arch }}" + pkg_fileType: ".tgz" + pkg_url: "https://github.com/djangoyi-yunify/keepalived-compiler/releases/download/keepalived-{{ keepalived_version }}_{{ os }}-{{ osv }}/keepalived-{{ keepalived_version }}-{{ arch }}.tgz" + +- name: decompress file + unarchive: + src: "{{ downloaded }}" + dest: "/" + creates: "/usr/local/sbin/keepalived" + extra_opts: + - --strip-components=1 + +- name: creates directory + file: + path: /etc/keepalived + state: directory + +- name: creates directory + file: + path: "{{ optPath }}/{{ keepalived_version }}" + state: directory + +- name: link for current + file: + src: "{{ optPath }}/{{ keepalived_version }}" + dest: "{{ optPath }}/current" + state: link + +- name: more path for keepalived + file: + path: "{{ optPath }}/current/etc/sysconfig" + state: directory + +- name: copy service files + copy: + src: files/lib/systemd/system/ + dest: /lib/systemd/system/ + owner: root + group: root + mode: preserve + +- name: mask services + systemd: + name: "{{ item }}" + enabled: no + masked: yes + state: stopped + loop: + - haproxy + - keepalived \ No newline at end of file diff --git a/ansible/roles/node-proxy/files/etc/confd/templates/keepalived.sh/01.keepalived.conf.tmpl b/ansible/roles/node-proxy/files/etc/confd/templates/keepalived.sh/01.keepalived.conf.tmpl deleted file mode 100644 index ee5bc20..0000000 --- a/ansible/roles/node-proxy/files/etc/confd/templates/keepalived.sh/01.keepalived.conf.tmpl +++ /dev/null @@ -1,34 +0,0 @@ -flush /etc/keepalived/keepalived.conf << KEEPALIVED_EOF -{{ $replicaIPs := split (getv "/cluster/endpoints/reserved_ips/vip/value") "." }} -global_defs { -} - -vrrp_script check_haproxy { - script "/usr/bin/killall -0 haproxy" - interval 2 - weight 2 -} - -vrrp_instance HAProxy_HA { - state BACKUP - interface eth0 - virtual_router_id {{ index $replicaIPs 3 }} - priority {{ getv "/host/sid" }} - advert_int 2 - nopreempt - unicast_src_ip {{ getv "/host/ip" }} - authentication { - auth_type PASS - auth_pass pwd - } - - virtual_ipaddress { #set VIP - {{ getv "/cluster/endpoints/reserved_ips/vip/value" }}/24 - } - - track_script { - check_haproxy - } -} - -KEEPALIVED_EOF diff --git a/ansible/roles/node-proxy/files/opt/app/bin/node/proxy-node.sh b/ansible/roles/node-proxy/files/opt/app/bin/node/proxy-node.sh deleted file mode 100644 index e378170..0000000 --- a/ansible/roles/node-proxy/files/opt/app/bin/node/proxy-node.sh +++ /dev/null @@ -1,7 +0,0 @@ -initNode() { - _initNode - mkdir -p /data/haproxy/logs - chown -R haproxy.haproxy /data/haproxy - mkdir -p /data/keepalived/logs - chown -R root.root /data/keepalived -} diff --git a/ansible/roles/node-proxy/tasks/main.yml b/ansible/roles/node-proxy/tasks/main.yml deleted file mode 100644 index cf955eb..0000000 --- a/ansible/roles/node-proxy/tasks/main.yml +++ /dev/null @@ -1,13 +0,0 @@ ---- -- name: copy conf files - copy: - src: files/opt/app/ - dest: /opt/app/ - owner: root - group: svc - mode: preserve - directory_mode: u=rwx,g=rx,o= - -- name: install confd files - include_role: - name: confd-files-1.0.2 \ No newline at end of file diff --git a/ansible/roles/node-rabbitmq/files/etc/confd/templates/caddy.sh/00.svc.env.tmpl b/ansible/roles/node-rabbitmq/files/etc/confd/templates/caddy.sh/00.svc.env.tmpl index 36ac4df..6bf6607 100644 --- a/ansible/roles/node-rabbitmq/files/etc/confd/templates/caddy.sh/00.svc.env.tmpl +++ b/ansible/roles/node-rabbitmq/files/etc/confd/templates/caddy.sh/00.svc.env.tmpl @@ -1,6 +1,6 @@ flush /opt/app/bin/envs/svc-caddy.env << CADDY_ENV_EOF -{{- if getvs "/host/role" | filter "(disc|ram)" }} -SERVICES="\$SERVICES caddy/{{ getv "/env/web_console_enabled" "false" }}/http:80" +{{- if getvs "/host/role" | filter "(disc|ram|haproxy)" }} +SERVICES="\$SERVICES caddy/{{ getv "/env/web_console_enabled" "true" }}/http:80" {{- end }} CADDY_ENV_EOF diff --git a/ansible/roles/node-rabbitmq/files/etc/confd/templates/caddy.sh/01.caddyfile.tmpl b/ansible/roles/node-rabbitmq/files/etc/confd/templates/caddy.sh/01.caddyfile.tmpl index fc1efd0..91c7352 100644 --- a/ansible/roles/node-rabbitmq/files/etc/confd/templates/caddy.sh/01.caddyfile.tmpl +++ b/ansible/roles/node-rabbitmq/files/etc/confd/templates/caddy.sh/01.caddyfile.tmpl @@ -1,13 +1,31 @@ +{{- if getvs "/host/role" | filter "(disc|ram|haproxy)" }} ln -s -f /opt/app/conf/caddy/index.html /data/index.html +flush /opt/app/conf/caddy/caddyfile << CADDYFILE_EOF +{ + admin off + auto_https off + persist_config off + order upload before file_server + log { + output file /data/caddy/caddy.log + } +} + +:80 { + vars { + rootDir "/data" + } + + root {vars.rootDir} + + handle /log* { + file_server browse + } -flush /opt/app/conf/caddy/Caddyfile << CADDYFILE_EOF -{{ getv "/host/ip" }}:80 { - {{- with getv "/env/web_console_password" "" }} - basicauth / "{{ getv "/env/web_console_username" "admin" }}" "{{ . }}" - {{- end }} - root /data - gzip - browse /log - tls off + file_server } CADDYFILE_EOF +{{- end }} + + + diff --git a/ansible/roles/node-rabbitmq/files/etc/confd/templates/caddy.sh/02.index.html.tmpl b/ansible/roles/node-rabbitmq/files/etc/confd/templates/caddy.sh/02.index.html.tmpl index cd76688..9c64cd3 100644 --- a/ansible/roles/node-rabbitmq/files/etc/confd/templates/caddy.sh/02.index.html.tmpl +++ b/ansible/roles/node-rabbitmq/files/etc/confd/templates/caddy.sh/02.index.html.tmpl @@ -64,12 +64,12 @@ flush /opt/app/conf/caddy/index.html << HTML_EOF Files -
文件查看器
+
日志查看器
{{- $discNodes := getvs "/hosts/disc/*/ip" }} - {{- $ramNodes := getvs "/hosts/ram/*/ip" }} + {{- $proxyNodes := getvs "/hosts/haproxy/*/ip" }}
- 共 {{ len $discNodes }} 个 disc 节点, {{ len $ramNodes }} 个 ram 节点 + 共 {{ len $discNodes }} 个 disc 节点, {{ len $proxyNodes }} 个 haproxy 节点