create cluster failed

[Joey-gvwal]

use templates/cluster/2nic/prod_stack/dynamic/f5_bigip_cluster_2_nic.yaml to create bigip cluster, it falied with error:
Resource CREATE failed: WaitConditionTimeout: resources.device_cluster.resources[1].resources.wait_condition_onboard_complete: 2 of 3 received - Onboard command exited without error.;Pre-onboard completed without error.

In bigip instance,
master:
2018-10-17T03:12:04.240Z info: [pid: 25241] [scripts/cluster.js] Waiting for BIG-IP to be active.
2018-10-17T03:12:04.251Z debug: [pid: 25241] [lib/bigIp.js]
2018-10-17T03:12:04.252Z debug: [pid: 25241] [lib/bigIp.js] list 10.10.3.8 /shared/identified-devices/config/device-info/available
2018-10-17T03:12:04.265Z debug: [pid: 25241] [lib/bigIp.js]
2018-10-17T03:12:04.267Z debug: [pid: 25241] [lib/bigIp.js] list 10.10.3.8 /tm/sys/available
2018-10-17T03:12:04.285Z debug: [pid: 25241] [lib/bigIp.js]
2018-10-17T03:12:04.287Z debug: [pid: 25241] [lib/bigIp.js] list 10.10.3.8 /tm/cm/available
2018-10-17T03:12:04.305Z debug: [pid: 25241] [lib/bigIp.js]
2018-10-17T03:12:04.307Z debug: [pid: 25241] [lib/bigIp.js] list 10.10.3.8 /tm/sys/mcp-state/
2018-10-17T03:12:04.327Z debug: [pid: 25241] [lib/bigIp.js] {"kind":"tm:sys:mcp-state:mcp-statestats","selfLink":"https://localhost/mgmt/tm/sys/mcp-state?ver=13.1.0.2","entries":{"https://localhost/mgmt/tm/sys/mcp-state/0":{"nestedStats":{"entries":{"endPlatformIdReceived":{"description":"true"},"lastLoad":{"description":"full-config-load-succeed"},"phase":{"description":"running"}}}}}}
2018-10-17T03:12:04.329Z debug: [pid: 25241] [lib/bigIp.js] list 10.10.3.8 /tm/cm/failover-status
2018-10-17T03:12:04.349Z debug: [pid: 25241] [lib/bigIp.js] {"kind":"tm:cm:failover-status:failover-statusstats","selfLink":"https://localhost/mgmt/tm/cm/failover-status?ver=13.1.0.2","entries":{"https://localhost/mgmt/tm/cm/failover-status/0":{"nestedStats":{"entries":{"color":{"description":"green"},"https://localhost/mgmt/tm/cm/failoverStatus/0/details":{"nestedStats":{"entries":{"https://localhost/mgmt/tm/cm/failoverStatus/0/details/0":{"nestedStats":{"entries":{"details":{"description":"active for /Common/traffic-group-1"}}}}}}},"status":{"description":"ACTIVE"},"summary":{"description":"1/1 active"}}}}}}
2018-10-17T03:12:04.350Z debug: [pid: 25241] [scripts/cluster.js]
2018-10-17T03:12:04.350Z debug: [pid: 25241] [lib/bigIp.js] Current state: ACTIVE

standby:
2018-10-17T03:12:27.189Z info: [pid: 25706] [scripts/cluster.js] BIG-IP is ready.
2018-10-17T03:12:27.190Z debug: [pid: 25706] [scripts/cluster.js]
2018-10-17T03:12:27.190Z debug: [pid: 25706] [scripts/cluster.js]
2018-10-17T03:12:27.191Z info: [pid: 25706] [scripts/cluster.js] Joining group.
2018-10-17T03:12:28.331Z debug: [pid: 25706] [lib/bigIp.js] list 10.10.3.5 /shared/echo-js/available
2018-10-17T03:12:28.330Z info: [pid: 25706] [lib/bigIpCluster.js] Checking remote host for cluster readiness.
2018-10-17T03:12:28.342Z debug: [pid: 25706] [lib/bigIp.js]
2018-10-17T03:12:28.344Z debug: [pid: 25706] [lib/bigIp.js] list 10.10.3.5 /shared/identified-devices/config/device-info/available
2018-10-17T03:12:28.349Z debug: [pid: 25706] [lib/bigIp.js]
2018-10-17T03:12:28.350Z debug: [pid: 25706] [lib/bigIp.js] list 10.10.3.5 /tm/sys/available
2018-10-17T03:12:28.359Z debug: [pid: 25706] [lib/bigIp.js]
2018-10-17T03:12:28.361Z debug: [pid: 25706] [lib/bigIp.js] list 10.10.3.5 /tm/cm/available
2018-10-17T03:12:28.368Z debug: [pid: 25706] [lib/bigIp.js]
2018-10-17T03:12:28.369Z debug: [pid: 25706] [lib/bigIp.js] list 10.10.3.5 /tm/sys/mcp-state/
2018-10-17T03:12:28.385Z debug: [pid: 25706] [lib/bigIp.js] {"kind":"tm:sys:mcp-state:mcp-statestats","selfLink":"https://localhost/mgmt/tm/sys/mcp-state?ver=13.1.0.2","entries":{"https://localhost/mgmt/tm/sys/mcp-state/0":{"nestedStats":{"entries":{"endPlatformIdReceived":{"description":"true"},"lastLoad":{"description":"full-config-load-succeed"},"phase":{"description":"running"}}}}}}
2018-10-17T03:12:28.387Z debug: [pid: 25706] [lib/bigIp.js] list 10.10.3.5 /shared/identified-devices/config/device-info
2018-10-17T03:12:28.398Z debug: [pid: 25706] [lib/bigIp.js] {"baseMac":"FA:16:3E:0F:CD:2A","hostMac":"FA:16:3E:0F:CD:2A","time":1539745948390,"halUuid":"f1b5f395-e581-9a4c-a228-74ecf72002d4","physicalMemory":8000,"platform":"Z100","chassisSerialNumber":"f1b5f395-e581-9a4c-74ecf72002d4","cpu":"Intel Core Processor (Skylake)","slots":[{"volume":"HD1.1","product":"BIG-IP","version":"13.1.0.2","build":"0.0.6","isActive":true}],"license":{"generation":0,"lastUpdateMicros":1539745884822663},"interfaces":["1.1","mgmt"],"isIControlRestSupported":true,"icrdPort":8100,"machineId":"a64c447d-9a08-46c6-b171-5da16da7eacf","address":"192.168.80.5","hostname":"host-10-10-3-5.openstacklocal","version":"13.1.0.2","product":"BIG-IP","platformMarketingName":"BIG-IP Virtual Edition","edition":"Point Release 2","build":"0.0.6","restFrameworkVersion":"13.1.0.2-0.0.6","managementAddress":"10.10.3.5","mcpDeviceName":"/Common/host-e1c2fe91-4d29-4a33-b3e9-ecb188949001.localhost","trustDomainGuid":"75a85c69-881f-4235-a47bfa163e0fcd2a","isClustered":false,"isVirtual":true,"generation":0,"lastUpdateMicros":0,"kind":"shared:resolver:device-groups:deviceinfostate","selfLink":"https://localhost/mgmt/shared/identified-devices/config/device-info"}
2018-10-17T03:12:28.427Z debug: [pid: 25706] [lib/bigIp.js] list 10.10.3.5 /tm/cm/device/Commonhost-10-10-3-5.openstacklocal
2018-10-17T03:12:38.449Z debug: [pid: 25706] [lib/bigIp.js] list 10.10.3.5 /shared/echo-js/available
2018-10-17T03:12:38.459Z debug: [pid: 25706] [lib/bigIp.js]
2018-10-17T03:12:38.461Z debug: [pid: 25706] [lib/bigIp.js] list 10.10.3.5 /shared/identified-devices/config/device-info/available
2018-10-17T03:12:38.468Z debug: [pid: 25706] [lib/bigIp.js]
2018-10-17T03:12:38.469Z debug: [pid: 25706] [lib/bigIp.js] list 10.10.3.5 /tm/sys/available
2018-10-17T03:12:38.483Z debug: [pid: 25706] [lib/bigIp.js]
2018-10-17T03:12:38.485Z debug: [pid: 25706] [lib/bigIp.js] list 10.10.3.5 /tm/cm/available
2018-10-17T03:12:38.492Z debug: [pid: 25706] [lib/bigIp.js]
2018-10-17T03:12:38.494Z debug: [pid: 25706] [lib/bigIp.js] list 10.10.3.5 /tm/sys/mcp-state/
2018-10-17T03:12:38.506Z debug: [pid: 25706] [lib/bigIp.js] {"kind":"tm:sys:mcp-state:mcp-statestats","selfLink":"https://localhost/mgmt/tm/sys/mcp-state?ver=13.1.0.2","entries":{"https://localhost/mgmt/tm/sys/mcp-state/0":{"nestedStats":{"entries":{"endPlatformIdReceived":{"description":"true"},"lastLoad":{"description":"full-config-load-succeed"},"phase":{"description":"running"}}}}}}
2018-10-17T03:12:38.509Z debug: [pid: 25706] [lib/bigIp.js] list 10.10.3.5 /shared/identified-devices/config/device-info
2018-10-17T03:12:38.518Z debug: [pid: 25706] [lib/bigIp.js] {"baseMac":"FA:16:3E:0F:CD:2A","hostMac":"FA:16:3E:0F:CD:2A","time":1539745958512,"halUuid":"f1b5f395-e581-9a4c-a228-74ecf72002d4","physicalMemory":8000,"platform":"Z100","chassisSerialNumber":"f1b5f395-e581-9a4c-74ecf72002d4","cpu":"Intel Core Processor (Skylake)","slots":[{"volume":"HD1.1","product":"BIG-IP","version":"13.1.0.2","build":"0.0.6","isActive":true}],"license":{"generation":0,"lastUpdateMicros":1539745884822663},"interfaces":["1.1","mgmt"],"isIControlRestSupported":true,"icrdPort":8100,"machineId":"a64c447d-9a08-46c6-b171-5da16da7eacf","address":"192.168.80.5","hostname":"host-10-10-3-5.openstacklocal","version":"13.1.0.2","product":"BIG-IP","platformMarketingName":"BIG-IP Virtual Edition","edition":"Point Release 2","build":"0.0.6","restFrameworkVersion":"13.1.0.2-0.0.6","managementAddress":"10.10.3.5","mcpDeviceName":"/Common/host-e1c2fe91-4d29-4a33-b3e9-ecb188949001.localhost","trustDomainGuid":"75a85c69-881f-4235-a47bfa163e0fcd2a","isClustered":false,"isVirtual":true,"generation":0,"lastUpdateMicros":0,"kind":"shared:resolver:device-groups:deviceinfostate","selfLink":"https://localhost/mgmt/shared/identified-devices/config/device-info"}
2018-10-17T03:12:38.528Z debug: [pid: 25706] [lib/bigIp.js] list 10.10.3.5 /tm/cm/device/Commonhost-10-10-3-5.openstacklocal
2018-10-17T03:12:48.545Z debug: [pid: 25706] [lib/bigIp.js] list 10.10.3.5 /shared/echo-js/available
...
...

I can not see any error in joining group, but it has no return.

[kaustriaf5]

Hi @guanzydev, based on the logs above, it seems that postOnboard.sh may have either failed, or that it ran but sending of heat signal by that script failed. Do you see anything in /var/log/cloud/openstack/postOnboard.log or /var/log/cloud/openstack/runScript.log?