Skip to content

Commit d63afea

Browse files
frhuelszCopilot
andauthored
engineering: storm-based e2e base config (#377)
* Progress * Install e2e * disable regular * Restrict template expansion * Fix typo * missing closing ) * Force type * Fix params * Fix apt install command * fix ob folder * fix package versions * Setup fixes * Fix socket override * Fix init * Use correct function * Install virt-firmware * re-enable regular e2e * add deve clarification * pr comments * pr comments * Update tools/storm/utils/cmd/cmd.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update tools/storm/utils/env/osrelease.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update tools/pkg/netfinder/netfinder.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update tools/storm/e2e/scenario/trident.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * pr comments * camelCase * Update tools/pkg/netlaunch/netlaunch.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update tools/pkg/netlaunch/netlaunch.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * pr comments and proper error wrapping * pr comments * Add readme --------- Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
1 parent 5000455 commit d63afea

File tree

23 files changed

+1582
-118
lines changed

23 files changed

+1582
-118
lines changed

.pipelines/templates/e2e-template.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -194,7 +194,7 @@ stages:
194194
micVersion: ${{ parameters.micVersion }}
195195
dependsOnStage: ${{ parameters.baseImageArtifactStage }}
196196

197-
# Build AzL installer ISO (attended and unattended)
197+
# Build AzL installer ISO (attended and unattended)
198198
- ${{ if or(eq(parameters.stageType, 'pr-e2e'), eq(parameters.stageType, 'ci'), eq(parameters.stageType, 'pr-e2e-azure')) }}:
199199
- template: stages/azl_installer/azl-installer.yml
200200
parameters:

.pipelines/templates/stages/testing_e2e/storm_e2e.yml

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -35,10 +35,10 @@ stages:
3535
displayName: Matrix of Trident configurations for E2E Tests
3636
3737
# TODO: enable once storm tests are ready
38-
# - template: test_execution_template.yml
39-
# parameters:
40-
# hardwareType: "VM"
41-
# runtimeType: "HOST"
38+
- template: test_execution_template.yml
39+
parameters:
40+
hardwareType: "VM"
41+
runtimeType: "HOST"
4242

4343
# - template: test_execution_template.yml
4444
# parameters:
@@ -49,8 +49,10 @@ stages:
4949
# parameters:
5050
# hardwareType: "BM"
5151
# runtimeType: "HOST"
52+
# maxParallel: 1
5253

5354
# - template: test_execution_template.yml
5455
# parameters:
5556
# hardwareType: "BM"
5657
# runtimeType: "CONTAINER"
58+
# maxParallel: 1

.pipelines/templates/stages/testing_e2e/test_execution_template.yml

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,10 @@ parameters:
1515
type: boolean
1616
default: false
1717

18+
- name: maxParallel
19+
type: string
20+
default: null
21+
1822
stages:
1923
- stage: E2ETesting_${{ parameters.hardwareType }}_${{ parameters.runtimeType }}
2024
displayName: E2E ${{ parameters.hardwareType }} ${{ parameters.runtimeType }} Testing
@@ -50,6 +54,8 @@ stages:
5054

5155
strategy:
5256
matrix: $[ variables['matrixJson'] ]
57+
${{ if and(parameters.maxParallel, ne(parameters.maxParallel, 'null'), ne(parameters.maxParallel, ''), ne(parameters.maxParallel, 0)) }}:
58+
maxParallel: ${{ parameters.maxParallel }}
5359

5460
condition: ne(variables['matrixJson'], '{}')
5561
variables:
@@ -81,6 +87,14 @@ stages:
8187
value: false
8288

8389
steps:
90+
- bash: |
91+
if [ ${{ variables['tridentConfigurationName'] }} == 'split' ]; then
92+
splitInstallerIsoName="trident-split-installer"
93+
echo "setting variable.installerISOName to $splitInstallerIsoName"
94+
echo "##vso[task.setvariable variable=installerISOName]$splitInstallerIsoName"
95+
fi
96+
displayName: Special handle for split test
97+
8498
# Download all test images
8599
- template: ../testing_common/download-test-images.yml
86100
parameters:
@@ -92,5 +106,9 @@ stages:
92106
- bash: |
93107
set -eux
94108
95-
./bin/storm-trident run "$(scenario)"
109+
mkdir -p $(ob_outputDirectory)
110+
111+
./bin/storm-trident run "$(scenario)" -- \
112+
--pipeline-run \
113+
--iso "./artifacts/iso/$(installerISOName).iso"
96114
displayName: "🧪 Run E2E Test"

tools/go.mod

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ require (
3636
)
3737

3838
require (
39+
github.com/Jeffail/gabs/v2 v2.7.0
3940
github.com/VictorLowther/simplexml v0.0.0-20180716164440-0bff93621230 // indirect
4041
github.com/VictorLowther/soap v0.0.0-20150314151524-8e36fca84b22 // indirect
4142
github.com/alecthomas/kong v1.8.1

tools/go.sum

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
github.com/Jeffail/gabs/v2 v2.7.0 h1:Y2edYaTcE8ZpRsR2AtmPu5xQdFDIthFG0jYhu5PY8kg=
2+
github.com/Jeffail/gabs/v2 v2.7.0/go.mod h1:dp5ocw1FvBBQYssgHsG7I1WYsiLRtkUaB1FEtSwvNUw=
13
github.com/VictorLowther/simplexml v0.0.0-20180716164440-0bff93621230 h1:t95Grn2mOPfb3+kPDWsNnj4dlNcxnvuR72IjY8eYjfQ=
24
github.com/VictorLowther/simplexml v0.0.0-20180716164440-0bff93621230/go.mod h1:t2EzW1qybnPDQ3LR/GgeF0GOzHUXT5IVMLP2gkW1cmc=
35
github.com/VictorLowther/soap v0.0.0-20150314151524-8e36fca84b22 h1:a0MBqYm44o0NcthLKCljZHe1mxlN6oahCQHHThnSwB4=

tools/pkg/netfinder/netfinder.go

Lines changed: 81 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,12 @@
11
package netfinder
22

33
import (
4+
"bytes"
45
"errors"
56
"fmt"
67
"net"
78

9+
log "github.com/sirupsen/logrus"
810
"github.com/vishvananda/netlink"
911
)
1012

@@ -17,7 +19,7 @@ func FindLocalIpForTargetIp(target string) (string, error) {
1719

1820
routes, err := netlink.RouteGet(ip)
1921
if err != nil {
20-
return "", fmt.Errorf("failed to get routes: %v", err)
22+
return "", fmt.Errorf("failed to get routes: %w", err)
2123
}
2224

2325
if len(routes) == 0 {
@@ -26,12 +28,12 @@ func FindLocalIpForTargetIp(target string) (string, error) {
2628

2729
link, err := netlink.LinkByIndex(routes[0].LinkIndex)
2830
if err != nil {
29-
return "", fmt.Errorf("failed to get link by index: %v", err)
31+
return "", fmt.Errorf("failed to get link by index: %w", err)
3032
}
3133

3234
addrs, err := netlink.AddrList(link, netlink.FAMILY_V4)
3335
if err != nil {
34-
return "", fmt.Errorf("failed to get addresses: %v", err)
36+
return "", fmt.Errorf("failed to get addresses: %w", err)
3537
}
3638

3739
if len(addrs) == 0 {
@@ -40,3 +42,79 @@ func FindLocalIpForTargetIp(target string) (string, error) {
4042

4143
return addrs[0].IPNet.IP.String(), nil
4244
}
45+
46+
// FindDefaultOutboundInterface attempts to find the default outbound interface
47+
// on the host.
48+
func FindDefaultOutboundInterface() (netlink.Link, error) {
49+
// Strategy:
50+
// 1. Enumerate IPv4 routes and look for the default route (Dst == nil).
51+
// 2. Prefer a default route that has a gateway (Gw != nil).
52+
// 3. Resolve the link name from the route's LinkIndex.
53+
// 4. If nothing found for IPv4, attempt IPv6 default as a fallback.
54+
55+
routes, err := netlink.RouteList(nil, netlink.FAMILY_V4)
56+
if err != nil {
57+
return nil, fmt.Errorf("listing routes failed: %w", err)
58+
}
59+
60+
isDefaultNet := func(network *net.IPNet) bool {
61+
if network == nil {
62+
return false
63+
}
64+
defaultNetwork := net.IPNet{
65+
IP: net.IPv4(0, 0, 0, 0),
66+
Mask: net.CIDRMask(0, 32),
67+
}
68+
return network.IP.Equal(defaultNetwork.IP) && bytes.Equal(network.Mask, defaultNetwork.Mask)
69+
}
70+
71+
var candidate *netlink.Route
72+
for i := range routes {
73+
r := &routes[i]
74+
log.Tracef("Checking Route: %+v", *r)
75+
if !isDefaultNet(r.Dst) {
76+
// Route does not target the default network
77+
log.Trace("Not a default route, skipping")
78+
continue
79+
}
80+
81+
if r.Gw == nil {
82+
// Skip routes without a gateway
83+
log.Trace("No gateway, skipping")
84+
continue
85+
}
86+
87+
candidate = r
88+
// Good enough
89+
break
90+
}
91+
92+
if candidate == nil {
93+
return nil, errors.New("no default route found")
94+
}
95+
96+
link, err := netlink.LinkByIndex(candidate.LinkIndex)
97+
if err != nil {
98+
return nil, fmt.Errorf("resolve link by index %d: %w", candidate.LinkIndex, err)
99+
}
100+
101+
return link, nil
102+
}
103+
104+
func FindDefaultOutboundIp() (string, error) {
105+
link, err := FindDefaultOutboundInterface()
106+
if err != nil {
107+
return "", fmt.Errorf("failed to find default outbound interface: %w", err)
108+
}
109+
110+
addrs, err := netlink.AddrList(link, netlink.FAMILY_V4)
111+
if err != nil {
112+
return "", fmt.Errorf("failed to get addresses: %w", err)
113+
}
114+
115+
if len(addrs) == 0 {
116+
return "", fmt.Errorf("no addresses found on interface %s", link.Attrs().Name)
117+
}
118+
119+
return addrs[0].IPNet.IP.String(), nil
120+
}

tools/pkg/netlaunch/netlaunch.go

Lines changed: 28 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ func RunNetlaunch(config *NetLaunchConfig) error {
2626
// Read the ISO
2727
iso, err := os.ReadFile(config.IsoPath)
2828
if err != nil {
29-
log.WithError(err).Fatalf("failed to find iso for testing")
29+
return fmt.Errorf("failed to read ISO file '%s': %w", config.IsoPath, err)
3030
}
3131

3232
localListenAddress := fmt.Sprintf(":%d", config.ListenPort)
@@ -58,13 +58,20 @@ func RunNetlaunch(config *NetLaunchConfig) error {
5858
if config.Netlaunch.AnnounceIp != nil {
5959
// If an IP is specified, use it.
6060
announceIp = *config.Netlaunch.AnnounceIp
61-
} else {
61+
} else if config.Netlaunch.Bmc != nil && config.Netlaunch.Bmc.Ip != "" {
6262
// Otherwise, try to be clever...
6363
// We need to find the IP of the local interface that can reach the BMC.
6464
log.Warn("No announce IP specified. Attempting to find local IP to announce based on BMC IP.")
6565
announceIp, err = netfinder.FindLocalIpForTargetIp(config.Netlaunch.Bmc.Ip)
6666
if err != nil {
67-
return fmt.Errorf("failed to find local IP for BMC: %v", err)
67+
return fmt.Errorf("failed to find local IP for BMC: %w", err)
68+
}
69+
} else {
70+
// If we have no BMC, find the default outbound IP.
71+
log.Warn("No announce IP specified. Attempting to find default outbound IP to announce.")
72+
announceIp, err = netfinder.FindDefaultOutboundIp()
73+
if err != nil {
74+
return fmt.Errorf("failed to find default outbound IP: %w", err)
6875
}
6976
}
7077

@@ -85,7 +92,7 @@ func RunNetlaunch(config *NetLaunchConfig) error {
8592
log.Info("Using Trident config file: ", config.HostConfigFile)
8693
tridentConfigContents, err := os.ReadFile(config.HostConfigFile)
8794
if err != nil {
88-
return fmt.Errorf("failed to read Trident config: %v", err)
95+
return fmt.Errorf("failed to read Trident config: %w", err)
8996
}
9097

9198
// Replace NETLAUNCH_HOST_ADDRESS with the address of the netlaunch server
@@ -94,7 +101,7 @@ func RunNetlaunch(config *NetLaunchConfig) error {
94101
trident := make(map[string]interface{})
95102
err = yaml.UnmarshalStrict([]byte(tridentConfigContentsStr), &trident)
96103
if err != nil {
97-
return fmt.Errorf("failed to unmarshal Trident config: %v", err)
104+
return fmt.Errorf("failed to unmarshal Trident config: %w", err)
98105
}
99106

100107
if _, ok := trident["trident"]; !ok {
@@ -105,27 +112,27 @@ func RunNetlaunch(config *NetLaunchConfig) error {
105112

106113
tridentConfig, err := yaml.Marshal(trident)
107114
if err != nil {
108-
return fmt.Errorf("failed to marshal Trident config: %v", err)
115+
return fmt.Errorf("failed to marshal Trident config: %w", err)
109116
}
110117

111118
err = isopatcher.PatchFile(iso, "/etc/trident/config.yaml", tridentConfig)
112119
if err != nil {
113-
return fmt.Errorf("failed to patch Trident config into ISO: %v", err)
120+
return fmt.Errorf("failed to patch Trident config into ISO: %w", err)
114121
}
115122

116123
if config.Iso.PreTridentScript != nil {
117124
log.Info("Patching in pre-trident script!")
118125
err = isopatcher.PatchFile(iso, "/trident_cdrom/pre-trident-script.sh", []byte(*config.Iso.PreTridentScript))
119126
if err != nil {
120-
return fmt.Errorf("failed to patch pre-trident script into ISO: %v", err)
127+
return fmt.Errorf("failed to patch pre-trident script into ISO: %w", err)
121128
}
122129
}
123130

124131
if config.Iso.ServiceOverride != nil {
125132
log.Info("Patching Trident service override!")
126133
err = isopatcher.PatchFile(iso, "/trident_cdrom/trident-override.conf", []byte(*config.Iso.ServiceOverride))
127134
if err != nil {
128-
return fmt.Errorf("failed to patch service override into ISO: %v", err)
135+
return fmt.Errorf("failed to patch service override into ISO: %w", err)
129136
}
130137
}
131138

@@ -153,14 +160,14 @@ func RunNetlaunch(config *NetLaunchConfig) error {
153160
// Set up listening for logstream
154161
logstreamFull, err := phonehome.SetupLogstream(config.LogstreamFile)
155162
if err != nil {
156-
return fmt.Errorf("failed to setup logstream: %v", err)
163+
return fmt.Errorf("failed to setup logstream: %w", err)
157164
}
158165
defer logstreamFull.Close()
159166

160167
// Set up listening for tracestream
161168
traceFile, err := phonehome.SetupTraceStream(config.TracestreamFile)
162169
if err != nil {
163-
return fmt.Errorf("failed to setup tracestream: %v", err)
170+
return fmt.Errorf("failed to setup tracestream: %w", err)
164171
}
165172
defer traceFile.Close()
166173

@@ -193,7 +200,7 @@ func RunNetlaunch(config *NetLaunchConfig) error {
193200
if config.Netlaunch.Bmc != nil && config.Netlaunch.Bmc.SerialOverSsh != nil {
194201
serial, err := config.Netlaunch.Bmc.ListenForSerialOutput()
195202
if err != nil {
196-
return fmt.Errorf("failed to open serial over SSH session: %v", err)
203+
return fmt.Errorf("failed to open serial over SSH session: %w", err)
197204
}
198205
defer serial.Close()
199206
}
@@ -218,27 +225,27 @@ func RunNetlaunch(config *NetLaunchConfig) error {
218225
log.Info("Connecting to BMC")
219226
client.Registry.Drivers = client.Registry.For("gofish")
220227
if err := client.Open(context.Background()); err != nil {
221-
return fmt.Errorf("failed to open connection to BMC: %v", err)
228+
return fmt.Errorf("failed to open connection to BMC: %w", err)
222229
}
223230

224231
log.Info("Shutting down machine")
225232
if _, err = client.SetPowerState(ctx, "off"); err != nil {
226-
return fmt.Errorf("failed to turn off machine: %v", err)
233+
return fmt.Errorf("failed to turn off machine: %w", err)
227234
}
228235

229236
log.WithField("url", iso_location).Info("Setting virtual media to ISO")
230237
if _, err = client.SetVirtualMedia(ctx, string(redfish.CDMediaType), iso_location); err != nil {
231-
return fmt.Errorf("failed to set virtual media: %v", err)
238+
return fmt.Errorf("failed to set virtual media: %w", err)
232239
}
233240

234241
log.Info("Setting boot media")
235242
if _, err = client.SetBootDevice(ctx, "cdrom", false, true); err != nil {
236-
return fmt.Errorf("failed to set boot media: %v", err)
243+
return fmt.Errorf("failed to set boot media: %w", err)
237244
}
238245

239246
log.Info("Turning on machine")
240247
if _, err = client.SetPowerState(ctx, "on"); err != nil {
241-
return fmt.Errorf("failed to turn on machine: %v", err)
248+
return fmt.Errorf("failed to turn on machine: %w", err)
242249
}
243250
}
244251

@@ -270,21 +277,21 @@ func startLocalVm(localVmUuidStr string, isoLocation string, secureBoot bool, si
270277
// TODO: Parse the UUID directly when reading the config file
271278
vmUuid, err := uuid.Parse(localVmUuidStr)
272279
if err != nil {
273-
return fmt.Errorf("failed to parse LocalVmUuid as UUID: %v", err)
280+
return fmt.Errorf("failed to parse LocalVmUuid as UUID: %w", err)
274281
}
275282

276283
vm, err := stormutils.InitializeVm(vmUuid)
277284
if err != nil {
278-
return fmt.Errorf("failed to initialize VM: %v", err)
285+
return fmt.Errorf("failed to initialize VM: %w", err)
279286
}
280287
defer vm.Disconnect()
281288

282289
if err = vm.SetFirmwareVars(isoLocation, secureBoot, signingCert); err != nil {
283-
return fmt.Errorf("failed to set UEFI variables: %v", err)
290+
return fmt.Errorf("failed to set UEFI variables: %w", err)
284291
}
285292

286293
if err = vm.Start(); err != nil {
287-
return fmt.Errorf("failed to start VM: %v", err)
294+
return fmt.Errorf("failed to start VM: %w", err)
288295
}
289296

290297
return nil

0 commit comments

Comments
 (0)