diff --git a/Package.resolved b/Package.resolved index cb0d8c5..5d616db 100644 --- a/Package.resolved +++ b/Package.resolved @@ -1,6 +1,50 @@ { - "originHash" : "ad1e11fc32cf3dd4130057a00a66ff5133a2eb146b272e199dd59131ab027200", + "originHash" : "3fdea633ce1bf54ea9439217287fd68d6551bf835fa2286e18d142cd9d8865c0", "pins" : [ + { + "identity" : "edge-agent-common", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apache-edge/edge-agent-common", + "state" : { + "revision" : "952035635c630d366dfbcff04af1934c7c051f23" + } + }, + { + "identity" : "grpc-swift", + "kind" : "remoteSourceControl", + "location" : "https://github.com/grpc/grpc-swift.git", + "state" : { + "revision" : "c4d6281784f50bf2e60d3af45e83be1194056062", + "version" : "2.1.2" + } + }, + { + "identity" : "grpc-swift-nio-transport", + "kind" : "remoteSourceControl", + "location" : "https://github.com/grpc/grpc-swift-nio-transport.git", + "state" : { + "revision" : "dac5f358f2ed36cc782f4c5476117398e62cb53c", + "version" : "1.0.2" + } + }, + { + "identity" : "grpc-swift-protobuf", + "kind" : "remoteSourceControl", + "location" : "https://github.com/grpc/grpc-swift-protobuf.git", + "state" : { + "revision" : "63982ca29f11d2c6a7e559c1899fee812dd55cd9", + "version" : "1.1.0" + } + }, + { + "identity" : "swift-algorithms", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-algorithms.git", + "state" : { + "revision" : "87e50f483c54e6efd60e885f7f5aa946cee68023", + "version" : "1.2.1" + } + }, { "identity" : "swift-argument-parser", "kind" : "remoteSourceControl", @@ -10,6 +54,60 @@ "version" : "1.5.0" } }, + { + "identity" : "swift-asn1", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-asn1.git", + "state" : { + "revision" : "ae33e5941bb88d88538d0a6b19ca0b01e6c76dcf", + "version" : "1.3.1" + } + }, + { + "identity" : "swift-atomics", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-atomics.git", + "state" : { + "revision" : "cd142fd2f64be2100422d658e7411e39489da985", + "version" : "1.2.0" + } + }, + { + "identity" : "swift-collections", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-collections.git", + "state" : { + "revision" : "671108c96644956dddcd89dd59c203dcdb36cec7", + "version" : "1.1.4" + } + }, + { + "identity" : "swift-crypto", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-crypto.git", + "state" : { + "revision" : "a6ce32a18b81b04ce7e897d1d98df6eb2da04786", + "version" : "3.12.2" + } + }, + { + "identity" : "swift-http-structured-headers", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-http-structured-headers.git", + "state" : { + "revision" : "d01361d32e14ae9b70ea5bd308a3794a198a2706", + "version" : "1.2.0" + } + }, + { + "identity" : "swift-http-types", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-http-types.git", + "state" : { + "revision" : "ef18d829e8b92d731ad27bb81583edd2094d1ce3", + "version" : "1.3.1" + } + }, { "identity" : "swift-log", "kind" : "remoteSourceControl", @@ -18,6 +116,78 @@ "revision" : "3d8596ed08bd13520157f0355e35caed215ffbfa", "version" : "1.6.3" } + }, + { + "identity" : "swift-nio", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-nio.git", + "state" : { + "revision" : "c51907a839e63ebf0ba2076bba73dd96436bd1b9", + "version" : "2.81.0" + } + }, + { + "identity" : "swift-nio-extras", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-nio-extras.git", + "state" : { + "revision" : "00f3f72d2f9942d0e2dc96057ab50a37ced150d4", + "version" : "1.25.0" + } + }, + { + "identity" : "swift-nio-http2", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-nio-http2.git", + "state" : { + "revision" : "170f4ca06b6a9c57b811293cebcb96e81b661310", + "version" : "1.35.0" + } + }, + { + "identity" : "swift-nio-ssl", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-nio-ssl.git", + "state" : { + "revision" : "0cc3528ff48129d64ab9cab0b1cd621634edfc6b", + "version" : "2.29.3" + } + }, + { + "identity" : "swift-nio-transport-services", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-nio-transport-services.git", + "state" : { + "revision" : "3c394067c08d1225ba8442e9cffb520ded417b64", + "version" : "1.23.1" + } + }, + { + "identity" : "swift-numerics", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-numerics.git", + "state" : { + "revision" : "e0ec0f5f3af6f3e4d5e7a19d2af26b481acb6ba8", + "version" : "1.0.3" + } + }, + { + "identity" : "swift-protobuf", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-protobuf.git", + "state" : { + "revision" : "d72aed98f8253ec1aa9ea1141e28150f408cf17f", + "version" : "1.29.0" + } + }, + { + "identity" : "swift-system", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-system.git", + "state" : { + "revision" : "a34201439c74b53f0fd71ef11741af7e7caf01e1", + "version" : "1.4.2" + } } ], "version" : 3 diff --git a/Package.swift b/Package.swift index 64326ce..698ccff 100644 --- a/Package.swift +++ b/Package.swift @@ -1,4 +1,4 @@ -// swift-tools-version: 6.0 +// swift-tools-version: 6.1 import PackageDescription let package = Package( @@ -6,9 +6,19 @@ let package = Package( platforms: [ .macOS(.v15) ], + products: [ + .executable(name: "edge", targets: ["edge"]) + ], dependencies: [ .package(url: "https://github.com/apple/swift-argument-parser.git", from: "1.5.0"), .package(url: "https://github.com/apple/swift-log.git", from: "1.6.3"), + .package(url: "https://github.com/grpc/grpc-swift-nio-transport.git", from: "1.0.0"), + .package(url: "https://github.com/apple/swift-nio.git", from: "2.81.0"), + .package(url: "https://github.com/apple/swift-crypto.git", from: "3.12.2"), + .package( + url: "https://github.com/apache-edge/edge-agent-common.git", + revision: "952035635c630d366dfbcff04af1934c7c051f23" + ), ], targets: [ /// The main executable provided by edge-cli. @@ -16,30 +26,23 @@ let package = Package( name: "edge", dependencies: [ .product(name: "ArgumentParser", package: "swift-argument-parser"), - .target(name: "EdgeCLI"), .product(name: "Logging", package: "swift-log"), + .product(name: "_NIOFileSystem", package: "swift-nio"), + .product(name: "GRPCNIOTransportHTTP2", package: "grpc-swift-nio-transport"), + .product(name: "EdgeAgentGRPC", package: "edge-agent-common"), + .target(name: "EdgeCLI"), ], resources: [ .copy("Resources") ] ), - /// The EdgeAgent executable. It's currently here for development purposes, and will be - /// moved to a separate package in the future. - .executableTarget( - name: "EdgeAgent", - dependencies: [ - .product(name: "ArgumentParser", package: "swift-argument-parser"), - .product(name: "Logging", package: "swift-log"), - ] - ), - /// Contains everything EdgeCLI, except for the command line interface. .target( name: "EdgeCLI", dependencies: [ .target(name: "ContainerBuilder"), - .target(name: "Shell"), + .product(name: "Shell", package: "edge-agent-common"), .product(name: "Logging", package: "swift-log"), ] ), @@ -48,15 +51,8 @@ let package = Package( .target( name: "ContainerBuilder", dependencies: [ - .target(name: "Shell") - ] - ), - - /// Utility for executing shell commands. - .target( - name: "Shell", - dependencies: [ - .product(name: "Logging", package: "swift-log") + .product(name: "Shell", package: "edge-agent-common"), + .product(name: "Crypto", package: "swift-crypto"), ] ), ] diff --git a/Scripts/Generate-Proto.sh b/Scripts/Generate-Proto.sh new file mode 100755 index 0000000..30176ba --- /dev/null +++ b/Scripts/Generate-Proto.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +# This scripts generates the EdgeAgentGRPC Swift code. It requires `protoc` to be available in the PATH. +# It should be run from the root of the project. + +# Get the binary path +BIN_PATH=$(swift build --show-bin-path) +PROTOC_GEN_GRPC_PATH="$BIN_PATH/protoc-gen-grpc-swift" +PROTOC_GEN_SWIFT_PATH="$BIN_PATH/protoc-gen-swift" + +# Check if protoc-gen-grpc-swift exists, and build it if needed +if [ ! -f "$PROTOC_GEN_GRPC_PATH" ]; then + echo "protoc-gen-grpc-swift not found. Building it now..." + swift build --product protoc-gen-grpc-swift +fi + +rm -rf Sources/EdgeAgentGRPC/Proto +mkdir -p Sources/EdgeAgentGRPC/Proto + +protoc \ + --plugin $PROTOC_GEN_GRPC_PATH \ + --grpc-swift_out=Sources/EdgeAgentGRPC/Proto \ + --grpc-swift_opt=Visibility=Public \ + --grpc-swift_opt=Server=True \ + --grpc-swift_opt=Client=True \ + --include_imports \ + --descriptor_set_out=Sources/EdgeAgentGRPC/Proto/edge_agent.protoset \ + --experimental_allow_proto3_optional \ + -I=Proto \ + Proto/edge/agent/services/v1/*.proto + +# Check if protoc-gen-swift exists +if [ ! -f "$PROTOC_GEN_SWIFT_PATH" ]; then + echo "protoc-gen-swift not found. Building it now..." + swift build --product protoc-gen-swift +fi + +protoc \ + --plugin $PROTOC_GEN_SWIFT_PATH \ + --swift_out=Sources/EdgeAgentGRPC/Proto \ + --swift_opt=Visibility=Public \ + --experimental_allow_proto3_optional \ + -I=Proto \ + Proto/edge/agent/services/v1/*.proto \ No newline at end of file diff --git a/Sources/ContainerBuilder/ContainerBuilder.swift b/Sources/ContainerBuilder/ContainerBuilder.swift index 13b60e2..17dfbb5 100644 --- a/Sources/ContainerBuilder/ContainerBuilder.swift +++ b/Sources/ContainerBuilder/ContainerBuilder.swift @@ -1,4 +1,4 @@ -import CryptoKit +import Crypto import Foundation import Shell @@ -136,7 +136,7 @@ public func buildDockerContainerImage( try FileManager.default.removeItem(at: tempDir) } -// Calculate SHA256 hash using CryptoKit +// Calculate SHA256 hash using Swift Crypto private func sha256(data: Data) -> String { let digest = SHA256.hash(data: data) return digest.map { String(format: "%02x", $0) }.joined() diff --git a/Sources/EdgeAgent/EdgeAgent.swift b/Sources/EdgeAgent/EdgeAgent.swift deleted file mode 100644 index a96ad8a..0000000 --- a/Sources/EdgeAgent/EdgeAgent.swift +++ /dev/null @@ -1,12 +0,0 @@ -import ArgumentParser -import Foundation - -@main -struct EdgeCLI: ParsableCommand { - static let configuration = CommandConfiguration( - commandName: "edge-agent", - abstract: "Edge Agent", - subcommands: [] - ) -} - diff --git a/Sources/Shell/Shell.swift b/Sources/Shell/Shell.swift deleted file mode 100644 index 32741df..0000000 --- a/Sources/Shell/Shell.swift +++ /dev/null @@ -1,111 +0,0 @@ -import Foundation -import Logging - -/// Utility for executing shell commands. -public enum Shell { - static let logger = Logger(label: "apache-edge.shell") - - /// Error thrown when a process execution fails. - public enum Error: Swift.Error, LocalizedError { - case nonZeroExit(command: [String], exitCode: Int32) - - public var errorDescription: String? { - switch self { - case .nonZeroExit(let command, let exitCode): - return - "Command '\(command.joined(separator: " "))' failed with exit code \(exitCode)" - } - } - } - - /// Run a CLI command. - /// - /// This method executes a command in a subprocess. If the command is not successful - /// (indicated by a non-zero exit code), an error is thrown. - /// - /// - Parameter arguments: An array of command-line arguments to execute. - /// - Returns: A string containing the command's standard output and standard error. - /// - Throws: An error if the command execution fails - @discardableResult public static func run(_ arguments: [String]) async throws -> String { - logger.info("Running command", metadata: ["command": .array(arguments.map { .string($0) })]) - - let process = Process() - - // Create pipes for stdout and stderr to both capture and display output - let stdoutPipe = Pipe() - let stderrPipe = Pipe() - let stdoutCapture = Pipe() - let stderrCapture = Pipe() - - process.executableURL = URL(fileURLWithPath: "/usr/bin/env") - process.arguments = arguments - - process.standardOutput = stdoutPipe - process.standardError = stderrPipe - process.environment = [ - // TODO: Don't hardcode the path to the Swift toolchain – manage our own toolchain instead. - "PATH": - "/Library/Developer/Toolchains/swift-6.0.3-RELEASE.xctoolchain/usr/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin", - "TOOLCHAINS": "org.swift.603202412101a", - ] - - stdoutPipe.fileHandleForReading.readabilityHandler = { fileHandle in - let data = fileHandle.availableData - if !data.isEmpty { - FileHandle.standardOutput.write(data) - stdoutCapture.fileHandleForWriting.write(data) - } - } - - stderrPipe.fileHandleForReading.readabilityHandler = { fileHandle in - let data = fileHandle.availableData - if !data.isEmpty { - FileHandle.standardError.write(data) - stderrCapture.fileHandleForWriting.write(data) - } - } - - try process.run() - - return try await withTaskCancellationHandler { - try await withCheckedThrowingContinuation { continuation in - process.terminationHandler = { proc in - // Clean up handlers - stdoutPipe.fileHandleForReading.readabilityHandler = nil - stderrPipe.fileHandleForReading.readabilityHandler = nil - - // Close write handles to ensure we can read all data - stdoutCapture.fileHandleForWriting.closeFile() - stderrCapture.fileHandleForWriting.closeFile() - - if process.terminationStatus == 0 { - // Read captured output - let stdoutData = stdoutCapture.fileHandleForReading.readDataToEndOfFile() - let stderrData = stderrCapture.fileHandleForReading.readDataToEndOfFile() - - // Combine stdout and stderr - let combinedData = stdoutData + stderrData - let output = String(data: combinedData, encoding: .utf8) ?? "" - continuation.resume(returning: output) - } else { - continuation.resume( - throwing: Error.nonZeroExit( - command: arguments, - exitCode: process.terminationStatus - ) - ) - } - } - } - } onCancel: { - // Kill the process when the task is cancelled - logger.trace( - "Task cancelled, terminating process", - metadata: [ - "command": .array(arguments.map { .string($0) }) - ] - ) - process.terminate() - } - } -} diff --git a/Sources/edge/AgentConnectionOptions.swift b/Sources/edge/AgentConnectionOptions.swift new file mode 100644 index 0000000..d3aa0d6 --- /dev/null +++ b/Sources/edge/AgentConnectionOptions.swift @@ -0,0 +1,9 @@ +import ArgumentParser + +struct AgentConnectionOptions: ParsableArguments { + @Option(name: .long, help: "The host of the Edge Agent to connect to.") + var agentHost: String + + @Option(name: .long, help: "The port of the Edge Agent to connect to.") + var agentPort: Int = 50051 +} diff --git a/Sources/edge/Commands/RunCommand.swift b/Sources/edge/Commands/RunCommand.swift index 13dda5f..e93d79d 100644 --- a/Sources/edge/Commands/RunCommand.swift +++ b/Sources/edge/Commands/RunCommand.swift @@ -1,8 +1,13 @@ import ArgumentParser import ContainerBuilder +import EdgeAgentGRPC import EdgeCLI import Foundation +import GRPCCore +import GRPCNIOTransportHTTP2 import Logging +import NIO +import NIOFileSystem import Shell struct RunCommand: AsyncParsableCommand { @@ -12,7 +17,7 @@ struct RunCommand: AsyncParsableCommand { var description: String { switch self { case .noExecutableTarget: - return String(localized: "No executable target found in package") + return "No executable target found in package" } } } @@ -25,6 +30,8 @@ struct RunCommand: AsyncParsableCommand { @Flag(name: .shortAndLong, help: "Attach a debugger to the container") var debug: Bool = false + @OptionGroup var agentConnectionOptions: AgentConnectionOptions + func run() async throws { let logger = Logger(label: "apache-edge.cli.run") @@ -83,30 +90,66 @@ struct RunCommand: AsyncParsableCommand { outputPath: outputPath ) - logger.info( - "Loading into Docker", - metadata: [ - "imageName": .string(imageName), - "path": .string(outputPath), - ] + let target = ResolvableTargets.DNS( + host: agentConnectionOptions.agentHost, + port: agentConnectionOptions.agentPort ) - try await Shell.run(["docker", "load", "-i", outputPath]) - - if debug { - logger.info( - "Running container with debugger", - metadata: ["imageName": .string(imageName)] + #if os(macOS) + let transport = try HTTP2ClientTransport.TransportServices( + target: target, + transportSecurity: .plaintext ) - try await Shell.run([ - "docker", "run", "--rm", "-it", "-p", "4242:4242", - "--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined", imageName, - "ds2", "gdbserver", "0.0.0.0:4242", "/bin/\(executableTarget.name)", - ]) - return - } + #else + let transport = try HTTP2ClientTransport.Posix( + target: target, + transportSecurity: .plaintext + ) + #endif + + try await withGRPCClient(transport: transport) { client in + let agent = Edge_Agent_Services_V1_EdgeAgentService.Client(wrapping: client) + try await agent.runContainer { writer in + // First, send the header. + try await writer.write( + .with { + $0.header.imageName = imageName + } + ) - logger.info("Running container", metadata: ["imageName": .string(imageName)]) - try await Shell.run(["docker", "run", "--rm", imageName]) + // Send the chunks + let fileHandle = try await FileSystem.shared.openFile( + forReadingAt: FilePath(outputPath) + ) + do { + for try await chunk in fileHandle.readChunks() { + try await writer.write( + .with { + $0.requestType = .chunk( + .with { $0.data = Data(chunk.readableBytesView) } + ) + } + ) + } + } catch { + try await fileHandle.close() + throw error + } + try await fileHandle.close() + + // Send the control command to start the container. + try await writer.write( + .with { + $0.requestType = .control( + .with { $0.command = .run(.with { $0.debug = debug }) } + ) + } + ) + } onResponse: { response in + for try await message in response.messages { + print(message) + } + } + } } }