diff --git a/.gitignore b/.gitignore index bf42ac8..8dd0903 100644 --- a/.gitignore +++ b/.gitignore @@ -27,6 +27,7 @@ go.work !.vscode/launch.json !.vscode/extensions.json !.vscode/*.code-snippets +.idea # Local History for Visual Studio Code .history/ diff --git a/cmd/main.go b/cmd/main.go index f5fb954..f4f9afb 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -7,11 +7,17 @@ import ( "math/rand" "time" - "github.com/liver/gordma" + "github.com/ivagulin/gordma" ) func main() { - c, err := gordma.NewRdmaContext("", 1, 0, gordma.IBV_MTU_4096) + isServer := flag.Bool("s", false, "init server") + server := flag.String("a", "localhost", "addr to bind/connect to") + port := flag.Int("p", 8008, "port to bind/connect to") + sgid := flag.Int("g", 0, "sgid to use") + flag.Parse() + + c, err := gordma.NewRdmaContext("", 1, *sgid, gordma.IBV_MTU_4096) if err != nil { panic(err) } @@ -34,24 +40,18 @@ func main() { panic(err) } - isServer := flag.Bool("s", false, "") - flag.Parse() - - server := "localhost" - port := 8008 - if *isServer { portSelection := make(chan int, 1) - go func (p chan int) { + go func(p chan int) { fmt.Printf("rdma server port: %d\n", <-p) }(portSelection) - c, err := gordma.ConnectQpServer(c, qp, mr, port, portSelection) + c, err := gordma.ConnectQpServer(c, qp, mr, *port, portSelection, uint8(*sgid)) if err != nil { panic(err) } defer c.Close() } else { - c, err := gordma.ConnectQpClient(c, qp, mr, server, port) + c, err := gordma.ConnectQpClient(c, qp, mr, *server, *port, uint8(*sgid)) if err != nil { panic(err) } @@ -113,7 +113,7 @@ func runServer(qp *gordma.QueuePair, mr *gordma.MemoryRegion) error { (*localNotice)[1] = byte(rand.Intn(9)) (*localNotice)[2] = byte(rand.Intn(9)) fmt.Printf("from server S: %d%d%d\n", (*mr.Notice())[0], (*mr.Notice())[1], (*mr.Notice())[2]) - + time.Sleep(time.Second) wr_id, err = qp.PostSend(swr) if err != nil { @@ -168,7 +168,7 @@ func runClient(qp *gordma.QueuePair, mr *gordma.MemoryRegion) error { rwr := gordma.NewReceiveWorkRequest(mr) defer rwr.Close() - + wr_id, err = qp.PostReceive(rwr) if err != nil { return fmt.Errorf("PostReceive failed: %v\n", err) @@ -184,7 +184,7 @@ func runClient(qp *gordma.QueuePair, mr *gordma.MemoryRegion) error { fmt.Printf("PostReceive WaitForCompletionId %d cs:%v\n", wr_id, cs) fmt.Printf("from server R: %d%d%d\n", (*mr.Notice())[0], (*mr.Notice())[1], (*mr.Notice())[2]) - + for { if (*mr.Notice())[0] == 1 { break diff --git a/go.mod b/go.mod index 14679cf..8e0a474 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github.com/liver/gordma +module github.com/ivagulin/gordma go 1.16 diff --git a/qp.go b/qp.go index 5f2900c..6609162 100644 --- a/qp.go +++ b/qp.go @@ -117,7 +117,7 @@ func (q *QueuePair) Init() error { } // Ready2Receive RTR -func (q *QueuePair) Ready2Receive(mtu uint32, destGidLocal uint16, destGidGlobal [16]byte, destQpn, destPsn uint32) error { +func (q *QueuePair) Ready2Receive(mtu uint32, destGidLocal uint16, destGidGlobal [16]byte, destQpn, destPsn uint32, sgid uint8) error { attr := C.struct_ibv_qp_attr{} attr.qp_state = C.IBV_QPS_RTR attr.path_mtu = mtu @@ -133,19 +133,19 @@ func (q *QueuePair) Ready2Receive(mtu uint32, destGidLocal uint16, destGidGlobal attr.ah_attr.src_path_bits = 0 attr.ah_attr.port_num = C.uint8_t(q.port) mask := C.IBV_QP_STATE | C.IBV_QP_AV | C.IBV_QP_PATH_MTU | C.IBV_QP_DEST_QPN | - C.IBV_QP_RQ_PSN | C.IBV_QP_MAX_DEST_RD_ATOMIC | C.IBV_QP_MIN_RNR_TIMER - + C.IBV_QP_RQ_PSN | C.IBV_QP_MAX_DEST_RD_ATOMIC | C.IBV_QP_MIN_RNR_TIMER + // for Soft-RoCE (aka RXE) attr.ah_attr.is_global = 1 attr.ah_attr.grh.dgid = destGidGlobal - attr.ah_attr.grh.flow_label = 0; - attr.ah_attr.grh.hop_limit = 1; - attr.ah_attr.grh.sgid_index = C.uint8_t(0) - attr.ah_attr.grh.traffic_class = 0; + attr.ah_attr.grh.flow_label = 0 + attr.ah_attr.grh.hop_limit = 1 + attr.ah_attr.grh.sgid_index = C.uint8_t(sgid) + attr.ah_attr.grh.traffic_class = 0 // return q.modify(&attr, mask) - + } // Ready2Send RTS @@ -241,7 +241,7 @@ func (q *QueuePair) PostWriteImm(wr *SendWorkRequest, memType Type, imm uint32, if imm > 0 { wr.sendWr.send_flags = 0 - }else { + } else { wr.sendWr.send_flags = IBV_SEND_SIGNALED } @@ -309,7 +309,7 @@ func (q *QueuePair) PostRead(wr *SendWorkRequest, memType Type, udl int) (uint64 binary.LittleEndian.PutUint64(wr.sendWr.wr[:8], wr.mr.NoticeRemoteAddr()) binary.LittleEndian.PutUint32(wr.sendWr.wr[8:12], wr.mr.NoticeRemoteKey()) } - + wr.sendWr.sg_list = wr.sge wr.sendWr.num_sge = 1 wr.sendWr.next = nil diff --git a/sock.go b/sock.go index 9da2dcc..7a35418 100644 --- a/sock.go +++ b/sock.go @@ -12,11 +12,11 @@ import ( var SockSyncMsg string = "sync" -func ConnectQpClient(ctx *RdmaContext, qp *QueuePair, mr *MemoryRegion, server string, port int) (net.Conn, error) { +func ConnectQpClient(ctx *RdmaContext, qp *QueuePair, mr *MemoryRegion, server string, port int, sgid uint8) (net.Conn, error) { if server == "" { server = "localhost" } - + c, err := net.Dial("tcp", net.JoinHostPort(server, fmt.Sprintf("%d", port))) if err != nil { return nil, err @@ -74,7 +74,7 @@ func ConnectQpClient(ctx *RdmaContext, qp *QueuePair, mr *MemoryRegion, server s MTU: NetToHostLong(bufQpInfo.MTU), } - err = modify_qp_to_rts(qp, mr.qp.MTU, mr.qp.Lid, mr.qp.Gid, mr.qp.QpNum, mr.qp.Psn) + err = modify_qp_to_rts(qp, mr.qp.MTU, mr.qp.Lid, mr.qp.Gid, mr.qp.QpNum, mr.qp.Psn, sgid) if err != nil { c.Close() return nil, err @@ -95,7 +95,7 @@ func ConnectQpClient(ctx *RdmaContext, qp *QueuePair, mr *MemoryRegion, server s return c, nil } -func ConnectQpServer(ctx *RdmaContext, qp *QueuePair, mr *MemoryRegion, port int, portSelection chan int) (net.Conn, error) { +func ConnectQpServer(ctx *RdmaContext, qp *QueuePair, mr *MemoryRegion, port int, portSelection chan int, sgid uint8) (net.Conn, error) { if port < 0 { port = 0 } @@ -167,7 +167,7 @@ func ConnectQpServer(ctx *RdmaContext, qp *QueuePair, mr *MemoryRegion, port int MTU: uint32(ctx.IBV_MTU), } - err = modify_qp_to_rts(qp, mr.qp.MTU, mr.qp.Lid, mr.qp.Gid, mr.qp.QpNum, mr.qp.Psn) + err = modify_qp_to_rts(qp, mr.qp.MTU, mr.qp.Lid, mr.qp.Gid, mr.qp.QpNum, mr.qp.Psn, sgid) if err != nil { c.Close() return nil, err @@ -188,13 +188,13 @@ func ConnectQpServer(ctx *RdmaContext, qp *QueuePair, mr *MemoryRegion, port int return c, nil } -func modify_qp_to_rts(qp *QueuePair, mtu uint32, destLid uint16, destGid [16]byte, destQpNum uint32, destPsn uint32) error { +func modify_qp_to_rts(qp *QueuePair, mtu uint32, destLid uint16, destGid [16]byte, destQpNum uint32, destPsn uint32, sgid uint8) error { err := qp.Init() if err != nil { return err } - err = qp.Ready2Receive(mtu, destLid, destGid, destQpNum, destPsn) + err = qp.Ready2Receive(mtu, destLid, destGid, destQpNum, destPsn, sgid) if err != nil { return err }