d/freebsd-nq
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
freebsd-nq/contrib/ofed/libibverbs/fixes/rocee_examples.patch
Jeff Roberson aa0a1e58f0 - Merge in OFED 1.5.3 from projects/ofed/head
2011-03-21 09:58:24 +00:00

1052 lines
34 KiB
Diff
Raw Blame History

[PATCH 4/4] libibverbs: Update examples
Since RDMAoE requires usage of GRH, update ibv_*_pinpong examples to accept
GIDs. GIDs are given as an index to the local port's table and are exchanged
between the client and the server through the socket connection.
Signed-off-by: Eli Cohen <eli@mellanox.co.il>
---
examples/devinfo.c | 14 ++++++++
examples/pingpong.c | 31 ++++++++++++++++++
examples/pingpong.h | 4 ++
examples/rc_pingpong.c | 79 +++++++++++++++++++++++++++++++++++-----------
examples/srq_pingpong.c | 72 ++++++++++++++++++++++++++++++++----------
examples/uc_pingpong.c | 70 ++++++++++++++++++++++++++++++++---------
examples/ud_pingpong.c | 69 ++++++++++++++++++++++++++++++-----------
7 files changed, 269 insertions(+), 70 deletions(-)
Index: libibverbs/examples/devinfo.c
===================================================================
--- libibverbs.orig/examples/devinfo.c 2010-02-08 15:04:24.369329000 +0200
+++ libibverbs/examples/devinfo.c 2010-03-17 14:08:48.404754000 +0200
@@ -184,6 +184,19 @@ static int print_all_port_gids(struct ib
return rc;
}
+static const char *link_layer_str(uint8_t link_layer)
+{
+ switch (link_layer) {
+ case IBV_LINK_LAYER_UNSPECIFIED:
+ case IBV_LINK_LAYER_INFINIBAND:
+ return "IB";
+ case IBV_LINK_LAYER_ETHERNET:
+ return "Ethernet";
+ default:
+ return "Unknown";
+ }
+}
+
static int print_hca_cap(struct ibv_device *ib_dev, uint8_t ib_port)
{
struct ibv_context *ctx;
@@ -284,6 +297,7 @@ static int print_hca_cap(struct ibv_devi
printf("\t\t\tsm_lid:\t\t\t%d\n", port_attr.sm_lid);
printf("\t\t\tport_lid:\t\t%d\n", port_attr.lid);
printf("\t\t\tport_lmc:\t\t0x%02x\n", port_attr.lmc);
+ printf("\t\t\tlink_layer:\t\t%s\n", link_layer_str(port_attr.link_layer));
if (verbose) {
printf("\t\t\tmax_msg_sz:\t\t0x%x\n", port_attr.max_msg_sz);
Index: libibverbs/examples/pingpong.c
===================================================================
--- libibverbs.orig/examples/pingpong.c 2010-02-08 15:04:24.372330000 +0200
+++ libibverbs/examples/pingpong.c 2010-03-17 14:08:48.433754000 +0200
@@ -31,6 +31,10 @@
*/
#include "pingpong.h"
+#include <arpa/inet.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
enum ibv_mtu pp_mtu_to_enum(int mtu)
{
@@ -53,3 +57,30 @@ uint16_t pp_get_local_lid(struct ibv_con
return attr.lid;
}
+
+int pp_get_port_info(struct ibv_context *context, int port,
+ struct ibv_port_attr *attr)
+{
+ return ibv_query_port(context, port, attr);
+}
+
+void wire_gid_to_gid(const char *wgid, union ibv_gid *gid)
+{
+ char tmp[9];
+ uint32_t v32;
+ int i;
+
+ for (tmp[8] = 0, i = 0; i < 4; ++i) {
+ memcpy(tmp, wgid + i * 8, 8);
+ sscanf(tmp, "%x", &v32);
+ *(uint32_t *)(&gid->raw[i * 4]) = ntohl(v32);
+ }
+}
+
+void gid_to_wire_gid(const union ibv_gid *gid, char wgid[])
+{
+ int i;
+
+ for (i = 0; i < 4; ++i)
+ sprintf(&wgid[i * 8], "%08x", htonl(*(uint32_t *)(gid->raw + i * 4)));
+}
Index: libibverbs/examples/pingpong.h
===================================================================
--- libibverbs.orig/examples/pingpong.h 2010-02-08 15:04:24.375328000 +0200
+++ libibverbs/examples/pingpong.h 2010-03-17 14:08:48.443756000 +0200
@@ -37,5 +37,9 @@
enum ibv_mtu pp_mtu_to_enum(int mtu);
uint16_t pp_get_local_lid(struct ibv_context *context, int port);
+int pp_get_port_info(struct ibv_context *context, int port,
+ struct ibv_port_attr *attr);
+void wire_gid_to_gid(const char *wgid, union ibv_gid *gid);
+void gid_to_wire_gid(const union ibv_gid *gid, char wgid[]);
#endif /* IBV_PINGPONG_H */
Index: libibverbs/examples/rc_pingpong.c
===================================================================
--- libibverbs.orig/examples/rc_pingpong.c 2010-02-08 15:04:24.378329000 +0200
+++ libibverbs/examples/rc_pingpong.c 2010-03-17 14:11:16.145313000 +0200
@@ -67,17 +67,19 @@ struct pingpong_context {
int size;
int rx_depth;
int pending;
+ struct ibv_port_attr portinfo;
};
struct pingpong_dest {
int lid;
int qpn;
int psn;
+ union ibv_gid gid;
};
static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn,
enum ibv_mtu mtu, int sl,
- struct pingpong_dest *dest)
+ struct pingpong_dest *dest, int sgid_idx)
{
struct ibv_qp_attr attr = {
.qp_state = IBV_QPS_RTR,
@@ -94,6 +96,13 @@ static int pp_connect_ctx(struct pingpon
.port_num = port
}
};
+
+ if (dest->gid.global.interface_id) {
+ attr.ah_attr.is_global = 1;
+ attr.ah_attr.grh.hop_limit = 1;
+ attr.ah_attr.grh.dgid = dest->gid;
+ attr.ah_attr.grh.sgid_index = sgid_idx;
+ }
if (ibv_modify_qp(ctx->qp, &attr,
IBV_QP_STATE |
IBV_QP_AV |
@@ -135,10 +144,11 @@ static struct pingpong_dest *pp_client_e
.ai_socktype = SOCK_STREAM
};
char *service;
- char msg[sizeof "0000:000000:000000"];
+ char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"];
int n;
int sockfd = -1;
struct pingpong_dest *rem_dest = NULL;
+ char gid[33];
if (asprintf(&service, "%d", port) < 0)
return NULL;
@@ -169,7 +179,8 @@ static struct pingpong_dest *pp_client_e
return NULL;
}
- sprintf(msg, "%04x:%06x:%06x", my_dest->lid, my_dest->qpn, my_dest->psn);
+ gid_to_wire_gid(&my_dest->gid, gid);
+ sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn, my_dest->psn, gid);
if (write(sockfd, msg, sizeof msg) != sizeof msg) {
fprintf(stderr, "Couldn't send local address\n");
goto out;
@@ -187,7 +198,8 @@ static struct pingpong_dest *pp_client_e
if (!rem_dest)
goto out;
- sscanf(msg, "%x:%x:%x", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn);
+ sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn, gid);
+ wire_gid_to_gid(gid, &rem_dest->gid);
out:
close(sockfd);
@@ -197,7 +209,8 @@ out:
static struct pingpong_dest *pp_server_exch_dest(struct pingpong_context *ctx,
int ib_port, enum ibv_mtu mtu,
int port, int sl,
- const struct pingpong_dest *my_dest)
+ const struct pingpong_dest *my_dest,
+ int sgid_idx)
{
struct addrinfo *res, *t;
struct addrinfo hints = {
@@ -206,10 +219,11 @@ static struct pingpong_dest *pp_server_e
.ai_socktype = SOCK_STREAM
};
char *service;
- char msg[sizeof "0000:000000:000000"];
+ char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"];
int n;
int sockfd = -1, connfd;
struct pingpong_dest *rem_dest = NULL;
+ char gid[33];
if (asprintf(&service, "%d", port) < 0)
return NULL;
@@ -263,16 +277,19 @@ static struct pingpong_dest *pp_server_e
if (!rem_dest)
goto out;
- sscanf(msg, "%x:%x:%x", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn);
+ sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn, gid);
+ wire_gid_to_gid(gid, &rem_dest->gid);
- if (pp_connect_ctx(ctx, ib_port, my_dest->psn, mtu, sl, rem_dest)) {
+ if (pp_connect_ctx(ctx, ib_port, my_dest->psn, mtu, sl, rem_dest, sgid_idx)) {
fprintf(stderr, "Couldn't connect to remote QP\n");
free(rem_dest);
rem_dest = NULL;
goto out;
}
- sprintf(msg, "%04x:%06x:%06x", my_dest->lid, my_dest->qpn, my_dest->psn);
+
+ gid_to_wire_gid(&my_dest->gid, gid);
+ sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn, my_dest->psn, gid);
if (write(connfd, msg, sizeof msg) != sizeof msg) {
fprintf(stderr, "Couldn't send local address\n");
free(rem_dest);
@@ -289,11 +306,11 @@ out:
static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size,
int rx_depth, int port,
- int use_event)
+ int use_event, int is_server)
{
struct pingpong_context *ctx;
- ctx = malloc(sizeof *ctx);
+ ctx = calloc(1, sizeof *ctx);
if (!ctx)
return NULL;
@@ -306,7 +323,7 @@ static struct pingpong_context *pp_init_
return NULL;
}
- memset(ctx->buf, 0, size);
+ memset(ctx->buf, 0x7b + is_server, size);
ctx->context = ibv_open_device(ib_dev);
if (!ctx->context) {
@@ -481,6 +498,7 @@ static void usage(const char *argv0)
printf(" -n, --iters=<iters> number of exchanges (default 1000)\n");
printf(" -l, --sl=<sl> service level value\n");
printf(" -e, --events sleep on CQ events (default poll)\n");
+ printf(" -g, --gid-idx=<gid index> local port gid index\n");
}
int main(int argc, char *argv[])
@@ -504,6 +522,8 @@ int main(int argc, char *argv[])
int rcnt, scnt;
int num_cq_events = 0;
int sl = 0;
+ int gidx = -1;
+ char gid[33];
srand48(getpid() * time(NULL));
@@ -520,10 +540,11 @@ int main(int argc, char *argv[])
{ .name = "iters", .has_arg = 1, .val = 'n' },
{ .name = "sl", .has_arg = 1, .val = 'l' },
{ .name = "events", .has_arg = 0, .val = 'e' },
+ { .name = "gid-idx", .has_arg = 1, .val = 'g' },
{ 0 }
};
- c = getopt_long(argc, argv, "p:d:i:s:m:r:n:l:e", long_options, NULL);
+ c = getopt_long(argc, argv, "p:d:i:s:m:r:n:l:eg:", long_options, NULL);
if (c == -1)
break;
@@ -576,6 +597,10 @@ int main(int argc, char *argv[])
++use_event;
break;
+ case 'g':
+ gidx = strtol(optarg, NULL, 0);
+ break;
+
default:
usage(argv[0]);
return 1;
@@ -615,7 +640,7 @@ int main(int argc, char *argv[])
}
}
- ctx = pp_init_ctx(ib_dev, size, rx_depth, ib_port, use_event);
+ ctx = pp_init_ctx(ib_dev, size, rx_depth, ib_port, use_event, !servername);
if (!ctx)
return 1;
@@ -631,30 +656,47 @@ int main(int argc, char *argv[])
return 1;
}
- my_dest.lid = pp_get_local_lid(ctx->context, ib_port);
- my_dest.qpn = ctx->qp->qp_num;
- my_dest.psn = lrand48() & 0xffffff;
- if (!my_dest.lid) {
+
+ if (pp_get_port_info(ctx->context, ib_port, &ctx->portinfo)) {
+ fprintf(stderr, "Couldn't get port info\n");
+ return 1;
+ }
+
+ my_dest.lid = ctx->portinfo.lid;
+ if (ctx->portinfo.link_layer == IBV_LINK_LAYER_INFINIBAND && !my_dest.lid) {
fprintf(stderr, "Couldn't get local LID\n");
return 1;
}
- printf(" local address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x\n",
- my_dest.lid, my_dest.qpn, my_dest.psn);
+ if (gidx >= 0) {
+ if (ibv_query_gid(ctx->context, ib_port, gidx, &my_dest.gid)) {
+ fprintf(stderr, "Could not get local gid for gid index %d\n", gidx);
+ return 1;
+ }
+ } else
+ memset(&my_dest.gid, 0, sizeof my_dest.gid);
+
+ my_dest.qpn = ctx->qp->qp_num;
+ my_dest.psn = lrand48() & 0xffffff;
+ inet_ntop(AF_INET6, &my_dest.gid, gid, sizeof gid);
+ printf(" local address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n",
+ my_dest.lid, my_dest.qpn, my_dest.psn, gid);
+
if (servername)
rem_dest = pp_client_exch_dest(servername, port, &my_dest);
else
- rem_dest = pp_server_exch_dest(ctx, ib_port, mtu, port, sl, &my_dest);
+ rem_dest = pp_server_exch_dest(ctx, ib_port, mtu, port, sl, &my_dest, gidx);
if (!rem_dest)
return 1;
- printf(" remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x\n",
- rem_dest->lid, rem_dest->qpn, rem_dest->psn);
+ inet_ntop(AF_INET6, &rem_dest->gid, gid, sizeof gid);
+ printf(" remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n",
+ rem_dest->lid, rem_dest->qpn, rem_dest->psn, gid);
if (servername)
- if (pp_connect_ctx(ctx, ib_port, my_dest.psn, mtu, sl, rem_dest))
+ if (pp_connect_ctx(ctx, ib_port, my_dest.psn, mtu, sl, rem_dest, gidx))
return 1;
ctx->pending = PINGPONG_RECV_WRID;
@@ -706,6 +748,7 @@ int main(int argc, char *argv[])
fprintf(stderr, "poll CQ failed %d\n", ne);
return 1;
}
+
} while (!use_event && ne < 1);
for (i = 0; i < ne; ++i) {
Index: libibverbs/examples/srq_pingpong.c
===================================================================
--- libibverbs.orig/examples/srq_pingpong.c 2010-02-08 15:04:24.382329000 +0200
+++ libibverbs/examples/srq_pingpong.c 2010-03-17 14:13:22.332220000 +0200
@@ -71,17 +71,19 @@ struct pingpong_context {
int num_qp;
int rx_depth;
int pending[MAX_QP];
+ struct ibv_port_attr portinfo;
};
struct pingpong_dest {
int lid;
int qpn;
int psn;
+ union ibv_gid gid;
};
static int pp_connect_ctx(struct pingpong_context *ctx, int port, enum ibv_mtu mtu,
int sl, const struct pingpong_dest *my_dest,
- const struct pingpong_dest *dest)
+ const struct pingpong_dest *dest, int sgid_idx)
{
int i;
@@ -101,6 +103,13 @@ static int pp_connect_ctx(struct pingpon
.port_num = port
}
};
+
+ if (dest->gid.global.interface_id) {
+ attr.ah_attr.is_global = 1;
+ attr.ah_attr.grh.hop_limit = 1;
+ attr.ah_attr.grh.dgid = dest->gid;
+ attr.ah_attr.grh.sgid_index = sgid_idx;
+ }
if (ibv_modify_qp(ctx->qp[i], &attr,
IBV_QP_STATE |
IBV_QP_AV |
@@ -143,12 +152,13 @@ static struct pingpong_dest *pp_client_e
.ai_socktype = SOCK_STREAM
};
char *service;
- char msg[sizeof "0000:000000:000000"];
+ char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"];
int n;
int r;
int i;
int sockfd = -1;
struct pingpong_dest *rem_dest = NULL;
+ char gid[33];
if (asprintf(&service, "%d", port) < 0)
return NULL;
@@ -180,7 +190,8 @@ static struct pingpong_dest *pp_client_e
}
for (i = 0; i < MAX_QP; ++i) {
- sprintf(msg, "%04x:%06x:%06x", my_dest[i].lid, my_dest[i].qpn, my_dest[i].psn);
+ gid_to_wire_gid(&my_dest[i].gid, gid);
+ sprintf(msg, "%04x:%06x:%06x:%s", my_dest[i].lid, my_dest[i].qpn, my_dest[i].psn, gid);
if (write(sockfd, msg, sizeof msg) != sizeof msg) {
fprintf(stderr, "Couldn't send local address\n");
goto out;
@@ -204,8 +215,9 @@ static struct pingpong_dest *pp_client_e
n += r;
}
- sscanf(msg, "%x:%x:%x",
- &rem_dest[i].lid, &rem_dest[i].qpn, &rem_dest[i].psn);
+ sscanf(msg, "%x:%x:%x:%s",
+ &rem_dest[i].lid, &rem_dest[i].qpn, &rem_dest[i].psn, gid);
+ wire_gid_to_gid(gid, &rem_dest[i].gid);
}
write(sockfd, "done", sizeof "done");
@@ -218,7 +230,8 @@ out:
static struct pingpong_dest *pp_server_exch_dest(struct pingpong_context *ctx,
int ib_port, enum ibv_mtu mtu,
int port, int sl,
- const struct pingpong_dest *my_dest)
+ const struct pingpong_dest *my_dest,
+ int sgid_idx)
{
struct addrinfo *res, *t;
struct addrinfo hints = {
@@ -227,12 +240,13 @@ static struct pingpong_dest *pp_server_e
.ai_socktype = SOCK_STREAM
};
char *service;
- char msg[sizeof "0000:000000:000000"];
+ char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"];
int n;
int r;
int i;
int sockfd = -1, connfd;
struct pingpong_dest *rem_dest = NULL;
+ char gid[33];
if (asprintf(&service, "%d", port) < 0)
return NULL;
@@ -292,11 +306,12 @@ static struct pingpong_dest *pp_server_e
n += r;
}
- sscanf(msg, "%x:%x:%x",
- &rem_dest[i].lid, &rem_dest[i].qpn, &rem_dest[i].psn);
+ sscanf(msg, "%x:%x:%x:%s",
+ &rem_dest[i].lid, &rem_dest[i].qpn, &rem_dest[i].psn, gid);
+ wire_gid_to_gid(gid, &rem_dest[i].gid);
}
- if (pp_connect_ctx(ctx, ib_port, mtu, sl, my_dest, rem_dest)) {
+ if (pp_connect_ctx(ctx, ib_port, mtu, sl, my_dest, rem_dest, sgid_idx)) {
fprintf(stderr, "Couldn't connect to remote QP\n");
free(rem_dest);
rem_dest = NULL;
@@ -304,7 +319,8 @@ static struct pingpong_dest *pp_server_e
}
for (i = 0; i < MAX_QP; ++i) {
- sprintf(msg, "%04x:%06x:%06x", my_dest[i].lid, my_dest[i].qpn, my_dest[i].psn);
+ gid_to_wire_gid(&my_dest[i].gid, gid);
+ sprintf(msg, "%04x:%06x:%06x:%s", my_dest[i].lid, my_dest[i].qpn, my_dest[i].psn, gid);
if (write(connfd, msg, sizeof msg) != sizeof msg) {
fprintf(stderr, "Couldn't send local address\n");
free(rem_dest);
@@ -327,7 +343,7 @@ static struct pingpong_context *pp_init_
struct pingpong_context *ctx;
int i;
- ctx = malloc(sizeof *ctx);
+ ctx = calloc(1, sizeof *ctx);
if (!ctx)
return NULL;
@@ -551,6 +567,7 @@ static void usage(const char *argv0)
printf(" -n, --iters=<iters> number of exchanges per QP(default 1000)\n");
printf(" -l, --sl=<sl> service level value\n");
printf(" -e, --events sleep on CQ events (default poll)\n");
+ printf(" -g, --gid-idx=<gid index> local port gid index\n");
}
int main(int argc, char *argv[])
@@ -578,6 +595,8 @@ int main(int argc, char *argv[])
int i;
int num_cq_events = 0;
int sl = 0;
+ int gidx = -1;
+ char gid[33];
srand48(getpid() * time(NULL));
@@ -595,10 +614,11 @@ int main(int argc, char *argv[])
{ .name = "iters", .has_arg = 1, .val = 'n' },
{ .name = "sl", .has_arg = 1, .val = 'l' },
{ .name = "events", .has_arg = 0, .val = 'e' },
+ { .name = "gid-idx", .has_arg = 1, .val = 'g' },
{ 0 }
};
- c = getopt_long(argc, argv, "p:d:i:s:m:q:r:n:l:e", long_options, NULL);
+ c = getopt_long(argc, argv, "p:d:i:s:m:q:r:n:l:eg:", long_options, NULL);
if (c == -1)
break;
@@ -655,6 +675,10 @@ int main(int argc, char *argv[])
++use_event;
break;
+ case 'g':
+ gidx = strtol(optarg, NULL, 0);
+ break;
+
default:
usage(argv[0]);
return 1;
@@ -722,33 +746,50 @@ int main(int argc, char *argv[])
memset(my_dest, 0, sizeof my_dest);
+ if (pp_get_port_info(ctx->context, ib_port, &ctx->portinfo)) {
+ fprintf(stderr, "Couldn't get port info\n");
+ return 1;
+ }
for (i = 0; i < num_qp; ++i) {
my_dest[i].qpn = ctx->qp[i]->qp_num;
my_dest[i].psn = lrand48() & 0xffffff;
- my_dest[i].lid = pp_get_local_lid(ctx->context, ib_port);
- if (!my_dest[i].lid) {
+ my_dest[i].lid = ctx->portinfo.lid;
+ if (ctx->portinfo.link_layer == IBV_LINK_LAYER_INFINIBAND && !my_dest[i].lid) {
fprintf(stderr, "Couldn't get local LID\n");
return 1;
}
- printf(" local address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x\n",
- my_dest[i].lid, my_dest[i].qpn, my_dest[i].psn);
+ if (gidx >= 0) {
+ if (ibv_query_gid(ctx->context, ib_port, gidx, &my_dest[i].gid)) {
+ fprintf(stderr, "Could not get local gid for gid index %d\n", gidx);
+ return 1;
+ }
+ } else
+ memset(&my_dest[i].gid, 0, sizeof my_dest[i].gid);
+
+ inet_ntop(AF_INET6, &my_dest[i].gid, gid, sizeof gid);
+ printf(" local address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n",
+ my_dest[i].lid, my_dest[i].qpn, my_dest[i].psn, gid);
}
if (servername)
rem_dest = pp_client_exch_dest(servername, port, my_dest);
else
- rem_dest = pp_server_exch_dest(ctx, ib_port, mtu, port, sl, my_dest);
+ rem_dest = pp_server_exch_dest(ctx, ib_port, mtu, port, sl, my_dest, gidx);
if (!rem_dest)
return 1;
- for (i = 0; i < num_qp; ++i)
- printf(" remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x\n",
- rem_dest[i].lid, rem_dest[i].qpn, rem_dest[i].psn);
+ inet_ntop(AF_INET6, &rem_dest->gid, gid, sizeof gid);
+
+ for (i = 0; i < num_qp; ++i) {
+ inet_ntop(AF_INET6, &rem_dest[i].gid, gid, sizeof gid);
+ printf(" remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n",
+ rem_dest[i].lid, rem_dest[i].qpn, rem_dest[i].psn, gid);
+ }
if (servername)
- if (pp_connect_ctx(ctx, ib_port, mtu, sl, my_dest, rem_dest))
+ if (pp_connect_ctx(ctx, ib_port, mtu, sl, my_dest, rem_dest, gidx))
return 1;
if (servername)
Index: libibverbs/examples/uc_pingpong.c
===================================================================
--- libibverbs.orig/examples/uc_pingpong.c 2010-02-08 15:04:24.386328000 +0200
+++ libibverbs/examples/uc_pingpong.c 2010-03-17 14:14:23.573114000 +0200
@@ -67,17 +67,19 @@ struct pingpong_context {
int size;
int rx_depth;
int pending;
+ struct ibv_port_attr portinfo;
};
struct pingpong_dest {
int lid;
int qpn;
int psn;
+ union ibv_gid gid;
};
static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn,
enum ibv_mtu mtu, int sl,
- struct pingpong_dest *dest)
+ struct pingpong_dest *dest, int sgid_idx)
{
struct ibv_qp_attr attr = {
.qp_state = IBV_QPS_RTR,
@@ -92,6 +94,14 @@ static int pp_connect_ctx(struct pingpon
.port_num = port
}
};
+
+ if (dest->gid.global.interface_id) {
+ attr.ah_attr.is_global = 1;
+ attr.ah_attr.grh.hop_limit = 1;
+ attr.ah_attr.grh.dgid = dest->gid;
+ attr.ah_attr.grh.sgid_index = sgid_idx;
+ }
+
if (ibv_modify_qp(ctx->qp, &attr,
IBV_QP_STATE |
IBV_QP_AV |
@@ -123,10 +133,11 @@ static struct pingpong_dest *pp_client_e
.ai_socktype = SOCK_STREAM
};
char *service;
- char msg[sizeof "0000:000000:000000"];
+ char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"];
int n;
int sockfd = -1;
struct pingpong_dest *rem_dest = NULL;
+ char gid[33];
if (asprintf(&service, "%d", port) < 0)
return NULL;
@@ -157,7 +168,8 @@ static struct pingpong_dest *pp_client_e
return NULL;
}
- sprintf(msg, "%04x:%06x:%06x", my_dest->lid, my_dest->qpn, my_dest->psn);
+ gid_to_wire_gid(&my_dest->gid, gid);
+ sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn, my_dest->psn, gid);
if (write(sockfd, msg, sizeof msg) != sizeof msg) {
fprintf(stderr, "Couldn't send local address\n");
goto out;
@@ -175,7 +187,8 @@ static struct pingpong_dest *pp_client_e
if (!rem_dest)
goto out;
- sscanf(msg, "%x:%x:%x", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn);
+ sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn, gid);
+ wire_gid_to_gid(gid, &rem_dest->gid);
out:
close(sockfd);
@@ -185,7 +198,8 @@ out:
static struct pingpong_dest *pp_server_exch_dest(struct pingpong_context *ctx,
int ib_port, enum ibv_mtu mtu,
int port, int sl,
- const struct pingpong_dest *my_dest)
+ const struct pingpong_dest *my_dest,
+ int sgid_idx)
{
struct addrinfo *res, *t;
struct addrinfo hints = {
@@ -194,10 +208,11 @@ static struct pingpong_dest *pp_server_e
.ai_socktype = SOCK_STREAM
};
char *service;
- char msg[sizeof "0000:000000:000000"];
+ char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"];
int n;
int sockfd = -1, connfd;
struct pingpong_dest *rem_dest = NULL;
+ char gid[33];
if (asprintf(&service, "%d", port) < 0)
return NULL;
@@ -251,16 +266,18 @@ static struct pingpong_dest *pp_server_e
if (!rem_dest)
goto out;
- sscanf(msg, "%x:%x:%x", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn);
+ sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn, gid);
+ wire_gid_to_gid(gid, &rem_dest->gid);
- if (pp_connect_ctx(ctx, ib_port, my_dest->psn, mtu, sl, rem_dest)) {
+ if (pp_connect_ctx(ctx, ib_port, my_dest->psn, mtu, sl, rem_dest, sgid_idx)) {
fprintf(stderr, "Couldn't connect to remote QP\n");
free(rem_dest);
rem_dest = NULL;
goto out;
}
- sprintf(msg, "%04x:%06x:%06x", my_dest->lid, my_dest->qpn, my_dest->psn);
+ gid_to_wire_gid(&my_dest->gid, gid);
+ sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn, my_dest->psn, gid);
if (write(connfd, msg, sizeof msg) != sizeof msg) {
fprintf(stderr, "Couldn't send local address\n");
free(rem_dest);
@@ -281,7 +298,7 @@ static struct pingpong_context *pp_init_
{
struct pingpong_context *ctx;
- ctx = malloc(sizeof *ctx);
+ ctx = calloc(1, sizeof *ctx);
if (!ctx)
return NULL;
@@ -469,6 +486,7 @@ static void usage(const char *argv0)
printf(" -n, --iters=<iters> number of exchanges (default 1000)\n");
printf(" -l, --sl=<sl> service level value\n");
printf(" -e, --events sleep on CQ events (default poll)\n");
+ printf(" -g, --gid-idx=<gid index> local port gid index\n");
}
int main(int argc, char *argv[])
@@ -492,6 +510,8 @@ int main(int argc, char *argv[])
int rcnt, scnt;
int num_cq_events = 0;
int sl = 0;
+ int gidx = -1;
+ char gid[33];
srand48(getpid() * time(NULL));
@@ -508,10 +528,11 @@ int main(int argc, char *argv[])
{ .name = "iters", .has_arg = 1, .val = 'n' },
{ .name = "sl", .has_arg = 1, .val = 'l' },
{ .name = "events", .has_arg = 0, .val = 'e' },
+ { .name = "gid-idx", .has_arg = 1, .val = 'g' },
{ 0 }
};
- c = getopt_long(argc, argv, "p:d:i:s:m:r:n:l:e", long_options, NULL);
+ c = getopt_long(argc, argv, "p:d:i:s:m:r:n:l:eg:", long_options, NULL);
if (c == -1)
break;
@@ -564,6 +585,10 @@ int main(int argc, char *argv[])
++use_event;
break;
+ case 'g':
+ gidx = strtol(optarg, NULL, 0);
+ break;
+
default:
usage(argv[0]);
return 1;
@@ -619,30 +644,45 @@ int main(int argc, char *argv[])
return 1;
}
- my_dest.lid = pp_get_local_lid(ctx->context, ib_port);
- my_dest.qpn = ctx->qp->qp_num;
- my_dest.psn = lrand48() & 0xffffff;
- if (!my_dest.lid) {
+ if (pp_get_port_info(ctx->context, ib_port, &ctx->portinfo)) {
+ fprintf(stderr, "Couldn't get port info\n");
+ return 1;
+ }
+
+ my_dest.lid = ctx->portinfo.lid;
+ if (ctx->portinfo.link_layer == IBV_LINK_LAYER_INFINIBAND && !my_dest.lid) {
fprintf(stderr, "Couldn't get local LID\n");
return 1;
}
- printf(" local address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x\n",
- my_dest.lid, my_dest.qpn, my_dest.psn);
+ if (gidx >= 0) {
+ if (ibv_query_gid(ctx->context, ib_port, gidx, &my_dest.gid)) {
+ fprintf(stderr, "Could not get local gid for gid index %d\n", gidx);
+ return 1;
+ }
+ } else
+ memset(&my_dest.gid, 0, sizeof my_dest.gid);
+
+ my_dest.qpn = ctx->qp->qp_num;
+ my_dest.psn = lrand48() & 0xffffff;
+ inet_ntop(AF_INET6, &my_dest.gid, gid, sizeof gid);
+ printf(" local address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n",
+ my_dest.lid, my_dest.qpn, my_dest.psn, gid);
if (servername)
rem_dest = pp_client_exch_dest(servername, port, &my_dest);
else
- rem_dest = pp_server_exch_dest(ctx, ib_port, mtu, port, sl, &my_dest);
+ rem_dest = pp_server_exch_dest(ctx, ib_port, mtu, port, sl, &my_dest, gidx);
if (!rem_dest)
return 1;
- printf(" remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x\n",
- rem_dest->lid, rem_dest->qpn, rem_dest->psn);
+ inet_ntop(AF_INET6, &rem_dest->gid, gid, sizeof gid);
+ printf(" remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n",
+ rem_dest->lid, rem_dest->qpn, rem_dest->psn, gid);
if (servername)
- if (pp_connect_ctx(ctx, ib_port, my_dest.psn, mtu, sl, rem_dest))
+ if (pp_connect_ctx(ctx, ib_port, my_dest.psn, mtu, sl, rem_dest, gidx))
return 1;
ctx->pending = PINGPONG_RECV_WRID;
Index: libibverbs/examples/ud_pingpong.c
===================================================================
--- libibverbs.orig/examples/ud_pingpong.c 2010-02-08 15:04:24.389329000 +0200
+++ libibverbs/examples/ud_pingpong.c 2010-03-17 14:08:48.502754000 +0200
@@ -68,16 +68,18 @@ struct pingpong_context {
int size;
int rx_depth;
int pending;
+ struct ibv_port_attr portinfo;
};
struct pingpong_dest {
int lid;
int qpn;
int psn;
+ union ibv_gid gid;
};
static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn,
- int sl, struct pingpong_dest *dest)
+ int sl, struct pingpong_dest *dest, int sgid_idx)
{
struct ibv_ah_attr ah_attr = {
.is_global = 0,
@@ -105,6 +107,13 @@ static int pp_connect_ctx(struct pingpon
return 1;
}
+ if (dest->gid.global.interface_id) {
+ ah_attr.is_global = 1;
+ ah_attr.grh.hop_limit = 1;
+ ah_attr.grh.dgid = dest->gid;
+ ah_attr.grh.sgid_index = sgid_idx;
+ }
+
ctx->ah = ibv_create_ah(ctx->pd, &ah_attr);
if (!ctx->ah) {
fprintf(stderr, "Failed to create AH\n");
@@ -123,10 +132,11 @@ static struct pingpong_dest *pp_client_e
.ai_socktype = SOCK_STREAM
};
char *service;
- char msg[sizeof "0000:000000:000000"];
+ char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"];
int n;
int sockfd = -1;
struct pingpong_dest *rem_dest = NULL;
+ char gid[33];
if (asprintf(&service, "%d", port) < 0)
return NULL;
@@ -157,7 +167,8 @@ static struct pingpong_dest *pp_client_e
return NULL;
}
- sprintf(msg, "%04x:%06x:%06x", my_dest->lid, my_dest->qpn, my_dest->psn);
+ gid_to_wire_gid(&my_dest->gid, gid);
+ sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn, my_dest->psn, gid);
if (write(sockfd, msg, sizeof msg) != sizeof msg) {
fprintf(stderr, "Couldn't send local address\n");
goto out;
@@ -175,7 +186,8 @@ static struct pingpong_dest *pp_client_e
if (!rem_dest)
goto out;
- sscanf(msg, "%x:%x:%x", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn);
+ sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn, gid);
+ wire_gid_to_gid(gid, &rem_dest->gid);
out:
close(sockfd);
@@ -184,7 +196,8 @@ out:
static struct pingpong_dest *pp_server_exch_dest(struct pingpong_context *ctx,
int ib_port, int port, int sl,
- const struct pingpong_dest *my_dest)
+ const struct pingpong_dest *my_dest,
+ int sgid_idx)
{
struct addrinfo *res, *t;
struct addrinfo hints = {
@@ -193,10 +206,11 @@ static struct pingpong_dest *pp_server_e
.ai_socktype = SOCK_STREAM
};
char *service;
- char msg[sizeof "0000:000000:000000"];
+ char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"];
int n;
int sockfd = -1, connfd;
struct pingpong_dest *rem_dest = NULL;
+ char gid[33];
if (asprintf(&service, "%d", port) < 0)
return NULL;
@@ -250,16 +264,18 @@ static struct pingpong_dest *pp_server_e
if (!rem_dest)
goto out;
- sscanf(msg, "%x:%x:%x", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn);
+ sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn, gid);
+ wire_gid_to_gid(gid, &rem_dest->gid);
- if (pp_connect_ctx(ctx, ib_port, my_dest->psn, sl, rem_dest)) {
+ if (pp_connect_ctx(ctx, ib_port, my_dest->psn, sl, rem_dest, sgid_idx)) {
fprintf(stderr, "Couldn't connect to remote QP\n");
free(rem_dest);
rem_dest = NULL;
goto out;
}
- sprintf(msg, "%04x:%06x:%06x", my_dest->lid, my_dest->qpn, my_dest->psn);
+ gid_to_wire_gid(&my_dest->gid, gid);
+ sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn, my_dest->psn, gid);
if (write(connfd, msg, sizeof msg) != sizeof msg) {
fprintf(stderr, "Couldn't send local address\n");
free(rem_dest);
@@ -474,10 +490,11 @@ static void usage(const char *argv0)
printf(" -p, --port=<port> listen on/connect to port <port> (default 18515)\n");
printf(" -d, --ib-dev=<dev> use IB device <dev> (default first device found)\n");
printf(" -i, --ib-port=<port> use port <port> of IB device (default 1)\n");
- printf(" -s, --size=<size> size of message to exchange (default 2048)\n");
+ printf(" -s, --size=<size> size of message to exchange (default 1024)\n");
printf(" -r, --rx-depth=<dep> number of receives to post at a time (default 500)\n");
printf(" -n, --iters=<iters> number of exchanges (default 1000)\n");
printf(" -e, --events sleep on CQ events (default poll)\n");
+ printf(" -g, --gid-idx=<gid index> local port gid index\n");
}
int main(int argc, char *argv[])
@@ -492,7 +509,7 @@ int main(int argc, char *argv[])
char *servername = NULL;
int port = 18515;
int ib_port = 1;
- int size = 2048;
+ int size = 1024;
int rx_depth = 500;
int iters = 1000;
int use_event = 0;
@@ -500,6 +517,8 @@ int main(int argc, char *argv[])
int rcnt, scnt;
int num_cq_events = 0;
int sl = 0;
+ int gidx = -1;
+ char gid[33];
srand48(getpid() * time(NULL));
@@ -515,10 +534,11 @@ int main(int argc, char *argv[])
{ .name = "iters", .has_arg = 1, .val = 'n' },
{ .name = "sl", .has_arg = 1, .val = 'l' },
{ .name = "events", .has_arg = 0, .val = 'e' },
+ { .name = "gid-idx", .has_arg = 1, .val = 'g' },
{ 0 }
};
- c = getopt_long(argc, argv, "p:d:i:s:r:n:l:e", long_options, NULL);
+ c = getopt_long(argc, argv, "p:d:i:s:r:n:l:eg:", long_options, NULL);
if (c == -1)
break;
@@ -563,6 +583,10 @@ int main(int argc, char *argv[])
++use_event;
break;
+ case 'g':
+ gidx = strtol(optarg, NULL, 0);
+ break;
+
default:
usage(argv[0]);
return 1;
@@ -618,30 +642,41 @@ int main(int argc, char *argv[])
return 1;
}
- my_dest.lid = pp_get_local_lid(ctx->context, ib_port);
- my_dest.qpn = ctx->qp->qp_num;
- my_dest.psn = lrand48() & 0xffffff;
- if (!my_dest.lid) {
- fprintf(stderr, "Couldn't get local LID\n");
+ if (pp_get_port_info(ctx->context, ib_port, &ctx->portinfo)) {
+ fprintf(stderr, "Couldn't get port info\n");
return 1;
}
+ my_dest.lid = ctx->portinfo.lid;
+
+ my_dest.qpn = ctx->qp->qp_num;
+ my_dest.psn = lrand48() & 0xffffff;
+
+ if (gidx >= 0) {
+ if (ibv_query_gid(ctx->context, ib_port, gidx, &my_dest.gid)) {
+ fprintf(stderr, "Could not get local gid for gid index %d\n", gidx);
+ return 1;
+ }
+ } else
+ memset(&my_dest.gid, 0, sizeof my_dest.gid);
- printf(" local address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x\n",
- my_dest.lid, my_dest.qpn, my_dest.psn);
+ inet_ntop(AF_INET6, &my_dest.gid, gid, sizeof gid);
+ printf(" local address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x: GID %s\n",
+ my_dest.lid, my_dest.qpn, my_dest.psn, gid);
if (servername)
rem_dest = pp_client_exch_dest(servername, port, &my_dest);
else
- rem_dest = pp_server_exch_dest(ctx, ib_port, port, sl, &my_dest);
+ rem_dest = pp_server_exch_dest(ctx, ib_port, port, sl, &my_dest, gidx);
if (!rem_dest)
return 1;
- printf(" remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x\n",
- rem_dest->lid, rem_dest->qpn, rem_dest->psn);
+ inet_ntop(AF_INET6, &rem_dest->gid, gid, sizeof gid);
+ printf(" remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n",
+ rem_dest->lid, rem_dest->qpn, rem_dest->psn, gid);
if (servername)
- if (pp_connect_ctx(ctx, ib_port, my_dest.psn, sl, rem_dest))
+ if (pp_connect_ctx(ctx, ib_port, my_dest.psn, sl, rem_dest, gidx))
return 1;
ctx->pending = PINGPONG_RECV_WRID;
Reference in New Issue View Git Blame Copy Permalink