From: Luis Kornblueh (luis.kornblueh_at_[hidden])
Date: 2007-05-16 10:08:53
Hi,
well we currently try to change over to ofed. The same time we try to change
to openmpi-1.2.1 from openpi-1.2b3, unfortunately we get a SEGV in mpiexec ;-)
Cheerio,
Luis
> Greetings and thanks for the patch!
>
> We actually made this change on the development trunk but never moved
> it to the release branch. See https://svn.open-mpi.org/trac/ompi/
> changeset/13709
>
> The problem is that we accidentally used a gcc-ism in the mvapi BTL
> (i.e., the older InfiniBand driver). We didn't think too many people
> were using the mvapi BTL, and therefore didn't move it over to the
> v1.2 branch. Sorry about that!
>
> Do you have a strong requirement for using mvapi? Specifically, is
> there any chance that you can upgrade to OFED? I ask because all
> current and future OMPI work for InfiniBand is being done in the
> openib BTL (not the mvapi BTL -- other than this fix, we haven't made
> any changes to the mvapi BTL for a long, long time).
>
> See http://www.open-mpi.org/faq/?category=openfabrics#vapi-support.
>
>
>
> On May 16, 2007, at 12:08 AM, Luis Kornblueh wrote:
>
> > Hi everybody,
> >
> > we tried to install openmpi with Sun cc on Linux. IT required some
> > patches. I add the necessary pathces here. I hope you can include
> > those.
> >
> > Cheerio,
> > Luis
> >
> > diff -Naur openmpi-1.2.1/ompi/mca/btl/mvapi/btl_mvapi.c
> > openmpi-1.2.1_patched/ompi/mca/btl/mvapi/btl_mvapi.c
> > --- openmpi-1.2.1/ompi/mca/btl/mvapi/btl_mvapi.c 2007-04-19
> > 18:30:54.000000000 +0200
> > +++ openmpi-1.2.1_patched/ompi/mca/btl/mvapi/btl_mvapi.c 2007-05-09
> > 14:33:24.000000000 +0200
> > @@ -463,7 +463,7 @@
> > mca_btl_mvapi_frag_t* frag = (mca_btl_mvapi_frag_t*)descriptor;
> > frag->endpoint = endpoint;
> > frag->hdr->tag = tag;
> > - frag->sr_desc.opcode = VAPI_SEND;
> > + frag->desc.sr_desc.opcode = VAPI_SEND;
> > return mca_btl_mvapi_endpoint_send(endpoint, frag);
> > }
> >
> > @@ -481,7 +481,7 @@
> >
> > /* setup for queued requests */
> > frag->endpoint = endpoint;
> > - frag->sr_desc.opcode = VAPI_RDMA_WRITE;
> > + frag->desc.sr_desc.opcode = VAPI_RDMA_WRITE;
> >
> > /* check for a send wqe */
> > if (OPAL_THREAD_ADD32(&endpoint->sd_wqe_lp,-1) < 0) {
> > @@ -494,12 +494,12 @@
> > /* post descriptor */
> > } else {
> >
> > - frag->sr_desc.remote_qp = endpoint->rem_info.rem_qp_num_lp;
> > - frag->sr_desc.remote_addr = (VAPI_virt_addr_t) frag-
> > >base.des_dst->seg_addr.lval;
> > - frag->sr_desc.r_key = frag->base.des_dst->seg_key.key32[0];
> > + frag->desc.sr_desc.remote_qp = endpoint-
> > >rem_info.rem_qp_num_lp;
> > + frag->desc.sr_desc.remote_addr = (VAPI_virt_addr_t) frag-
> > >base.des_dst->seg_addr.lval;
> > + frag->desc.sr_desc.r_key = frag->base.des_dst-
> > >seg_key.key32[0];
> > frag->sg_entry.addr = (VAPI_virt_addr_t) (MT_virt_addr_t)
> > frag->base.des_src->seg_addr.pval;
> > frag->sg_entry.len = frag->base.des_src->seg_len;
> > - if(VAPI_OK != VAPI_post_sr(mvapi_btl->nic, endpoint-
> > >lcl_qp_hndl_lp, &frag->sr_desc)) {
> > + if(VAPI_OK != VAPI_post_sr(mvapi_btl->nic, endpoint-
> > >lcl_qp_hndl_lp, &frag->desc.sr_desc)) {
> > rc = OMPI_ERROR;
> > } else {
> > rc = OMPI_SUCCESS;
> > @@ -531,7 +531,7 @@
> > mca_btl_mvapi_frag_t* frag = (mca_btl_mvapi_frag_t*) descriptor;
> >
> > frag->endpoint = endpoint;
> > - frag->sr_desc.opcode = VAPI_RDMA_READ;
> > + frag->desc.sr_desc.opcode = VAPI_RDMA_READ;
> >
> > /* check for a send wqe */
> > if (OPAL_THREAD_ADD32(&endpoint->sd_wqe_lp,-1) < 0) {
> > @@ -555,13 +555,13 @@
> >
> > } else {
> >
> > - frag->sr_desc.remote_qp = endpoint->rem_info.rem_qp_num_lp;
> > - frag->sr_desc.remote_addr = (VAPI_virt_addr_t) frag-
> > >base.des_src->seg_addr.lval;
> > - frag->sr_desc.r_key = frag->base.des_src->seg_key.key32[0];
> > + frag->desc.sr_desc.remote_qp = endpoint-
> > >rem_info.rem_qp_num_lp;
> > + frag->desc.sr_desc.remote_addr = (VAPI_virt_addr_t) frag-
> > >base.des_src->seg_addr.lval;
> > + frag->desc.sr_desc.r_key = frag->base.des_src-
> > >seg_key.key32[0];
> > frag->sg_entry.addr = (VAPI_virt_addr_t) (MT_virt_addr_t)
> > frag->base.des_dst->seg_addr.pval;
> > frag->sg_entry.len = frag->base.des_dst->seg_len;
> >
> > - if(VAPI_OK != VAPI_post_sr(mvapi_btl->nic, endpoint-
> > >lcl_qp_hndl_lp, &frag->sr_desc)) {
> > + if(VAPI_OK != VAPI_post_sr(mvapi_btl->nic, endpoint-
> > >lcl_qp_hndl_lp, &frag->desc.sr_desc)) {
> > rc = OMPI_ERROR;
> > } else {
> > rc = OMPI_SUCCESS;
> > diff -Naur openmpi-1.2.1/ompi/mca/btl/mvapi/btl_mvapi_endpoint.c
> > openmpi-1.2.1_patched/ompi/mca/btl/mvapi/btl_mvapi_endpoint.c
> > --- openmpi-1.2.1/ompi/mca/btl/mvapi/btl_mvapi_endpoint.c
> > 2007-04-19 18:30:54.000000000 +0200
> > +++ openmpi-1.2.1_patched/ompi/mca/btl/mvapi/btl_mvapi_endpoint.c
> > 2007-05-09 14:45:31.000000000 +0200
> > @@ -143,7 +143,7 @@
> > }
> > }
> >
> > - frag->sr_desc.remote_qkey = 0;
> > + frag->desc.sr_desc.remote_qkey = 0;
> > frag->sg_entry.addr = (VAPI_virt_addr_t) (MT_virt_addr_t) frag-
> > >hdr;
> > frag->sg_entry.len =
> > frag->segment.seg_len + sizeof(mca_btl_mvapi_header_t) +
> > @@ -153,14 +153,14 @@
> > mca_btl_mvapi_footer_t* ftr =
> > (mca_btl_mvapi_footer_t*)(((char*)frag-
> > >segment.seg_addr.pval) +
> > frag->segment.seg_len);
> > - frag->sr_desc.opcode = VAPI_RDMA_WRITE;
> > + frag->desc.sr_desc.opcode = VAPI_RDMA_WRITE;
> > MCA_BTL_MVAPI_RDMA_FRAG_SET_SIZE(ftr, frag->sg_entry.len);
> > MCA_BTL_MVAPI_RDMA_MAKE_LOCAL(ftr);
> > #ifdef OMPI_ENABLE_DEBUG
> > ftr->seq = endpoint->eager_rdma_remote.seq++;
> > #endif
> > - frag->sr_desc.r_key = (VAPI_rkey_t)endpoint-
> > >eager_rdma_remote.rkey;
> > - frag->sr_desc.remote_addr = (VAPI_virt_addr_t)
> > + frag->desc.sr_desc.r_key = (VAPI_rkey_t)endpoint-
> > >eager_rdma_remote.rkey;
> > + frag->desc.sr_desc.remote_addr = (VAPI_virt_addr_t)
> > endpoint->eager_rdma_remote.base.lval +
> > endpoint->eager_rdma_remote.head *
> > mvapi_btl->eager_rdma_frag_size +
> > @@ -168,17 +168,17 @@
> > sizeof(mca_btl_mvapi_header_t) +
> > frag->size +
> > sizeof(mca_btl_mvapi_footer_t);
> > - frag->sr_desc.remote_addr -= frag->sg_entry.len;
> > + frag->desc.sr_desc.remote_addr -= frag->sg_entry.len;
> > MCA_BTL_MVAPI_RDMA_NEXT_INDEX (endpoint-
> > >eager_rdma_remote.head);
> > } else {
> > - frag->sr_desc.opcode = VAPI_SEND;
> > + frag->desc.sr_desc.opcode = VAPI_SEND;
> > }
> >
> >
> > if(frag->sg_entry.len <= mvapi_btl->ib_inline_max) {
> > - ret = EVAPI_post_inline_sr(mvapi_btl->nic, qp_hndl, &frag-
> > >sr_desc);
> > + ret = EVAPI_post_inline_sr(mvapi_btl->nic, qp_hndl, &frag-
> > >desc.sr_desc);
> > } else {
> > - ret = VAPI_post_sr(mvapi_btl->nic, qp_hndl, &frag->sr_desc);
> > + ret = VAPI_post_sr(mvapi_btl->nic, qp_hndl, &frag-
> > >desc.sr_desc);
> > }
> >
> > if(VAPI_OK != ret) {
> > @@ -1072,15 +1072,15 @@
> > OPAL_THREAD_ADD32(&endpoint->rd_credits_lp, -frag->hdr->credits);
> > ((mca_btl_mvapi_control_header_t *)frag-
> > >segment.seg_addr.pval)->type = MCA_BTL_MVAPI_CONTROL_NOOP;
> >
> > - frag->sr_desc.opcode = VAPI_SEND;
> > + frag->desc.sr_desc.opcode = VAPI_SEND;
> > frag->sg_entry.addr = (VAPI_virt_addr_t) (MT_virt_addr_t) frag-
> > >hdr;
> > frag->sg_entry.len = sizeof(mca_btl_mvapi_header_t) +
> > sizeof(mca_btl_mvapi_control_header_t);
> >
> > if(sizeof(mca_btl_mvapi_header_t) <= mvapi_btl->ib_inline_max) {
> > - ret = EVAPI_post_inline_sr(mvapi_btl->nic, endpoint-
> > >lcl_qp_hndl_lp, &frag->sr_desc);
> > + ret = EVAPI_post_inline_sr(mvapi_btl->nic, endpoint-
> > >lcl_qp_hndl_lp, &frag->desc.sr_desc);
> > } else {
> > - ret = VAPI_post_sr(mvapi_btl->nic, endpoint-
> > >lcl_qp_hndl_lp, &frag->sr_desc);
> > + ret = VAPI_post_sr(mvapi_btl->nic, endpoint-
> > >lcl_qp_hndl_lp, &frag->desc.sr_desc);
> > }
> > if(ret != VAPI_SUCCESS) {
> > OPAL_THREAD_ADD32(&endpoint->sd_credits_lp, -1);
> > @@ -1151,15 +1151,15 @@
> > ((mca_btl_mvapi_control_header_t *)frag-
> > >segment.seg_addr.pval)->type = MCA_BTL_MVAPI_CONTROL_NOOP;
> >
> >
> > - frag->sr_desc.opcode = VAPI_SEND;
> > + frag->desc.sr_desc.opcode = VAPI_SEND;
> > frag->sg_entry.addr = (VAPI_virt_addr_t) (MT_virt_addr_t) frag-
> > >hdr;
> > frag->sg_entry.len = sizeof(mca_btl_mvapi_header_t) +
> > sizeof(mca_btl_mvapi_control_header_t);
> >
> > if(sizeof(mca_btl_mvapi_header_t) <= mvapi_btl->ib_inline_max) {
> > - ret = EVAPI_post_inline_sr(mvapi_btl->nic, endpoint-
> > >lcl_qp_hndl_hp, &frag->sr_desc);
> > + ret = EVAPI_post_inline_sr(mvapi_btl->nic, endpoint-
> > >lcl_qp_hndl_hp, &frag->desc.sr_desc);
> > } else {
> > - ret = VAPI_post_sr(mvapi_btl->nic, endpoint-
> > >lcl_qp_hndl_hp, &frag->sr_desc);
> > + ret = VAPI_post_sr(mvapi_btl->nic, endpoint-
> > >lcl_qp_hndl_hp, &frag->desc.sr_desc);
> > }
> > if(ret != VAPI_SUCCESS) {
> > OPAL_THREAD_ADD32(&endpoint->sd_credits_lp, -1);
> > diff -Naur openmpi-1.2.1/ompi/mca/btl/mvapi/btl_mvapi_endpoint.h
> > openmpi-1.2.1_patched/ompi/mca/btl/mvapi/btl_mvapi_endpoint.h
> > --- openmpi-1.2.1/ompi/mca/btl/mvapi/btl_mvapi_endpoint.h
> > 2007-04-19 18:30:54.000000000 +0200
> > +++ openmpi-1.2.1_patched/ompi/mca/btl/mvapi/btl_mvapi_endpoint.h
> > 2007-05-09 14:38:43.000000000 +0200
> > @@ -234,7 +234,7 @@
> > frag->sg_entry.len = frag->size + \
> > ((unsigned char*) frag->segment.seg_addr.pval- \
> > (unsigned char*) frag->hdr); \
> > - desc_post[i] = frag->rr_desc; \
> > + desc_post[i] = frag->desc.rr_desc; \
> > }\
> > rc = EVAPI_post_rr_list( nic, \
> > qp, \
> > diff -Naur openmpi-1.2.1/ompi/mca/btl/mvapi/btl_mvapi_frag.c
> > openmpi-1.2.1_patched/ompi/mca/btl/mvapi/btl_mvapi_frag.c
> > --- openmpi-1.2.1/ompi/mca/btl/mvapi/btl_mvapi_frag.c 2007-04-19
> > 18:30:54.000000000 +0200
> > +++ openmpi-1.2.1_patched/ompi/mca/btl/mvapi/btl_mvapi_frag.c
> > 2007-05-09 14:47:38.000000000 +0200
> > @@ -43,12 +43,12 @@
> > frag->base.des_dst = NULL;
> > frag->base.des_dst_cnt = 0;
> >
> > - frag->sr_desc.comp_type = VAPI_SIGNALED;
> > - frag->sr_desc.opcode = VAPI_SEND;
> > - frag->sr_desc.remote_qkey = 0;
> > - frag->sr_desc.sg_lst_len = 1;
> > - frag->sr_desc.sg_lst_p = &frag->sg_entry;
> > - frag->sr_desc.id = (VAPI_virt_addr_t) (MT_virt_addr_t) frag;
> > + frag->desc.sr_desc.comp_type = VAPI_SIGNALED;
> > + frag->desc.sr_desc.opcode = VAPI_SEND;
> > + frag->desc.sr_desc.remote_qkey = 0;
> > + frag->desc.sr_desc.sg_lst_len = 1;
> > + frag->desc.sr_desc.sg_lst_p = &frag->sg_entry;
> > + frag->desc.sr_desc.id = (VAPI_virt_addr_t) (MT_virt_addr_t) frag;
> >
> > }
> >
> > @@ -61,11 +61,11 @@
> > frag->base.des_src = NULL;
> > frag->base.des_src_cnt = 0;
> >
> > - frag->rr_desc.comp_type = VAPI_SIGNALED;
> > - frag->rr_desc.opcode = VAPI_RECEIVE;
> > - frag->rr_desc.sg_lst_len = 1;
> > - frag->rr_desc.sg_lst_p = &frag->sg_entry;
> > - frag->rr_desc.id = (VAPI_virt_addr_t) (MT_virt_addr_t) frag;
> > + frag->desc.rr_desc.comp_type = VAPI_SIGNALED;
> > + frag->desc.rr_desc.opcode = VAPI_RECEIVE;
> > + frag->desc.rr_desc.sg_lst_len = 1;
> > + frag->desc.rr_desc.sg_lst_p = &frag->sg_entry;
> > + frag->desc.rr_desc.id = (VAPI_virt_addr_t) (MT_virt_addr_t) frag;
> >
> >
> > }
> > diff -Naur openmpi-1.2.1/ompi/mca/btl/mvapi/btl_mvapi_frag.h
> > openmpi-1.2.1_patched/ompi/mca/btl/mvapi/btl_mvapi_frag.h
> > --- openmpi-1.2.1/ompi/mca/btl/mvapi/btl_mvapi_frag.h 2007-04-19
> > 18:30:54.000000000 +0200
> > +++ openmpi-1.2.1_patched/ompi/mca/btl/mvapi/btl_mvapi_frag.h
> > 2007-05-09 14:31:08.000000000 +0200
> > @@ -92,7 +92,7 @@
> > union{
> > VAPI_rr_desc_t rr_desc;
> > VAPI_sr_desc_t sr_desc;
> > - };
> > + } desc;
> > VAPI_sg_lst_entry_t sg_entry;
> > mca_btl_mvapi_header_t *hdr;
> > mca_btl_mvapi_footer_t *ftr;
> > @@ -165,7 +165,7 @@
> >
> > #define MCA_BTL_IB_FRAG_PROGRESS(frag) \
> > do { \
> > - switch(frag->sr_desc.opcode) { \
> > + switch(frag->desc.sr_desc.opcode) { \
> > case VAPI_SEND: \
> > if(OMPI_SUCCESS != mca_btl_mvapi_endpoint_send(frag-
> > >endpoint, frag)) { \
> > BTL_ERROR(("error in posting pending send\n")); \
> > @@ -186,7 +186,7 @@
> > } \
> > break; \
> > default: \
> > - BTL_ERROR(("error in posting pending operation, invalide
> > opcode %d\n", frag->sr_desc.opcode)); \
> > + BTL_ERROR(("error in posting pending operation, invalide
> > opcode %d\n", frag->desc.sr_desc.opcode)); \
> > break; \
> > } \
> > } while (0)
> > diff -Naur openmpi-1.2.1/ompi/mca/btl/mvapi/btl_mvapi.h
> > openmpi-1.2.1_patched/ompi/mca/btl/mvapi/btl_mvapi.h
> > --- openmpi-1.2.1/ompi/mca/btl/mvapi/btl_mvapi.h 2007-04-19
> > 18:30:54.000000000 +0200
> > +++ openmpi-1.2.1_patched/ompi/mca/btl/mvapi/btl_mvapi.h 2007-05-09
> > 14:37:23.000000000 +0200
> > @@ -260,7 +260,7 @@
> > frag->sg_entry.len = frag->size + \
> > ((unsigned char*) frag->segment.seg_addr.pval- \
> > (unsigned char*) frag->hdr); \
> > - desc_post[i] = frag->rr_desc; \
> > + desc_post[i] = frag->desc.rr_desc; \
> > }\
> > ret = VAPI_post_srq( nic, \
> > srq_hndl, \
> >
> >
> > --
> > \\\\\\
> > (-0^0-)
> > --------------------------oOO--(_)--OOo-----------------------------
> >
> > Luis Kornblueh Tel. : +49-40-41173289
> > Max-Planck-Institute for Meteorology Fax. : +49-40-41173298
> > Bundesstr. 53
> > D-20146 Hamburg Email: luis.kornblueh_at_[hidden]
> > Federal Republic of Germany
> > _______________________________________________
> > users mailing list
> > users_at_[hidden]
> > http://www.open-mpi.org/mailman/listinfo.cgi/users
>
>
> --
> Jeff Squyres
> Cisco Systems
>
> _______________________________________________
> users mailing list
> users_at_[hidden]
> http://www.open-mpi.org/mailman/listinfo.cgi/users
--
\\\\\\
(-0^0-)
--------------------------oOO--(_)--OOo-----------------------------
Luis Kornblueh Tel. : +49-40-41173289
Max-Planck-Institute for Meteorology Fax. : +49-40-41173298
Bundesstr. 53
D-20146 Hamburg Email: luis.kornblueh_at_[hidden]
Federal Republic of Germany