What you’ve got is pretty much “best practice”; it’s just a bit confusing until you get used to it.
Two things, though:
First, be careful with this: sizeof(MPI_CHAR) is, I assume, 4 bytes, not 1. MPI_CHAR is an (integer) constant that describes (to the MPI library) a character. You probably want sizeof(char), or SIZE/2*sizeof(char), or anything else convenient. But the basic idea of doing a resize is right.
Second, I think you’re stuck using MPI_Scatterv, though, because there’s no easy way to make the offset between each block the same size. That is, the first element in the first block is at a[0], the second is at a[SIZE/2] (jump of size/2), the next is at a[SIZE*(SIZE/2)] (jump of (SIZE-1)*(SIZE/2)). So you need to be able to manually generate the offsets.
The following seems to work for me (I generalized it a little bit to make it clearer when “size” means “number of rows” vs “number of columns”, etc):
#include <stdio.h>
#include <stdlib.h>
#include <mpi.h>
#define COLS 12
#define ROWS 8
int main(int argc, char **argv) {
MPI_Init(&argc, &argv);
int p, rank;
MPI_Comm_size(MPI_COMM_WORLD, &p);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
char i;
char a[ROWS*COLS];
const int NPROWS=2; /* number of rows in _decomposition_ */
const int NPCOLS=3; /* number of cols in _decomposition_ */
const int BLOCKROWS = ROWS/NPROWS; /* number of rows in _block_ */
const int BLOCKCOLS = COLS/NPCOLS; /* number of cols in _block_ */
if (rank == 0) {
for (int ii=0; ii<ROWS*COLS; ii++) {
a[ii] = (char)ii;
}
}
if (p != NPROWS*NPCOLS) {
fprintf(stderr,"Error: number of PEs %d != %d x %d\n", p, NPROWS, NPCOLS);
MPI_Finalize();
exit(-1);
}
char b[BLOCKROWS*BLOCKCOLS];
for (int ii=0; ii<BLOCKROWS*BLOCKCOLS; ii++) b[ii] = 0;
MPI_Datatype blocktype;
MPI_Datatype blocktype2;
MPI_Type_vector(BLOCKROWS, BLOCKCOLS, COLS, MPI_CHAR, &blocktype2);
MPI_Type_create_resized( blocktype2, 0, sizeof(char), &blocktype);
MPI_Type_commit(&blocktype);
int disps[NPROWS*NPCOLS];
int counts[NPROWS*NPCOLS];
for (int ii=0; ii<NPROWS; ii++) {
for (int jj=0; jj<NPCOLS; jj++) {
disps[ii*NPCOLS+jj] = ii*COLS*BLOCKROWS+jj*BLOCKCOLS;
counts [ii*NPCOLS+jj] = 1;
}
}
MPI_Scatterv(a, counts, disps, blocktype, b, BLOCKROWS*BLOCKCOLS, MPI_CHAR, 0, MPI_COMM_WORLD);
/* each proc prints it's "b" out, in order */
for (int proc=0; proc<p; proc++) {
if (proc == rank) {
printf("Rank = %d\n", rank);
if (rank == 0) {
printf("Global matrix: \n");
for (int ii=0; ii<ROWS; ii++) {
for (int jj=0; jj<COLS; jj++) {
printf("%3d ",(int)a[ii*COLS+jj]);
}
printf("\n");
}
}
printf("Local Matrix:\n");
for (int ii=0; ii<BLOCKROWS; ii++) {
for (int jj=0; jj<BLOCKCOLS; jj++) {
printf("%3d ",(int)b[ii*BLOCKCOLS+jj]);
}
printf("\n");
}
printf("\n");
}
MPI_Barrier(MPI_COMM_WORLD);
}
MPI_Finalize();
return 0;
}
Running:
$ mpirun -np 6 ./matrix
Rank = 0
Global matrix:
0 1 2 3 4 5 6 7 8 9 10 11
12 13 14 15 16 17 18 19 20 21 22 23
24 25 26 27 28 29 30 31 32 33 34 35
36 37 38 39 40 41 42 43 44 45 46 47
48 49 50 51 52 53 54 55 56 57 58 59
60 61 62 63 64 65 66 67 68 69 70 71
72 73 74 75 76 77 78 79 80 81 82 83
84 85 86 87 88 89 90 91 92 93 94 95
Local Matrix:
0 1 2 3
12 13 14 15
24 25 26 27
36 37 38 39
Rank = 1
Local Matrix:
4 5 6 7
16 17 18 19
28 29 30 31
40 41 42 43
Rank = 2
Local Matrix:
8 9 10 11
20 21 22 23
32 33 34 35
44 45 46 47
Rank = 3
Local Matrix:
48 49 50 51
60 61 62 63
72 73 74 75
84 85 86 87
Rank = 4
Local Matrix:
52 53 54 55
64 65 66 67
76 77 78 79
88 89 90 91
Rank = 5
Local Matrix:
56 57 58 59
68 69 70 71
80 81 82 83
92 93 94 95