adding command line utilities and README

README.txt

+Requirements: python with numpy and sympy (eg. anaconda)
+
+Suppose we want to find problem sizes and configurations for a HACC run on
+all of Sequoia with 1 MPI rank per processor:
+
+nranks = 16*1024*96 = 1572864
+
+
+
+1) Find ng^3 problem sizes for which co-commensurate decompositions exist.
+
+$ python cl_ng.py 
+USAGE: cl_ng.py <nranks> <ng0> <ng1> [maxAspectRatio]
+
+nranks: total number of MPI ranks
+ng0: min ng to check
+ng1: max ng to check
+maxAspectRatio: the (optional) maximum ratio between the largest and
+smallest number of MPI ranks along a dimension in the 3D decomposition.
+
+$ python cl_ng.py 1572864 10000 30000 5.0
+12288
+15360
+18432
+21504
+24576
+27648
+
+
+
+2) List potential 3D decompositions for a particular problem size for which
+co-commensurate pencil decompositions exist.
+
+$ python cl_decomp3d.py 
+USAGE: cl_decomp3d.py <nranks> <ng> [maxAspectRatio]
+
+nranks: total number of MPI ranks
+ng: grid points along one side of 3D FFT
+maxAspectRatio: the (optional) maximum ratio between the largest and
+smallest number of MPI ranks along a dimension in the 3D decomposition.
+
+$ python cl_decomp3d.py 1572864 15360 5.0
+(96, 128, 128) 1.333333
+(64, 128, 192) 3.000000
+(64, 96, 256) 4.000000
+
+
+
+3) Enumerate pencil decompositions that are co-commensurate with a particular
+3D decomposition for a particular problem size.
+
+$ python cl_pencils.py 
+USAGE: cl_pencils.py <ng> <nrankx> <nranky> <nrankz>
+
+ng: grid points along one side of 3D FFT
+nrankx: number of MPI ranks in the x dimension
+nranky: number of MPI ranks in the y dimension
+nrankz: number of MPI ranks in the z dimension
+
+$ python cl_pencils.py 15360 96 128 128
+ng = 15360
+
+3D
+(96, 128, 128)
+
+X-PENCILS
+(1, 1024, 1536)
+(1, 1536, 1024)
+(1, 512, 3072)
+(1, 3072, 512)
+
+Y-PENCILS
+(1536, 1, 1024)
+(3072, 1, 512)
+
+Z-PENCILS
+(1536, 1024, 1)
+(3072, 512, 1)
+
+
+
+4) This can be compared with the output of the CheckDecompsition utility 
+available in HACC and SWFFT that is based on the actual decomposition.c code.
+
+$ ./CheckDecomposition 15360 15360 15360 1572864 96 128 128
+distribution 1D: [1572864:1:1]
+distribution 3D: [96:128:128]
+  2d_z: 1536, 1024, 1.
+distribution 2z: [1536:1024:1]
+  2d_x: 1, 1536, 1024.
+distribution 2x: [1:1536:1024]
+  2d_y: 1536, 1, 1024.
+distribution 2y: [1536:1:1024]

cl_decomp3d.py

+import sys
+import decomp as D
+
+if len(sys.argv) < 3:
+    print('USAGE: %s <nranks> <ng> [maxAspectRatio]'%sys.argv[0])
+    sys.exit(-1)
+
+nranks = int(sys.argv[1])
+ng = int(sys.argv[2])
+if len(sys.argv) > 3:
+    maxAspectRatio = float(sys.argv[3])
+else:
+    maxAspectRation = 0.0
+
+decompList = D.filterNgNranks(ng, nranks, maxAspectRatio)
+
+for decomp in decompList:
+    print('%s %f'%(decomp[1],decomp[0]))

cl_ng.py

+import sys
+import decomp as D
+
+if len(sys.argv) < 4:
+    print('USAGE: %s <nranks> <ng0> <ng1> [maxAspectRatio]'%sys.argv[0])
+    sys.exit(-1)
+
+nranks = int(sys.argv[1])
+ng0 = int(sys.argv[2])
+ng1 = int(sys.argv[3])
+if len(sys.argv) > 4:
+    maxAspectRatio = float(sys.argv[4])
+else:
+    maxAspectRatio = 0.0
+
+ngList = D.ngCandidatesNranks(nranks, ng0, ng1, maxAspectRatio)
+
+for ng in ngList:
+    print('%d'%ng)

cl_pencils.py

+import sys
+import decomp as D
+
+if len(sys.argv) < 5:
+    print('USAGE: %s <ng> <nrankx> <nranky> <nrankz>'%sys.argv[0])
+    sys.exit(-1)
+
+ng = int(sys.argv[1])
+nrankx = int(sys.argv[2])
+nranky = int(sys.argv[3])
+nrankz = int(sys.argv[4])
+
+t3d = (nrankx,nranky,nrankz)
+pencils = D.enumeratePencilsNg3d(ng, t3d)
+
+print('ng = %d\n'%ng)
+
+print('3D')
+print('%s' % str(t3d))
+
+print('')
+
+print('X-PENCILS')
+for pencil in pencils[0]:
+    print('%s' % str(pencil))
+
+print('')
+
+print('Y-PENCILS')
+for pencil in pencils[1]:
+    print('%s' % str(pencil))
+
+print('')
+
+print('Z-PENCILS')
+for pencil in pencils[2]:
+    print('%s' % str(pencil))