[Mitgcm-support] Re: problem with MPI on BU SGI O2000

mitgcm-support at dev.mitgcm.org mitgcm-support at dev.mitgcm.org
Wed Jul 9 15:57:16 EDT 2003


----------
X-Sun-Data-Type: text
X-Sun-Data-Description: text
X-Sun-Data-Name: text
X-Sun-Charset: us-ascii
X-Sun-Content-Lines: 19


Hi Sonya,

We've had to customize genmake for the BU SGI and this might be related to 
your problem.

Attached is the genmake that I have been using on the BU SGI. You can copy 
and paste the part relative to this machine -- case lego.bu.edu+mpi. We 
probably need to check this in at some point.

When you invoke genmake use the option -platform=lego.bu.edu+mpi.

Otherwise, I remember another fix, which I think should be included now.
The fix was to change the value of exchUsesBarrier from .TRUE. to .FALSE. in
ini_communication_patterns.F. But this was a customized fix too I think.

Hope this helps,

Daniel
----------
X-Sun-Data-Type: default-app
X-Sun-Data-Description: default
X-Sun-Data-Name: genmake
X-Sun-Charset: us-ascii
X-Sun-Content-Lines: 391

#!/bin/csh -f
#
# $Header: /u/gcmpack/models/MITgcmUV/tools/genmake,v 1.11 1999/05/07 18:12:29 adcroft Exp $
#
# Makefile generator for MITgcm UV codes
#   created by cnh 03/98
#   adapted by aja 06/98

# Process command-line arguments
set allargs=( $argv )
while ($#allargs)
 set arg = $allargs[1]
 switch ($arg)
 case -makefile:
   set mfile = ( Makefile )
   breaksw
 case -makefile=*:
   set mfile = ( `echo $arg | sed 's/-makefile=//' `)
   breaksw
 case -platform:
 case -platform=:
   echo "To change platform you must specify one with -platform="
   echo "eg. -platform=sparc  or  -platform=mips"
   exit
   breaksw
 case -platform*:
   set platform = ( `echo $arg | sed 's/-platform=//' `)
   breaksw
 case -mpi:
   echo "Enabling MPI options"
   set USEMPI
   breaksw
 case -help:
   echo "usage: $0 [-help] [-makefile[=...]] [-platform=...] [-mpi]"
   exit
   breaksw
 default:
   echo "Unknown command-line option: " $arg
   echo $0 "-help to show usage"
   exit
   breaksw
 endsw
 shift allargs
end

if ($?platform == 0) then
 set platform = (`uname`)
endif
if ($?mfile == 0) set mfile = ( Makefile.$platform )
set mach  = ( `uname -a` )
echo Operating system: $mach

# Directories for source, includes, binaries and executables
# Note
# o If you prefer/need everything under a single directory
#   copy everything in ../eesupp/src, ../model/src,
#   ../eesupp/inc and ../model/inc into a directory and then
#   edit the paths below to ./
set SOURCEDIRS  = (  ../eesupp/src/            \
                     ../model/src/             \
                     ../diags/src/             \
                  )
set INCLUDEDIRS = (  ../eesupp/inc/            \
                     ../model/inc/             \
                     ../diags/inc/             \
                  )
set BUILDDIR    = ( ../bin/ )
set EXEDIR      = ( ../exe/ )
set EXECUTABLE  = ( mitgcmuv )

# This is the generic configuration.
# Platform specific options are chosen below
set LN         = ( 'ln -sf' )
set CPP        = ( '/lib/cpp -P' )
set KPP        = (  )
set FC         = ( 'f77' )
set LINK       = ( 'f77' )
set DEFINES    = (  )
set INCLUDES   = (  )
set FFLAGS     = (  )
set FOPTIM     = (  )
set KFLAGS1    = (  )
set KFLAGS2    = (  )
set LIBS       = (  )
set KPPFILES   = (  )
set NOOPTFILES = (  )
set NOOPTFLAGS = (  )
set RMFILES    = (  )

# We often want to use different compile/link options is using MPI
if ($?USEMPI) then
  set USEMPI = ( '+mpi' )
  set DEFINES = ( ${DEFINES} '-DALLOW_USE_MPI -DALWAYS_USE_MPI' )
else
  set USEMPI
# set DEFINES = ( ${DEFINES} '-UALLOW_USE_MPI -UALWAYS_USE_MPI' )
endif

# Platform specific options
switch ($platform$USEMPI)
  case OSF1:
  case OSF1+mpi:
    echo "Configuring for DEC Alpha"
    set CPP        = ( '/usr/bin/cpp -P' )
    set DEFINES    = ( ${DEFINES}  '-DTARGET_DEC -DWORDLENGTH=1' )
    set KPP        = ( 'kapf' )
    set KPPFILES   = ( 'main.F' )
    set KFLAGS1    = ( '-scan=132 -noconc -cmp=' )
    set FC         = ( 'f77' )
    set FFLAGS     = ( '-convert big_endian -r8 -extend_source -u -automatic -call_shared -notransform_loops -align dcommons' )
    set FOPTIM     = ( '-O5 -fast -tune host -inline all' )
    set NOOPTFLAGS = ( '-O0' )
    set LIBS       = ( '-lfmpi -lmpi -lkmp_osfp10 -pthread' )
    set NOOPTFILES = ( 'barrier.F different_multiple.F load_external_fields.F')
    set RMFILES    = ( '*.p.out' )
    breaksw
  case IRIX64+mpi:
    echo "Configuring for SGI Mips with MPI"
    set DEFINES    = ( ${DEFINES} '-DTARGET_SGI -DWORDLENGTH=4' )
    set INCLUDES   = ( '-I/usr/local/mpi/include' )
    set FC         = ( 'mpif77' )
    set LINK       = ( 'mpif77' )
    set FFLAGS     = ( '-extend_source -bytereclen -r10000 -mips4' )
    set FOPTIM     = ( '-O3' )
    set RMFILES    = ( 'rii_files' )
    breaksw
  case lego.bu.edu+mpi:    /* Boston Uni. SGI with SGI MPI - coupled run */
    echo "Configuring for lego.bu.edu Mips with MPI - coupled setup"
    set DEFINES    = ( ${DEFINES} '-DTARGET_SGI -DWORDLENGTH=1 -DCOMPONENT_MODULE' )
    set INCLUDES   = ( '-I/usr/include' )
    set FC         = ( 'f77' )
    set LINK       = ( 'f77' )
    set FFLAGS     = ( '-r8 -64 -LANG:recursive=on -extend_source' )
    set FOPTIM     = ( '-O3' )
    set RMFILES    = ( 'rii_files' )
    set LIBS       = ( '-lmpi' )
    breaksw
  case IRIX64:
    echo "Configuring for SGI Mips"
    set DEFINES    = ( ${DEFINES} '-DTARGET_SGI -DWORDLENGTH=4' )
    set INCLUDES   = ( '-I/usr/local/mpi/include' )
    set FFLAGS     = ( '-extend_source -mp -mpio -bytereclen -r10000 -mips4' )
    set FOPTIM     = ( '-O3' )
#   set NOOPTFLAGS = ( '-O0' )
#   set NOOPTFILES = ( 'barrier.F different_multiple.F ' \
#                      'load_external_fields.F' )
    set RMFILES    = ( 'rii_files' )
    breaksw
  case SunOS:
    set LN         = ( '/usr/bin/ln -fs' )
    set CPP        = ( '/usr/ccs/lib/cpp -P' )
    set DEFINES    = ( ${DEFINES} '-DTARGET_SUN -DWORDLENGTH=4' )
    set FFLAGS     = ( '-stackvar -explicitpar -vpara -e -u -noautopar')
    set FOPTIM     = ( '-fast -O4' )
    set NOOPTFLAGS = ( '-O0' )
    set NOOPTFILES = ( 'barrier.F different_multiple.F load_external_fields.F ini_vertical_grid.F')
    breaksw
  case SunOS+mpi:
    set LN         = ( '/usr/bin/ln -fs' )
    set CPP        = ( '/usr/ccs/lib/cpp -P' )
    set DEFINES    = ( ${DEFINES} '-DTARGET_SUN -DWORDLENGTH=4' )
    set INCLUDES   = ( '-I/usr/local/mpi/include' )
    set FFLAGS     = ( '-stackvar -explicitpar -vpara -e -u -noautopar')
    set FOPTIM     = ( '-fast -O4' )
    set NOOPTFLAGS = ( '-O0' )
    set LIBS       = ( '-L/usr/local/mpi/lib/solaris/ch_shmem -lmpi -lthread' \
                       '-lsocket -lnsl' )
    set NOOPTFILES = ( 'barrier.F different_multiple.F load_external_fields.F ini_vertical_grid.F')
    breaksw
  case IRIX32:
    echo "Configuring for SGI ONYX running IRIX64"
    set DEFINES    = ( ${DEFINES} '-DTARGET_SGI -DWORDLENGTH=4' )
    set INCLUDES   = ( '-I/usr/include' )
    set FFLAGS     = ( '-extend_source -bytereclen -r10000 -64' )
    set FOPTIM     = ( '-O2' )
    set NOOPTFLAGS = ( '-O0' )
    set NOOPTFILES = ( 'barrier.F different_multiple.F ' \
                       'load_external_fields.F' )
    set LIBS       = ( '-lmpi' )
    breaksw
  case HP-UX+mpi:
    set FC         = ( 'mpif77' )
    set LINK       = ( 'mpif77' )
    set INCLUDES   = ( '-I/opt/mpi/include' )
  case HP-UX:
    echo "Configuring for HP Exemplar"
    set CPP        = ( '/usr/ccs/lbin/cpp -P' )
    set DEFINES    = ( ${DEFINES}  '-DTARGET_HP -DWORDLENGTH=4' )
    set FFLAGS     = ( '+es +U77 +Onoautopar +Oexemplar_model' \
                       '+Okernel_threads' )
    set FOPTIM     = ( '+O2' )
    set NOOPTFLAGS = ( '+O0' )
    set NOOPTFILES = ( 'barrier.F different_multiple.F' \
                       'load_external_fields.F' )
    breaksw
  case Linux+mpi:
    set LIBS       = ( '-L/usr/local/lib/LINUX/ch_p4/ -lfmpich -lmpich' )
  case Linux:
    set LN         = ( '/bin/ln -fs' )
    set CPP        = ( '/usr/lib/gcc-lib/i386-redhat-linux/2.7.2.3/cpp  -traditional -P' )
    set DEFINES    = ( ${DEFINES} '-D_BYTESWAPIO -DWORDLENGTH=4' )
    set INCLUDES   = ( '-I/usr/local/include' )
    set FC         = ( 'g77' )
    set FFLAGS     = ( ' ' )
    set FOPTIM     = ( '-O3 -funroll-loops ' )
    set LINK       = ( 'g77' )
    breaksw
  case Linux+pgi:
    set LN         = ( '/bin/ln -fs' )
    set CPP        = ( '/usr/lib/gcc-lib/i386-redhat-linux/2.7.2.3/cpp -P' )
    set DEFINES    = ( ${DEFINES} '-DWORDLENGTH=4' )
    set INCLUDES   = ( '-I/usr/local/include' )
    set FC         = ( 'pgf77' )
    set FFLAGS     = ( '-byteswapio' )
    set FOPTIM     = ( '-tp p6 -v -O2 -Munroll -Mvect=cachesize:512000,transform -Kieee' )
    set LINK       = ( 'pgf77' )
    set LIBS       = ( '-L/usr/local/lib/LINUX/ch_p4/ -lfmpich -lmpich -Kieee' )
    breaksw
  default:
    echo "Error: platform not recognized: uname -p = " $platform$USEMPI
    exit
    breaksw
endsw      

###############################################################################
##                                                                           ##
##  Everything below here should not need to be changed. Platform specific   ##
##  changes and code specific changes should be configured above this line.  ##
##                                                                           ##
###############################################################################

# Create list of files
set flist = `ls -1 ${SOURCEDIRS} | grep '.*\.[F]$'`
if ( $#flist ) then
 echo -n "SRCFILES = " > srclist.inc
 echo -n "F77FILES = " > f77list.inc
 echo -n "OBJFILES = " > objlist.inc
 foreach ff ( ${flist} )
  set fname = ( ${ff:t}       )
  echo ' \' >> srclist.inc
  echo -n "           " ${fname:r}.F >> srclist.inc
  echo ' \' >> f77list.inc
  echo -n "           " ${fname:r}.f >> f77list.inc
  echo ' \' >> objlist.inc
  echo -n "           " ${fname:r}.o >> objlist.inc
 end
else
 echo No source files found...\!
 exit
endif
set flist = `ls -1 ${SOURCEDIRS} | grep '.*\.[c]$'`
if ( $#flist ) then
 foreach ff ( ${flist} )
  set fname = ( ${ff:t}       )
  echo ' \' >> objlist.inc
  echo -n "           " ${fname:r}.o >> objlist.inc
endif
echo " " >> srclist.inc
echo " " >> f77list.inc
echo " " >> objlist.inc

# Convert lists of directories into command-line options
foreach inc ($INCLUDEDIRS)
 set INCLUDES = ($INCLUDES -I$inc)
end
set SRCSTARS = ( )
foreach dr ($SOURCEDIRS)
 set SRCSTARS = ("${SRCSTARS}" $dr\*.\[Fc\] )
end

set THISHOSTNAME = ( `hostname` )
set THISCWD = ( `pwd` )
set THISDATE = ( `date` )

###########################################
## This is the template for the makefile ##
###########################################
echo Creating makefile: $mfile
echo "# Multithreaded + multi-processing makefile for $mach" > ${mfile}
echo "# This makefile was generated automatically pn" >> ${mfile}
echo "#    $THISDATE" >> ${mfile}
echo "# by the command:" >> ${mfile}
echo "#    ${0} $argv" >> ${mfile}
echo "# executed by:" >> ${mfile}
echo "#    $USER@${THISHOSTNAME}:${THISCWD}" >> ${mfile}
cat >> ${mfile} <<EOF
# 
# BUILDDIR     : Directory where object files are written
# SOURCEDIRS   : Directories containing the source (.F) files
# INCLUDEDIRS  : Directories containing the header-source (.h) files
# EXEDIR       : Directory where executable that is generated is written
# EXECUTABLE   : Full path of executable binary
#
# CPP          : C-preprocessor command
# INCLUDES     : Directories searched for header files
# DEFINES      : Macro definitions for CPP
# KPP          : Special preprocessor command (specific to platform)
# KFLAGS       : Flags for KPP
# FC           : Fortran compiler command
# FFLAGS       : Configuration/debugging options for FC
# FOPTIM       : Optimization options for FC
# LINK         : Command for link editor program
# LIBS         : Library flags /or/ additional optimization/debugging flags

BUILDDIR   = ${BUILDDIR}  
SOURCEDIRS = ${SOURCEDIRS}
INCLUDEDIRS = ${INCLUDEDIRS}
EXEDIR     = ${EXEDIR}
EXECUTABLE = \$(EXEDIR)${EXECUTABLE}

# Unix ln (link)
LN = ${LN}
# C preprocessor
CPP = cat \$< | ../tools/set64bitConst.sh | ${CPP}
# Special preprocessor (KAP on DECs, FPP on Crays)
KPP = ${KPP}
# Fortran compiler
FC = ${FC}
# Link editor
LINK = ${LINK}

# Defines for CPP
DEFINES = ${DEFINES}
# Includes for CPP
INCLUDES = ${INCLUDES}
# Flags for KPP
KFLAGS1 = ${KFLAGS1}
KFLAGS2 = ${KFLAGS2}
# Optim./debug for FC
FFLAGS = ${FFLAGS}
FOPTIM = ${FOPTIM}
# Files that should not be optimized
NOOPTFILES = ${NOOPTFILES}
NOOPTFLAGS = ${NOOPTFLAGS}
# Flags and libraries needed for linking
LIBS = ${LIBS}

EOF

cat srclist.inc >> ${mfile}
cat f77list.inc >> ${mfile}
cat objlist.inc >> ${mfile}
rm -f srclist.inc f77list.inc objlist.inc

cat >> ${mfile} <<EOF

.SUFFIXES:
.SUFFIXES: .o .f .p .F .c

all: \$(EXECUTABLE)
\$(EXECUTABLE): \$(OBJFILES)
	\$(LINK) -o \$@ \$(FFLAGS) \$(FOPTIM) \$(OBJFILES) \$(LIBS)
links:
	@-\$(LN) $SRCSTARS .
depend:
	@make links
	makedepend -o .f \$(INCLUDES) \$(SRCFILES)
clean:
	-rm -rf *.o *.f *.p ${RMFILES}
Clean:
	@make clean
	-find . -type l -exec rm {} \;
	-rm Makefile.bak

# The normal chain of rules is (  .F - .f - .o  )
.F.f:
	\$(CPP) \$(DEFINES) \$(INCLUDES) > \$@
.f.o:
	\$(FC) \$(FFLAGS) \$(FOPTIM) -c \$<

# Special exceptions that use the ( .F - .p - .f - .o ) rule-chain
.F.p:
	\$(CPP) \$(DEFINES) \$(INCLUDES) > \$@
.p.f:
	\$(KPP) \$(KFLAGS1)\$@ \$(KFLAGS2) \$<
EOF

# Make list of "exceptions" that need ".p" files
foreach sf ($KPPFILES)
 set fname=( ${sf:t} )
 echo "${fname:r}.f: ${fname:r}.p" >> ${mfile}
end
foreach sf ($NOOPTFILES)
 set fname=( ${sf:t} )
 echo "${fname:r}.o: ${fname:r}.f" >> ${mfile}
 echo '	$(FC) $(FFLAGS) $(NOOPTFLAGS) -c $<' >> ${mfile}
end
echo >> ${mfile}
echo "# DO NOT DELETE" >> ${mfile}

exit



More information about the MITgcm-support mailing list