Source code

001/*
002 * $Id: MPIEngine.java 4604 2013-08-25 16:59:02Z kredel $
003 */
004
005package edu.jas.kern;
006
007
008import java.io.IOException;
009import java.util.ArrayList;
010import java.util.Arrays;
011
012import mpi.Comm;
013import mpi.Intracomm;
014import mpi.MPI;
015import mpi.MPIException;
016import mpi.Status;
017
018import org.apache.log4j.Logger;
019
020
021/**
022 * MPI engine, provides global MPI service. <b>Note:</b> could eventually be
023 * done directly with MPI, but provides logging. <b>Usage:</b> To obtain a
024 * reference to the MPI service communicator use
025 * <code>MPIEngine.getComminicator()</code>. Once an engine has been created it
026 * must be shutdown to exit JAS with <code>MPIEngine.terminate()</code>.
027 * @author Heinz Kredel
028 */
029
030public final class MPIEngine {
031
032
033    private static final Logger logger = Logger.getLogger(MPIEngine.class);
034
035
036    private static final boolean debug = logger.isDebugEnabled();
037
038
039    /**
040     * Command line arguments. Required for MPI runtime system.
041     */
042    protected static String[] cmdline;
043
044
045    /**
046     * Hostnames of MPI partners.
047     */
048    public static ArrayList<String> hostNames = new ArrayList<String>();
049
050
051    /**
052     * Flag for MPI usage. <b>Note:</b> Only introduced because Google app
053     * engine does not support MPI.
054     */
055    public static boolean NO_MPI = false;
056
057
058    /**
059     * Number of processors.
060     */
061    public static final int N_CPUS = Runtime.getRuntime().availableProcessors();
062
063
064    /*
065     * Core number of threads.
066     * N_CPUS x 1.5, x 2, x 2.5, min 3, ?.
067     */
068    public static final int N_THREADS = (N_CPUS < 3 ? 3 : N_CPUS + N_CPUS / 2);
069
070
071    /**
072     * MPI communicator engine.
073     */
074    static Intracomm mpiComm;
075
076
077    /**
078     * MPI engine base tag number.
079     */
080    public static final int TAG = 11;
081
082
083    /**
084     * Hostname suffix.
085     */
086    public static final String hostSuf = "-ib";
087
088
089    // /*
090    //  * Send locks per tag.
091    //  */
092    // private static SortedMap<Integer,Object> sendLocks = new TreeMap<Integer,Object>();
093
094
095    // /*
096    //  * receive locks per tag.
097    //  */
098    // private static SortedMap<Integer,Object> recvLocks = new TreeMap<Integer,Object>();
099
100
101    /**
102     * No public constructor.
103     */
104    private MPIEngine() {
105    }
106
107
108    /**
109     * Set the commandline.
110     * @param args the command line to use for the MPI runtime system.
111     */
112    public static synchronized void setCommandLine(String[] args) {
113        cmdline = args;
114    }
115
116
117    /**
118     * Test if a pool is running.
119     * @return true if a thread pool has been started or is running, else false.
120     */
121    public static synchronized boolean isRunning() {
122        if (mpiComm == null) {
123            return false;
124        }
125        return true;
126    }
127
128
129    /**
130     * Get the MPI communicator.
131     * @return a Communicator constructed for cmdline.
132     */
133    public static synchronized Comm getCommunicator() throws IOException, MPIException {
134        if (cmdline == null) {
135            throw new IllegalArgumentException("command line not set");
136        }
137        return getCommunicator(cmdline);
138    }
139
140
141    /**
142     * Get the MPI communicator.
143     * @param args the command line to use for the MPI runtime system.
144     * @return a Communicator.
145     */
146    public static synchronized Comm getCommunicator(String[] args) throws IOException, MPIException {
147        if (NO_MPI) {
148            return null;
149        }
150        if (mpiComm == null) {
151            //String[] args = new String[] { }; //"-np " + N_THREADS };
152            if (args == null) {
153                throw new IllegalArgumentException("command line is null");
154            }
155            cmdline = args;
156            args = MPI.Init(args);
157            //int tl = MPI.Init_thread(args,MPI.THREAD_MULTIPLE);
158            logger.info("MPI initialized on " + MPI.Get_processor_name());
159            //logger.info("thread level MPI.THREAD_MULTIPLE: " + MPI.THREAD_MULTIPLE 
160            //            + ", provided: " + tl);
161            if (debug) {
162                logger.debug("remaining args: " + Arrays.toString(args));
163            }
164            mpiComm = MPI.COMM_WORLD;
165            int size = mpiComm.Size();
166            int rank = mpiComm.Rank();
167            logger.info("MPI size = " + size + ", rank = " + rank);
168            // maintain list of hostnames of partners
169            hostNames.ensureCapacity(size);
170            for (int i = 0; i < size; i++) {
171                hostNames.add("");
172            }
173            String myhost = MPI.Get_processor_name();
174            if ( myhost.matches("\\An\\d*") ) { // bwGRiD node names n010207
175                myhost += hostSuf;
176            }
177            if ( myhost.matches("kredel.*") ) { 
178                myhost = "localhost";
179            }
180            hostNames.set(rank, myhost);
181            if (rank == 0) {
182                String[] va = new String[1];
183                va[0] = hostNames.get(0);
184                mpiComm.Bcast(va, 0, va.length, MPI.OBJECT, 0);
185                for (int i = 1; i < size; i++) {
186                    Status stat = mpiComm.Recv(va, 0, va.length, MPI.OBJECT, i, TAG);
187                    if (stat == null) {
188                        throw new IOException("no Status received");
189                        //throw new MPIException("no Status received");
190                    }
191                    int cnt = stat.Get_count(MPI.OBJECT);
192                    if (cnt == 0) {
193                        throw new IOException("no Object received");
194                        //throw new MPIException("no object received");
195                    }
196                    String v = va[0];
197                    hostNames.set(i, v);
198                }
199                logger.info("MPI partner host names = " + hostNames);
200            } else {
201                String[] va = new String[1];
202                mpiComm.Bcast(va, 0, va.length, MPI.OBJECT, 0);
203                hostNames.set(0, va[0]);
204                va[0] = hostNames.get(rank);
205                mpiComm.Send(va, 0, va.length, MPI.OBJECT, 0, TAG);
206            }
207        }
208        return mpiComm;
209    }
210
211
212    /**
213     * Stop execution.
214     */
215    public static synchronized void terminate() {
216        if (mpiComm == null) {
217            return;
218        }
219        try {
220            logger.info("terminating MPI on rank = " + mpiComm.Rank());
221            mpiComm = null;
222            MPI.Finalize();
223        } catch (MPIException e) {
224            e.printStackTrace();
225        }
226    }
227
228
229    /**
230     * Set no MPI usage.
231     */
232    public static synchronized void setNoMPI() {
233        NO_MPI = true;
234        terminate();
235    }
236
237
238    /**
239     * Set MPI usage.
240     */
241    public static synchronized void setMPI() {
242        NO_MPI = false;
243    }
244
245
246    // /*
247    //  * Get send lock per tag.
248    //  * @param tag message tag.
249    //  * @return a lock for sends.
250    //  */
251    // public static synchronized Object getSendLock(int tag) {
252    //     tag = 11; // one global lock
253    //     Object lock = sendLocks.get(tag);
254    //     if ( lock == null ) {
255    //         lock = new Object();
256    //         sendLocks.put(tag,lock);
257    //     }
258    //     return lock;
259    // }
260
261
262    // /*
263    //  * Get receive lock per tag.
264    //  * @param tag message tag.
265    //  * @return a lock for receives.
266    //  */
267    // public static synchronized Object getRecvLock(int tag) {
268    //     Object lock = recvLocks.get(tag);
269    //     if ( lock == null ) {
270    //         lock = new Object();
271    //         recvLocks.put(tag,lock);
272    //     }
273    //     return lock;
274    // }
275
276
277    // /*
278    //  * Wait for termination of a mpi Request.
279    //  * @param req a Request.
280    //  * @return a Status after termination of req.Wait().
281    //  */
282    // public static Status waitRequest(final Request req) throws MPIException {
283    //     if ( req == null || req.Is_null() ) {
284    //         throw new IllegalArgumentException("null request");
285    //     }
286    //     int delay = 50;
287    //     int delcnt = 0;
288    //     Status stat = null;
289    //     while (true) {
290    //         synchronized (MPIEngine.class) { // global static lock
291    //             stat = req.Test(); 
292    //             logger.info("Request: " + req + ", Status: " + stat);
293    //             if (stat != null) {
294    //                 logger.info("Status: index = " + stat.index + ", source = " + stat.source
295    //                                   + ", tag = " + stat.tag);
296    //                 if (!stat.Test_cancelled()) {
297    //                     logger.info("enter req.Wait(): " + Thread.currentThread().toString());
298    //                     return req.Wait(); // should terminate immediately
299    //                 }
300    //             }
301    //         }
302    //         try {
303    //             Thread.currentThread().sleep(delay); // varied a bit
304    //         } catch (InterruptedException e) {
305    //             logger.info("sleep interrupted");
306    //             e.printStackTrace();
307    //         }
308    //         delcnt++; 
309    //         if ( delcnt % 7 != 0 ) {
310    //             delay++;
311    //             logger.info("delay(" + delay + "): " + Thread.currentThread().toString());
312    //         } 
313    //     }
314    // }
315
316}