Source code

001/*
002 * $Id: MPJEngine.java 4748 2014-02-12 23:25:15Z kredel $
003 */
004
005package edu.jas.kern;
006
007
008import java.io.IOException;
009import java.util.ArrayList;
010import java.util.Arrays;
011
012import mpi.Comm;
013import mpi.Intracomm;
014import mpi.MPI;
015import mpi.MPIException;
016import mpi.Status;
017
018import org.apache.log4j.Logger;
019
020
021/**
022 * MPI engine, provides global MPI service. <b>Note:</b> could eventually be
023 * done directly with MPI, but provides logging. <b>Usage:</b> To obtain a
024 * reference to the MPI service communicator use
025 * <code>MPJEngine.getComminicator()</code>. Once an engine has been created it
026 * must be shutdown to exit JAS with <code>MPJEngine.terminate()</code>.
027 * @author Heinz Kredel
028 */
029
030public final class MPJEngine {
031
032
033    private static final Logger logger = Logger.getLogger(MPJEngine.class);
034
035
036    private static final boolean debug = logger.isDebugEnabled();
037
038
039    /**
040     * Command line arguments. Required for MPI runtime system.
041     */
042    protected static String[] cmdline;
043
044
045    /**
046     * Hostnames of MPI partners.
047     */
048    public static ArrayList<String> hostNames = new ArrayList<String>();
049
050
051    /**
052     * Flag for MPI usage. <b>Note:</b> Only introduced because Google app
053     * engine does not support MPI.
054     */
055    public static boolean NO_MPI = false;
056
057
058    /**
059     * Number of processors.
060     */
061    public static final int N_CPUS = Runtime.getRuntime().availableProcessors();
062
063
064    /*
065     * Core number of threads.
066     * N_CPUS x 1.5, x 2, x 2.5, min 3, ?.
067     */
068    public static final int N_THREADS = (N_CPUS < 3 ? 3 : N_CPUS + N_CPUS / 2);
069
070
071    /**
072     * MPI communicator engine.
073     */
074    static Intracomm mpiComm;
075
076
077    /**
078     * MPI engine base tag number.
079     */
080    public static final int TAG = 11;
081
082
083    /**
084     * Hostname suffix.
085     */
086    public static final String hostSuf = "-ib";
087
088
089    // /*
090    //  * Send locks per tag.
091    //  */
092    // private static SortedMap<Integer,Object> sendLocks = new TreeMap<Integer,Object>();
093
094
095    // /*
096    //  * receive locks per tag.
097    //  */
098    // private static SortedMap<Integer,Object> recvLocks = new TreeMap<Integer,Object>();
099
100
101    /**
102     * No public constructor.
103     */
104    private MPJEngine() {
105    }
106
107
108    /**
109     * Set the commandline.
110     * @param args the command line to use for the MPI runtime system.
111     */
112    public static synchronized void setCommandLine(String[] args) {
113        cmdline = args;
114    }
115
116
117    /**
118     * Test if a pool is running.
119     * @return true if a thread pool has been started or is running, else false.
120     */
121    public static synchronized boolean isRunning() {
122        if (mpiComm == null) {
123            return false;
124        }
125        //if (MPI.Finalized()) { // FMPJ only
126        //    return false;
127        //}
128        return true;
129    }
130
131
132    /**
133     * Get the MPI communicator.
134     * @return a Communicator constructed for cmdline.
135     */
136    public static synchronized Comm getCommunicator() throws IOException {
137        if (cmdline == null) {
138            throw new IllegalArgumentException("command line not set");
139        }
140        return getCommunicator(cmdline);
141    }
142
143
144    /**
145     * Get the MPI communicator.
146     * @param args the command line to use for the MPI runtime system.
147     * @return a Communicator.
148     */
149    public static synchronized Comm getCommunicator(String[] args) throws IOException {
150        if (NO_MPI) {
151            return null;
152        }
153        if (mpiComm == null) {
154            //String[] args = new String[] { }; //"-np " + N_THREADS };
155            if (!MPI.Initialized()) {
156                if (args == null) {
157                    throw new IllegalArgumentException("command line is null");
158                }
159                cmdline = args;
160                args = MPI.Init(args);
161                //int tl = MPI.Init_thread(args,MPI.THREAD_MULTIPLE);
162                logger.info("MPI initialized on " + MPI.Get_processor_name());
163                //logger.info("thread level MPI.THREAD_MULTIPLE: " + MPI.THREAD_MULTIPLE 
164                //            + ", provided: " + tl);
165                if (debug) {
166                    logger.debug("remaining args: " + Arrays.toString(args));
167                }
168            }
169            mpiComm = MPI.COMM_WORLD;
170            int size = mpiComm.Size();
171            int rank = mpiComm.Rank();
172            logger.info("MPI size = " + size + ", rank = " + rank);
173            // maintain list of hostnames of partners
174            hostNames.ensureCapacity(size);
175            for (int i = 0; i < size; i++) {
176                hostNames.add("");
177            }
178            String myhost = MPI.Get_processor_name();
179            if (myhost.matches("\\An\\d*")) { // bwGRiD node names n010207
180                myhost += hostSuf;
181            }
182            if (myhost.matches("kredel.*")) {
183                myhost = "localhost";
184            }
185            hostNames.set(rank, myhost);
186            if (rank == 0) {
187                String[] va = new String[1];
188                va[0] = hostNames.get(0);
189                mpiComm.Bcast(va, 0, va.length, MPI.OBJECT, 0);
190                for (int i = 1; i < size; i++) {
191                    Status stat = mpiComm.Recv(va, 0, va.length, MPI.OBJECT, i, TAG);
192                    if (stat == null) {
193                        throw new IOException("no Status received");
194                        //throw new MPIException("no Status received");
195                    }
196                    int cnt = stat.Get_count(MPI.OBJECT);
197                    if (cnt == 0) {
198                        throw new IOException("no Object received");
199                        //throw new MPIException("no object received");
200                    }
201                    String v = va[0];
202                    hostNames.set(i, v);
203                }
204                logger.info("MPI partner host names = " + hostNames);
205            } else {
206                String[] va = new String[1];
207                mpiComm.Bcast(va, 0, va.length, MPI.OBJECT, 0);
208                hostNames.set(0, va[0]);
209                va[0] = hostNames.get(rank);
210                mpiComm.Send(va, 0, va.length, MPI.OBJECT, 0, TAG);
211            }
212        }
213        return mpiComm;
214    }
215
216
217    /**
218     * Stop execution.
219     */
220    public static synchronized void terminate() {
221        if (mpiComm == null) {
222            return;
223        }
224        //if (MPI.Finalized()) { // FMPJ only
225        //    return;
226        //}
227        try {
228            logger.info("terminating MPI on rank = " + mpiComm.Rank());
229            mpiComm = null;
230            MPI.Finalize();
231        } catch (MPIException e) {
232            e.printStackTrace();
233        }
234    }
235
236
237    /**
238     * Set no MPI usage.
239     */
240    public static synchronized void setNoMPI() {
241        NO_MPI = true;
242        terminate();
243    }
244
245
246    /**
247     * Set MPI usage.
248     */
249    public static synchronized void setMPI() {
250        NO_MPI = false;
251    }
252
253
254    // /*
255    //  * Get send lock per tag.
256    //  * @param tag message tag.
257    //  * @return a lock for sends.
258    //  */
259    // public static synchronized Object getSendLock(int tag) {
260    //     tag = 11; // one global lock
261    //     Object lock = sendLocks.get(tag);
262    //     if ( lock == null ) {
263    //         lock = new Object();
264    //         sendLocks.put(tag,lock);
265    //     }
266    //     return lock;
267    // }
268
269
270    // /*
271    //  * Get receive lock per tag.
272    //  * @param tag message tag.
273    //  * @return a lock for receives.
274    //  */
275    // public static synchronized Object getRecvLock(int tag) {
276    //     Object lock = recvLocks.get(tag);
277    //     if ( lock == null ) {
278    //         lock = new Object();
279    //         recvLocks.put(tag,lock);
280    //     }
281    //     return lock;
282    // }
283
284
285    // /*
286    //  * Wait for termination of a mpj Request.
287    //  * @param req a Request.
288    //  * @return a Status after termination of req.Wait().
289    //  */
290    // public static Status waitRequest(final Request req) {
291    //     if ( req == null ) {
292    //         throw new IllegalArgumentException("null request");
293    //     }
294    //     int delay = 10;
295    //     int delcnt = 0;
296    //     Status stat = null;
297    //     while (true) {
298    //         synchronized (MPJEngine.class) { // global static lock
299    //             stat = req.Get_status(); // should be non destructive, but is not
300    //             if ( stat != null ) {
301    //                 return req.Wait(); // should terminate immediately
302    //             }
303    //         }
304    //         try {
305    //             Thread.currentThread().sleep(delay); // varied a bit
306    //         } catch (InterruptedException e) {
307    //             logger.info("sleep interrupted");
308    //             e.printStackTrace();
309    //         }
310    //         delcnt++; 
311    //         if ( delcnt % 7 == 0 ) {
312    //             delay++;
313    //             System.out.println("delay(" + delay + "): " + Thread.currentThread().toString());
314    //         } 
315    //     }
316    // }
317
318}