/**
 * JOnAS: Java(TM) Open Application Server
 * Copyright (C) 2006-2007 Bull S.A.S.
 * Contact: jonas-team@ow2.org
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
 * USA
 *
 * --------------------------------------------------------------------------
 * $Id: ClusterDaemon.java 19896 2010-06-01 21:13:42Z pelletib $
 * --------------------------------------------------------------------------
 */
package org.ow2.jonas.cluster.daemon;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.lang.management.ClassLoadingMXBean;
import java.lang.management.CompilationMXBean;
import java.lang.management.GarbageCollectorMXBean;
import java.lang.management.ManagementFactory;
import java.lang.management.MemoryMXBean;
import java.lang.management.MemoryManagerMXBean;
import java.lang.management.MemoryPoolMXBean;
import java.lang.management.OperatingSystemMXBean;
import java.lang.management.RuntimeMXBean;
import java.lang.management.ThreadMXBean;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Hashtable;
import java.util.List;
import java.util.Map;
import java.util.Vector;

import javax.management.MBeanAttributeInfo;
import javax.management.MBeanInfo;
import javax.management.MBeanServer;
import javax.management.MalformedObjectNameException;
import javax.management.ObjectName;
import javax.management.remote.JMXConnectorFactory;
import javax.management.remote.JMXConnectorServer;
import javax.management.remote.JMXServiceURL;
import javax.management.remote.MBeanServerForwarder;

import org.objectweb.util.monolog.api.BasicLevel;
import org.objectweb.util.monolog.api.Logger;
import org.ow2.carol.jndi.ns.NameServiceException;
import org.ow2.carol.jndi.ns.NameServiceManager;
import org.ow2.carol.util.configuration.TraceCarol;
import org.ow2.jonas.cluster.daemon.mbean.JMXRemoteException;
import org.ow2.jonas.cluster.daemon.mbean.JMXRemoteHelper;
import org.ow2.jonas.cluster.daemon.mbean.MBeanServerException;
import org.ow2.jonas.cluster.daemon.mbean.MBeanServerHelper;
import org.ow2.jonas.deployment.clusterd.xml.Server;
import org.ow2.jonas.deployment.common.xml.JLinkedList;
import org.ow2.jonas.discovery.jgroups.JgroupsDiscoveryServiceImpl;
import org.ow2.jonas.discovery.jgroups.utils.JGroupsDiscoveryUtils;
import org.ow2.jonas.lib.bootstrap.JProp;
import org.ow2.jonas.lib.util.ConfigurationConstants;
import org.ow2.jonas.lib.util.Env;
import org.ow2.jonas.lib.util.Log;
import org.ow2.jonas.lib.windows.WinSysEnv;
import org.ow2.jonas.security.auth.spi.RoleBasedAuthorizationModule;

/**
 * ClusterDaemon is provided to control JOnAS instances remotely. The
 * ClusterDaemon is an independent Java program, and runs on every node
 * (machine) in the cluster. It receives remote commands (JMX), and does
 * something such as starting or stopping JOnAS instances.
 * @author zhengzz (initial version for PKUAS)
 * @author Benoit Pelletier (integration in JOnAS)
 * @author eyindanga (Administration)
 * @author S. Ali Tokmen (JMX security)
 */

public class ClusterDaemon implements ClusterDaemonMBean {

    /**
     * Allow to trace errors/output of a process.
     */
    class CmdReader implements Runnable {

        /**
         * Input stream containing information.
         */
        private InputStream is;

        /**
         * Should send as error or debug message.
         */
        private boolean isErrorMessage = false;

        /**
         * Logger to use.
         */
        private Logger logger = null;

        /**
         * Cmd associated with the reader.
         */
        private String cmd = null;

        /**
         * Constructor.
         * @param logger logger
         * @param cmd command
         * @param is given input stream
         * @param isErrorMessage Should send as error or debug message
         */
        public CmdReader(final Logger logger, final String cmd, final InputStream is, final boolean isErrorMessage) {
            this.is = is;
            this.isErrorMessage = isErrorMessage;
            this.logger = logger;
            this.cmd = cmd;
        }

        /**
         * Thread execution printing information received.
         */
        public void run() {
            try {
                BufferedReader br = new BufferedReader(new InputStreamReader(is));
                String str = null;
                while ((str = br.readLine()) != null) {
                    if (isErrorMessage) {
                        this.logger.log(BasicLevel.ERROR, "<" + cmd + "> " + str);
                    } else {
                        this.logger.log(BasicLevel.DEBUG, "<" + cmd + "> " + str);
                    }
                }
                // close input stream
                is.close();
            } catch (Exception e) {
                this.logger.log(BasicLevel.ERROR, e.getMessage());
                e.printStackTrace();
            }
        }
    }

    /**
     * Allow stop the cmd childs
     */
    class ShutdownHook extends Thread {

        /**
         * Action at the shutdown : destroy the process.
         */
        public static final int ACTION_DESTROY = 0;

        /**
         * Action at the shutdown : stop the JOnAS instance.
         */
        public static final int ACTION_JONAS_STOP = 1;

        /**
         * Cmdkey.
         */
        private String keyCmd = null;

        /**
         * Action.
         */
        private int action = -1;

        /**
         * JOnAS instance name.
         */
        private String serverName = null;

        /**
         * Constructor.
         * @param keyCmd key command
         */
        public ShutdownHook(final String serverName, final String keyCmd, final int action) {
            super();
            this.keyCmd = keyCmd;
            this.action = action;
            this.serverName = serverName;
        }

        /**
         * Thread execution printing information received.
         */
        @Override
        public void run() {
            try {
                if (action == ACTION_DESTROY) {
                    ((Process) ClusterDaemon.this.processMap.get(this.keyCmd)).destroy();
                    ClusterDaemon.this.logger.log(BasicLevel.DEBUG, "destroy cmd" + keyCmd);
                } else if (action == ACTION_JONAS_STOP) {
                    ClusterDaemon.this.logger.log(BasicLevel.DEBUG, "stop JOnAS instance " + serverName);
                    ClusterDaemon.this.doStopJOnAS(serverName);
                    ClusterDaemon.this.logger.log(BasicLevel.DEBUG, "destroy cmd" + keyCmd);
                    ((Process) ClusterDaemon.this.processMap.get(this.keyCmd)).destroy();
                }

            } catch (Exception e) {
                TraceCarol.error("ShutdownHook problem for key=" + this.keyCmd, e);
            }
        }
    }

    /**
     * Monitor is a tool that collects node's performance information regularly.
     * if needed (only supporting CPU and memory information now).
     * @author zhengzz
     */
    class Monitor extends Thread {

        private static final int FACTOR = 10;

        /**
         * Logger to use.
         */
        private Logger logger = null;

        private int cpurate;

        private long totalM, availM, percentM;

        private BufferedWriter out;

        private int count = 0;

        /**
         * Initializes log monitor.
         * @param logger
         */
        public Monitor(final Logger logger) {
            this.logger = logger;
            try {
                out = new BufferedWriter(new FileWriter("monitor.txt", true));
            } catch (Exception e) {
                this.logger.log(BasicLevel.ERROR, "Fail to new FileWrite from monitor.txt" + e);
            }
        }

        /*
         * (non-Javadoc)
         * @see java.lang.Thread#run()
         */
        @Override
        @SuppressWarnings("static-access")
        public void run() {
            this.logger.log(BasicLevel.INFO, "Start monitoring thread....");
            while (true) {
                try {
                    Thread.currentThread().sleep(FACTOR * SLEEP_TIME / 2);
                    count = count + FACTOR;
                } catch (Exception e) {
                    e.printStackTrace();
                }
            }
        }
    }

    /**
     * Sleep time to wait.
     */
    private static final int SLEEP_TIME = 2000;

    /**
     * JOnAS interaction mode : tighly coupled.
     */
    private static final String TIGHLY_COUPLED = "tighly-coupled";

    /**
     * JOnAS interaction mode : loosely coupled.
     */
    private static final String LOOSELY_COUPLED = "loosely-coupled";

    /**
     * jonas admin ping short timeout.
     */
    private static final int PING_SHORT_TIMEOUT = 5;

    /**
     * jonas admin ping long timeout.
     */
    private static final int PING_LONG_TIMEOUT = 120;

    /**
     * Sleep before exit of JVM.
     */
    public static final int SLEEP_DELAY = 2000;

    /**
     * Used to exclude attribute that are not Long typed when getting. Operating
     * System dynamic attributes
     */

    public static final String LONG_TYPE_DYN_INFO = "long";

    /**
     * JOnAS command name in windows environmment.
     */
    public static final String JONAS_CMD_NAME_WIN_DEF = "jonas.bat";

    /**
     * JOnAS command name in unix environmment.
     */
    public static final String JONAS_CMD_NAME_UNIX_DEF = "jonas";

    /**
     * trace.properties file to use instead of the default file.
     */
    private static String csTraceFile = null;

    /**
     * carol.properties file to use instead of the default file.
     */
    private static String carolFile = null;

    /**
     * clusterd.xml filename (by default $JONAS_ROOT/conf/clusterd.xml).
     */
    private static String confFile = null;

    /**
     * J2EE Management domain.
     */
    private static String domainName = null;

    /**
     * Logger to use.
     */
    private Logger logger = null;

    /**
     * Object name.
     */
    private String objectName = null;

    /**
     * Cluster Daemon name.
     */
    private String name = null;

    /**
     * Process list.
     */
    private Map processMap = new HashMap();

    /**
     * server state.
     */
    private boolean isStarted = false;

    /**
     * Controled servers url list.
     */
    private ArrayList ControlledServersNames;

    /**
     * the clusterd jmx url.
     */
    private String jmxUrl = null;

    /**
     * MXbean for runtime.
     */

    private static RuntimeMXBean runtimeMxbean = null;

    /**
     * MBean for class loading.
     */

    private static ClassLoadingMXBean classLoadingMxBean = null;

    /**
     * MBean for operating system.
     */

    private static OperatingSystemMXBean operatingSystemMxBean = null;

    /**
     * MBean for threads.
     */

    private static ThreadMXBean threadMxBean = null;

    /**
     * MBean for compilation.
     */

    @SuppressWarnings("unused")
    private static CompilationMXBean compilationMxBean = null;

    /**
     * MBean for garbage collector.
     */

    @SuppressWarnings("unused")
    private static List<GarbageCollectorMXBean> garbageCollectorMXBean = null;

    /**
     * MBean for memory manager.
     */
    @SuppressWarnings("unused")
    private static List<MemoryManagerMXBean> memoryManagerMXBean = null;

    /**
     * MBean for memory.
     */
    private static MemoryMXBean memoryMXBean = null;

    /**
     * MBean for memory pool.
     */
    @SuppressWarnings("unused")
    private static List<MemoryPoolMXBean> memoryPoolMXBean = null;

    // the platform MbeanSever
    /**
     * MBean server of the platform.
     */
    private static MBeanServer platFormMbeanServer = null;

    // Management attributtes
    // Runtime Mbean
    /**
     * Spec vendor..
     */
    private String runTimeSpecVendor = null;

    /**
     * Spec version.
     */
    private String runTimeSpecVersion = null;

    /**
     * Virtual machine name.
     */
    private String runTimeVmName = null;

    /**
     * Virtual machine vendor.
     */
    private String runTimeVmVendor = null;

    /**
     * Runtime version.
     */
    private String runTimeVmVersion = null;

    // OperatingSystem MBean
    /**
     * OS avalaibale processors.
     */
    private String operatingSystemAvailableProcessors = null;

    /**
     * OS name.
     */
    private String operatingSystemName = null;

    /**
     * OS version.
     */
    private String operatingSystemVersion = null;

    /**
     * OS architecture.
     */
    private String operatingSystemArch = null;

    /**
     * OS dynamic attributes.
     */
    private Hashtable<String, String> dynamicHostAttributes = null;

    /**
     * JGroups discovery instance.
     */
    private JgroupsDiscoveryServiceImpl discovery = null;

    /**
     * Constructor.
     */
    private ClusterDaemon() {
    }

    /**
     * Execute a JOnAS command in a separate JVM.
     * @param server JOnAS server to pass the command
     * @param cmd command
     * @param keyCmd id the of command
     * @return Process associated with the command
     * @throws Exception exception if an error occurs
     */
    @SuppressWarnings("unchecked")
    private Process execJOnASCmd(final Server server, final String cmd, final String keyCmd) throws Exception {
        // check if the server exists
        if (server == null) {
            throw new ClusterDaemonException("server=null");
        }

        // Check that a same process in not running
        Process previousP = (Process) processMap.get(keyCmd);
        if (previousP != null) {
            try {
                this.logger.log(BasicLevel.INFO, "Cmd =" + cmd + " Process=" + previousP.exitValue());
            } catch (IllegalThreadStateException exc) {
                // there is already a process running
                this.logger.log(BasicLevel.INFO, "Cmd =" + cmd + " already running (pid=" + previousP.toString() + ").");
                throw new IllegalStateException("Cmd =" + cmd + " already running (pid=" + previousP.toString() + ").");
            }
        }

        // Set the environment
        Vector envv = new Vector();
        envv.addElement("JAVA_HOME=" + server.getJavaHome());
        envv.addElement("JONAS_ROOT=" + server.getJonasRoot());
        envv.addElement("JONAS_BASE=" + server.getJonasBase());

        String catalinaHome = System.getProperty("catalina.home");
        if (catalinaHome != null) {
            envv.addElement("CATALINA_HOME=" + catalinaHome);
            envv.addElement("CATALINA_BASE=" + server.getJonasBase());
        }

        String jettyHome = System.getProperty("jetty.home");
        if (jettyHome != null) {
            envv.addElement("JETTY_HOME=" + jettyHome);
        }

        // if OS is of Windows family, set the SystemRoot variable
        if (Env.isOsWindows()) {
            envv.addElement("SystemRoot=" + WinSysEnv.get("SystemRoot"));
        }

        String[] enva = new String[envv.size()];
        envv.copyInto(enva);

        // set the bin dir according to the platform (only if it's not a user
        // command)
        String binDir = null;
        if (isDefinedUserJonasCmd(server)) {
            // a full path must be set in the configuration
            this.logger.log(BasicLevel.INFO, "User cmd - don't add the jonas-root prefix dir");
            binDir = "";
        } else {
            this.logger.log(BasicLevel.INFO, "Default cmd - add the jonas-root prefix dir");
            binDir = server.getJonasRoot();
            if (File.separatorChar == '/') { // unix system
                binDir += "/bin/";
            } else if (File.separatorChar == '\\') { // windows system
                binDir += "\\bin\\";
            }
        }

        String[] cmda = null;
        if (cmd != null) {
            cmda = (binDir + cmd).split("\\s+");
        }
        // Set the working directory to the current directory
        File workingDirectory = new File(".");

        // Launch the command
        this.logger.log(BasicLevel.INFO, "Execute the command <" + binDir + cmd + "> with the env " + envv);

        Process p = Runtime.getRuntime().exec(cmda, enva, workingDirectory);

        // if synchronous mode, wait for the end of the execution

        processMap.put(keyCmd, p);

        // wait for starting
        Thread.sleep(SLEEP_TIME);

        // trace the start execution
        InputStream cmdError = p.getErrorStream();
        InputStream cmdOut = p.getInputStream();
        Thread err = new Thread(new CmdReader(this.logger, keyCmd, cmdError, true));
        Thread out = new Thread(new CmdReader(this.logger, keyCmd, cmdOut, false));
        out.start();
        err.start();

        // add a shutdown hook for this process
        // if loosely coupled mode, only the current commands are destroyed -
        // the JOnAS instances are not stopped
        int action = ShutdownHook.ACTION_DESTROY;
        if (keyCmd.startsWith("start") && isTighlyCoupled()) {
            action = ShutdownHook.ACTION_JONAS_STOP;
        }
        Runtime.getRuntime().addShutdownHook(new ShutdownHook(server.getName(), keyCmd, action));

        this.logger.log(BasicLevel.INFO, "Command " + cmd + " with the env " + envv + " launched");

        return p;
    }

    /**
     * @param server JOnAS server configuration.
     * @return true if a user command is defined.
     */
    private boolean isDefinedUserJonasCmd(final Server server) {
        if (server.getJonasCmd() == null) {
            return false;
        }
        if (server.getJonasCmd().trim().equals("")) {
            return false;
        }
        return true;
    }

    /**
     * @param server the server name.
     * @return JOnAS command name according to the os.
     */
    private String getJOnASCmdName(final Server server) {
        // Test whether a user command is set
        if (isDefinedUserJonasCmd(server)) {
            // user command
            return server.getJonasCmd().trim();
        } else {
            // default command
            if (Env.isOsWindows()) {
                return JONAS_CMD_NAME_WIN_DEF;
            } else {
                return JONAS_CMD_NAME_UNIX_DEF;
            }
        }

    }

    /**
     * Start a JOnAS instance.
     * @param name server name to start
     * @param domainName domain name
     * @param prm extra parameters
     * @param sync wait until the complete starting
     * @throws ClusterDaemonException if an error occurs
     */
    private void doStartJOnAS(final String name, final String domainName, final String prm, final boolean sync)
            throws ClusterDaemonException {

        // Get the Server
        Server server = getServer(name);

        // check if the server exists
        if (server == null) {
            // throw new ClusterDaemonException("JOnAS instance " + name + "
            // doesn't exist");
            throw new ClusterDaemonException("JOnAS server " + name + " is not known by the cluster daemon " + getName());
        }

        // get prm if needed
        String xprm = null;
        if (prm == null) {
            xprm = "";
        } else {
            xprm = prm;
        }

        // add the parameters given in the configuration
        if (server.getXprm() != null) {
            xprm += server.getXprm();
        }

        String d = null;
        if (domainName == null) {
            d = ClusterDaemon.domainName;
        } else {
            d = domainName;
        }

        // Compose the command
        String cmd = getJOnASCmdName(server) + " start -n " + name + " -Ddomain.name=" + d + " " + xprm;

        // If the interaction mode with JOnAS is tighly coupled, the instance is
        // launched in the foreground
        if (isTighlyCoupled()) {
            cmd += " -fg";
        }

        this.logger.log(BasicLevel.INFO, "JOnAS instance " + name + " is starting ...");

        String keyCmd = "start/" + name;

        Process p = null;
        try {
            p = execJOnASCmd(server, cmd, keyCmd);
        } catch (Throwable t) {
            this.logger.log(BasicLevel.ERROR, "Unable to start the JOnAS instance <" + name + ">, cmd=<" + cmd + ">", t);
            throw new ClusterDaemonException("Unable to start the JOnAS instance <" + name + ">, cmd=<" + cmd + ">", t);
        }

        this.logger.log(BasicLevel.INFO, "JOnAS instance " + name + " launched");

        // If sync is required, wait for the end of the starting
        if (sync) {
            if (isLooselyCoupled()) {
                @SuppressWarnings("unused")
                int exitCode = waitEndProcess(p);
            } else {
                if (doPingJOnAS(name, PING_LONG_TIMEOUT) == 1) {
                    throw new ClusterDaemonException("Unable to start the JOnAS instance " + name + " - unreachable server");
                }
            }
            this.logger.log(BasicLevel.INFO, "JOnAS instance " + name + "  started");
        }
    }

    /**
     * Stop a JOnAS instance.
     * @param name instance name
     * @throws ClusterDaemonException if an error occurs
     */
    private void doStopJOnAS(final String name) throws ClusterDaemonException {
        // Get the Server
        Server server = getServer(name);

        // check if the server exists
        if (server == null) {
            throw new ClusterDaemonException("JOnAS instance " + name + " doesn't exist");
        }

        String cmd = getJOnASCmdName(server) + " stop -n " + server.getName();

        this.logger.log(BasicLevel.INFO, "JOnAS instance " + name + " is stopping ...");

        String keyCmd = "stop/" + server.getName();

        Process p = null;
        try {
            p = execJOnASCmd(server, cmd, keyCmd);
        } catch (Throwable t) {
            this.logger.log(BasicLevel.ERROR, "Unable to stop the JOnAS instance <" + name + ">, cmd=<" + cmd + ">", t);
            throw new ClusterDaemonException("Unable to stop the JOnAS instance <" + name + ">, cmd=<" + cmd + ">", t);
        }
        // Wait until command termination and get the exit code
        @SuppressWarnings("unused")
        int exitCode = waitEndProcess(p);

        this.logger.log(BasicLevel.INFO, "JOnAS instance " + name + " stopped");
    }

    /**
     * Halt a JOnAS instance.
     * @param name instance name
     * @throws ClusterDaemonException if an error occurs
     */
    private void doHaltJOnAS(final String name) throws ClusterDaemonException {
        // Get the Server
        Server server = getServer(name);

        // check if the server exists
        if (server == null) {
            throw new ClusterDaemonException("JOnAS instance " + name + " doesn't exist");
        }

        String cmd = getJOnASCmdName(server) + " halt -n " + server.getName();

        this.logger.log(BasicLevel.INFO, "JOnAS instance " + name + " is halting ...");

        String keyCmd = "halt/" + server.getName();

        Process p = null;
        try {
            p = execJOnASCmd(server, cmd, keyCmd);
        } catch (Throwable t) {
            this.logger.log(BasicLevel.ERROR, "Unable to halt the JOnAS instance <" + name + ">, cmd=<" + cmd + ">", t);
            throw new ClusterDaemonException("Unable to halt the JOnAS instance <" + name + ">, cmd=<" + cmd + ">", t);
        }
        // Wait until command termination and get the exit code
        @SuppressWarnings("unused")
        int exitCode = waitEndProcess(p);

        this.logger.log(BasicLevel.INFO, "JOnAS instance " + name + " halted");
    }

    /**
     * Kill a JOnAS instance (only in tighly coupled mode).
     * @param name instance name
     * @throws ClusterDaemonException if an error occurs
     */
    private void doKillJOnAS(final String name) throws ClusterDaemonException {
        // Get the Server
        Server server = getServer(name);

        // check if the server exists
        if (server == null) {
            throw new ClusterDaemonException("JOnAS instance " + name + " doesn't exist");
        }

        // check the mode
        if (isLooselyCoupled()) {
            throw new ClusterDaemonException("Command not compatible with the loosely-coupled mode");
        }

        String keyCmd = "start/" + server.getName();

        // get the process
        Process p = (Process) processMap.get(keyCmd);

        // check that the process is alive
        boolean alive = false;
        try {
            this.logger.log(BasicLevel.DEBUG, "Process=" + p.exitValue());

            // Process not alived
            this.logger.log(BasicLevel.DEBUG, "JOnAS instance " + name + " is already dead");

        } catch (IllegalThreadStateException exc) {
            // there is already a process running
            this.logger.log(BasicLevel.DEBUG, "JOnAS instance " + server.getName() + " is going to be killed");
            alive = true;
        }

        if (alive) {
            // kill the process
            p.destroy();
            this.logger.log(BasicLevel.INFO, "JOnAS instance " + name + " killed");
        } else {
            this.logger.log(BasicLevel.INFO, "JOnAS instance " + name + " already killed");
        }

    }

    /**
     * Wait until the end of the execution of the process.
     * @param p process
     * @return exit code of the process
     */
    private int waitEndProcess(final Process p) {

        if (p == null) {
            return -1;
        }
        boolean isRunning = true;
        int exitCode = -1;
        while (isRunning) {
            this.logger.log(BasicLevel.DEBUG, "check if the process " + p + " is running");
            try {
                exitCode = p.exitValue();
                this.logger.log(BasicLevel.DEBUG, "cmd " + p + " is finished");
                isRunning = false;
            } catch (IllegalThreadStateException exc) {
                // there is already a process running
                this.logger.log(BasicLevel.DEBUG, "cmd " + p + " is still running");
                try {
                    Thread.sleep(SLEEP_TIME / 2);
                } catch (InterruptedException e) {
                    this.logger.log(BasicLevel.DEBUG, e);
                }
            }
        }
        return exitCode;
    }

    /**
     * Ping a JOnAS instance.
     * @param name instance name
     * @param timeout timeout
     * @return exit code of the ping (0 ok, 1 ko)
     * @throws ClusterDaemonException if an error occurs
     */
    private int doPingJOnAS(final String name, final int timeout) throws ClusterDaemonException {

        // Get the Server
        Server server = getServer(name);

        // check if the server exists
        if (server == null) {
            throw new ClusterDaemonException("JOnAS instance " + name + " doesn't exist");
        }

        String cmd = getJOnASCmdName(server) + " admin -n " + server.getName() + " -ping -timeout " + timeout;

        this.logger.log(BasicLevel.INFO, "Ping JOnAS instance " + name + " ...");

        String keyCmd = "ping/" + server.getName();

        try {
            execJOnASCmd(server, cmd, keyCmd);
        } catch (Throwable t) {
            this.logger.log(BasicLevel.ERROR, "Unable to ping the JOnAS instance <" + name + ">, cmd=<" + cmd + ">", t);
            throw new ClusterDaemonException("Unable to ping the JOnAS instance <" + name + ">, cmd=<" + cmd + ">", t);
        }

        // Wait until command termination and get the exit code
        Process p = (Process) processMap.get(keyCmd);
        int exitCode = waitEndProcess(p);
        this.logger.log(BasicLevel.INFO, "Ping JOnAS instance " + name + " return " + exitCode);
        return exitCode;
    }

    /**
     * Check jonas instance state.
     * @param name server name
     * @return true if the server is running.
     * @throws ClusterDaemonException if an error occurs.
     */
    @SuppressWarnings("unused")
    private boolean doCheckState(final String name) throws ClusterDaemonException {
        // Get the Server
        Server server = getServer(name);
        // check if the server exists
        if (server == null) {
            throw new ClusterDaemonException("JOnAS instance " + name + " doesn't exist");
        }

        // check the mode
        if (isLooselyCoupled()) {
            throw new ClusterDaemonException("Command not compatible with the loosely-coupled mode");
        }

        String keyCmd = "start/" + server.getName();

        // get the process
        Process p = (Process) processMap.get(keyCmd);

        // check that the process is alive
        boolean alive = false;
        try {
            this.logger.log(BasicLevel.DEBUG, "Process=" + p.exitValue());

            // Process not alive
            this.logger.log(BasicLevel.DEBUG, "JOnAS instance " + name + " is STOPPED");

        } catch (IllegalThreadStateException exc) {
            // there is already a process running
            this.logger.log(BasicLevel.DEBUG, "JOnAS instance " + server.getName() + " is RUNNING");
            alive = true;
        }
        return alive;

    }

    /**
     * Retrieve the state of the given controlled server.
     * @param srvName server's name.
     * @return true if the server is running
     * @throws ClusterDaemonException any.
     */
    public boolean checkServerState(final String srvName) throws ClusterDaemonException{
        boolean running = false;
        try {
            running = doCheckState(name);
        } catch (ClusterDaemonException e) {
            this.logger.log(BasicLevel.DEBUG, "Unabale to get Server State" + e);
            throw e;
        }
        return running;
    }

    /**
     * Call the Log class to instanciate the cluster daemon logger.
     * @throws ClusterDaemonException if an error occurs.
     */
    private void initLogger() throws ClusterDaemonException {

        if (csTraceFile != null) {
            File tClient = new File(csTraceFile);

            if (!tClient.exists()) {
                throw new ClusterDaemonException("The file '" + csTraceFile + "' was not found.");
            }

            if (!tClient.isFile()) {
                throw new ClusterDaemonException("The file '" + csTraceFile + "' is not a valid file. Maybe a directory ?");
            }

            // Configure log
            System.setProperty("jonas.client.trace.file", csTraceFile);
            Log.reset();
        } else {
            csTraceFile = "trace";
        }
        // Allow tracing ejb/jms code
        try {
            System.setProperty("jonas.client.trace.file", csTraceFile);
            Log.configure(csTraceFile);
        } catch (NoClassDefFoundError ncdfe) {
            // 'normal exception' thrown outside the JOnAS containers
            ncdfe.printStackTrace();
        }
        // init the logger
        this.logger = Log.getLogger(Log.JONAS_CLUSTER_DAEMON);
    }

    /**
     * Starts the name service
     * @throws ClusterDaemonException any.
     */
    private void startNameService() throws ClusterDaemonException {
        // start registry
        try {
            NameServiceManager.getNameServiceManager().startNS();
        } catch (NameServiceException e) {
            throw new ClusterDaemonException("Cannot start registry", e);
        }
    }

    /**
     * Init MBeans.
     * @throws ClusterDaemonException if an error occurs.
     */
    private void initMBeans() throws ClusterDaemonException {

        // JonasObjectName.setDomain(domainName);
        try {
            MBeanServerHelper.startMBeanServer(domainName);
        } catch (MBeanServerException e) {
            throw new ClusterDaemonException("Cannot start MBean server", e);
        }

        try {
            Map props = null;
            this.name = ClusterDaemonTools.getCurrentConfiguration().getClusterDaemon().getName();
            String url = ClusterDaemonTools.getJmxUrl(name);
            String objName = ClusterDaemonTools.getObjectName();

            // Set JMX authentication and authorization options
            // We'll test later if JMX has really been secured
            boolean jmxSecured = ClusterDaemonTools.getCurrentConfiguration().getClusterDaemon().isJmxSecured();
            ;
            MBeanServerForwarder forwarder = null;
            if (jmxSecured) {
                props = new HashMap();
                String fileSeparator = System.getProperty("file.separator");

                // Authentication, avoid NullPointerExceptions
                String loginModule = ClusterDaemonTools.getCurrentConfiguration().getClusterDaemon()
                        .getJmxAuthenticationMethod();
                String loginParam = ClusterDaemonTools.getCurrentConfiguration().getClusterDaemon()
                        .getJmxAuthenticationParameter();
                if (loginModule != null && loginParam != null) {
                    if ("jmx.remote.x.password.file".equals(loginModule)) {
                        // Make sure files get read using JONAS_BASE
                        props.put(loginModule, JProp.getJonasBase() + fileSeparator + loginParam);
                    } else {
                        props.put(loginModule, loginParam);
                    }
                }

                // Authorization, avoid NullPointerExceptions
                String authModule = ClusterDaemonTools.getCurrentConfiguration().getClusterDaemon().getJmxAuthorizationMethod();
                String authParam = ClusterDaemonTools.getCurrentConfiguration().getClusterDaemon()
                        .getJmxAuthorizationParameter();
                if (authModule != null && authParam != null) {
                    if (authModule.startsWith("jmx.remote.x.access.rolebased")) {
                        forwarder = RoleBasedAuthorizationModule.newProxyInstance(authModule
                                .substring("jmx.remote.x.access.rolebased".length() + 1), authParam);
                    } else if ("jmx.remote.x.access.file".equals(authModule)) {
                        // Make sure files get read using JONAS_BASE
                        props.put(authModule, JProp.getJonasBase() + fileSeparator + authParam);
                    } else {
                        props.put(authModule, authParam);
                    }
                }
            }

            JMXConnectorServer connectorServer = JMXRemoteHelper.startConnector(url, objName, props);
            if (forwarder != null) {
                connectorServer.setMBeanServerForwarder(forwarder);
            }
            this.logger.log(BasicLevel.INFO, "JMX remote connector (" + url + ", " + objName + " started ");

            // set management attribute
            setJmxUrl(url);

            if (jmxSecured) {
                // Test security: try to connect without credentials
                try {
                    JMXConnectorFactory.connect(new JMXServiceURL(url)).close();
                    logger.log(BasicLevel.WARN, "JMX security is enabled but anonymous logins are still accepted! "
                            + "Please check your JMX security configuration.");
                } catch (SecurityException e) {
                    // It's working: connection was refused
                    logger.log(BasicLevel.INFO, "JMX security is enabled and active");
                } catch (Exception e) {
                    throw new ClusterDaemonException("Cannot test JMX security", e);
                }
            } else {
                logger.log(BasicLevel.WARN, "JMX security is DISABLED");
            }
        } catch (JMXRemoteException e) {
            throw new ClusterDaemonException("Cannot start JMX Remote connector", e);
        }

        // register local MBeans
        StringBuffer sb = new StringBuffer(domainName);
        // TODO add the cd name when registering its mbean
        sb.append(":type=ClusterDaemon");
        ObjectName on = null;
        try {
            on = new ObjectName(sb.toString());

        } catch (MalformedObjectNameException e) {
            throw new ClusterDaemonException("Cannot build ObjectName", e);
        }
        try {
            MBeanServerHelper.getMBeanServer().registerMBean(this, on);
            this.logger.log(BasicLevel.DEBUG, "MBean (" + on + ") registered");

        } catch (Exception e) {
            throw new ClusterDaemonException("Cannot register MBean '" + on + "' in MBeanServer", e);
        }

        // set management attribute
        setobjectName(on.toString());

        this.logger.log(BasicLevel.INFO, "MBeans initialized");
    }

    /**
     * Gets runtime info of current platform.
     */
    public void buildRuntimeInfo() {
        this.logger.log(BasicLevel.INFO, "getting Runtime Mbean");
        RuntimeMXBean mxbean = ManagementFactory.getRuntimeMXBean();
        @SuppressWarnings("unused")
        String vendor = mxbean.getVmVendor();
        return;

    }

    /**
     * @return true is the loosely-coupled mode is enabled
     */
    private boolean isLooselyCoupled() {
        String mode = ClusterDaemonTools.getCurrentConfiguration().getClusterDaemon().getJonasInteractionMode();
        return ClusterDaemon.LOOSELY_COUPLED.equals(mode);
    }

    /**
     * @return true is the tighly-coupled mode is enabled.
     */
    private boolean isTighlyCoupled() {
        String mode = ClusterDaemonTools.getCurrentConfiguration().getClusterDaemon().getJonasInteractionMode();
        return ClusterDaemon.TIGHLY_COUPLED.equals(mode);
    }

    /**
     * @param name server name.
     * @return Configuration of the server
     */
    private Server getServer(final String name) throws ClusterDaemonException {
        try {
            JLinkedList servers = ClusterDaemonTools.getCurrentConfiguration().getClusterDaemon().getServerList();

            if (servers != null) {
                for (int i = 0; i < servers.size(); i++) {
                    Server serverElement = (Server) servers.get(i);
                    if (serverElement.getName().equals(name)) {
                        return serverElement;
                    }
                }
            }
        } catch (Exception e) {
            throw new ClusterDaemonException(e);
        }
        return null;
    }

    /**
     * Start all the JOnAS instances configured with auto-reboot
     * @param domainName domain name
     * @param prm extra parameters
     * @param force the starting
     * @param sync synchronous starting
     * @return the nodes list with an indicator started/starting failed
     * @throws ClusterDaemonException any.
     */
    private String doStartAllJOnAS(final String domainName, final String prm, final boolean force, final boolean sync)
            throws ClusterDaemonException {
        try {
            // This feature is valid only if the jonas-interaction-mode is
            // loosely-coupled
            if (!force) {
                if (isLooselyCoupled()) {
                    String result = "JOnAS interaction mode is set to loosely-coupled => don't start the JOnAS instances";
                    this.logger.log(BasicLevel.INFO, result);
                    return result;
                } else {
                    this.logger.log(BasicLevel.INFO,
                            "JOnAS interaction mode is set to tighly-coupled => start the JOnAS instances");
                }
            }
            int nbInst = 0;
            int nbInstStarted = 0;
            String instStarted = "";
            int nbInstStartingError = 0;
            String instStartingError = "";

            JLinkedList servers = ClusterDaemonTools.getCurrentConfiguration().getClusterDaemon().getServerList();

            if (servers != null) {
                for (int i = 0; i < servers.size(); i++) {
                    nbInst++;
                    Server serverElement = (Server) servers.get(i);
                    if (serverElement.isAutoBoot() || force) {
                        try {
                            doStartJOnAS(serverElement.getName(), domainName, prm, sync);
                            nbInstStarted++;
                            instStarted += serverElement.getName() + " ";
                        } catch (ClusterDaemonException e) {
                            this.logger.log(BasicLevel.ERROR, "Error during the launching of JOnAS instance "
                                    + serverElement.getName(), e);
                            nbInstStartingError++;
                            instStartingError += serverElement.getName() + " ex=" + e + " ";
                        }
                    }
                }
            }
            String result = "Instances started (" + nbInstStarted + "/" + nbInst + ", " + instStarted + ")";
            result += " Instances starting failed (" + nbInstStartingError + "/" + nbInst + ", " + instStartingError + ")";

            this.logger.log(BasicLevel.INFO, result);
            return result;
        } catch (Exception e) {
            if (e instanceof ClusterDaemonException) {
                throw (ClusterDaemonException) e;
            }
            throw new ClusterDaemonException(e);
        }

    }

    /**
     * Stop all the JOnAS instances
     * @return the nodes list with an indicator stopped/stopping failed
     */
    private String doStopAllJOnAS() {

        int nbInst = 0;
        int nbInstStopped = 0;
        String instStopped = "";
        int nbInstStoppingError = 0;
        String instStoppingError = "";

        JLinkedList servers = ClusterDaemonTools.getCurrentConfiguration().getClusterDaemon().getServerList();

        if (servers != null) {
            for (int i = 0; i < servers.size(); i++) {
                Server serverElement = (Server) servers.get(i);
                try {
                    doStopJOnAS(serverElement.getName());
                    nbInstStopped++;
                    instStopped += serverElement.getName() + " ";
                } catch (ClusterDaemonException e) {
                    this.logger.log(BasicLevel.ERROR, "Error during the stopping of JOnAS instance " + serverElement.getName(),
                            e);
                    nbInstStoppingError++;
                    instStoppingError += serverElement.getName() + " ex=" + e + " ";
                }
            }
        }
        String result = "Instances stopped (" + nbInstStopped + "/" + nbInst + ", " + instStopped + ")";
        result += " Instances stopping failed (" + nbInstStoppingError + "/" + nbInst + ", " + instStoppingError + ")";

        this.logger.log(BasicLevel.INFO, result);
        return result;
    }

    /**
     * Kill all the JOnAS instances
     * @return the nodes list with an indicator killed/killing failed
     */
    @SuppressWarnings("unused")
    private String doKillAllJOnAS() {

        int nbInst = 0;
        int nbInstKilled = 0;
        String instKilled = "";
        int nbInstKillingError = 0;
        String instKillingError = "";

        JLinkedList servers = ClusterDaemonTools.getCurrentConfiguration().getClusterDaemon().getServerList();

        if (servers != null) {
            for (int i = 0; i < servers.size(); i++) {
                Server serverElement = (Server) servers.get(i);
                try {
                    doKillJOnAS(serverElement.getName());
                    nbInstKilled++;
                    instKilled += serverElement.getName() + " ";
                } catch (ClusterDaemonException e) {
                    this.logger.log(BasicLevel.ERROR, "Error during the killing of JOnAS instance " + serverElement.getName(),
                            e);
                    nbInstKillingError++;
                    instKillingError += serverElement.getName() + " ex=" + e + " ";
                }
            }
        }
        String result = "Instances killed (" + nbInstKilled + "/" + nbInst + ", " + instKilled + ")";
        result += " Instances killing failed (" + nbInstKillingError + "/" + nbInst + ", " + instKillingError + ")";

        this.logger.log(BasicLevel.INFO, result);
        return result;
    }

    /**
     * Main function.
     * @param args arguments list
     * @throws ClusterDaemonException if an error occurs
     */
    public void start(final String[] args) throws ClusterDaemonException {

        boolean useMonitor = false;

        /**
         * Analyse the arguments list
         */
        for (int argn = 0; argn < args.length; argn++) {
            String arg = args[argn];

            try {
                if (arg.equals("-useMonitor")) {
                    useMonitor = true;
                    continue;
                }

                if (arg.equals("-traceFile")) {
                    csTraceFile = args[++argn];
                    continue;
                }

                if (arg.equals("-carolFile")) {
                    carolFile = args[++argn];
                    continue;
                }

                if (arg.equals("-confFile")) {
                    confFile = args[++argn];
                    continue;
                }

                if (arg.equals("--help") || arg.equals("-help") || arg.equals("-h") || arg.equals("-?")) {
                    usage();
                    System.exit(1);
                }

            } catch (ArrayIndexOutOfBoundsException aioobe) {
                // The next argument is not in the array
                throw new ClusterDaemonException("A required parameter was missing after the argument" + arg);
            }
        }

        /**
         * Init the trace logger
         */
        initLogger();

        /**
         * Load clusterd.xml
         */
        ClusterDaemonTools.loadClusterDaemonConfiguration(confFile);
        /**
         * get controlled server names from configuration
         */
        buildControlledServersNames();
        /**
         * getting cluster daemon Host info for monitoring
         */
        initHostInfos();
        /**
         * Get the domain name
         */
        domainName = System.getProperty(ConfigurationConstants.DOMAIN_NAME_PROP, ConfigurationConstants.DEFAULT_DOMAIN_NAME);
        String confDomainName = ClusterDaemonTools.getCurrentConfiguration().getClusterDaemon().getDomainName();

        if (domainName.equals(ConfigurationConstants.DEFAULT_DOMAIN_NAME)) {
            if (confDomainName == null) {
                this.logger.log(BasicLevel.INFO, "Domain set with default value : " + domainName);
            } else {
                domainName = confDomainName;
                this.logger.log(BasicLevel.INFO, "Domain set with the value defined in the clusterd.xml : " + domainName);
            }
        } else if (domainName.equals(confDomainName)) {
            this.logger.log(BasicLevel.INFO, "Domain set with the value defined in the clusterd.xml : " + domainName);
        } else {
            throw new ClusterDaemonException("Incorrect domain name set in clusterd.xml file and in the property "
                    + confDomainName + "/" + domainName);
        }
        /**
         * Init Carol
         */
        ClusterDaemonTools.initCarol();
        /**
         * Start the name service.
         */
        startNameService();

        this.logger.log(BasicLevel.INFO, "Carol initialized!");
        /**
         * Init MBean server
         */
        initMBeans();

        this.logger.log(BasicLevel.INFO, "JOnAS Cluster Daemon Started!");

        /**
         * start the monitor thread if required
         */
        if (useMonitor) {
            new Monitor(this.logger).start();
        }

        /**
         * Launches the JOnAS instances configured with auto-reboot
         */
        doStartAllJOnAS(domainName, null, false, false);
        getDynamicHostAttributes();

        if (ClusterDaemonTools.getCurrentConfiguration().getClusterDaemon().getDiscovery().getStartDiscovery()) {
            /**
             * Starts JGroups discovery
             */
            startDiscovery();
        }

        /**
         * The server is started now
         */
        isStarted = true;

    }

    /**
     * Starts JGroups discovery.
     */
    @SuppressWarnings("unchecked")
    private void startDiscovery() {
        discovery = new JgroupsDiscoveryServiceImpl();
        HashMap<String, Object> discoveryEnv = new HashMap<String, Object>();
        discoveryEnv.put("group.name", ClusterDaemonTools.getCurrentConfiguration().getClusterDaemon().getDiscovery()
                .getDiscoveryGroupName());
        discoveryEnv.put("jgroups.conf", ClusterDaemonTools.getCurrentConfiguration().getClusterDaemon().getDiscovery()
                .getDiscoveryStackFile());
        discoveryEnv.put("host.name", ClusterDaemonTools.getCurrentConfiguration().getClusterDaemon().getName());
        discoveryEnv.put("domain.name", domainName);
        try {
            String[] urls = {ClusterDaemonTools.getJmxUrl(name)};
            discoveryEnv.put("connector.urls", urls);
            discovery.start(discoveryEnv, MBeanServerHelper.getMBeanServer(), JGroupsDiscoveryUtils.DISCOVERY_IS_CLUSTERD);
        } catch (Exception e) {
            logger.log(BasicLevel.DEBUG, "Cluster daemon was unable to start JGroups discovery \n", e);
        }
        logger.log(BasicLevel.INFO, "JGroups discovery started \n");
    }

    /**
     * Usage.
     */
    private static void usage() {
        System.out
                .println("Usage : jclusterd [-DdomainName=<name>] [-useMonitor] [-confFile my_clusterd.xml] [-carolFile=<my_carol.properties>]");
    }

    /**
     * displaying dynamic host attributes.
     */
    public void displayDynamicAttributes() {
        Hashtable<String, String> myhash = dynamicHostAttributes;
        for (String str : myhash.keySet()) {
            String val = myhash.get(str);
            this.logger.log(BasicLevel.DEBUG, "Key : " + str + " Value " + val);
        }
    }

    /**
     * Main method
     * @param args the arguments of the cluster daemon
     */
    public static void main(final String[] args) {
        // Retrieve command line parameters
        ClusterDaemon cs = new ClusterDaemon();

        try {
            cs.start(args);
        } catch (Exception e) {
            System.err.println("There was the following exception : " + e.getMessage());
            e.printStackTrace();
            System.exit(-1);
        }
    }

    // ////////////////////////////////////////////////////////////////////////////////////////
    // /
    // / MBEAN INTERFACES
    // /
    // ////////////////////////////////////////////////////////////////////////////////////////
    /**
     * @return Object Name.
     */
    public String getobjectName() {
        return objectName;
    }

    /**
     * Sets the object name of this mbean.
     * @param name the Object Name
     */
    public void setobjectName(final String name) {
        this.objectName = name;
    }

    /**
     * @return true if it is an event provider
     */
    public boolean iseventProvider() {
        return false;
    }

    /**
     * @return true if this managed object implements J2EE State Management
     *         Model
     */
    public boolean isstateManageable() {
        return false;
    }

    /**
     * @return true if this managed object implements the J2EE StatisticProvider
     *         Model
     */
    public boolean isstatisticsProvider() {
        return false;
    }

    /**
     * (MBean interface).
     * @return the JAVA_HOME for a specified server name
     * @param name JOnAS instance name
     * @throws ClusterDaemonException any.
     */
    public String getJavaHome4Server(final String name) throws ClusterDaemonException {
        Server server = getServer(name);
        if (server != null) {
            return server.getJavaHome();
        } else {
            return null;
        }
    }

    /**
     * (MBean interface).
     * @return the JONAS_BASE for a specified server name
     * @param name JOnAS instance name
     * @throws ClusterDaemonException any.
     */
    public String getJonasBase4Server(final String name) throws ClusterDaemonException {
        Server server = getServer(name);
        if (server != null) {
            return server.getJonasBase();
        } else {
            return null;
        }
    }

    /**
     * (MBean interface).
     * @return the JONAS_ROOT for a specified server name
     * @param name JOnAS instance name
     * @throws ClusterDaemonException any.
     */
    public String getJonasRoot4Server(final String name) throws ClusterDaemonException {
        Server server = getServer(name);
        if (server != null) {
            return server.getJonasRoot();
        } else {
            return null;
        }
    }

    /**
     * @return the user command for a specified server name.
     * @param name JOnAS instance name
     * @throws ClusterDaemonException any.
     */
    public String getJonasCmd4Server(final String name) throws ClusterDaemonException {
        Server server = getServer(name);
        if (server != null) {
            return server.getJonasCmd();
        } else {
            return null;
        }
    }

    /**
     * @return the xprm for a specified server name
     * @param name JOnAS instance name
     * @throws ClusterDaemonException any.
     */
    public String getXprm4Server(final String name) throws ClusterDaemonException {
        Server server = getServer(name);
        if (server != null) {
            return server.getXprm();
        } else {
            return null;
        }
    }

    /**
     * @return the autoboot value for a specified server name
     * @param name JOnAS instance name
     * @throws ClusterDaemonException any.
     */
    public String getAutoBoot4Server(final String name) throws ClusterDaemonException {
        Server server = getServer(name);
        if (server != null) {
            return server.getAutoBoot();
        } else {
            return null;
        }
    }

    /**
     * Gets servers that are controlled by the clusterd.
     * @throws ClusterDaemonException any.
     */
    @SuppressWarnings("unchecked")
    private void buildControlledServersNames() throws ClusterDaemonException {
        try {
            JLinkedList servers = ClusterDaemonTools.getCurrentConfiguration().getClusterDaemon().getServerList();
            if (ControlledServersNames == null) {
                ControlledServersNames = new ArrayList();
            }

            if (servers != null) {
                for (int i = 0; i < servers.size(); i++) {
                    Server serverElement = (Server) servers.get(i);
                    ControlledServersNames.add(serverElement.getName());
                }
            }
        } catch (Exception e) {
            throw new ClusterDaemonException(e);
        }

    }

    /**
     * Flush the configuration to the clusterd.xml file.
     * @throws ClusterDaemonException if an error occurs
     */
    private void flushConfiguration() throws ClusterDaemonException {
        String buf = ClusterDaemonTools.getCurrentConfiguration().toXML();
        FileWriter fw;
        if (confFile == null) {
            confFile = ClusterDaemonTools.getCurrentConfigurationFileName();
        }
        try {
            fw = new FileWriter(confFile);
            fw.write(buf);
            fw.close();
        } catch (IOException e) {
            throw new ClusterDaemonException(e);
        }

        this.logger.log(BasicLevel.DEBUG, "Configuration flushed in the file " + confFile + "\n" + buf);
    }

    /**
     * Remove a server configuration (MBean interface).
     * @param name server name
     * @throws ClusterDaemonException if an error occurs
     */
    public void removeServer(final String serverName) throws ClusterDaemonException {

        // Get the Server
        Server server = getServer(serverName);
        // check if the server exists
        if (server == null) {
            logger.log(BasicLevel.INFO, "Server named " + serverName + " can't be removed from cluster daemon " + this.name
                    + "  control as unknown");
            return;
        }
        // Remove the server
        ClusterDaemonTools.getCurrentConfiguration().getClusterDaemon().getServerList().remove(server);
        ControlledServersNames.remove(serverName);
        this.logger.log(BasicLevel.DEBUG, "Server " + name + " removed");
    }

    /**
     * Modify a server configuration (MBean interface).
     * @param name server name
     * @param description server description
     * @param javaHome JAVA_HOME dir
     * @param jonasBase JONAS_BASE dir
     * @param jonasRoot JONAS_ROOT dir
     * @param xprm extra JVM parameters
     * @param autoBoot automatic start
     * @param jonasCmd user command
     * @throws ClusterDaemonException if an error occurs
     */
    public void modifyServer(final String name, final String description, final String javaHome, final String jonasRoot,
            final String jonasBase, final String xprm, final String autoBoot, final String jonasCmd)
            throws ClusterDaemonException {

        // Get the Server
        Server myServer = getServer(name);

        // check if the server exists
        if (myServer == null) {
            throw new ClusterDaemonException("JOnAS instance " + name + " doesn't exist");
        }

        // Update the server
        myServer.setDescription(description);
        myServer.setJavaHome(javaHome);
        myServer.setJonasBase(jonasBase);
        myServer.setJonasRoot(jonasRoot);
        myServer.setXprm(xprm);
        myServer.setAutoBoot(autoBoot);
        myServer.setJonasCmd(jonasCmd);

        // flush the configuration
        flushConfiguration();

        this.logger.log(BasicLevel.DEBUG, "Server " + name + " updated");
    }

    /**
     * Reload the configuration (MBean interface).
     * @throws ClusterDaemonException if an error occurs
     */
    public void reloadConfiguration() throws ClusterDaemonException {

        ClusterDaemonTools.loadClusterDaemonConfiguration(confFile);

    }

    /**
     * Start a JOnAS instance (MBean interface).
     * @param name instance name
     * @param domainName domain name
     * @param prm extra parameters
     * @throws ClusterDaemonException if an error occurs
     */
    public void startJOnAS(final String name, final String domainName, final String prm) {
        try {
            doStartJOnAS(name, domainName, prm, true);
        } catch (ClusterDaemonException e) {
            this.logger.log(BasicLevel.ERROR, "Error during the launching of JOnAS instance " + name, e);
        }
    }

    /**
     * Stop a JOnAS instance (MBean interface).
     * @param name instance name
     * @throws ClusterDaemonException if an error occurs
     */
    public void stopJOnAS(final String name) throws ClusterDaemonException {
        doStopJOnAS(name);
    }

    /**
     * Halt a JOnAS instance (MBean interface).
     * @param name instance name
     * @throws ClusterDaemonException if an error occurs
     */
    public void haltJOnAS(final String name) throws ClusterDaemonException {
        doHaltJOnAS(name);
    }

    /**
     * Ping a JOnAS instance (MBean interface).
     * @param name instance name
     * @return exit code of the ping (0 ok, 1 ko)
     * @throws ClusterDaemonException if an error occurs
     */
    public int pingJOnAS(final String name) throws ClusterDaemonException {
        return doPingJOnAS(name, PING_SHORT_TIMEOUT);
    }

    /**
     * Start all the JOnAS instances configured with auto-reboot.
     * @param domainName domain name
     * @param prm extra parameters
     * @return the nodes list with an indicator started/starting failed
     * @throws ClusterDaemonException any.
     */
    public String startAllJOnAS(final String domainName, final String prm) throws ClusterDaemonException {
        return doStartAllJOnAS(domainName, prm, true, true);
    }

    /**
     * Stop all the JOnAS instances.
     * @return the nodes list with an indicator stopped/stopping failed
     * @throws ClusterDaemonException any.
     */
    public String stopAllJOnAS() {
        return doStopAllJOnAS();
    }

    /**
     * Stop the current instance.
     */
    public void stopClusterDaemon() {

        if (isStarted) {
            this.logger.log(BasicLevel.DEBUG, "Server not started");
        }
        // Stop JGroups discovery
        if (discovery != null) {
            discovery.stop();
        }
        isStarted = false;

        // Delay the exit of the JVM so the link client/server is not broken.
        // client call will return before the end of the JVM on server side.
        new Thread(new Runnable() {

            public void run() {
                try {
                    // Wait before exit
                    Thread.sleep(SLEEP_DELAY);
                } catch (InterruptedException ie) {
                    ie.printStackTrace();
                    throw new IllegalStateException("Cannot wait: " + ie.getMessage());
                }
                System.exit(0);
            }
        }).start();

        this.logger.log(BasicLevel.INFO, "Server stopped");
    }

    /**
     * @return clusterd name.
     */
    public String getName() {
        return name;
    }

    /**
     * Controlled servers.
     * @return cluster daemon controlled server names.
     * @throws ClusterDaemonException any.
     */
    public ArrayList getControlledServersNames() throws ClusterDaemonException {
        if (ControlledServersNames == null) {
            buildControlledServersNames();
        }
        return this.ControlledServersNames;
    }

    /**
     * Get jmx url of the cluster daemon.
     * @return clusterd jmx url
     * @throws ClusterDaemonException any.
     */
    public ArrayList serversNames() throws ClusterDaemonException {
        return getControlledServersNames();

    }

    /**
     * Get jmx url of the cluster daemon.
     * @return clusterd jmx url
     */
    public String getJmxUrl() {
        return jmxUrl;
    }

    /**
     * @param url the url used by the JMX Remote connector
     */
    public void setJmxUrl(final String url) {
        jmxUrl = url;
    }

    /**
     * Get available processors of the OS.
     * @return Operating system processors number
     */
    public String getOperatingSystemAvailableProcessors() {
        return operatingSystemAvailableProcessors;
    }

    /**
     * @param operatingSystemAvailableProcessors.
     */
    public void setOperatingSystemAvailableProcessors(final String operatingSystemAvailableProcessors) {
        this.operatingSystemAvailableProcessors = operatingSystemAvailableProcessors;
    }

    /**
     * Get OS name.
     * @return OS name
     * @throws ClusterDaemonException any.
     */
    public String getOperatingSystemName() {
        return operatingSystemName;
    }

    /**
     * Sets operating system name.
     * @param operatingSystemName
     */
    public void setOperatingSystemName(final String operatingSystemName) {
        this.operatingSystemName = operatingSystemName;
    }

    /**
     * Get OS version.
     * @return OS version
     * @throws ClusterDaemonException any.
     */
    public String getOperatingSystemVersion() {
        return operatingSystemVersion;
    }

    /**
     * Sets OS version
     * @param operatingSystemVersion
     */
    public void setOperatingSystemVersion(final String operatingSystemVersion) {
        this.operatingSystemVersion = operatingSystemVersion;
    }

    /**
     * Get spec. vendor.
     * @return Spec vendor
     * @throws ClusterDaemonException any.
     */
    public String getRunTimeSpecVendor() {
        return runTimeSpecVendor;
    }

    /**
     * Sets Runtime vendor.
     * @param runTimeSpecVendor
     */
    public void setRunTimeSpecVendor(final String runTimeSpecVendor) {
        this.runTimeSpecVendor = runTimeSpecVendor;
    }

    /**
     * Get runtime spec. version.
     * @return Spec version
     * @throws ClusterDaemonException any.
     */
    public String getRunTimeSpecVersion() {
        return runTimeSpecVersion;
    }

    /**
     * Sets Runtime spec. version.
     * @param runTimeSpecVersion
     */
    public void setRunTimeSpecVersion(final String runTimeSpecVersion) {
        this.runTimeSpecVersion = runTimeSpecVersion;
    }

    /**
     * Get runtime vm name.
     * @return the Vm name
     * @throws ClusterDaemonException any.
     */
    public String getRunTimeVmName() {
        return runTimeVmName;
    }

    /**
     * @param runTimeVmName
     */
    public void setRunTimeVmName(final String runTimeVmName) {
        this.runTimeVmName = runTimeVmName;
    }

    /**
     * Get runtime vendor.
     * @return Vm vendor
     * @throws ClusterDaemonException any.
     */
    public String getRunTimeVmVendor() {
        return runTimeVmVendor;
    }

    /**
     * Sets Runtime VM vendor.
     * @param runTimeVmVendor
     */
    public void setRunTimeVmVendor(final String runTimeVmVendor) {
        this.runTimeVmVendor = runTimeVmVendor;
    }

    /**
     * Get runtime vm version.
     * @return Vm version
     * @throws ClusterDaemonException any.
     */
    public String getRunTimeVmVersion() {
        return runTimeVmVersion;
    }

    /**
     * Sets runtime VM version.
     * @param runTimeVmVersion
     */
    public void setRunTimeVmVersion(final String runTimeVmVersion) {
        this.runTimeVmVersion = runTimeVmVersion;
    }

    /**
     * Get dynamic host attributes.
     * @return hashtable with dynamic attributes keys and values.
     * @throws ClusterDaemonException any.
     */
    public Hashtable<String, String> getDynamicHostAttributes() throws ClusterDaemonException {
        if (dynamicHostAttributes == null) {
            dynamicHostAttributes = new Hashtable<String, String>();
        }
        this.logger.log(BasicLevel.DEBUG, "retrieving cluster daemon's host dynamic information");
        try {
            // ClassLoading Mbean
            dynamicHostAttributes.put("loadedClassCount", new Integer(classLoadingMxBean.getLoadedClassCount()).toString());
            dynamicHostAttributes.put("unloadedClassCount", new Long(classLoadingMxBean.getUnloadedClassCount()).toString());
            // Threading Mbean
            dynamicHostAttributes.put("threadCount", new Integer(threadMxBean.getThreadCount()).toString());
            dynamicHostAttributes
                    .put("totalStartedThreadCount", new Long(threadMxBean.getTotalStartedThreadCount()).toString());
            // heap memory mxBeans
            // "vm" is added to specify that the memory attribute is for virtual
            // machine # Operating system memory
            dynamicHostAttributes.put("initHeapMemoryVm", new Long(memoryMXBean.getHeapMemoryUsage().getInit()).toString());
            dynamicHostAttributes.put("usedHeapMemoryVm", new Long(memoryMXBean.getHeapMemoryUsage().getUsed()).toString());
            dynamicHostAttributes.put("maxHeapMemoryVm", new Long(memoryMXBean.getHeapMemoryUsage().getMax()).toString());
            dynamicHostAttributes.put("heapCommittedMemoryVm", new Long(memoryMXBean.getHeapMemoryUsage().getCommitted())
                    .toString());
            // non heap
            dynamicHostAttributes.put("initNonHeapMemoryVm", new Long(memoryMXBean.getNonHeapMemoryUsage().getInit())
                    .toString());
            dynamicHostAttributes.put("usedNonHeapMemoryVm", new Long(memoryMXBean.getNonHeapMemoryUsage().getUsed())
                    .toString());
            dynamicHostAttributes.put("maxNonHeapMemoryVm", new Long(memoryMXBean.getNonHeapMemoryUsage().getMax()).toString());
            dynamicHostAttributes.put("CommittedNonHeapMemoryVm", new Long(memoryMXBean.getNonHeapMemoryUsage().getCommitted())
                    .toString());
            // retrieve operating system dynamic infos.
            buildOperatingSystemdynInfo();
            // used in debug mode..
            displayDynamicAttributes();
        } catch (Exception e) {
            this.logger.log(BasicLevel.DEBUG, "Cannot retrieve cluster daemon's host dynamic information: " + e);
            throw new ClusterDaemonException(e);
        } finally {
            return dynamicHostAttributes;
        }
    }

    /**
     * Used to get management infos thare are not available from MXbeans.
     * @throws ClusterDaemonException any.
     */
    private void buildOperatingSystemdynInfo() throws ClusterDaemonException {
        ObjectName osOn;
        try {
            osOn = new ObjectName("java.lang:type=OperatingSystem");

            if ((platFormMbeanServer != null) && (platFormMbeanServer.isRegistered(osOn))) {

                MBeanInfo mbeaInfo = platFormMbeanServer.getMBeanInfo(osOn);
                MBeanAttributeInfo[] atts = mbeaInfo.getAttributes();
                String attrName = null;
                String attrType = null;
                Long attrValue;
                for (int i = 0; i < atts.length; i++) {
                    attrName = atts[i].getName();
                    attrType = atts[i].getType();
                    if (attrType.equals(LONG_TYPE_DYN_INFO)) {
                        logger.log(BasicLevel.DEBUG, "**** Attribute Info ****");
                        logger.log(BasicLevel.DEBUG, "Name " + atts[i].getName());
                        logger.log(BasicLevel.DEBUG, "Type " + atts[i].getType());
                        logger.log(BasicLevel.DEBUG, "isIs " + atts[i].isIs());
                        logger.log(BasicLevel.DEBUG, "isReadable " + atts[i].isReadable());
                        logger.log(BasicLevel.DEBUG, "isWritable " + atts[i].isWritable());
                        attrValue = (Long) platFormMbeanServer.getAttribute(osOn, atts[i].getName());
                        logger.log(BasicLevel.DEBUG, "value " + attrValue);
                        dynamicHostAttributes.put(attrName, attrValue.toString());

                    }
                }

            }
        } catch (Exception e) {
            logger.log(BasicLevel.DEBUG, "Unable to get dynamic dynamic infos for the Operating System " + e);
            throw new ClusterDaemonException(e);
        }

    }

    /**
     * Initializes platform infos.
     * @throws ClusterDaemonException any.
     */
    public void initHostInfos() throws ClusterDaemonException {
        this.logger.log(BasicLevel.INFO, "Initializing Host monitoring informations");
        try {
            // MXbeans initialization
            runtimeMxbean = ManagementFactory.getRuntimeMXBean();
            classLoadingMxBean = ManagementFactory.getClassLoadingMXBean();
            threadMxBean = ManagementFactory.getThreadMXBean();
            compilationMxBean = ManagementFactory.getCompilationMXBean();
            garbageCollectorMXBean = ManagementFactory.getGarbageCollectorMXBeans();
            memoryManagerMXBean = ManagementFactory.getMemoryManagerMXBeans();
            memoryMXBean = ManagementFactory.getMemoryMXBean();
            memoryPoolMXBean = ManagementFactory.getMemoryPoolMXBeans();
            platFormMbeanServer = ManagementFactory.getPlatformMBeanServer();
            //
            runTimeVmVendor = runtimeMxbean.getVmVendor();
            runTimeSpecVendor = runtimeMxbean.getSpecVendor();
            runTimeSpecVersion = runtimeMxbean.getSpecName();
            runTimeVmName = runtimeMxbean.getVmName();
            runTimeVmVersion = runtimeMxbean.getVmVersion();
            // OperatingSystem mbean
            operatingSystemMxBean = ManagementFactory.getOperatingSystemMXBean();
            operatingSystemAvailableProcessors = new Integer(operatingSystemMxBean.getAvailableProcessors()).toString();
            operatingSystemName = operatingSystemMxBean.getName();
            operatingSystemVersion = operatingSystemMxBean.getVersion();
            operatingSystemArch = operatingSystemMxBean.getArch();
            dynamicHostAttributes = new Hashtable<String, String>();

        } catch (Exception e) {
            this.logger.log(BasicLevel.DEBUG, "cannot get host monitoring informations : " + e);
            throw new ClusterDaemonException(e);
        }

    }

    /**
     * Gets OS architecture
     * @return operating system architecture
     */
    public String getOperatingSystemArch() {
        return operatingSystemArch;
    }

    /**
     *Sets OS Architecture
     * @param operatingSystemArch
     */
    public void setOperatingSystemArch(final String operatingSystemArch) {
        this.operatingSystemArch = operatingSystemArch;
    }

    /**
     * Getting remote Vm's Current used non Heap memory.
     * @return the value of Vm's Current used non Heap memory
     * @throws ClusterDaemonException any.
     */
    public String getVmCurrentUsedNonHeapMemory() throws ClusterDaemonException {
        try {
            long nonheap = memoryMXBean.getNonHeapMemoryUsage().getUsed();
            return new Long(nonheap).toString();
        } catch (Exception e) {
            throw new ClusterDaemonException(e);
        }

    }

    /**
     * Getting Operating system Current used space.
     * @return the value of Operating system Current used space
     * @throws ClusterDaemonException any.
     */
    public String getOsCurrentUsedSpace() throws ClusterDaemonException {
        try {
            long totalPhys = getOsAtributeValue("TotalPhysical");
            long totalFree = getOsAtributeValue("FreePhysical");
            return new Long(totalPhys - totalFree).toString();
        } catch (Exception e) {
            throw new ClusterDaemonException(e);
        }

    }

    /**
     * Getting Operating system Current used space.
     * @return the value of Operating system Total space
     * @throws ClusterDaemonException any.
     */
    public String getOsTotalSpace() throws ClusterDaemonException {
        try {
            return getOsAtributeValue("TotalPhysical").toString();
        } catch (Exception e) {
            throw new ClusterDaemonException(e);
        }

    }

    /**
     * Getting remote Vm's Current used Heap memory.
     * @return the value of Vm's Current used Heap memory
     * @throws ClusterDaemonException any.
     */
    public String getVmCurrentUsedHeapMemory() throws ClusterDaemonException {
        try {
            long memusage = memoryMXBean.getHeapMemoryUsage().getUsed();
            return new Long(memusage).toString();
        } catch (Exception e) {
            throw new ClusterDaemonException(e);
        }

    }

    /**
     * Get remote Vm used Memory.
     * @return the value of current used memory
     * @throws ClusterDaemonException any.
     */
    public String getVmCurrentUsedMemory() throws ClusterDaemonException {
        try {
            long heapmem = memoryMXBean.getHeapMemoryUsage().getUsed();
            long nonHeapmem = memoryMXBean.getNonHeapMemoryUsage().getUsed();
            return new Long(heapmem + nonHeapmem).toString();
        } catch (Exception e) {
            throw new ClusterDaemonException(e);
        }

    }

    /**
     * Getting remote Vm Total Memory.
     * @return the value of Vm Total memory
     * @throws ClusterDaemonException any.
     */
    public String getVmTotalMemory() throws ClusterDaemonException {
        try {
            long committed = memoryMXBean.getHeapMemoryUsage().getCommitted();
            long usage = memoryMXBean.getNonHeapMemoryUsage().getCommitted();
            return new Long(committed + usage).toString();
        } catch (Exception e) {
            throw new ClusterDaemonException(e);
        }

    }

    /**
     * Use this method to get an attribute value. Can't use direct call to the
     * getAttribute method because some attributes are not exposed to management
     * @param key get value for this key
     * @return OS attribute value for given key
     * @throws ClusterDaemonException if an error occurs.
     */
    @SuppressWarnings("unused")
    private Long getOsAtributeValue(final String key) throws ClusterDaemonException {
        ObjectName osOn;
        try {
            osOn = new ObjectName("java.lang:type=OperatingSystem");

            if ((platFormMbeanServer != null) && (platFormMbeanServer.isRegistered(osOn))) {

                MBeanInfo mbeaInfo = platFormMbeanServer.getMBeanInfo(osOn);
                MBeanAttributeInfo[] atts = mbeaInfo.getAttributes();
                String attrName = null;
                int i = 0;
                while ((!atts[i].getName().contains(key)) && (i < atts.length)) {
                    i++;
                }
                if (i < atts.length) {
                    return (Long) platFormMbeanServer.getAttribute(osOn, atts[i].getName());
                } else {
                    throw new ClusterDaemonException("Key {0} in Operating System mbean" + key);
                }

            }
        } catch (Exception e) {
            logger.log(BasicLevel.DEBUG, "Unable to get dynamic dynamic infos for the Operating System " + e);
            throw new ClusterDaemonException(e);
        }
        return null;

    }

    /**
     * Add a server to cluster daemon control.
     * @param name the server name
     * @param description server description
     * @param javaHome path to JRE
     * @param jonasRoot path to bin repository
     * @param JonasBase path to lib repository
     * @param xprem extra parameter e.g: -Djava.net.preferIPv4Stack=true
     * @param autoBoot true if the server is launched when cluster daemon starts
     * @param jonasCmd user command
     * @param saveIt boolean
     * @throws ClusterDaemonException any.
     */
    @SuppressWarnings("unchecked")
    public void addServer(final String name, final String description, final String jonasRoot, final String jonasBase,
            final String javaHome, final String xprem, final String autoBoot, final String jonasCmd, final String saveIt)
            throws ClusterDaemonException {
        try {
            JLinkedList servers = ClusterDaemonTools.getCurrentConfiguration().getClusterDaemon().getServerList();
            Server srv = new Server();
            srv.setName(name);
            srv.setJonasRoot(jonasRoot);
            srv.setJonasBase(jonasBase);
            srv.setJavaHome(javaHome);
            srv.setXprm(xprem);
            srv.setAutoBoot(autoBoot);
            srv.setJonasCmd(jonasCmd);
            servers.add(srv);
            if (saveIt.equals("true")) {
                try {
                    flushConfiguration();
                } catch (Exception e) {
                    logger.log(BasicLevel.DEBUG, "Cannot flush configuration for cluster daemon named" + this.name);
                    return;
                }
            }
            // Update controled servers list
            ControlledServersNames.add(srv.getName());
            logger.log(BasicLevel.DEBUG, "Server named " + name + " successfully added to cluster daemon " + this.name
                    + "control");
        } catch (Exception e) {
            throw new ClusterDaemonException(e);
        }

    }

    /**
     * Remove This server from cluster daemon control.
     * @param serverName The server to remove
     * @param saveIt True to flush clusterd configuration
     * @throws ClusterDaemonException any.
     */
    public void removeServer(final String serverName, final String saveIt) throws ClusterDaemonException {

        Server server = getServer(serverName);
        if (server != null) {

            try {
                ClusterDaemonTools.getCurrentConfiguration().getClusterDaemon().getServerList().remove(server);

                if (saveIt.equals("true")) {
                    flushConfiguration();
                }
            } catch (ClusterDaemonException e) {
                logger.log(BasicLevel.DEBUG, "Cannot flush configuration for cluster daemon named" + this.name);
                return;
            }
            ControlledServersNames.remove(serverName);
            logger.log(BasicLevel.INFO, "Server named " + serverName + " successfully removed from cluster daemon " + this.name
                    + "  control");
        } else {
            logger.log(BasicLevel.INFO, "Server named " + serverName + " can't be removed from cluster daemon " + this.name
                    + "  control as not known");
        }

    }

    /**
     * Save current configuration
     * @throws ClusterDaemonException any.
     */
    public void saveConfiguration() throws ClusterDaemonException {
        flushConfiguration();
    }

}
