package org.protempa;

/*
 * #%L
 * Protempa Framework
 * %%
 * Copyright (C) 2012 - 2015 Emory University
 * %%
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 *      http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * #L%
 */
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Queue;
import java.util.Set;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.apache.commons.lang3.StringUtils;
import org.arp.javautil.arrays.Arrays;
import org.arp.javautil.collections.Iterators;
import org.arp.javautil.log.Logging;
import org.protempa.backend.dsb.filter.Filter;
import org.protempa.dest.Destination;
import org.protempa.dest.GetSupportedPropositionIdsException;
import org.protempa.dest.QueryResultsHandler;
import org.protempa.dest.QueryResultsHandlerCloseException;
import org.protempa.dest.QueryResultsHandlerInitException;
import org.protempa.dest.QueryResultsHandlerProcessingException;
import org.protempa.dest.QueryResultsHandlerValidationFailedException;
import org.protempa.proposition.Proposition;
import org.protempa.proposition.UniqueId;
import org.protempa.query.And;
import org.protempa.query.Query;

/**
 *
 * @author Andrew Post
 */
final class Executor implements AutoCloseable {

    private static final Logger LOGGER = ProtempaUtil.logger();
    private final Set<String> keyIds;
    private final Set<String> propIds;
    private Set<String> propIdsToRetain;
    private final Set<And<String>> termIds;
    private final Filter filters;
    private final PropositionDefinition[] propDefs;
    private final KnowledgeSource ks;
    private final Query query;
    private final QuerySession qs;
    private DerivationsBuilder derivationsBuilder;
    private final ExecutorStrategy strategy;
    private Collection<PropositionDefinition> allNarrowerDescendants;
    private final AbstractionFinder abstractionFinder;
    private ExecutionStrategy executionStrategy = null;
    private final ExecutorCounter counter = new ExecutorCounter();
    private final List<QueryException> exceptions;
    private final Destination destination;
    private QueryResultsHandler resultsHandler;
    private boolean failed;
    private final MessageFormat logMessageFormat;
    private Thread handleQueryResultThread;
    private boolean canceled;

    Executor(Query query, Destination resultsHandlerFactory, QuerySession querySession, AbstractionFinder abstractionFinder) throws QueryException {
        this(query, resultsHandlerFactory, querySession, null, abstractionFinder);
    }

    Executor(Query query, Destination resultsHandlerFactory, QuerySession querySession, ExecutorStrategy strategy, AbstractionFinder abstractionFinder) throws QueryException {
        this.abstractionFinder = abstractionFinder;
        assert query != null : "query cannot be null";
        assert resultsHandlerFactory != null : "resultsHandlerFactory cannot be null";
        assert abstractionFinder != null : "abstractionFinder cannot be null";
        if (abstractionFinder.isClosed()) {
            throw new QueryException(query.getName(), new ProtempaAlreadyClosedException());
        }
        this.keyIds = Arrays.asSet(query.getKeyIds());
        this.propIds = Arrays.asSet(query.getPropositionIds());
        this.termIds = Arrays.asSet(query.getTermIds());
        this.filters = query.getFilters();
        this.propDefs = query.getPropositionDefinitions();
        if (propDefs != null && propDefs.length > 0) {
            ks = new KnowledgeSourceImplWrapper(abstractionFinder.getKnowledgeSource(), propDefs);
        } else {
            ks = abstractionFinder.getKnowledgeSource();
        }
        this.query = query;
        this.qs = querySession;
        this.derivationsBuilder = new DerivationsBuilder();
        this.strategy = strategy;
        this.destination = resultsHandlerFactory;
        this.exceptions = Collections.synchronizedList(new ArrayList<QueryException>());
        this.logMessageFormat = new MessageFormat("Query " + this.query.getName() + ": {0}");
    }

    void init() throws QueryException {
        log(Level.FINE, "Initializing query results handler...");
        try {
            this.resultsHandler = this.destination.getQueryResultsHandler(getQuery(), this.abstractionFinder.getDataSource(), getKnowledgeSource());
            log(Level.FINE, "Got query results handler {0}", this.resultsHandler.getId());
            log(Level.FINE, "Validating query results handler");
            this.resultsHandler.validate();
            log(Level.FINE, "Query results handler validated successfully");
            String[] supportedPropositionIds = destination.getSupportedPropositionIds(this.abstractionFinder.getDataSource(), this.abstractionFinder.getKnowledgeSource());
            setPropIdsToRetain(supportedPropositionIds);

            if (isLoggable(Level.FINE)) {
                log(Level.FINE, "Propositions to be queried are {0}", StringUtils.join(this.propIds, ", "));
            }
            retain(this.ks.collectPropDefDescendantsUsingAllNarrower(false, this.propIds.toArray(new String[this.propIds.size()])));

            if (isLoggable(Level.FINE)) {
                Set<String> allNarrowerDescendantsPropIds = new HashSet<>();
                for (PropositionDefinition pd : this.allNarrowerDescendants) {
                    allNarrowerDescendantsPropIds.add(pd.getId());
                }
                log(Level.FINE, "Proposition details: {0}", StringUtils.join(allNarrowerDescendantsPropIds, ", "));
            }

            if (strategy != null) {
                log(Level.FINE, "Setting execution strategy...");
                switch (strategy) {
                    case STATELESS:
                        executionStrategy = newStatelessStrategy();
                        break;
                    case STATEFUL:
                        executionStrategy = newStatefulStrategy();
                        break;
                    default:
                        throw new AssertionError("Invalid execution strategy: " + strategy);
                }
                log(Level.FINE, "Execution strategy is set to {0}", strategy);
            }

            log(Level.FINE, "Calling query results handler start...");
            this.resultsHandler.start(getAllNarrowerDescendants());
            log(Level.FINE, "Query results handler started");
            log(Level.FINE, "Query results handler waiting for results...");
        } catch (KnowledgeSourceReadException | QueryResultsHandlerValidationFailedException | QueryResultsHandlerInitException | QueryResultsHandlerProcessingException | GetSupportedPropositionIdsException | Error | RuntimeException ex) {
            this.failed = true;
            throw new QueryException(this.query.getName(), ex);
        }
    }

    void cancel() {
        synchronized (this) {
            if (this.handleQueryResultThread != null) {
                this.handleQueryResultThread.interrupt();
            }
            this.canceled = true;
        }
        log(Level.INFO, "Canceled");
    }

    void execute() throws QueryException {
        try {
            Thread retrieveDataThread;
            Thread doProcessThread;
            synchronized (this) {
                if (this.canceled) {
                    return;
                }
                log(Level.INFO, "Processing data");
                DataStreamingEvent doProcessPoisonPill = new DataStreamingEvent("poison", Collections.emptyList());
                QueueObject hqrPoisonPill = new QueueObject();
                BlockingQueue<DataStreamingEvent> doProcessQueue = new ArrayBlockingQueue<>(1000);
                BlockingQueue<QueueObject> hqrQueue = new ArrayBlockingQueue<>(1000);
                retrieveDataThread = new RetrieveDataThread(doProcessQueue, doProcessPoisonPill);
                doProcessThread = new DoProcessThread(doProcessQueue, hqrQueue, doProcessPoisonPill, hqrPoisonPill, retrieveDataThread);
                this.handleQueryResultThread = new HandleQueryResultThread(hqrQueue, hqrPoisonPill, doProcessThread);
                retrieveDataThread.start();
                doProcessThread.start();
                this.handleQueryResultThread.start();
            }

            try {
                retrieveDataThread.join();
                log(Level.INFO, "Done retrieving data");
            } catch (InterruptedException ex) {
                log(Level.FINER, "Protempa producer thread join interrupted", ex);
            }
            try {
                doProcessThread.join();
                log(Level.INFO, "Done processing data");
            } catch (InterruptedException ex) {
                log(Level.FINER, "Protempa consumer thread join interrupted", ex);
            }
            try {
                this.handleQueryResultThread.join();
                log(Level.INFO, "Done outputting results");
            } catch (InterruptedException ex) {
                log(Level.FINER, "Protempa consumer thread join interrupted", ex);
            }

            if (!exceptions.isEmpty()) {
                throw exceptions.get(0);
            }
        } catch (QueryException ex) {
            this.failed = true;
            throw ex;
        }
    }

    @Override
    public void close() throws CloseException {
        if (executionStrategy != null) {
            executionStrategy.cleanup();
        }
        try {
            // Might be null if init() fails.
            if (this.resultsHandler != null) {
                if (!this.failed) {
                    this.resultsHandler.finish();
                }
                this.resultsHandler.close();
                this.resultsHandler = null;
            }
        } catch (QueryResultsHandlerProcessingException | QueryResultsHandlerCloseException ex) {
            throw new CloseException(ex);
        } finally {
            if (this.resultsHandler != null) {
                try {
                    this.resultsHandler.close();
                } catch (QueryResultsHandlerCloseException ignore) {

                }
            }
        }
    }

    ExecutionStrategy getExecutionStrategy() {
        return this.executionStrategy;
    }

    int getCount() {
        return counter.getCount();
    }

    Query getQuery() {
        return query;
    }

    Set<String> getPropIds() {
        return this.propIds;
    }

    Collection<PropositionDefinition> getAllNarrowerDescendants() {
        return allNarrowerDescendants;
    }

    Filter getFilters() {
        return this.filters;
    }

    Set<String> getKeyIds() {
        return this.keyIds;
    }

    void setPropIdsToRetain(String[] propIds) {
        if (propIds == null || propIds.length == 0) {
            this.propIdsToRetain = null;
        } else {
            this.propIdsToRetain = Arrays.asSet(propIds);
        }
        this.allNarrowerDescendants = null;
    }

    DerivationsBuilder getDerivationsBuilder() {
        return this.derivationsBuilder;
    }

    KnowledgeSource getKnowledgeSource() {
        return ks;
    }

    QuerySession getQuerySession() {
        return qs;
    }

    boolean isLoggable(Level level) {
        return LOGGER.isLoggable(level);
    }

    void log(Level level, String msg, Object[] params) {
        if (isLoggable(level)) {
            LOGGER.log(level, this.logMessageFormat.format(new Object[]{msg}), params);
        }
    }

    void log(Level level, String msg, Object param) {
        if (isLoggable(level)) {
            LOGGER.log(level, this.logMessageFormat.format(new Object[]{msg}), param);
        }
    }

    void log(Level level, String msg, Throwable throwable) {
        if (isLoggable(level)) {
            LOGGER.log(level, this.logMessageFormat.format(new Object[]{msg}), throwable);
        }
    }

    void log(Level level, String msg) {
        if (isLoggable(level)) {
            LOGGER.log(level, this.logMessageFormat.format(new Object[]{msg}));
        }
    }

    void logCount(Level level, int count, String singularMsg, String pluralMsg, Object[] singularParams, Object[] pluralParams) {
        if (isLoggable(level)) {
            Logging.logCount(LOGGER, level, count, this.logMessageFormat.format(new Object[]{singularMsg}), this.logMessageFormat.format(new Object[]{pluralMsg}), singularParams, pluralParams);
        }
    }

    private class ExecutorCounter {

        private int count;

        ExecutorCounter() {
        }

        void incr() throws QueryException {
            if (++this.count % 1000 == 0) {
                logNumProcessed(this.count);
            }
        }

        int getCount() {
            return this.count;
        }

        private void logNumProcessed(int numProcessed) throws QueryException {
            if (isLoggable(Level.FINE)) {
                try {
                    String keyTypeSingDisplayName = abstractionFinder.getDataSource().getKeyTypeDisplayName();
                    String keyTypePluralDisplayName = abstractionFinder.getDataSource().getKeyTypePluralDisplayName();
                    logCount(Level.FINE, numProcessed, "Processed {0} {1}", "Processed {0} {1}", new Object[]{keyTypeSingDisplayName}, new Object[]{keyTypePluralDisplayName});
                } catch (DataSourceReadException ex) {
                    throw new QueryException(Executor.this.query.getName(), ex);
                }
            }
        }
    }

    private class RetrieveDataThread extends Thread {

        private final BlockingQueue<DataStreamingEvent> queue;
        private final DataStreamingEvent poisonPill;
        private final DataStreamingEventIterator<Proposition> itr;

        RetrieveDataThread(BlockingQueue<DataStreamingEvent> queue, DataStreamingEvent poisonPill) throws QueryException {
            super("protempa.executor.RetrieveDataThread");
            this.queue = queue;
            this.poisonPill = poisonPill;
            this.itr = newDataIterator();
        }

        @Override
        public void run() {
            log(Level.FINER, "Start retrieve data thread");
            boolean itrClosed = false;
            try {
                while (!isInterrupted() && itr.hasNext()) {
                    queue.put(itr.next());
                }
                itr.close();
                queue.put(poisonPill);
                itrClosed = true;
            } catch (DataSourceReadException ex) {
                exceptions.add(new QueryException(Executor.this.query.getName(), ex));
                try {
                    queue.put(poisonPill);
                } catch (InterruptedException ignore) {
                    log(Level.SEVERE, "Failed to send stop message to the do process thread; the query may be hung", ignore);
                }
            } catch (Error | RuntimeException ex) {
                exceptions.add(new QueryException(Executor.this.query.getName(), ex));
                try {
                    queue.put(poisonPill);
                } catch (InterruptedException ignore) {
                    log(Level.SEVERE, "Failed to send stop message to the do process thread; the query may be hung", ignore);
                }
            } catch (InterruptedException ex) { // by DoProcessThread
                log(Level.FINER, "Retrieve data thread interrupted", ex);
            } finally {
                if (!itrClosed) {
                    try {
                        itr.close();
                    } catch (DataSourceReadException ignore) {
                    }
                }
            }
            log(Level.FINER, "End retrieve data thread");
        }
    }

    private class DoProcessThread extends Thread {

        private final BlockingQueue<DataStreamingEvent> doProcessQueue;
        private final BlockingQueue<QueueObject> hqrQueue;
        private final QueueObject hqrPoisonPill;
        private final DataStreamingEvent doProcessPoisonPill;
        private final Thread producer;

        DoProcessThread(BlockingQueue<DataStreamingEvent> doProcessQueue, BlockingQueue<QueueObject> hqrQueue, DataStreamingEvent doProcessPoisonPill, QueueObject hqrPoisonPill, Thread producer) {
            super("protempa.executor.DoProcessThread");
            this.doProcessQueue = doProcessQueue;
            this.hqrQueue = hqrQueue;
            this.doProcessPoisonPill = doProcessPoisonPill;
            this.producer = producer;
            this.hqrPoisonPill = hqrPoisonPill;
        }

        @Override
        public void run() {
            log(Level.FINER, "Start do process thread");
            try {
                DataStreamingEvent dse;
                while (!isInterrupted() && ((dse = doProcessQueue.take()) != doProcessPoisonPill)) {
                    String keyId = dse.getKeyId();
                    Iterator<Proposition> resultsItr;
                    ExecutionStrategy strategy = getExecutionStrategy();
                    if (strategy != null) {
                        resultsItr = strategy.execute(
                                keyId, propIds, dse.getData(),
                                null);
                    } else {
                        resultsItr = dse.getData().iterator();
                    }
                    Map<Proposition, List<Proposition>> forwardDerivations = derivationsBuilder.toForwardDerivations();
                    Map<Proposition, List<Proposition>> backwardDerivations = derivationsBuilder.toBackwardDerivations();
                    QuerySession qs = getQuerySession();
                    if (qs.isCachingEnabled()) {
                        List<Proposition> props = Iterators.asList(resultsItr);
                        addToCache(qs, Collections.unmodifiableList(props), Collections.unmodifiableMap(forwardDerivations), Collections.unmodifiableMap(backwardDerivations));
                        resultsItr = props.iterator();
                    }
                    Map<UniqueId, Proposition> refs = new HashMap<>();
                    List<Proposition> filteredPropositions = extractRequestedPropositions(resultsItr, refs);
                    if (isLoggable(Level.FINEST)) {
                        log(Level.FINEST, "Proposition ids: {0}", propIds);
                        log(Level.FINEST, "Filtered propositions: {0}", filteredPropositions);
                        log(Level.FINEST, "Forward derivations: {0}", forwardDerivations);
                        log(Level.FINEST, "Backward derivations: {0}", backwardDerivations);
                        log(Level.FINEST, "References: {0}", refs);
                    }
                    this.hqrQueue.put(new QueueObject(keyId, filteredPropositions, forwardDerivations, backwardDerivations, refs));
                    log(Level.FINER, "Results put on query result handler queue");
                    counter.incr();
                    derivationsBuilder.reset();
                }
                this.hqrQueue.put(this.hqrPoisonPill);
            } catch (QueryException ex) {
                log(Level.FINER, "Do process thread threw ExecutorExecuteException", ex);
                exceptions.add(ex);
                producer.interrupt();
                try {
                    hqrQueue.put(hqrPoisonPill);
                } catch (InterruptedException ignore) {
                    log(Level.SEVERE, "Failed to stop the query results handler queue; the query may be hung", ignore);
                }
            } catch (InterruptedException ex) { // by the HQR thread
                log(Level.FINER, "Do process thread interrupted", ex);
                producer.interrupt();
            } catch (Error | RuntimeException t) {
                log(Level.SEVERE, "Do process thread threw exception; the query may be hung", t);
                throw t;
            }
            log(Level.FINER, "End do process thread");
        }

        private List<Proposition> extractRequestedPropositions(
                Iterator<Proposition> propositions,
                Map<UniqueId, Proposition> refs) {
            List<Proposition> result = new ArrayList<>();
            while (!isInterrupted() && propositions.hasNext()) {
                Proposition prop = propositions.next();
                refs.put(prop.getUniqueId(), prop);
                if (propIds.contains(prop.getId())) {
                    result.add(prop);
                }
            }
            return result;
        }

        private void addToCache(QuerySession qs,
                List<Proposition> propositions,
                Map<Proposition, List<Proposition>> forwardDerivations,
                Map<Proposition, List<Proposition>> backwardDerivations) {
            qs.addPropositionsToCache(propositions);
            for (Map.Entry<Proposition, List<Proposition>> me
                    : forwardDerivations.entrySet()) {
                qs.addDerivationsToCache(me.getKey(), me.getValue());
            }
            for (Map.Entry<Proposition, List<Proposition>> me
                    : backwardDerivations.entrySet()) {
                qs.addDerivationsToCache(me.getKey(), me.getValue());
            }
        }
    }

    private class HandleQueryResultThread extends Thread {

        private final BlockingQueue<QueueObject> queue;
        private final Thread producerThread;
        private final QueueObject poisonPill;

        HandleQueryResultThread(BlockingQueue<QueueObject> queue, QueueObject poisonPill, Thread producerThread) {
            super("protempa.executor.HandleQueryResultThread");
            this.queue = queue;
            this.producerThread = producerThread;
            this.poisonPill = poisonPill;
        }

        @Override
        public void run() {
            log(Level.FINER, "Start handle query results thread");
            QueueObject qo;
            try {
                while ((qo = queue.take()) != poisonPill) {
                    log(Level.FINER, "Handling some results");
                    try {
                        resultsHandler.handleQueryResult(qo.keyId, qo.propositions, qo.forwardDerivations, qo.backwardDerivations, qo.refs);
                    } catch (QueryResultsHandlerProcessingException ex) {
                        log(Level.FINER, "Handle query results threw QueryResultsHandlerProcessingException", ex);
                        exceptions.add(new QueryException(Executor.this.query.getName(), ex));
                        producerThread.interrupt();
                        break;
                    } catch (Error | RuntimeException t) {
                        log(Level.FINER, "Handle query results threw exception", t);
                        exceptions.add(new QueryException(Executor.this.query.getName(), new QueryResultsHandlerProcessingException(t)));
                        producerThread.interrupt();
                        break;
                    }
                    log(Level.FINER, "Results passed to query result handler");
                }
            } catch (InterruptedException ex) {
                log(Level.FINER, "Handle query results thread interrupted", ex);
                producerThread.interrupt();
            }
            log(Level.FINER, "End handle query results thread");
        }

    };

    private DataStreamingEventIterator<Proposition> newDataIterator() throws QueryException {
        log(Level.INFO, "Retrieving data");
        Set<String> inDataSourcePropIds = new HashSet<>();
        for (PropositionDefinition pd : allNarrowerDescendants) {
            if (pd.getInDataSource()) {
                inDataSourcePropIds.add(pd.getId());
            }
        }
        if (isLoggable(Level.FINER)) {
            log(Level.FINER, "Asking data source for {0}", StringUtils.join(inDataSourcePropIds, ", "));
        }
        DataStreamingEventIterator<Proposition> itr;
        try {
            itr = abstractionFinder.getDataSource().readPropositions(this.keyIds, inDataSourcePropIds, this.filters, getQuerySession(), this.resultsHandler);
        } catch (DataSourceReadException ex) {
            throw new QueryException(this.query.getName(), ex);
        }
        return itr;
    }

    private void retain(Set<PropositionDefinition> propDefs) {
        if (this.propIdsToRetain != null) {
            Map<String, PropositionDefinition> propDefMap = new HashMap<>();
            for (PropositionDefinition and : propDefs) {
                propDefMap.put(and.getId(), and);
            }
            Queue<String> propIdsQueue = new LinkedList<>(this.propIds);
            String pid;
            Set<PropositionDefinition> propDefsToKeep = new HashSet<>();
            while ((pid = propIdsQueue.poll()) != null) {
                if (this.propIdsToRetain.contains(pid)) {
                    Queue<String> propIdsToKeep = new LinkedList<>();
                    propIdsToKeep.add(pid);
                    String pid2;
                    while ((pid2 = propIdsToKeep.poll()) != null) {
                        PropositionDefinition get = propDefMap.get(pid2);
                        propDefsToKeep.add(get);
                        Arrays.addAll(propIdsToKeep, get.getChildren());
                    }
                } else {
                    PropositionDefinition get = propDefMap.get(pid);
                    Arrays.addAll(propIdsQueue, get.getChildren());
                }
            }
            allNarrowerDescendants = propDefsToKeep;
        } else {
            allNarrowerDescendants = propDefs;
        }
    }

    private StatelessExecutionStrategy newStatelessStrategy() throws QueryException {
        StatelessExecutionStrategy result = new StatelessExecutionStrategy(abstractionFinder, abstractionFinder.getAlgorithmSource());
        try {
            createRuleBase(result);
        } catch (CreateRuleBaseException ex) {
            throw new QueryException(this.query.getName(), ex);
        }
        result.initialize();
        return result;
    }

    private StatefulExecutionStrategy newStatefulStrategy() throws QueryException {
        StatefulExecutionStrategy result = new StatefulExecutionStrategy(abstractionFinder.getAlgorithmSource());
        try {
            createRuleBase(result);
        } catch (CreateRuleBaseException ex) {
            throw new QueryException(this.query.getName(), ex);
        }
        result.initialize();
        return result;
    }

    private void createRuleBase(ExecutionStrategy result) throws CreateRuleBaseException {
        log(Level.FINEST, "Initializing rule base");
        result.createRuleBase(allNarrowerDescendants, derivationsBuilder, qs);
        abstractionFinder.clear();
        log(Level.FINEST, "Rule base initialized");
    }

}
