001/*
002 * The contents of this file are subject to the license and copyright
003 * detailed in the LICENSE and NOTICE files at the root of the source
004 * tree.
005 */
006package org.fcrepo.persistence.ocfl.impl;
007
008import static org.slf4j.LoggerFactory.getLogger;
009
010import java.time.Duration;
011import java.time.Instant;
012import java.util.ArrayList;
013import java.util.Iterator;
014import java.util.List;
015import java.util.concurrent.TimeUnit;
016import java.util.concurrent.atomic.AtomicInteger;
017import java.util.stream.Stream;
018
019import org.fcrepo.common.db.DbTransactionExecutor;
020import org.fcrepo.config.OcflPropsConfig;
021import org.fcrepo.kernel.api.Transaction;
022import org.fcrepo.kernel.api.TransactionManager;
023
024import org.slf4j.Logger;
025
026/**
027 * Class to coordinate the index rebuilding tasks.
028 * @author whikloj
029 * @since 6.0.0
030 */
031public class ReindexManager {
032
033    private static final Logger LOGGER = getLogger(ReindexManager.class);
034
035    private static final long REPORTING_INTERVAL_SECS = 300;
036
037    private final List<ReindexWorker> workers;
038
039    private final Iterator<String> ocflIter;
040
041    private final Stream<String> ocflStream;
042
043    private final AtomicInteger completedCount;
044
045    private final AtomicInteger errorCount;
046
047    private final ReindexService reindexService;
048
049    private final long batchSize;
050
051    private final boolean failOnError;
052
053    private TransactionManager txManager;
054
055    private DbTransactionExecutor dbTransactionExecutor;
056
057    private Transaction transaction = null;
058
059    /**
060     * Basic constructor
061     * @param ids stream of ocfl ids.
062     * @param reindexService the reindexing service.
063     * @param config OCFL property config object.
064     * @param manager the transaction manager object.
065     * @param dbTransactionExecutor manages db transactions
066     */
067    public ReindexManager(final Stream<String> ids,
068                          final ReindexService reindexService,
069                          final OcflPropsConfig config,
070                          final TransactionManager manager,
071                          final DbTransactionExecutor dbTransactionExecutor) {
072        this.ocflStream = ids;
073        this.ocflIter = ocflStream.iterator();
074        this.reindexService = reindexService;
075        this.batchSize = config.getReindexBatchSize();
076        this.failOnError = config.isReindexFailOnError();
077        txManager = manager;
078        this.dbTransactionExecutor = dbTransactionExecutor;
079        workers = new ArrayList<>();
080        completedCount = new AtomicInteger(0);
081        errorCount = new AtomicInteger(0);
082
083        final var workerCount = config.getReindexingThreads();
084
085        if (workerCount < 1) {
086            throw new IllegalStateException(String.format("Reindexing requires at least 1 thread. Found: %s",
087                    workerCount));
088        }
089
090        for (var i = 0; i < workerCount; i += 1) {
091            workers.add(new ReindexWorker("ReindexWorker-" + i, this,
092                    this.reindexService, txManager, this.dbTransactionExecutor, this.failOnError));
093        }
094    }
095
096    /**
097     * Start reindexing.
098     * @throws InterruptedException on an indexing error in a thread.
099     */
100    public void start() throws InterruptedException {
101        final var reporter = startReporter();
102        try {
103            workers.forEach(ReindexWorker::start);
104            for (final var worker : workers) {
105                worker.join();
106            }
107            if (!failOnError || errorCount.get() == 0) {
108                indexMembership();
109            } else {
110                LOGGER.error("Reindex did not complete successfully");
111            }
112        } catch (final Exception e) {
113            LOGGER.error("Error while rebuilding index", e);
114            stop();
115            throw e;
116        } finally {
117            reporter.interrupt();
118        }
119    }
120
121    /**
122     * Stop all threads.
123     */
124    public void stop() {
125        LOGGER.debug("Stop worker threads");
126        workers.forEach(ReindexWorker::stopThread);
127    }
128
129    /**
130     * Return a batch of OCFL ids to reindex.
131     * @return list of OCFL ids.
132     */
133    public synchronized List<String> getIds() {
134        int counter = 0;
135        final List<String> ids = new ArrayList<>((int) batchSize);
136        while (ocflIter.hasNext() && counter < batchSize) {
137            ids.add(ocflIter.next());
138            counter += 1;
139        }
140        return ids;
141    }
142
143    /**
144     * Update the master list of reindexing states.
145     * @param batchSuccessful how many items were completed successfully in the last batch.
146     * @param batchErrors how many items had an error in the last batch.
147     */
148    public void updateComplete(final int batchSuccessful, final int batchErrors) {
149        completedCount.addAndGet(batchSuccessful);
150        errorCount.addAndGet(batchErrors);
151    }
152
153    /**
154     * @return the count of items that completed successfully.
155     */
156    public int getCompletedCount() {
157        return completedCount.get();
158    }
159
160    /**
161     * @return the count of items that had errors.
162     */
163    public int getErrorCount() {
164        return errorCount.get();
165    }
166
167    /**
168     * Index the membership relationships
169     */
170    private void indexMembership() {
171        final var tx = transaction();
172        LOGGER.info("Starting membership indexing");
173        reindexService.indexMembership(tx);
174        tx.commit();
175        LOGGER.debug("Completed membership indexing");
176    }
177
178    /**
179     * Close stream.
180     */
181    public void shutdown() {
182        ocflStream.close();
183    }
184
185    private Thread startReporter() {
186        final var reporter = new Thread(() -> {
187            final var startTime = Instant.now();
188            try {
189                while (true) {
190                    TimeUnit.SECONDS.sleep(REPORTING_INTERVAL_SECS);
191                    final var complete = completedCount.get();
192                    final var errored = errorCount.get();
193                    final var now = Instant.now();
194                    final var duration = Duration.between(startTime, now);
195                    LOGGER.info("Index rebuild progress: Complete: {}; Errored: {}; Time: {}; Rate: {}/s",
196                            complete, errored, getDurationMessage(duration),
197                            (complete + errored) / duration.getSeconds());
198                }
199            } catch (final InterruptedException e) {
200                // processing has completed exit normally
201            }
202        });
203
204        reporter.start();
205        return reporter;
206    }
207
208    private String getDurationMessage(final Duration duration) {
209        String message = String.format("%d secs", duration.toSecondsPart());
210        if (duration.getSeconds() > 60) {
211            message = String.format("%d mins, ", duration.toMinutesPart()) + message;
212        }
213        if (duration.getSeconds() > 3600) {
214            message = String.format("%d hours, ", duration.getSeconds() / 3600) + message;
215        }
216        return message;
217    }
218
219    private Transaction transaction() {
220        if (transaction == null) {
221            transaction = txManager.create();
222            transaction.setShortLived(true);
223        }
224        return transaction;
225    }
226}