001/* 002 * The contents of this file are subject to the license and copyright 003 * detailed in the LICENSE and NOTICE files at the root of the source 004 * tree. 005 */ 006package org.fcrepo.persistence.ocfl.impl; 007 008import static org.slf4j.LoggerFactory.getLogger; 009 010import java.time.Duration; 011import java.time.Instant; 012import java.util.ArrayList; 013import java.util.Iterator; 014import java.util.List; 015import java.util.concurrent.TimeUnit; 016import java.util.concurrent.atomic.AtomicInteger; 017import java.util.stream.Stream; 018 019import org.fcrepo.common.db.DbTransactionExecutor; 020import org.fcrepo.config.OcflPropsConfig; 021import org.fcrepo.kernel.api.Transaction; 022import org.fcrepo.kernel.api.TransactionManager; 023 024import org.slf4j.Logger; 025 026/** 027 * Class to coordinate the index rebuilding tasks. 028 * @author whikloj 029 * @since 6.0.0 030 */ 031public class ReindexManager { 032 033 private static final Logger LOGGER = getLogger(ReindexManager.class); 034 035 private static final long REPORTING_INTERVAL_SECS = 300; 036 037 private final List<ReindexWorker> workers; 038 039 private final Iterator<String> ocflIter; 040 041 private final Stream<String> ocflStream; 042 043 private final AtomicInteger completedCount; 044 045 private final AtomicInteger errorCount; 046 047 private final ReindexService reindexService; 048 049 private final long batchSize; 050 051 private final boolean failOnError; 052 053 private TransactionManager txManager; 054 055 private DbTransactionExecutor dbTransactionExecutor; 056 057 private Transaction transaction = null; 058 059 /** 060 * Basic constructor 061 * @param ids stream of ocfl ids. 062 * @param reindexService the reindexing service. 063 * @param config OCFL property config object. 064 * @param manager the transaction manager object. 065 * @param dbTransactionExecutor manages db transactions 066 */ 067 public ReindexManager(final Stream<String> ids, 068 final ReindexService reindexService, 069 final OcflPropsConfig config, 070 final TransactionManager manager, 071 final DbTransactionExecutor dbTransactionExecutor) { 072 this.ocflStream = ids; 073 this.ocflIter = ocflStream.iterator(); 074 this.reindexService = reindexService; 075 this.batchSize = config.getReindexBatchSize(); 076 this.failOnError = config.isReindexFailOnError(); 077 txManager = manager; 078 this.dbTransactionExecutor = dbTransactionExecutor; 079 workers = new ArrayList<>(); 080 completedCount = new AtomicInteger(0); 081 errorCount = new AtomicInteger(0); 082 083 final var workerCount = config.getReindexingThreads(); 084 085 if (workerCount < 1) { 086 throw new IllegalStateException(String.format("Reindexing requires at least 1 thread. Found: %s", 087 workerCount)); 088 } 089 090 for (var i = 0; i < workerCount; i += 1) { 091 workers.add(new ReindexWorker("ReindexWorker-" + i, this, 092 this.reindexService, txManager, this.dbTransactionExecutor, this.failOnError)); 093 } 094 } 095 096 /** 097 * Start reindexing. 098 * @throws InterruptedException on an indexing error in a thread. 099 */ 100 public void start() throws InterruptedException { 101 final var reporter = startReporter(); 102 try { 103 workers.forEach(ReindexWorker::start); 104 for (final var worker : workers) { 105 worker.join(); 106 } 107 if (!failOnError || errorCount.get() == 0) { 108 indexMembership(); 109 } else { 110 LOGGER.error("Reindex did not complete successfully"); 111 } 112 } catch (final Exception e) { 113 LOGGER.error("Error while rebuilding index", e); 114 stop(); 115 throw e; 116 } finally { 117 reporter.interrupt(); 118 } 119 } 120 121 /** 122 * Stop all threads. 123 */ 124 public void stop() { 125 LOGGER.debug("Stop worker threads"); 126 workers.forEach(ReindexWorker::stopThread); 127 } 128 129 /** 130 * Return a batch of OCFL ids to reindex. 131 * @return list of OCFL ids. 132 */ 133 public synchronized List<String> getIds() { 134 int counter = 0; 135 final List<String> ids = new ArrayList<>((int) batchSize); 136 while (ocflIter.hasNext() && counter < batchSize) { 137 ids.add(ocflIter.next()); 138 counter += 1; 139 } 140 return ids; 141 } 142 143 /** 144 * Update the master list of reindexing states. 145 * @param batchSuccessful how many items were completed successfully in the last batch. 146 * @param batchErrors how many items had an error in the last batch. 147 */ 148 public void updateComplete(final int batchSuccessful, final int batchErrors) { 149 completedCount.addAndGet(batchSuccessful); 150 errorCount.addAndGet(batchErrors); 151 } 152 153 /** 154 * @return the count of items that completed successfully. 155 */ 156 public int getCompletedCount() { 157 return completedCount.get(); 158 } 159 160 /** 161 * @return the count of items that had errors. 162 */ 163 public int getErrorCount() { 164 return errorCount.get(); 165 } 166 167 /** 168 * Index the membership relationships 169 */ 170 private void indexMembership() { 171 final var tx = transaction(); 172 LOGGER.info("Starting membership indexing"); 173 reindexService.indexMembership(tx); 174 tx.commit(); 175 LOGGER.debug("Completed membership indexing"); 176 } 177 178 /** 179 * Close stream. 180 */ 181 public void shutdown() { 182 ocflStream.close(); 183 } 184 185 private Thread startReporter() { 186 final var reporter = new Thread(() -> { 187 final var startTime = Instant.now(); 188 try { 189 while (true) { 190 TimeUnit.SECONDS.sleep(REPORTING_INTERVAL_SECS); 191 final var complete = completedCount.get(); 192 final var errored = errorCount.get(); 193 final var now = Instant.now(); 194 final var duration = Duration.between(startTime, now); 195 LOGGER.info("Index rebuild progress: Complete: {}; Errored: {}; Time: {}; Rate: {}/s", 196 complete, errored, getDurationMessage(duration), 197 (complete + errored) / duration.getSeconds()); 198 } 199 } catch (final InterruptedException e) { 200 // processing has completed exit normally 201 } 202 }); 203 204 reporter.start(); 205 return reporter; 206 } 207 208 private String getDurationMessage(final Duration duration) { 209 String message = String.format("%d secs", duration.toSecondsPart()); 210 if (duration.getSeconds() > 60) { 211 message = String.format("%d mins, ", duration.toMinutesPart()) + message; 212 } 213 if (duration.getSeconds() > 3600) { 214 message = String.format("%d hours, ", duration.getSeconds() / 3600) + message; 215 } 216 return message; 217 } 218 219 private Transaction transaction() { 220 if (transaction == null) { 221 transaction = txManager.create(); 222 transaction.setShortLived(true); 223 } 224 return transaction; 225 } 226}