Source code

001/*
002 * The contents of this file are subject to the license and copyright
003 * detailed in the LICENSE and NOTICE files at the root of the source
004 * tree.
005 */
006
007package org.fcrepo.persistence.ocfl.impl;
008
009import java.util.Collections;
010import java.util.List;
011import java.util.Map;
012import java.util.concurrent.TimeUnit;
013
014import javax.annotation.Nonnull;
015import javax.annotation.PostConstruct;
016import javax.inject.Inject;
017import javax.sql.DataSource;
018
019import org.fcrepo.common.db.DbPlatform;
020import org.fcrepo.kernel.api.Transaction;
021import org.fcrepo.config.OcflPropsConfig;
022import org.fcrepo.kernel.api.exception.InvalidResourceIdentifierException;
023import org.fcrepo.kernel.api.exception.RepositoryRuntimeException;
024import org.fcrepo.kernel.api.identifiers.FedoraId;
025import org.fcrepo.persistence.ocfl.api.FedoraOcflMappingNotFoundException;
026import org.fcrepo.persistence.ocfl.api.FedoraToOcflObjectIndex;
027import org.fcrepo.storage.ocfl.cache.Cache;
028import org.fcrepo.storage.ocfl.cache.CaffeineCache;
029
030import org.slf4j.Logger;
031import org.slf4j.LoggerFactory;
032import org.springframework.beans.factory.annotation.Autowired;
033import org.springframework.dao.DataIntegrityViolationException;
034import org.springframework.dao.DuplicateKeyException;
035import org.springframework.dao.EmptyResultDataAccessException;
036import org.springframework.jdbc.BadSqlGrammarException;
037import org.springframework.jdbc.core.RowMapper;
038import org.springframework.jdbc.core.namedparam.MapSqlParameterSource;
039import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate;
040import org.springframework.stereotype.Component;
041import org.springframework.transaction.annotation.Propagation;
042import org.springframework.transaction.annotation.Transactional;
043
044import com.github.benmanes.caffeine.cache.Caffeine;
045
046/**
047 * Maps Fedora IDs to the OCFL IDs of the OCFL objects the Fedora resource is stored in. This implementation is backed
048 * by a relational database.
049 *
050 * @author pwinckles
051 */
052@Component("ocflIndexImpl")
053public class DbFedoraToOcflObjectIndex implements FedoraToOcflObjectIndex {
054
055    private static final Logger LOGGER = LoggerFactory.getLogger(DbFedoraToOcflObjectIndex.class);
056
057    private static final String MAPPING_TABLE = "ocfl_id_map";
058
059    private static final String FEDORA_ID_COLUMN = "fedora_id";
060
061    private static final String FEDORA_ROOT_ID_COLUMN = "fedora_root_id";
062
063    private static final String OCFL_ID_COLUMN = "ocfl_id";
064
065    private static final String TRANSACTION_OPERATIONS_TABLE = "ocfl_id_map_session_operations";
066
067    private static final String TRANSACTION_ID_COLUMN = "session_id";
068
069    private static final String OPERATION_COLUMN = "operation";
070
071    /*
072     * Lookup all mappings for the resource id.
073     */
074    private static final String LOOKUP_MAPPING = "SELECT " + FEDORA_ROOT_ID_COLUMN + ", " + OCFL_ID_COLUMN + " FROM " +
075            MAPPING_TABLE + " WHERE " + FEDORA_ID_COLUMN + " = :fedoraId";
076
077    /*
078     * Lookup all mappings from the mapping table as well as any new 'add's and excluding any 'delete's in this
079     * transaction.
080     */
081    private static final String LOOKUP_MAPPING_IN_TRANSACTION = "SELECT x." + FEDORA_ROOT_ID_COLUMN + "," +
082            " x." + OCFL_ID_COLUMN + " FROM" +
083            " (SELECT " + FEDORA_ROOT_ID_COLUMN + ", " + OCFL_ID_COLUMN + " FROM " + MAPPING_TABLE + " WHERE " +
084            FEDORA_ID_COLUMN + " = :fedoraId" +
085            " UNION SELECT " + FEDORA_ROOT_ID_COLUMN + ", " + OCFL_ID_COLUMN + " FROM " + TRANSACTION_OPERATIONS_TABLE +
086            " WHERE " + FEDORA_ID_COLUMN + " = :fedoraId AND " + TRANSACTION_ID_COLUMN + " = :transactionId" +
087            " AND " + OPERATION_COLUMN + " = 'add') x";
088
089    /*
090     * Add an 'add' operation to the transaction table.
091     */
092    private static final String UPSERT_MAPPING_TX_POSTGRESQL = "INSERT INTO " + TRANSACTION_OPERATIONS_TABLE +
093            " ( " + FEDORA_ID_COLUMN + ", " + FEDORA_ROOT_ID_COLUMN + ", " + OCFL_ID_COLUMN + ", " +
094            TRANSACTION_ID_COLUMN + ", " + OPERATION_COLUMN + ") VALUES (:fedoraId, :fedoraRootId, :ocflId," +
095            " :transactionId, :operation) ON CONFLICT (" + FEDORA_ID_COLUMN + ", " + TRANSACTION_ID_COLUMN + ")" +
096            " DO UPDATE SET " + FEDORA_ROOT_ID_COLUMN + " = EXCLUDED." + FEDORA_ROOT_ID_COLUMN + ", " +
097            OCFL_ID_COLUMN + " = EXCLUDED." + OCFL_ID_COLUMN + ", " + OPERATION_COLUMN + " = EXCLUDED." +
098            OPERATION_COLUMN;
099
100    private static final String UPSERT_MAPPING_TX_MYSQL_MARIA = "INSERT INTO " + TRANSACTION_OPERATIONS_TABLE +
101            " (" + FEDORA_ID_COLUMN + ", " + FEDORA_ROOT_ID_COLUMN + ", " + OCFL_ID_COLUMN + ", " +
102            TRANSACTION_ID_COLUMN + ", " + OPERATION_COLUMN + ")" +
103            " VALUES (:fedoraId, :fedoraRootId, :ocflId, :transactionId, :operation) ON DUPLICATE KEY UPDATE " +
104            FEDORA_ROOT_ID_COLUMN + " = VALUES(" + FEDORA_ROOT_ID_COLUMN + "), " + OCFL_ID_COLUMN + " = VALUES(" +
105            OCFL_ID_COLUMN + "), " + OPERATION_COLUMN + " = VALUES(" + OPERATION_COLUMN + ")";
106
107    private static final String UPSERT_MAPPING_TX_H2 = "MERGE INTO " + TRANSACTION_OPERATIONS_TABLE +
108            " (" + FEDORA_ID_COLUMN + ", " + FEDORA_ROOT_ID_COLUMN + ", " + OCFL_ID_COLUMN + ", " +
109            TRANSACTION_ID_COLUMN + ", " + OPERATION_COLUMN + ")" +
110            " KEY (" + FEDORA_ID_COLUMN + ", " + TRANSACTION_ID_COLUMN + ")" +
111            " VALUES (:fedoraId, :fedoraRootId, :ocflId, :transactionId, :operation)";
112
113    private static final String DIRECT_INSERT_MAPPING = "INSERT INTO " + MAPPING_TABLE +
114            " (" + FEDORA_ID_COLUMN + ", " + FEDORA_ROOT_ID_COLUMN + ", " + OCFL_ID_COLUMN + ")" +
115            " VALUES (:fedoraId, :fedoraRootId, :ocflId)";
116
117    private static final String DIRECT_INSERT_POSTGRESQL = " ON CONFLICT (" + FEDORA_ID_COLUMN + ")" +
118            " DO UPDATE SET " + FEDORA_ROOT_ID_COLUMN + " = EXCLUDED." + FEDORA_ROOT_ID_COLUMN + ", " +
119            OCFL_ID_COLUMN + " = EXCLUDED." + OCFL_ID_COLUMN;
120
121    private static final String DIRECT_INSERT_MYSQL_MARIA = " ON DUPLICATE KEY UPDATE " +
122            FEDORA_ROOT_ID_COLUMN + " = VALUES(" + FEDORA_ROOT_ID_COLUMN + "), " + OCFL_ID_COLUMN +
123            " = VALUES(" + OCFL_ID_COLUMN + ")";
124
125    private static final String DIRECT_INSERT_H2 = "MERGE INTO " + MAPPING_TABLE +
126            " (" + FEDORA_ID_COLUMN + ", " + FEDORA_ROOT_ID_COLUMN + ", " + OCFL_ID_COLUMN + ") " +
127            " KEY (" + FEDORA_ID_COLUMN + ")" +
128            " VALUES (:fedoraId, :fedoraRootId, :ocflId)";
129
130    private static final Map<DbPlatform, String> DIRECT_INSERT_MAP = Map.of(
131        DbPlatform.H2, DIRECT_INSERT_H2,
132        DbPlatform.MYSQL, DIRECT_INSERT_MAPPING + DIRECT_INSERT_MYSQL_MARIA,
133        DbPlatform.MARIADB, DIRECT_INSERT_MAPPING + DIRECT_INSERT_MYSQL_MARIA,
134        DbPlatform.POSTGRESQL, DIRECT_INSERT_MAPPING + DIRECT_INSERT_POSTGRESQL
135    );
136
137    /**
138     * Map of database product to UPSERT into operations table SQL.
139     */
140    private static final Map<DbPlatform, String> UPSERT_MAPPING_TX_MAP = Map.of(
141            DbPlatform.MYSQL, UPSERT_MAPPING_TX_MYSQL_MARIA,
142            DbPlatform.H2, UPSERT_MAPPING_TX_H2,
143            DbPlatform.POSTGRESQL, UPSERT_MAPPING_TX_POSTGRESQL,
144            DbPlatform.MARIADB, UPSERT_MAPPING_TX_MYSQL_MARIA
145    );
146
147    private static final String DIRECT_DELETE_MAPPING = "DELETE FROM ocfl_id_map WHERE fedora_id = :fedoraId";
148
149    private static final String COMMIT_ADD_MAPPING_POSTGRESQL = "INSERT INTO " + MAPPING_TABLE +
150            " ( " + FEDORA_ID_COLUMN + ", " + FEDORA_ROOT_ID_COLUMN + ", " + OCFL_ID_COLUMN + ") SELECT " +
151            FEDORA_ID_COLUMN + ", " + FEDORA_ROOT_ID_COLUMN + ", " + OCFL_ID_COLUMN + " FROM " +
152            TRANSACTION_OPERATIONS_TABLE + " WHERE " + OPERATION_COLUMN + " = 'add' AND " + TRANSACTION_ID_COLUMN +
153            " = :transactionId ON CONFLICT ( " +  FEDORA_ID_COLUMN + " )" +
154            " DO UPDATE SET " + FEDORA_ROOT_ID_COLUMN + " = EXCLUDED." + FEDORA_ROOT_ID_COLUMN + ", " +
155            OCFL_ID_COLUMN + " = EXCLUDED." + OCFL_ID_COLUMN;
156
157    private static final String COMMIT_ADD_MAPPING_MYSQL_MARIA = "INSERT INTO " + MAPPING_TABLE +
158            " (" + FEDORA_ID_COLUMN + ", " + FEDORA_ROOT_ID_COLUMN + ", " + OCFL_ID_COLUMN + ") SELECT " +
159            FEDORA_ID_COLUMN + ", " + FEDORA_ROOT_ID_COLUMN + ", " + OCFL_ID_COLUMN + " FROM " +
160            TRANSACTION_OPERATIONS_TABLE + " WHERE " + OPERATION_COLUMN + " = 'add' AND " + TRANSACTION_ID_COLUMN +
161            " = :transactionId ON DUPLICATE KEY UPDATE " +
162            FEDORA_ROOT_ID_COLUMN + " = VALUES(" + FEDORA_ROOT_ID_COLUMN + "), " + OCFL_ID_COLUMN + " = VALUES(" +
163            OCFL_ID_COLUMN + ")";
164
165    private static final String COMMIT_ADD_MAPPING_H2 = "MERGE INTO " + MAPPING_TABLE +
166            " (" + FEDORA_ID_COLUMN + ", " + FEDORA_ROOT_ID_COLUMN + ", " + OCFL_ID_COLUMN + ")" +
167            " SELECT " + FEDORA_ID_COLUMN + ", " + FEDORA_ROOT_ID_COLUMN + ", " + OCFL_ID_COLUMN + " FROM " +
168            TRANSACTION_OPERATIONS_TABLE + " WHERE " + OPERATION_COLUMN + " = 'add'";
169
170    /**
171     * Map of database product name to COMMIT to mapping table from operations table
172     */
173    private static final Map<DbPlatform, String> COMMIT_ADD_MAPPING_MAP = Map.of(
174            DbPlatform.MYSQL, COMMIT_ADD_MAPPING_MYSQL_MARIA,
175            DbPlatform.H2, COMMIT_ADD_MAPPING_H2,
176            DbPlatform.POSTGRESQL, COMMIT_ADD_MAPPING_POSTGRESQL,
177            DbPlatform.MARIADB, COMMIT_ADD_MAPPING_MYSQL_MARIA
178    );
179
180    /*
181     * Delete records from the mapping table that are to be deleted in this transaction.
182     */
183    private static final String COMMIT_DELETE_RECORDS = "DELETE FROM " + MAPPING_TABLE + " WHERE " +
184            "EXISTS (SELECT * FROM " + TRANSACTION_OPERATIONS_TABLE + " WHERE " +
185            TRANSACTION_ID_COLUMN + " = :transactionId AND " +  OPERATION_COLUMN + " = 'delete' AND " +
186            MAPPING_TABLE + "." + FEDORA_ID_COLUMN + " = " + TRANSACTION_OPERATIONS_TABLE + "." + FEDORA_ID_COLUMN +
187            ")";
188
189    /*
190     * Collect IDs to invalidate on transaction commit.
191     */
192    private static final String GET_DELETE_IDS = "SELECT " + FEDORA_ID_COLUMN + " FROM " +
193            TRANSACTION_OPERATIONS_TABLE + " WHERE " + TRANSACTION_ID_COLUMN + " = :transactionId AND " +
194            OPERATION_COLUMN + " = 'delete'";
195
196    private static final String TRUNCATE_MAPPINGS = "TRUNCATE TABLE " + MAPPING_TABLE;
197
198    private static final String TRUNCATE_TRANSACTIONS = "TRUNCATE TABLE " + TRANSACTION_OPERATIONS_TABLE;
199
200    /*
201     * Delete all records from the transaction table for the specified transaction.
202     */
203    private static final String DELETE_ENTIRE_TRANSACTION = "DELETE FROM " + TRANSACTION_OPERATIONS_TABLE + " WHERE " +
204            TRANSACTION_ID_COLUMN + " = :transactionId";
205
206    /*
207     * Row mapper for the Lookup queries.
208     */
209    private static final RowMapper<FedoraOcflMapping> GET_MAPPING_ROW_MAPPER = (resultSet, i) -> new FedoraOcflMapping(
210            FedoraId.create(resultSet.getString(1)),
211            resultSet.getString(2)
212    );
213
214    private Cache<String, FedoraOcflMapping> mappingCache;
215
216    private final DataSource dataSource;
217
218    private final NamedParameterJdbcTemplate jdbcTemplate;
219
220    private DbPlatform dbPlatform;
221
222    @Inject
223    private OcflPropsConfig ocflPropsConfig;
224
225    public DbFedoraToOcflObjectIndex(@Autowired final DataSource dataSource) {
226        this.dataSource = dataSource;
227        this.jdbcTemplate = new NamedParameterJdbcTemplate(dataSource);
228    }
229
230    @PostConstruct
231    public void setup() {
232        dbPlatform = DbPlatform.fromDataSource(dataSource);
233        final var cache = Caffeine.newBuilder()
234                .maximumSize(ocflPropsConfig.getFedoraToOcflCacheSize())
235                .expireAfterAccess(ocflPropsConfig.getFedoraToOcflCacheTimeout(), TimeUnit.MINUTES)
236                .build();
237        this.mappingCache = new CaffeineCache<>(cache);
238    }
239
240    @Override
241    public FedoraOcflMapping getMapping(final Transaction transaction, final FedoraId fedoraId)
242            throws FedoraOcflMappingNotFoundException {
243        try {
244            if (transaction.isOpenLongRunning()) {
245                final MapSqlParameterSource parameterSource = new MapSqlParameterSource();
246                parameterSource.addValue("fedoraId", fedoraId.getResourceId());
247                parameterSource.addValue("transactionId", transaction.getId());
248                return jdbcTemplate.queryForObject(LOOKUP_MAPPING_IN_TRANSACTION, parameterSource,
249                        GET_MAPPING_ROW_MAPPER);
250            } else {
251                return this.mappingCache.get(fedoraId.getResourceId(), key ->
252                        jdbcTemplate.queryForObject(LOOKUP_MAPPING, Map.of("fedoraId", key), GET_MAPPING_ROW_MAPPER)
253                );
254            }
255        } catch (final EmptyResultDataAccessException e) {
256            throw new FedoraOcflMappingNotFoundException("No OCFL mapping found for " + fedoraId);
257        }
258    }
259
260    @Override
261    public FedoraOcflMapping addMapping(@Nonnull final Transaction transaction, final FedoraId fedoraId,
262                                        final FedoraId fedoraRootId, final String ocflId) {
263        transaction.doInTx(() -> {
264            if (!transaction.isShortLived()) {
265                upsert(transaction, fedoraId, "add", fedoraRootId, ocflId);
266            } else {
267                directInsert(fedoraId, fedoraRootId, ocflId);
268            }
269        });
270
271        return new FedoraOcflMapping(fedoraRootId, ocflId);
272    }
273
274    @Override
275    public void removeMapping(@Nonnull final Transaction transaction, final FedoraId fedoraId) {
276        transaction.doInTx(() -> {
277            if (!transaction.isShortLived()) {
278                upsert(transaction, fedoraId, "delete");
279            } else {
280                final MapSqlParameterSource parameterSource = new MapSqlParameterSource();
281                parameterSource.addValue("fedoraId", fedoraId.getResourceId());
282                jdbcTemplate.update(DIRECT_DELETE_MAPPING, parameterSource);
283                this.mappingCache.invalidate(fedoraId.getResourceId());
284            }
285        });
286    }
287
288    private void upsert(final Transaction transaction, final FedoraId fedoraId, final String operation) {
289        upsert(transaction, fedoraId, operation, null, null);
290    }
291
292    /**
293     * Perform the upsert to the operations table.
294     *
295     * @param transaction the transaction/session id.
296     * @param fedoraId the resource id.
297     * @param operation the operation we are performing (add or delete)
298     * @param fedoraRootId the fedora root id (for add only)
299     * @param ocflId the ocfl id (for add only).
300     */
301    private void upsert(final Transaction transaction, final FedoraId fedoraId, final String operation,
302                        final FedoraId fedoraRootId, final String ocflId) {
303        final MapSqlParameterSource parameterSource = new MapSqlParameterSource();
304        parameterSource.addValue("fedoraId", fedoraId.getResourceId());
305        parameterSource.addValue("fedoraRootId", fedoraRootId == null ? null : fedoraRootId.getResourceId());
306        parameterSource.addValue("ocflId", ocflId);
307        parameterSource.addValue("transactionId", transaction.getId());
308        parameterSource.addValue("operation", operation);
309        try {
310            jdbcTemplate.update(UPSERT_MAPPING_TX_MAP.get(dbPlatform), parameterSource);
311        } catch (final DataIntegrityViolationException | BadSqlGrammarException e) {
312            handleInsertException(fedoraId, e);
313        }
314    }
315
316    private void directInsert(final FedoraId fedoraId, final FedoraId fedoraRootId, final String ocflId) {
317        final MapSqlParameterSource parameterSource = new MapSqlParameterSource();
318        parameterSource.addValue("fedoraId", fedoraId.getResourceId());
319        parameterSource.addValue("fedoraRootId", fedoraRootId == null ? null : fedoraRootId.getResourceId());
320        parameterSource.addValue("ocflId", ocflId);
321        try {
322            jdbcTemplate.update(DIRECT_INSERT_MAP.get(dbPlatform), parameterSource);
323        } catch (final DataIntegrityViolationException | BadSqlGrammarException e) {
324            handleInsertException(fedoraId, e);
325        }
326    }
327
328    @Override
329    public void reset() {
330        try {
331            jdbcTemplate.update(TRUNCATE_MAPPINGS, Collections.emptyMap());
332            jdbcTemplate.update(TRUNCATE_TRANSACTIONS, Collections.emptyMap());
333            this.mappingCache.invalidateAll();
334        } catch (final Exception e) {
335            throw new RepositoryRuntimeException("Failed to truncate FedoraToOcfl index tables", e);
336        }
337    }
338
339    @Override
340    public void commit(@Nonnull final Transaction transaction) {
341        if (!transaction.isShortLived()) {
342            transaction.ensureCommitting();
343
344            LOGGER.debug("Committing FedoraToOcfl index changes from transaction {}", transaction.getId());
345            final Map<String, String> map = Map.of("transactionId", transaction.getId());
346            try {
347                final List<String> deleteIds = jdbcTemplate.queryForList(GET_DELETE_IDS, map, String.class);
348                jdbcTemplate.update(COMMIT_DELETE_RECORDS, map);
349                jdbcTemplate.update(COMMIT_ADD_MAPPING_MAP.get(dbPlatform), map);
350                jdbcTemplate.update(DELETE_ENTIRE_TRANSACTION, map);
351                this.mappingCache.invalidateAll(deleteIds);
352            } catch (final Exception e) {
353                LOGGER.warn("Unable to commit FedoraToOcfl index transaction {}: {}", transaction, e.getMessage());
354                throw new RepositoryRuntimeException("Unable to commit FedoraToOcfl index transaction", e);
355            }
356        }
357    }
358
359    @Transactional(propagation = Propagation.NOT_SUPPORTED)
360    @Override
361    public void rollback(@Nonnull final Transaction transaction) {
362        if (!transaction.isShortLived()) {
363            jdbcTemplate.update(DELETE_ENTIRE_TRANSACTION, Map.of("transactionId", transaction.getId()));
364        }
365    }
366
367    private void handleInsertException(final FedoraId fedoraId, final Exception e) {
368        if (e.getMessage().contains("too long for")) {
369            throw new InvalidResourceIdentifierException("Database error - Fedora ID path too long",e);
370        } else if (e instanceof DuplicateKeyException) {
371            throw new RepositoryRuntimeException("Database error - primary key already exists for Fedora ID: " +
372                                                 fedoraId, e);
373        } else {
374            throw new RepositoryRuntimeException("Database error - error during upsert",e);
375        }
376    }
377
378    @Override
379    public void clearAllTransactions() {
380        try {
381            jdbcTemplate.update(TRUNCATE_TRANSACTIONS, Collections.emptyMap());
382        } catch (final Exception e) {
383            throw new RepositoryRuntimeException("Failed to truncate FedoraToOcfl transactions index tables", e);
384        }
385    }
386}