Source code

001/*
002 * The contents of this file are subject to the license and copyright
003 * detailed in the LICENSE and NOTICE files at the root of the source
004 * tree.
005 */
006
007package org.fcrepo.persistence.ocfl.impl;
008
009import java.util.Collections;
010import java.util.List;
011import java.util.Map;
012import java.util.concurrent.TimeUnit;
013
014import javax.annotation.Nonnull;
015import javax.annotation.PostConstruct;
016import javax.inject.Inject;
017import javax.sql.DataSource;
018
019import org.fcrepo.common.db.DbPlatform;
020import org.fcrepo.kernel.api.Transaction;
021import org.fcrepo.config.OcflPropsConfig;
022import org.fcrepo.kernel.api.exception.InvalidResourceIdentifierException;
023import org.fcrepo.kernel.api.exception.RepositoryRuntimeException;
024import org.fcrepo.kernel.api.identifiers.FedoraId;
025import org.fcrepo.persistence.ocfl.api.FedoraOcflMappingNotFoundException;
026import org.fcrepo.persistence.ocfl.api.FedoraToOcflObjectIndex;
027import org.fcrepo.storage.ocfl.cache.Cache;
028import org.fcrepo.storage.ocfl.cache.CaffeineCache;
029
030import org.slf4j.Logger;
031import org.slf4j.LoggerFactory;
032import org.springframework.beans.factory.annotation.Autowired;
033import org.springframework.dao.DataIntegrityViolationException;
034import org.springframework.dao.DuplicateKeyException;
035import org.springframework.dao.EmptyResultDataAccessException;
036import org.springframework.jdbc.BadSqlGrammarException;
037import org.springframework.jdbc.core.RowMapper;
038import org.springframework.jdbc.core.namedparam.MapSqlParameterSource;
039import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate;
040import org.springframework.stereotype.Component;
041import org.springframework.transaction.annotation.Propagation;
042import org.springframework.transaction.annotation.Transactional;
043
044import com.github.benmanes.caffeine.cache.Caffeine;
045
046/**
047 * Maps Fedora IDs to the OCFL IDs of the OCFL objects the Fedora resource is stored in. This implementation is backed
048 * by a relational database.
049 *
050 * @author pwinckles
051 */
052@Component("ocflIndexImpl")
053public class DbFedoraToOcflObjectIndex implements FedoraToOcflObjectIndex {
054
055    private static final Logger LOGGER = LoggerFactory.getLogger(DbFedoraToOcflObjectIndex.class);
056
057    private static final String MAPPING_TABLE = "ocfl_id_map";
058
059    private static final String FEDORA_ID_COLUMN = "fedora_id";
060
061    private static final String FEDORA_ROOT_ID_COLUMN = "fedora_root_id";
062
063    private static final String OCFL_ID_COLUMN = "ocfl_id";
064
065    private static final String TRANSACTION_OPERATIONS_TABLE = "ocfl_id_map_session_operations";
066
067    private static final String TRANSACTION_ID_COLUMN = "session_id";
068
069    private static final String OPERATION_COLUMN = "operation";
070
071    /*
072     * Lookup all mappings for the resource id.
073     */
074    private static final String LOOKUP_MAPPING = "SELECT " + FEDORA_ROOT_ID_COLUMN + ", " + OCFL_ID_COLUMN + " FROM " +
075            MAPPING_TABLE + " WHERE " + FEDORA_ID_COLUMN + " = :fedoraId";
076
077    /*
078     * Lookup all mappings from the mapping table as well as any new 'add's and excluding any 'delete's in this
079     * transaction.
080     */
081    private static final String LOOKUP_MAPPING_IN_TRANSACTION = "SELECT x." + FEDORA_ROOT_ID_COLUMN + "," +
082            " x." + OCFL_ID_COLUMN + " FROM" +
083            " (SELECT " + FEDORA_ROOT_ID_COLUMN + ", " + OCFL_ID_COLUMN + " FROM " + MAPPING_TABLE + " WHERE " +
084            FEDORA_ID_COLUMN + " = :fedoraId" +
085            " UNION SELECT " + FEDORA_ROOT_ID_COLUMN + ", " + OCFL_ID_COLUMN + " FROM " + TRANSACTION_OPERATIONS_TABLE +
086            " WHERE " + FEDORA_ID_COLUMN + " = :fedoraId AND " + TRANSACTION_ID_COLUMN + " = :transactionId" +
087            " AND " + OPERATION_COLUMN + " = 'add') x";
088
089    /*
090     * Add an 'add' operation to the transaction table.
091     */
092    private static final String UPSERT_MAPPING_TX_POSTGRESQL = "INSERT INTO " + TRANSACTION_OPERATIONS_TABLE +
093            " ( " + FEDORA_ID_COLUMN + ", " + FEDORA_ROOT_ID_COLUMN + ", " + OCFL_ID_COLUMN + ", " +
094            TRANSACTION_ID_COLUMN + ", " + OPERATION_COLUMN + ") VALUES (:fedoraId, :fedoraRootId, :ocflId," +
095            " :transactionId, :operation) ON CONFLICT (" + FEDORA_ID_COLUMN + ", " + TRANSACTION_ID_COLUMN + ")" +
096            " DO UPDATE SET " + FEDORA_ROOT_ID_COLUMN + " = EXCLUDED." + FEDORA_ROOT_ID_COLUMN + ", " +
097            OCFL_ID_COLUMN + " = EXCLUDED." + OCFL_ID_COLUMN + ", " + OPERATION_COLUMN + " = EXCLUDED." +
098            OPERATION_COLUMN;
099
100    private static final String UPSERT_MAPPING_TX_MYSQL_MARIA = "INSERT INTO " + TRANSACTION_OPERATIONS_TABLE +
101            " (" + FEDORA_ID_COLUMN + ", " + FEDORA_ROOT_ID_COLUMN + ", " + OCFL_ID_COLUMN + ", " +
102            TRANSACTION_ID_COLUMN + ", " + OPERATION_COLUMN + ")" +
103            " VALUES (:fedoraId, :fedoraRootId, :ocflId, :transactionId, :operation) ON DUPLICATE KEY UPDATE " +
104            FEDORA_ROOT_ID_COLUMN + " = VALUES(" + FEDORA_ROOT_ID_COLUMN + "), " + OCFL_ID_COLUMN + " = VALUES(" +
105            OCFL_ID_COLUMN + "), " + OPERATION_COLUMN + " = VALUES(" + OPERATION_COLUMN + ")";
106
107    private static final String UPSERT_MAPPING_TX_H2 = "MERGE INTO " + TRANSACTION_OPERATIONS_TABLE +
108            " (" + FEDORA_ID_COLUMN + ", " + FEDORA_ROOT_ID_COLUMN + ", " + OCFL_ID_COLUMN + ", " +
109            TRANSACTION_ID_COLUMN + ", " + OPERATION_COLUMN + ")" +
110            " KEY (" + FEDORA_ID_COLUMN + ", " + TRANSACTION_ID_COLUMN + ")" +
111            " VALUES (:fedoraId, :fedoraRootId, :ocflId, :transactionId, :operation)";
112
113    private static final String DIRECT_INSERT_MAPPING = "INSERT INTO " + MAPPING_TABLE +
114            " (" + FEDORA_ID_COLUMN + ", " + FEDORA_ROOT_ID_COLUMN + ", " + OCFL_ID_COLUMN + ")" +
115            " VALUES (:fedoraId, :fedoraRootId, :ocflId)";
116
117    /**
118     * Map of database product to UPSERT into operations table SQL.
119     */
120    private static final Map<DbPlatform, String> UPSERT_MAPPING_TX_MAP = Map.of(
121            DbPlatform.MYSQL, UPSERT_MAPPING_TX_MYSQL_MARIA,
122            DbPlatform.H2, UPSERT_MAPPING_TX_H2,
123            DbPlatform.POSTGRESQL, UPSERT_MAPPING_TX_POSTGRESQL,
124            DbPlatform.MARIADB, UPSERT_MAPPING_TX_MYSQL_MARIA
125    );
126
127    private static final String DIRECT_DELETE_MAPPING = "DELETE FROM ocfl_id_map WHERE fedora_id = :fedoraId";
128
129    private static final String COMMIT_ADD_MAPPING_POSTGRESQL = "INSERT INTO " + MAPPING_TABLE +
130            " ( " + FEDORA_ID_COLUMN + ", " + FEDORA_ROOT_ID_COLUMN + ", " + OCFL_ID_COLUMN + ") SELECT " +
131            FEDORA_ID_COLUMN + ", " + FEDORA_ROOT_ID_COLUMN + ", " + OCFL_ID_COLUMN + " FROM " +
132            TRANSACTION_OPERATIONS_TABLE + " WHERE " + OPERATION_COLUMN + " = 'add' AND " + TRANSACTION_ID_COLUMN +
133            " = :transactionId ON CONFLICT ( " +  FEDORA_ID_COLUMN + " )" +
134            " DO UPDATE SET " + FEDORA_ROOT_ID_COLUMN + " = EXCLUDED." + FEDORA_ROOT_ID_COLUMN + ", " +
135            OCFL_ID_COLUMN + " = EXCLUDED." + OCFL_ID_COLUMN;
136
137    private static final String COMMIT_ADD_MAPPING_MYSQL_MARIA = "INSERT INTO " + MAPPING_TABLE +
138            " (" + FEDORA_ID_COLUMN + ", " + FEDORA_ROOT_ID_COLUMN + ", " + OCFL_ID_COLUMN + ") SELECT " +
139            FEDORA_ID_COLUMN + ", " + FEDORA_ROOT_ID_COLUMN + ", " + OCFL_ID_COLUMN + " FROM " +
140            TRANSACTION_OPERATIONS_TABLE + " WHERE " + OPERATION_COLUMN + " = 'add' AND " + TRANSACTION_ID_COLUMN +
141            " = :transactionId ON DUPLICATE KEY UPDATE " +
142            FEDORA_ROOT_ID_COLUMN + " = VALUES(" + FEDORA_ROOT_ID_COLUMN + "), " + OCFL_ID_COLUMN + " = VALUES(" +
143            OCFL_ID_COLUMN + ")";
144
145    private static final String COMMIT_ADD_MAPPING_H2 = "MERGE INTO " + MAPPING_TABLE +
146            " (" + FEDORA_ID_COLUMN + ", " + FEDORA_ROOT_ID_COLUMN + ", " + OCFL_ID_COLUMN + ")" +
147            " SELECT " + FEDORA_ID_COLUMN + ", " + FEDORA_ROOT_ID_COLUMN + ", " + OCFL_ID_COLUMN + " FROM " +
148            TRANSACTION_OPERATIONS_TABLE + " WHERE " + OPERATION_COLUMN + " = 'add'";
149
150    /**
151     * Map of database product name to COMMIT to mapping table from operations table
152     */
153    private static final Map<DbPlatform, String> COMMIT_ADD_MAPPING_MAP = Map.of(
154            DbPlatform.MYSQL, COMMIT_ADD_MAPPING_MYSQL_MARIA,
155            DbPlatform.H2, COMMIT_ADD_MAPPING_H2,
156            DbPlatform.POSTGRESQL, COMMIT_ADD_MAPPING_POSTGRESQL,
157            DbPlatform.MARIADB, COMMIT_ADD_MAPPING_MYSQL_MARIA
158    );
159
160    /*
161     * Delete records from the mapping table that are to be deleted in this transaction.
162     */
163    private static final String COMMIT_DELETE_RECORDS = "DELETE FROM " + MAPPING_TABLE + " WHERE " +
164            "EXISTS (SELECT * FROM " + TRANSACTION_OPERATIONS_TABLE + " WHERE " +
165            TRANSACTION_ID_COLUMN + " = :transactionId AND " +  OPERATION_COLUMN + " = 'delete' AND " +
166            MAPPING_TABLE + "." + FEDORA_ID_COLUMN + " = " + TRANSACTION_OPERATIONS_TABLE + "." + FEDORA_ID_COLUMN +
167            ")";
168
169    /*
170     * Collect IDs to invalidate on transaction commit.
171     */
172    private static final String GET_DELETE_IDS = "SELECT " + FEDORA_ID_COLUMN + " FROM " +
173            TRANSACTION_OPERATIONS_TABLE + " WHERE " + TRANSACTION_ID_COLUMN + " = :transactionId AND " +
174            OPERATION_COLUMN + " = 'delete'";
175
176    private static final String TRUNCATE_MAPPINGS = "TRUNCATE TABLE " + MAPPING_TABLE;
177
178    private static final String TRUNCATE_TRANSACTIONS = "TRUNCATE TABLE " + TRANSACTION_OPERATIONS_TABLE;
179
180    /*
181     * Delete all records from the transaction table for the specified transaction.
182     */
183    private static final String DELETE_ENTIRE_TRANSACTION = "DELETE FROM " + TRANSACTION_OPERATIONS_TABLE + " WHERE " +
184            TRANSACTION_ID_COLUMN + " = :transactionId";
185
186    /*
187     * Row mapper for the Lookup queries.
188     */
189    private static final RowMapper<FedoraOcflMapping> GET_MAPPING_ROW_MAPPER = (resultSet, i) -> new FedoraOcflMapping(
190            FedoraId.create(resultSet.getString(1)),
191            resultSet.getString(2)
192    );
193
194    private Cache<String, FedoraOcflMapping> mappingCache;
195
196    private final DataSource dataSource;
197
198    private final NamedParameterJdbcTemplate jdbcTemplate;
199
200    private DbPlatform dbPlatform;
201
202    @Inject
203    private OcflPropsConfig ocflPropsConfig;
204
205    public DbFedoraToOcflObjectIndex(@Autowired final DataSource dataSource) {
206        this.dataSource = dataSource;
207        this.jdbcTemplate = new NamedParameterJdbcTemplate(dataSource);
208    }
209
210    @PostConstruct
211    public void setup() {
212        dbPlatform = DbPlatform.fromDataSource(dataSource);
213        final var cache = Caffeine.newBuilder()
214                .maximumSize(ocflPropsConfig.getFedoraToOcflCacheSize())
215                .expireAfterAccess(ocflPropsConfig.getFedoraToOcflCacheTimeout(), TimeUnit.MINUTES)
216                .build();
217        this.mappingCache = new CaffeineCache<>(cache);
218    }
219
220    @Override
221    public FedoraOcflMapping getMapping(final Transaction transaction, final FedoraId fedoraId)
222            throws FedoraOcflMappingNotFoundException {
223        try {
224            if (transaction.isOpenLongRunning()) {
225                final MapSqlParameterSource parameterSource = new MapSqlParameterSource();
226                parameterSource.addValue("fedoraId", fedoraId.getResourceId());
227                parameterSource.addValue("transactionId", transaction.getId());
228                return jdbcTemplate.queryForObject(LOOKUP_MAPPING_IN_TRANSACTION, parameterSource,
229                        GET_MAPPING_ROW_MAPPER);
230            } else {
231                return this.mappingCache.get(fedoraId.getResourceId(), key ->
232                        jdbcTemplate.queryForObject(LOOKUP_MAPPING, Map.of("fedoraId", key), GET_MAPPING_ROW_MAPPER)
233                );
234            }
235        } catch (final EmptyResultDataAccessException e) {
236            throw new FedoraOcflMappingNotFoundException("No OCFL mapping found for " + fedoraId);
237        }
238    }
239
240    @Override
241    public FedoraOcflMapping addMapping(@Nonnull final Transaction transaction, final FedoraId fedoraId,
242                                        final FedoraId fedoraRootId, final String ocflId) {
243        transaction.doInTx(() -> {
244            if (!transaction.isShortLived()) {
245                upsert(transaction, fedoraId, "add", fedoraRootId, ocflId);
246            } else {
247                directInsert(fedoraId, fedoraRootId, ocflId);
248            }
249        });
250
251        return new FedoraOcflMapping(fedoraRootId, ocflId);
252    }
253
254    @Override
255    public void removeMapping(@Nonnull final Transaction transaction, final FedoraId fedoraId) {
256        transaction.doInTx(() -> {
257            if (!transaction.isShortLived()) {
258                upsert(transaction, fedoraId, "delete");
259            } else {
260                final MapSqlParameterSource parameterSource = new MapSqlParameterSource();
261                parameterSource.addValue("fedoraId", fedoraId.getResourceId());
262                jdbcTemplate.update(DIRECT_DELETE_MAPPING, parameterSource);
263                this.mappingCache.invalidate(fedoraId.getResourceId());
264            }
265        });
266    }
267
268    private void upsert(final Transaction transaction, final FedoraId fedoraId, final String operation) {
269        upsert(transaction, fedoraId, operation, null, null);
270    }
271
272    /**
273     * Perform the upsert to the operations table.
274     *
275     * @param transaction the transaction/session id.
276     * @param fedoraId the resource id.
277     * @param operation the operation we are performing (add or delete)
278     * @param fedoraRootId the fedora root id (for add only)
279     * @param ocflId the ocfl id (for add only).
280     */
281    private void upsert(final Transaction transaction, final FedoraId fedoraId, final String operation,
282                        final FedoraId fedoraRootId, final String ocflId) {
283        final MapSqlParameterSource parameterSource = new MapSqlParameterSource();
284        parameterSource.addValue("fedoraId", fedoraId.getResourceId());
285        parameterSource.addValue("fedoraRootId", fedoraRootId == null ? null : fedoraRootId.getResourceId());
286        parameterSource.addValue("ocflId", ocflId);
287        parameterSource.addValue("transactionId", transaction.getId());
288        parameterSource.addValue("operation", operation);
289        try {
290            jdbcTemplate.update(UPSERT_MAPPING_TX_MAP.get(dbPlatform), parameterSource);
291        } catch (final DataIntegrityViolationException | BadSqlGrammarException e) {
292            handleInsertException(fedoraId, e);
293        }
294    }
295
296    private void directInsert(final FedoraId fedoraId, final FedoraId fedoraRootId, final String ocflId) {
297        final MapSqlParameterSource parameterSource = new MapSqlParameterSource();
298        parameterSource.addValue("fedoraId", fedoraId.getResourceId());
299        parameterSource.addValue("fedoraRootId", fedoraRootId == null ? null : fedoraRootId.getResourceId());
300        parameterSource.addValue("ocflId", ocflId);
301        try {
302            jdbcTemplate.update(DIRECT_INSERT_MAPPING, parameterSource);
303        } catch (final DataIntegrityViolationException | BadSqlGrammarException e) {
304            handleInsertException(fedoraId, e);
305        }
306    }
307
308    @Override
309    public void reset() {
310        try {
311            jdbcTemplate.update(TRUNCATE_MAPPINGS, Collections.emptyMap());
312            jdbcTemplate.update(TRUNCATE_TRANSACTIONS, Collections.emptyMap());
313            this.mappingCache.invalidateAll();
314        } catch (final Exception e) {
315            throw new RepositoryRuntimeException("Failed to truncate FedoraToOcfl index tables", e);
316        }
317    }
318
319    @Override
320    public void commit(@Nonnull final Transaction transaction) {
321        if (!transaction.isShortLived()) {
322            transaction.ensureCommitting();
323
324            LOGGER.debug("Committing FedoraToOcfl index changes from transaction {}", transaction.getId());
325            final Map<String, String> map = Map.of("transactionId", transaction.getId());
326            try {
327                final List<String> deleteIds = jdbcTemplate.queryForList(GET_DELETE_IDS, map, String.class);
328                jdbcTemplate.update(COMMIT_DELETE_RECORDS, map);
329                jdbcTemplate.update(COMMIT_ADD_MAPPING_MAP.get(dbPlatform), map);
330                jdbcTemplate.update(DELETE_ENTIRE_TRANSACTION, map);
331                this.mappingCache.invalidateAll(deleteIds);
332            } catch (final Exception e) {
333                LOGGER.warn("Unable to commit FedoraToOcfl index transaction {}: {}", transaction, e.getMessage());
334                throw new RepositoryRuntimeException("Unable to commit FedoraToOcfl index transaction", e);
335            }
336        }
337    }
338
339    @Transactional(propagation = Propagation.NOT_SUPPORTED)
340    @Override
341    public void rollback(@Nonnull final Transaction transaction) {
342        if (!transaction.isShortLived()) {
343            jdbcTemplate.update(DELETE_ENTIRE_TRANSACTION, Map.of("transactionId", transaction.getId()));
344        }
345    }
346
347    private void handleInsertException(final FedoraId fedoraId, final Exception e) {
348        if (e.getMessage().contains("too long for")) {
349            throw new InvalidResourceIdentifierException("Database error - Fedora ID path too long",e);
350        } else if (e instanceof DuplicateKeyException) {
351            throw new RepositoryRuntimeException("Database error - primary key already exists for Fedora ID: " +
352                                                 fedoraId, e);
353        } else {
354            throw new RepositoryRuntimeException("Database error - error during upsert",e);
355        }
356    }
357
358}