001/*
002 * Licensed to DuraSpace under one or more contributor license agreements.
003 * See the NOTICE file distributed with this work for additional information
004 * regarding copyright ownership.
005 *
006 * DuraSpace licenses this file to you under the Apache License,
007 * Version 2.0 (the "License"); you may not use this file except in
008 * compliance with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.fcrepo.kernel.impl.services;
019
020import static org.fcrepo.kernel.api.FedoraTypes.FEDORA_ID_PREFIX;
021import static org.slf4j.LoggerFactory.getLogger;
022
023import javax.annotation.Nonnull;
024import javax.annotation.PostConstruct;
025import javax.inject.Inject;
026import javax.sql.DataSource;
027import javax.transaction.Transactional;
028
029import java.util.List;
030import java.util.Map;
031import java.util.function.Predicate;
032import java.util.stream.Collectors;
033import java.util.stream.Stream;
034
035import com.google.common.base.Preconditions;
036import org.apache.jena.graph.Node;
037import org.apache.jena.graph.NodeFactory;
038import org.apache.jena.graph.Triple;
039import org.apache.jena.sparql.core.Quad;
040import org.fcrepo.common.db.DbPlatform;
041import org.fcrepo.kernel.api.ContainmentIndex;
042import org.fcrepo.kernel.api.RdfStream;
043import org.fcrepo.kernel.api.exception.RepositoryRuntimeException;
044import org.fcrepo.kernel.api.identifiers.FedoraId;
045import org.fcrepo.kernel.api.models.FedoraResource;
046import org.fcrepo.kernel.api.models.NonRdfSourceDescription;
047import org.fcrepo.kernel.api.observer.EventAccumulator;
048import org.fcrepo.kernel.api.rdf.DefaultRdfStream;
049import org.fcrepo.kernel.api.services.ReferenceService;
050import org.fcrepo.kernel.impl.operations.ReferenceOperation;
051import org.fcrepo.kernel.impl.operations.ReferenceOperationBuilder;
052import org.slf4j.Logger;
053import org.springframework.beans.factory.annotation.Autowired;
054import org.springframework.beans.factory.annotation.Qualifier;
055import org.springframework.core.io.DefaultResourceLoader;
056import org.springframework.jdbc.core.RowMapper;
057import org.springframework.jdbc.core.namedparam.MapSqlParameterSource;
058import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate;
059import org.springframework.jdbc.datasource.init.DatabasePopulatorUtils;
060import org.springframework.jdbc.datasource.init.ResourceDatabasePopulator;
061import org.springframework.stereotype.Component;
062
063/**
064 * Implementation of reference service.
065 * @author whikloj
066 * @since 6.0.0
067 */
068@Component("referenceServiceImpl")
069public class ReferenceServiceImpl implements ReferenceService {
070
071    private static final Logger LOGGER = getLogger(ReferenceServiceImpl.class);
072
073    @Inject
074    private DataSource dataSource;
075
076    @Inject
077    private EventAccumulator eventAccumulator;
078
079    @Autowired
080    @Qualifier("containmentIndex")
081    private ContainmentIndex containmentIndex;
082
083    private NamedParameterJdbcTemplate jdbcTemplate;
084
085    private static final String TABLE_NAME = "reference";
086
087    private static final String TRANSACTION_TABLE = "reference_transaction_operations";
088
089    private static final String RESOURCE_COLUMN = "fedora_id";
090
091    private static final String SUBJECT_COLUMN = "subject_id";
092
093    private static final String PROPERTY_COLUMN = "property";
094
095    private static final String TARGET_COLUMN = "target_id";
096
097    private static final String OPERATION_COLUMN = "operation";
098
099    private static final String TRANSACTION_COLUMN = "transaction_id";
100
101    private static final String SELECT_INBOUND = "SELECT " + SUBJECT_COLUMN + ", " + PROPERTY_COLUMN + " FROM " +
102            TABLE_NAME + " WHERE " + TARGET_COLUMN + " = :targetId";
103
104    private static final String SELECT_INBOUND_IN_TRANSACTION = "SELECT x." + SUBJECT_COLUMN + ", x." +
105            PROPERTY_COLUMN + " FROM " + "(SELECT " + SUBJECT_COLUMN + ", " + PROPERTY_COLUMN + " FROM " + TABLE_NAME +
106            " WHERE " + TARGET_COLUMN + " = :targetId UNION " + "SELECT " + SUBJECT_COLUMN + ", " + PROPERTY_COLUMN +
107            " FROM " + TRANSACTION_TABLE + " WHERE " + TARGET_COLUMN + " = :targetId AND "
108            + TRANSACTION_COLUMN + " = :transactionId AND " + OPERATION_COLUMN + " = 'add') x WHERE NOT EXISTS " +
109            "(SELECT 1 FROM " + TRANSACTION_TABLE + " WHERE " + TARGET_COLUMN + " = :targetId AND " +
110            OPERATION_COLUMN + " = 'delete')";
111
112    private static final String SELECT_OUTBOUND = "SELECT " + SUBJECT_COLUMN + ", " + TARGET_COLUMN + ", " +
113            PROPERTY_COLUMN + " FROM " + TABLE_NAME + " WHERE " + RESOURCE_COLUMN + " = :resourceId";
114
115    private static final String SELECT_OUTBOUND_IN_TRANSACTION = "SELECT x." + SUBJECT_COLUMN + ", x." + TARGET_COLUMN +
116            ", x." + PROPERTY_COLUMN + " FROM " + "(SELECT " + SUBJECT_COLUMN + ", " + TARGET_COLUMN + ", " +
117            PROPERTY_COLUMN + " FROM " + TABLE_NAME + " WHERE " + RESOURCE_COLUMN + " = :resourceId UNION " +
118            "SELECT " + SUBJECT_COLUMN + ", " + TARGET_COLUMN + ", " + PROPERTY_COLUMN + " FROM " + TRANSACTION_TABLE +
119            " WHERE " + RESOURCE_COLUMN + " = :resourceId " + "AND " + TRANSACTION_COLUMN + " = :transactionId AND " +
120            OPERATION_COLUMN + " = 'add') x WHERE NOT EXISTS (SELECT 1 FROM " + TRANSACTION_TABLE + " WHERE " +
121            RESOURCE_COLUMN + " = :resourceId AND " + OPERATION_COLUMN + " = 'delete')";
122
123    private static final String INSERT_REFERENCE_IN_TRANSACTION = "INSERT INTO " + TRANSACTION_TABLE + "(" +
124            RESOURCE_COLUMN + ", " + SUBJECT_COLUMN + ", " + PROPERTY_COLUMN + ", " + TARGET_COLUMN + ", " +
125            TRANSACTION_COLUMN + ", " + OPERATION_COLUMN + ") VALUES (:resourceId, :subjectId, :property, :targetId, " +
126            ":transactionId, 'add')";
127
128    private static final String UNDO_INSERT_REFERENCE_IN_TRANSACTION = "DELETE FROM " + TRANSACTION_TABLE + " WHERE " +
129            RESOURCE_COLUMN + " = :resourceId AND " + SUBJECT_COLUMN + " = :subjectId AND " + PROPERTY_COLUMN +
130            " = :property AND " + TARGET_COLUMN + " = :targetId AND " + TRANSACTION_COLUMN + " = :transactionId AND " +
131            OPERATION_COLUMN + " = 'add'";
132
133    private static final String DELETE_REFERENCE_IN_TRANSACTION = "INSERT INTO " + TRANSACTION_TABLE + "(" +
134            RESOURCE_COLUMN + ", " + SUBJECT_COLUMN + ", " + PROPERTY_COLUMN + ", " + TARGET_COLUMN + ", " +
135            TRANSACTION_COLUMN + ", " + OPERATION_COLUMN + ") VALUES (:resourceId, :subjectId, :property, :targetId, " +
136            ":transactionId, 'delete')";
137
138    private static final String UNDO_DELETE_REFERENCE_IN_TRANSACTION = "DELETE FROM " + TRANSACTION_TABLE + " WHERE " +
139            RESOURCE_COLUMN + " = :resourceId AND " + SUBJECT_COLUMN + " = :subjectId AND " + PROPERTY_COLUMN +
140            " = :property AND " + TARGET_COLUMN + " = :targetId AND " + TRANSACTION_COLUMN + " = :transactionId AND " +
141            OPERATION_COLUMN + " = 'delete'";
142
143    private static final String IS_REFERENCE_ADDED_IN_TRANSACTION = "SELECT TRUE FROM " + TRANSACTION_TABLE + " WHERE "
144            + RESOURCE_COLUMN + " = :resourceId AND " + SUBJECT_COLUMN + " = :subjectId AND " + PROPERTY_COLUMN +
145            " = :property AND " + TARGET_COLUMN + " = :targetId AND " + TRANSACTION_COLUMN + " = :transactionId AND " +
146            OPERATION_COLUMN + " = 'add'";
147
148    private static final String IS_REFERENCE_DELETED_IN_TRANSACTION = "SELECT TRUE FROM " + TRANSACTION_TABLE +
149            " WHERE " + RESOURCE_COLUMN + " = :resourceId AND " + SUBJECT_COLUMN + " = :subjectId AND " +
150            PROPERTY_COLUMN + " = :property AND " + TARGET_COLUMN + " = :targetId AND " + TRANSACTION_COLUMN +
151            " = :transactionId AND " + OPERATION_COLUMN + " = 'delete'";
152
153    private static final String COMMIT_ADD_RECORDS = "INSERT INTO " + TABLE_NAME + " ( " + RESOURCE_COLUMN + ", " +
154            SUBJECT_COLUMN + ", " + PROPERTY_COLUMN + ", " + TARGET_COLUMN + " ) SELECT " + RESOURCE_COLUMN + ", " +
155            SUBJECT_COLUMN + ", " + PROPERTY_COLUMN + ", " + TARGET_COLUMN + " FROM " + TRANSACTION_TABLE + " WHERE " +
156            TRANSACTION_COLUMN + " = :transactionId AND " + OPERATION_COLUMN + " = 'add'";
157
158    private static final String COMMIT_DELETE_RECORDS = "DELETE FROM " + TABLE_NAME + " WHERE " +
159            "EXISTS (SELECT * FROM " + TRANSACTION_TABLE + " t WHERE t." +
160            TRANSACTION_COLUMN + " = :transactionId AND t." +  OPERATION_COLUMN + " = 'delete' AND" +
161            " t." + RESOURCE_COLUMN + " = " + TABLE_NAME + "." + RESOURCE_COLUMN + " AND" +
162            " t." + SUBJECT_COLUMN + " = " + TABLE_NAME + "." + SUBJECT_COLUMN +
163            " AND t." + PROPERTY_COLUMN + " = " + TABLE_NAME + "." + PROPERTY_COLUMN +
164            " AND t." + TARGET_COLUMN + " = " + TABLE_NAME + "." + TARGET_COLUMN + ")";
165
166    private static final String DELETE_TRANSACTION = "DELETE FROM " + TRANSACTION_TABLE + " WHERE " +
167            TRANSACTION_COLUMN + " = :transactionId";
168
169    private static final String TRUNCATE_TABLE = "TRUNCATE TABLE " + TABLE_NAME;
170
171    private static final Map<DbPlatform, String> DDL_MAP = Map.of(
172            DbPlatform.MYSQL, "sql/mysql-references.sql",
173            DbPlatform.H2, "sql/default-references.sql",
174            DbPlatform.POSTGRESQL, "sql/default-references.sql",
175            DbPlatform.MARIADB, "sql/default-references.sql"
176    );
177
178    @PostConstruct
179    public void setUp() {
180        jdbcTemplate = new NamedParameterJdbcTemplate(getDataSource());
181
182        final var dbPlatform = DbPlatform.fromDataSource(dataSource);
183
184        Preconditions.checkArgument(DDL_MAP.containsKey(dbPlatform),
185                "Missing DDL mapping for %s", dbPlatform);
186
187        final var ddl = DDL_MAP.get(dbPlatform);
188        LOGGER.info("Applying ddl: {}", ddl);
189        DatabasePopulatorUtils.execute(
190                new ResourceDatabasePopulator(new DefaultResourceLoader().getResource("classpath:" + ddl)),
191                dataSource);
192    }
193
194    @Override
195    public RdfStream getInboundReferences(final String txId, final FedoraResource resource) {
196        final String resourceId = resource.getFedoraId().getFullId();
197        final Node subject = NodeFactory.createURI(resourceId);
198        final Stream<Triple> stream = getReferencesInternal(txId, resourceId);
199        if (resource instanceof NonRdfSourceDescription) {
200            final Stream<Triple> stream2 = getReferencesInternal(txId, resource.getFedoraId().getBaseId());
201            return new DefaultRdfStream(subject, Stream.concat(stream, stream2));
202        }
203        return new DefaultRdfStream(subject, stream);
204    }
205
206    /**
207     * Get the inbound references for the resource Id and the transaction id.
208     * @param txId transaction id or null for none.
209     * @param targetId the id that will be the target of references.
210     * @return RDF stream of inbound references
211     */
212    private Stream<Triple> getReferencesInternal(final String txId, final String targetId) {
213        final MapSqlParameterSource parameterSource = new MapSqlParameterSource();
214        parameterSource.addValue("targetId", targetId);
215        final Node targetNode = NodeFactory.createURI(targetId);
216
217        final RowMapper<Triple> inboundMapper = (rs, rowNum) ->
218                Triple.create(NodeFactory.createURI(rs.getString(SUBJECT_COLUMN)),
219                        NodeFactory.createURI(rs.getString(PROPERTY_COLUMN)),
220                        targetNode);
221
222        final List<Triple> references;
223        if (txId != null) {
224            // we are in a transaction
225            parameterSource.addValue("transactionId", txId);
226            references = jdbcTemplate.query(SELECT_INBOUND_IN_TRANSACTION, parameterSource, inboundMapper);
227        } else {
228            // not in a transaction
229            references = jdbcTemplate.query(SELECT_INBOUND, parameterSource, inboundMapper);
230        }
231        LOGGER.debug("getInboundReferences for {} in transaction {} found {} references",
232                targetId, txId, references.size());
233        return references.stream();
234    }
235
236    @Override
237    public void deleteAllReferences(@Nonnull final String txId, final FedoraId resourceId) {
238        final List<Quad> deleteReferences = getOutboundReferences(txId, resourceId);
239        if (resourceId.isDescription()) {
240            // Also get the binary references
241            deleteReferences.addAll(getOutboundReferences(txId, resourceId.asBaseId()));
242        }
243        // Remove all the existing references.
244        deleteReferences.forEach(t -> removeReference(txId, t));
245    }
246
247    /**
248     * Get a stream of quads of resources being referenced from the provided resource, the graph of the quad is the
249     * URI of the resource the reference is from.
250     * @param txId transaction Id or null if none.
251     * @param resourceId the resource Id.
252     * @return list of Quads
253     */
254    private List<Quad> getOutboundReferences(final String txId, final FedoraId resourceId) {
255        final MapSqlParameterSource parameterSource = new MapSqlParameterSource();
256        parameterSource.addValue("resourceId", resourceId.getFullId());
257        final Node subjectNode = NodeFactory.createURI(resourceId.getFullId());
258
259        final RowMapper<Quad> outboundMapper = (rs, rowNum) ->
260                Quad.create(subjectNode,
261                        NodeFactory.createURI(rs.getString(SUBJECT_COLUMN)),
262                        NodeFactory.createURI(rs.getString(PROPERTY_COLUMN)),
263                        NodeFactory.createURI(rs.getString(TARGET_COLUMN)));
264
265        final List<Quad> references;
266        if (txId != null) {
267            // we are in a transaction
268            parameterSource.addValue("transactionId", txId);
269            references = jdbcTemplate.query(SELECT_OUTBOUND_IN_TRANSACTION, parameterSource, outboundMapper);
270        } else {
271            // not in a transaction
272            references = jdbcTemplate.query(SELECT_OUTBOUND, parameterSource, outboundMapper);
273        }
274        LOGGER.debug("getOutboundReferences for {} in transaction {} found {} references",
275                resourceId, txId, references.size());
276        return references;
277    }
278
279    @Override
280    @Transactional
281    public void updateReferences(@Nonnull final String txId, final FedoraId resourceId, final String userPrincipal,
282                                 final RdfStream rdfStream) {
283        try {
284            final List<Triple> addReferences = getReferencesFromRdf(rdfStream).collect(Collectors.toList());
285            // This predicate checks for items we are adding, so we don't bother to delete and then re-add them.
286            final Predicate<Quad> notInAdds = q -> !addReferences.contains(q.asTriple());
287            // References from this resource.
288            final List<Quad> existingReferences = getOutboundReferences(txId, resourceId);
289            if (resourceId.isDescription()) {
290                // Resource is a binary description so also get the binary references.
291                existingReferences.addAll(getOutboundReferences(txId, resourceId.asBaseId()));
292            }
293            // Remove any existing references not being re-added.
294            existingReferences.stream().filter(notInAdds).forEach(t -> removeReference(txId, t));
295            final Node resourceNode = NodeFactory.createURI(resourceId.getFullId());
296            // This predicate checks for references that didn't already exist in the database.
297            final Predicate<Triple> alreadyExists = t -> !existingReferences.contains(Quad.create(resourceNode, t));
298            // Add the new references.
299            addReferences.stream().filter(alreadyExists).forEach(r ->
300                    addReference(txId, Quad.create(resourceNode, r), userPrincipal));
301        } catch (final Exception e) {
302            LOGGER.warn("Unable to update reference index for resource {} in transaction {}: {}",
303                    resourceId.getFullId(), txId, e.getMessage());
304            throw new RepositoryRuntimeException("Unable to update reference index", e);
305        }
306    }
307
308    @Override
309    @Transactional
310    public void commitTransaction(final String txId) {
311        try {
312            final Map<String, String> parameterSource = Map.of("transactionId", txId);
313            jdbcTemplate.update(COMMIT_DELETE_RECORDS, parameterSource);
314            jdbcTemplate.update(COMMIT_ADD_RECORDS, parameterSource);
315            jdbcTemplate.update(DELETE_TRANSACTION, parameterSource);
316        } catch (final Exception e) {
317            LOGGER.warn("Unable to commit reference index transaction {}: {}", txId, e.getMessage());
318            throw new RepositoryRuntimeException("Unable to commit reference index transaction", e);
319        }
320    }
321
322    @Override
323    @Transactional
324    public void rollbackTransaction(final String txId) {
325        try {
326            final Map<String, String> parameterSource = Map.of("transactionId", txId);
327            jdbcTemplate.update(DELETE_TRANSACTION, parameterSource);
328        } catch (final Exception e) {
329            LOGGER.warn("Unable to rollback reference index transaction {}: {}", txId, e.getMessage());
330            throw new RepositoryRuntimeException("Unable to rollback reference index transaction", e);
331        }
332    }
333
334    @Override
335    public void reset() {
336        try {
337            jdbcTemplate.update(TRUNCATE_TABLE, Map.of());
338        } catch (final Exception e) {
339            LOGGER.warn("Unable to reset reference index: {}", e.getMessage());
340            throw new RepositoryRuntimeException("Unable to reset reference index", e);
341        }
342    }
343
344    /**
345     * Remove a reference.
346     * @param txId transaction Id.
347     * @param reference the quad with the reference, is Quad(resourceId, subjectId, propertyId, targetId)
348     */
349    private void removeReference(@Nonnull final String txId, final Quad reference) {
350        final Map<String, String> parameterSource = Map.of("transactionId", txId,
351                "resourceId", reference.getGraph().getURI(),
352                "subjectId", reference.getSubject().getURI(),
353                "property", reference.getPredicate().getURI(),
354                "targetId", reference.getObject().getURI());
355        final boolean addedInTx = !jdbcTemplate.queryForList(IS_REFERENCE_ADDED_IN_TRANSACTION, parameterSource)
356                .isEmpty();
357        if (addedInTx) {
358            jdbcTemplate.update(UNDO_INSERT_REFERENCE_IN_TRANSACTION, parameterSource);
359        } else {
360            jdbcTemplate.update(DELETE_REFERENCE_IN_TRANSACTION, parameterSource);
361        }
362    }
363
364    /**
365     * Add a reference
366     * @param txId the transaction Id.
367     * @param reference the quad with the reference, is is Quad(resourceId, subjectId, propertyId, targetId)
368     * @param userPrincipal the user adding the reference.
369     */
370    private void addReference(@Nonnull final String txId, final Quad reference, final String userPrincipal) {
371        final String targetId = reference.getObject().getURI();
372        final Map<String, String> parameterSource = Map.of("transactionId", txId,
373                "resourceId", reference.getGraph().getURI(),
374                "subjectId", reference.getSubject().getURI(),
375                "property", reference.getPredicate().getURI(),
376                "targetId", targetId);
377        final boolean addedInTx = !jdbcTemplate.queryForList(IS_REFERENCE_DELETED_IN_TRANSACTION, parameterSource)
378                .isEmpty();
379        if (addedInTx) {
380            jdbcTemplate.update(UNDO_DELETE_REFERENCE_IN_TRANSACTION, parameterSource);
381        } else {
382            jdbcTemplate.update(INSERT_REFERENCE_IN_TRANSACTION, parameterSource);
383            recordEvent(txId, targetId, userPrincipal);
384        }
385    }
386
387    /**
388     * Record the inbound reference event if the target exists.
389     * @param txId the transaction id.
390     * @param resourceId the id of the target of the inbound reference.
391     * @param userPrincipal the user making the reference.
392     */
393    private void recordEvent(final String txId, final String resourceId, final String userPrincipal) {
394        final FedoraId fedoraId = FedoraId.create(resourceId);
395        if (this.containmentIndex.resourceExists(txId, fedoraId, false)) {
396            this.eventAccumulator.recordEventForOperation(txId, fedoraId, getOperation(fedoraId, userPrincipal));
397        }
398    }
399
400    /**
401     * Create a ReferenceOperation for the current add.
402     * @param id the target resource of the reference.
403     * @param user the user making the change
404     * @return a ReferenceOperation
405     */
406    private static ReferenceOperation getOperation(final FedoraId id, final String user) {
407        final ReferenceOperationBuilder builder = new ReferenceOperationBuilder(id);
408        builder.userPrincipal(user);
409        return builder.build();
410    }
411
412    /**
413     * Utility to filter a RDFStream to just the URIs from subjects and objects within the repository.
414     * @param stream the provided stream
415     * @return stream of triples with internal references.
416     */
417    private Stream<Triple> getReferencesFromRdf(final RdfStream stream) {
418        final Predicate<Triple> isInternalReference = t -> {
419            final Node s = t.getSubject();
420            final Node o = t.getObject();
421            return (s.isURI() && s.getURI().startsWith(FEDORA_ID_PREFIX) && o.isURI() &&
422                    o.getURI().startsWith(FEDORA_ID_PREFIX));
423        };
424        return stream.filter(isInternalReference);
425    }
426
427    /**
428     * Set the JDBC datastore.
429     * @param dataSource the dataStore.
430     */
431    public void setDataSource(final DataSource dataSource) {
432        this.dataSource = dataSource;
433    }
434
435    /**
436     * Get the JDBC datastore.
437     * @return the dataStore.
438     */
439    public DataSource getDataSource() {
440        return dataSource;
441    }
442}