Source code

001/*
002 * Licensed to DuraSpace under one or more contributor license agreements.
003 * See the NOTICE file distributed with this work for additional information
004 * regarding copyright ownership.
005 *
006 * DuraSpace licenses this file to you under the Apache License,
007 * Version 2.0 (the "License"); you may not use this file except in
008 * compliance with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.fcrepo.kernel.impl.services;
019
020import static org.slf4j.LoggerFactory.getLogger;
021
022import java.sql.ResultSet;
023import java.sql.SQLException;
024import java.sql.Timestamp;
025import java.time.Instant;
026import java.util.Map;
027import java.util.NoSuchElementException;
028import java.util.Queue;
029import java.util.Spliterator;
030import java.util.Spliterators;
031import java.util.concurrent.ConcurrentLinkedQueue;
032import java.util.function.Consumer;
033import java.util.stream.Stream;
034import java.util.stream.StreamSupport;
035
036import javax.annotation.PostConstruct;
037import javax.inject.Inject;
038import javax.sql.DataSource;
039import javax.transaction.Transactional;
040import org.apache.jena.graph.Node;
041import org.apache.jena.graph.NodeFactory;
042import org.apache.jena.graph.Triple;
043import org.fcrepo.common.db.DbPlatform;
044import org.fcrepo.kernel.api.identifiers.FedoraId;
045import org.slf4j.Logger;
046import org.springframework.core.io.DefaultResourceLoader;
047import org.springframework.jdbc.core.RowCallbackHandler;
048import org.springframework.jdbc.core.RowMapper;
049import org.springframework.jdbc.core.namedparam.MapSqlParameterSource;
050import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate;
051import org.springframework.jdbc.datasource.init.DatabasePopulatorUtils;
052import org.springframework.jdbc.datasource.init.ResourceDatabasePopulator;
053import org.springframework.stereotype.Component;
054
055import com.google.common.base.Preconditions;
056
057/**
058 * Manager for the membership index
059 *
060 * @author bbpennel
061 */
062@Component
063public class MembershipIndexManager {
064    private static final Logger log = getLogger(MembershipIndexManager.class);
065
066    private static final Timestamp NO_END_TIMESTAMP = Timestamp.from(MembershipServiceImpl.NO_END_INSTANT);
067    private static final Timestamp NO_START_TIMESTAMP = Timestamp.from(Instant.parse("1000-01-01T00:00:00.000Z"));
068
069    private static final String ADD_OPERATION = "add";
070    private static final String DELETE_OPERATION = "delete";
071    private static final String FORCE_FLAG = "force";
072
073    private static final String TX_ID_PARAM = "txId";
074    private static final String SUBJECT_ID_PARAM = "subjectId";
075    private static final String NO_END_TIME_PARAM = "noEndTime";
076    private static final String ADD_OP_PARAM = "addOp";
077    private static final String DELETE_OP_PARAM = "deleteOp";
078    private static final String MEMENTO_TIME_PARAM = "mementoTime";
079    private static final String PROPERTY_PARAM = "property";
080    private static final String TARGET_ID_PARAM = "targetId";
081    private static final String SOURCE_ID_PARAM = "sourceId";
082    private static final String START_TIME_PARAM = "startTime";
083    private static final String END_TIME_PARAM = "endTime";
084    private static final String LAST_UPDATED_PARAM = "lastUpdated";
085    private static final String OPERATION_PARAM = "operation";
086    private static final String FORCE_PARAM = "forceFlag";
087    private static final String OBJECT_ID_PARAM = "objectId";
088    private static final String LIMIT_PARAM = "limit";
089    private static final String OFFSET_PARAM = "offSet";
090
091    private static final String SELECT_ALL_MEMBERSHIP = "SELECT * FROM membership";
092
093    private static final String SELECT_ALL_OPERATIONS = "SELECT * FROM membership_tx_operations";
094
095    private static final String SELECT_MEMBERSHIP_IN_TX =
096            "SELECT property, object_id" +
097            " FROM membership m" +
098            " WHERE subject_id = :subjectId" +
099                " AND end_time = :noEndTime" +
100                " AND NOT EXISTS (" +
101                    " SELECT 1" +
102                    " FROM membership_tx_operations mto" +
103                    " WHERE mto.subject_id = :subjectId" +
104                        " AND mto.source_id = m.source_id" +
105                        " AND mto.object_id = m.object_id" +
106                        " AND mto.tx_id = :txId" +
107                        " AND mto.operation = :deleteOp)" +
108            " UNION" +
109            " SELECT property, object_id" +
110            " FROM membership_tx_operations" +
111            " WHERE subject_id = :subjectId" +
112                " AND tx_id = :txId" +
113                " AND end_time = :noEndTime" +
114                " AND operation = :addOp" +
115            " ORDER BY property, object_id" +
116            " LIMIT :limit OFFSET :offSet";
117
118    private static final String SELECT_MEMBERSHIP_MEMENTO_IN_TX =
119            "SELECT property, object_id" +
120            " FROM membership m" +
121            " WHERE m.subject_id = :subjectId" +
122                " AND m.start_time <= :mementoTime" +
123                " AND m.end_time > :mementoTime" +
124                " AND NOT EXISTS (" +
125                    " SELECT 1" +
126                    " FROM membership_tx_operations mto" +
127                    " WHERE mto.subject_id = :subjectId" +
128                        " AND mto.source_id = m.source_id" +
129                        " AND mto.property = m.property" +
130                        " AND mto.object_id = m.object_id" +
131                        " AND mto.end_time <= :mementoTime" +
132                        " AND mto.tx_id = :txId" +
133                        " AND mto.operation = :deleteOp)" +
134            " UNION" +
135            " SELECT property, object_id" +
136            " FROM membership_tx_operations" +
137            " WHERE subject_id = :subjectId" +
138                " AND tx_id = :txId" +
139                " AND start_time <= :mementoTime" +
140                " AND end_time > :mementoTime" +
141                " AND operation = :addOp" +
142            " ORDER BY property, object_id" +
143            " LIMIT :limit OFFSET :offSet";
144
145    private static final String SELECT_LAST_UPDATED =
146            "SELECT max(last_updated) as last_updated" +
147            " FROM membership" +
148            " WHERE subject_id = :subjectId";
149
150    // For mementos, use the start_time instead of last_updated as the
151    // end_time reflects when the next version starts
152    private static final String SELECT_LAST_UPDATED_MEMENTO =
153            "SELECT max(start_time)" +
154            " FROM membership" +
155            " WHERE subject_id = :subjectId" +
156                " AND start_time <= :mementoTime" +
157                " AND end_time > :mementoTime";
158
159    private static final String SELECT_LAST_UPDATED_IN_TX =
160            "SELECT max(combined.updated) as last_updated" +
161            " FROM (" +
162                " SELECT max(last_updated) as updated" +
163                " FROM membership m" +
164                " WHERE subject_id = :subjectId" +
165                    " AND NOT EXISTS (" +
166                        " SELECT 1" +
167                        " FROM membership_tx_operations mto" +
168                        " WHERE mto.subject_id = :subjectId" +
169                            " AND mto.source_id = m.source_id" +
170                            " AND mto.object_id = m.object_id" +
171                            " AND mto.tx_id = :txId" +
172                            " AND mto.operation = :deleteOp)" +
173                " UNION" +
174                " SELECT max(last_updated) as updated" +
175                " FROM membership_tx_operations" +
176                " WHERE subject_id = :subjectId" +
177                    " AND tx_id = :txId" +
178            ") combined";
179
180    private static final String INSERT_MEMBERSHIP_IN_TX =
181            "INSERT INTO membership_tx_operations" +
182            " (subject_id, property, object_id, source_id, start_time, end_time, last_updated, tx_id, operation)" +
183            " VALUES" +
184            " (:subjectId, :property, :targetId, :sourceId, :startTime, :endTime, :lastUpdated, :txId, :operation)";
185
186    private static final String END_EXISTING_MEMBERSHIP =
187            "INSERT INTO membership_tx_operations" +
188            " (subject_id, property, object_id, source_id, start_time, end_time, last_updated, tx_id, operation)" +
189            " SELECT m.subject_id, m.property, m.object_id, m.source_id, m.start_time," +
190                    " :endTime, :endTime, :txId, :deleteOp" +
191            " FROM membership m" +
192            " WHERE m.source_id = :sourceId" +
193                " AND m.end_time = :noEndTime" +
194                " AND m.subject_id = :subjectId" +
195                " AND m.property = :property" +
196                " AND m.object_id = :objectId";
197
198    private static final String CLEAR_ENTRY_IN_TX =
199            "DELETE FROM membership_tx_operations" +
200            " WHERE source_id = :sourceId" +
201                " AND tx_id = :txId" +
202                " AND subject_id = :subjectId" +
203                " AND property = :property" +
204                " AND object_id = :objectId" +
205                " AND operation = :operation" +
206                " AND force_flag IS NULL";
207
208    private static final String CLEAR_ALL_ADDED_FOR_SOURCE_IN_TX =
209            "DELETE FROM membership_tx_operations" +
210            " WHERE source_id = :sourceId" +
211                " AND tx_id = :txId" +
212                " AND operation = :addOp";
213
214    // Add "delete" entries for all existing membership from the given source, if not already deleted
215    private static final String END_EXISTING_FOR_SOURCE =
216            "INSERT INTO membership_tx_operations" +
217            " (subject_id, property, object_id, source_id, start_time, end_time, last_updated, tx_id, operation)" +
218            " SELECT subject_id, property, object_id, source_id, start_time, :endTime, :endTime, :txId, :deleteOp" +
219            " FROM membership m" +
220            " WHERE source_id = :sourceId" +
221                " AND end_time = :noEndTime" +
222                " AND NOT EXISTS (" +
223                    " SELECT TRUE" +
224                    " FROM membership_tx_operations mtx" +
225                    " WHERE mtx.subject_id = m.subject_id" +
226                        " AND mtx.property = m.property" +
227                        " AND mtx.object_id = m.object_id" +
228                        " AND mtx.source_id = m.source_id" +
229                        " AND mtx.operation = :deleteOp" +
230                    ")";
231
232    private static final String DELETE_EXISTING_FOR_SOURCE_AFTER =
233            "INSERT INTO membership_tx_operations(subject_id, property, object_id, source_id," +
234                    " start_time, end_time, last_updated, tx_id, operation, force_flag)" +
235            " SELECT subject_id, property, object_id, source_id, start_time, end_time," +
236                    " last_updated, :txId, :deleteOp, :forceFlag" +
237            " FROM membership m" +
238            " WHERE m.source_id = :sourceId" +
239                " AND (m.start_time >= :startTime" +
240                " OR m.end_time >= :startTime)";
241
242    private static final String PURGE_ALL_REFERENCES_MEMBERSHIP =
243            "DELETE from membership" +
244            " where source_id = :targetId" +
245                " OR subject_id = :targetId" +
246                " OR object_id = :targetId";
247
248    private static final String PURGE_ALL_REFERENCES_TRANSACTION =
249            "DELETE from membership_tx_operations" +
250            " WHERE tx_id = :txId" +
251                " AND (source_id = :targetId" +
252                " OR subject_id = :targetId" +
253                " OR object_id = :targetId)";
254
255    private static final String COMMIT_DELETES =
256            "DELETE from membership" +
257            " WHERE EXISTS (" +
258                " SELECT TRUE" +
259                " FROM membership_tx_operations mto" +
260                " WHERE mto.tx_id = :txId" +
261                    " AND mto.operation = :deleteOp" +
262                    " AND mto.force_flag = :forceFlag" +
263                    " AND membership.source_id = mto.source_id" +
264                    " AND membership.subject_id = mto.subject_id" +
265                    " AND membership.property = mto.property" +
266                    " AND membership.object_id = mto.object_id" +
267                " )";
268
269    private static final String COMMIT_ENDS_H2 =
270            "UPDATE membership m" +
271            " SET end_time = (" +
272                " SELECT mto.end_time" +
273                " FROM membership_tx_operations mto" +
274                " WHERE mto.tx_id = :txId" +
275                    " AND m.source_id = mto.source_id" +
276                    " AND m.subject_id = mto.subject_id" +
277                    " AND m.property = mto.property" +
278                    " AND m.object_id = mto.object_id" +
279                    " AND mto.operation = :deleteOp" +
280                " )," +
281                " last_updated = (" +
282                    " SELECT mto.end_time" +
283                    " FROM membership_tx_operations mto" +
284                    " WHERE mto.tx_id = :txId" +
285                        " AND m.source_id = mto.source_id" +
286                        " AND m.subject_id = mto.subject_id" +
287                        " AND m.property = mto.property" +
288                        " AND m.object_id = mto.object_id" +
289                        " AND mto.operation = :deleteOp" +
290                    " )" +
291            " WHERE EXISTS (" +
292                "SELECT TRUE" +
293                " FROM membership_tx_operations mto" +
294                " WHERE mto.tx_id = :txId" +
295                    " AND mto.operation = :deleteOp" +
296                    " AND m.source_id = mto.source_id" +
297                    " AND m.subject_id = mto.subject_id" +
298                    " AND m.property = mto.property" +
299                    " AND m.object_id = mto.object_id" +
300                " )";
301
302    private static final String COMMIT_ENDS_POSTGRES =
303            "UPDATE membership" +
304            " SET end_time = mto.end_time, last_updated = mto.end_time" +
305            " FROM membership_tx_operations mto" +
306            " WHERE mto.tx_id = :txId" +
307                " AND mto.operation = :deleteOp" +
308                " AND membership.source_id = mto.source_id" +
309                " AND membership.subject_id = mto.subject_id" +
310                " AND membership.property = mto.property" +
311                " AND membership.object_id = mto.object_id";
312
313    private static final String COMMIT_ENDS_MYSQL =
314            "UPDATE membership m" +
315            " INNER JOIN membership_tx_operations mto ON" +
316                " m.source_id = mto.source_id" +
317                " AND m.subject_id = mto.subject_id" +
318                " AND m.property = mto.property" +
319                " AND m.object_id = mto.object_id" +
320            " SET m.end_time = mto.end_time, m.last_updated = mto.end_time" +
321            " WHERE mto.tx_id = :txId" +
322                " AND mto.operation = :deleteOp";
323
324    private static final Map<DbPlatform, String> COMMIT_ENDS_MAP = Map.of(
325            DbPlatform.MYSQL, COMMIT_ENDS_MYSQL,
326            DbPlatform.MARIADB, COMMIT_ENDS_MYSQL,
327            DbPlatform.POSTGRESQL, COMMIT_ENDS_POSTGRES,
328            DbPlatform.H2, COMMIT_ENDS_H2
329    );
330
331    // Transfer all "add" operations from tx to committed membership, unless the entry already exists
332    private static final String COMMIT_ADDS =
333            "INSERT INTO membership" +
334            " (subject_id, property, object_id, source_id, start_time, end_time, last_updated)" +
335            " SELECT subject_id, property, object_id, source_id, start_time, end_time, last_updated" +
336            " FROM membership_tx_operations mto" +
337            " WHERE mto.tx_id = :txId" +
338                " AND mto.operation = :addOp" +
339                " AND NOT EXISTS (" +
340                    " SELECT TRUE" +
341                    " FROM membership m" +
342                    " WHERE m.source_id = mto.source_id" +
343                        " AND m.subject_id = mto.subject_id" +
344                        " AND m.property = mto.property" +
345                        " AND m.object_id = mto.object_id" +
346                        " AND m.start_time = mto.start_time" +
347                        " AND m.end_time = mto.end_time" +
348                " )";
349
350    private static final String DELETE_TRANSACTION =
351            "DELETE FROM membership_tx_operations" +
352            " WHERE tx_id = :txId";
353
354    private static final String TRUNCATE_MEMBERSHIP = "TRUNCATE TABLE membership";
355
356    private static final String TRUNCATE_MEMBERSHIP_TX = "TRUNCATE TABLE membership_tx_operations";
357
358    @Inject
359    private DataSource dataSource;
360
361    private NamedParameterJdbcTemplate jdbcTemplate;
362
363    private DbPlatform dbPlatform;
364
365    private static final Map<DbPlatform, String> DDL_MAP = Map.of(
366            DbPlatform.MYSQL, "sql/mysql-membership.sql",
367            DbPlatform.H2, "sql/default-membership.sql",
368            DbPlatform.POSTGRESQL, "sql/default-membership.sql",
369            DbPlatform.MARIADB, "sql/mariadb-membership.sql"
370    );
371
372    private static final int MEMBERSHIP_LIMIT = 50000;
373
374    @PostConstruct
375    public void setUp() {
376        jdbcTemplate = new NamedParameterJdbcTemplate(getDataSource());
377
378        dbPlatform = DbPlatform.fromDataSource(dataSource);
379
380        Preconditions.checkArgument(DDL_MAP.containsKey(dbPlatform),
381                "Missing DDL mapping for %s", dbPlatform);
382
383        final var ddl = DDL_MAP.get(dbPlatform);
384        log.debug("Applying ddl: {}", ddl);
385        DatabasePopulatorUtils.execute(
386                new ResourceDatabasePopulator(new DefaultResourceLoader().getResource("classpath:" + ddl)),
387                dataSource);
388    }
389
390    /**
391     * End a membership entry, setting an end time if committed, or clearing from the current tx
392     * if it was newly added.
393     *
394     * @param txId transaction id
395     * @param sourceId ID of the direct/indirect container whose membership should be ended
396     * @param membership membership triple to end
397     * @param endTime the time the resource was deleted, generally its last modified
398     */
399    @Transactional
400    public void endMembership(final String txId,  final FedoraId sourceId, final Triple membership,
401            final Instant endTime) {
402        final Map<String, Object> parameterSource = Map.of(
403                TX_ID_PARAM, txId,
404                SOURCE_ID_PARAM, sourceId.getFullId(),
405                SUBJECT_ID_PARAM, membership.getSubject().getURI(),
406                PROPERTY_PARAM, membership.getPredicate().getURI(),
407                OBJECT_ID_PARAM, membership.getObject().getURI(),
408                OPERATION_PARAM, ADD_OPERATION);
409
410        final int affected = jdbcTemplate.update(CLEAR_ENTRY_IN_TX, parameterSource);
411
412        // If no rows were deleted, then assume we need to delete permanent entry
413        if (affected == 0) {
414            final Map<String, Object> parameterSource2 = Map.of(
415                    TX_ID_PARAM, txId,
416                    SOURCE_ID_PARAM, sourceId.getFullId(),
417                    SUBJECT_ID_PARAM, membership.getSubject().getURI(),
418                    PROPERTY_PARAM, membership.getPredicate().getURI(),
419                    OBJECT_ID_PARAM, membership.getObject().getURI(),
420                    END_TIME_PARAM, formatInstant(endTime),
421                    NO_END_TIME_PARAM, NO_END_TIMESTAMP,
422                    DELETE_OP_PARAM, DELETE_OPERATION);
423            jdbcTemplate.update(END_EXISTING_MEMBERSHIP, parameterSource2);
424        }
425    }
426
427    /**
428     * End all membership properties resulting from the specified source container
429     * @param txId transaction id
430     * @param sourceId ID of the direct/indirect container whose membership should be ended
431     * @param endTime the time the resource was deleted, generally its last modified
432     */
433    @Transactional
434    public void endMembershipForSource(final String txId, final FedoraId sourceId, final Instant endTime) {
435        final Map<String, Object> parameterSource = Map.of(
436                TX_ID_PARAM, txId,
437                SOURCE_ID_PARAM, sourceId.getFullId(),
438                ADD_OP_PARAM, ADD_OPERATION);
439
440        jdbcTemplate.update(CLEAR_ALL_ADDED_FOR_SOURCE_IN_TX, parameterSource);
441
442        final Map<String, Object> parameterSource2 = Map.of(
443                TX_ID_PARAM, txId,
444                SOURCE_ID_PARAM, sourceId.getFullId(),
445                END_TIME_PARAM, formatInstant(endTime),
446                NO_END_TIME_PARAM, NO_END_TIMESTAMP,
447                DELETE_OP_PARAM, DELETE_OPERATION);
448        jdbcTemplate.update(END_EXISTING_FOR_SOURCE, parameterSource2);
449    }
450
451    /**
452     * Delete membership entries that are active at or after the given timestamp for the specified source
453     * @param txId transaction id
454     * @param sourceId ID of the direct/indirect container
455     * @param afterTime time at or after which membership should be removed
456     */
457    @Transactional
458    public void deleteMembershipForSourceAfter(final String txId, final FedoraId sourceId, final Instant afterTime) {
459        // Clear all membership added in this transaction
460        final Map<String, Object> parameterSource = Map.of(
461                TX_ID_PARAM, txId,
462                SOURCE_ID_PARAM, sourceId.getFullId(),
463                ADD_OP_PARAM, ADD_OPERATION);
464
465        jdbcTemplate.update(CLEAR_ALL_ADDED_FOR_SOURCE_IN_TX, parameterSource);
466
467        final var afterTimestamp = afterTime == null ? NO_START_TIMESTAMP : formatInstant(afterTime);
468
469        // Delete all existing membership entries that start after or end after the given timestamp
470        final Map<String, Object> parameterSource2 = Map.of(
471                TX_ID_PARAM, txId,
472                SOURCE_ID_PARAM, sourceId.getFullId(),
473                START_TIME_PARAM, afterTimestamp,
474                FORCE_PARAM, FORCE_FLAG,
475                DELETE_OP_PARAM, DELETE_OPERATION);
476        jdbcTemplate.update(DELETE_EXISTING_FOR_SOURCE_AFTER, parameterSource2);
477    }
478
479    /**
480     * Clean up any references to the target id, in transactions and outside
481     * @param txId transaction id
482     * @param targetId identifier of the resource to cleanup membership references for
483     */
484    @Transactional
485    public void deleteMembershipReferences(final String txId, final FedoraId targetId) {
486        final Map<String, Object> parameterSource = Map.of(
487                TARGET_ID_PARAM, targetId.getFullId(),
488                TX_ID_PARAM, txId);
489
490        jdbcTemplate.update(PURGE_ALL_REFERENCES_TRANSACTION, parameterSource);
491        jdbcTemplate.update(PURGE_ALL_REFERENCES_MEMBERSHIP, parameterSource);
492    }
493
494    /**
495     * Add new membership property to the index, clearing any delete
496     * operations for the property if necessary.
497     * @param txId transaction id
498     * @param sourceId ID of the direct/indirect container which produced the membership
499     * @param membership membership triple
500     * @param startTime time the membership triple was added
501     */
502    @Transactional
503    public void addMembership(final String txId, final FedoraId sourceId, final Triple membership,
504            final Instant startTime) {
505        // Clear any existing delete operation for this membership
506        final Map<String, Object> parametersDelete = Map.of(
507                TX_ID_PARAM, txId,
508                SOURCE_ID_PARAM, sourceId.getFullId(),
509                SUBJECT_ID_PARAM, membership.getSubject().getURI(),
510                PROPERTY_PARAM, membership.getPredicate().getURI(),
511                OBJECT_ID_PARAM, membership.getObject().getURI(),
512                OPERATION_PARAM, DELETE_OPERATION);
513
514        jdbcTemplate.update(CLEAR_ENTRY_IN_TX, parametersDelete);
515
516        // Add the new membership operation
517        addMembership(txId, sourceId, membership, startTime, null);
518    }
519
520    /**
521     * Add new membership property to the index
522     * @param txId transaction id
523     * @param sourceId ID of the direct/indirect container which produced the membership
524     * @param membership membership triple
525     * @param startTime time the membership triple was added
526     * @param endTime time the membership triple ends, or never if not provided
527     */
528    public void addMembership(final String txId, final FedoraId sourceId, final Triple membership,
529            final Instant startTime, final Instant endTime) {
530        final Timestamp endTimestamp;
531        final Timestamp lastUpdated;
532        final Timestamp startTimestamp = formatInstant(startTime);
533        if (endTime == null) {
534            endTimestamp = NO_END_TIMESTAMP;
535            lastUpdated = startTimestamp;
536        } else {
537            endTimestamp = formatInstant(endTime);
538            lastUpdated = endTimestamp;
539        }
540        // Add the new membership operation
541        final Map<String, Object> parameterSource = Map.of(
542                SUBJECT_ID_PARAM, membership.getSubject().getURI(),
543                PROPERTY_PARAM, membership.getPredicate().getURI(),
544                TARGET_ID_PARAM, membership.getObject().getURI(),
545                SOURCE_ID_PARAM, sourceId.getFullId(),
546                START_TIME_PARAM, startTimestamp,
547                END_TIME_PARAM, endTimestamp,
548                LAST_UPDATED_PARAM, lastUpdated,
549                TX_ID_PARAM, txId,
550                OPERATION_PARAM, ADD_OPERATION);
551
552        jdbcTemplate.update(INSERT_MEMBERSHIP_IN_TX, parameterSource);
553    }
554
555    /**
556     * Get a stream of membership triples with
557     * @param txId transaction from which membership will be retrieved, or null for no transaction
558     * @param subjectId ID of the subject
559     * @return Stream of membership triples
560     */
561    public Stream<Triple> getMembership(final String txId, final FedoraId subjectId) {
562        final Node subjectNode = NodeFactory.createURI(subjectId.getBaseId());
563
564        final RowMapper<Triple> membershipMapper = (rs, rowNum) ->
565                Triple.create(subjectNode,
566                        NodeFactory.createURI(rs.getString("property")),
567                        NodeFactory.createURI(rs.getString("object_id")));
568
569        final MapSqlParameterSource parameterSource = new MapSqlParameterSource();
570        parameterSource.addValue(TX_ID_PARAM, txId);
571
572        final String query;
573        if (subjectId.isMemento()) {
574            parameterSource.addValue(SUBJECT_ID_PARAM, subjectId.getBaseId());
575            parameterSource.addValue(MEMENTO_TIME_PARAM, formatInstant(subjectId.getMementoInstant()));
576            query = SELECT_MEMBERSHIP_MEMENTO_IN_TX;
577        } else {
578            parameterSource.addValue(SUBJECT_ID_PARAM, subjectId.getFullId());
579            parameterSource.addValue(NO_END_TIME_PARAM, NO_END_TIMESTAMP);
580            query = SELECT_MEMBERSHIP_IN_TX;
581        }
582
583        return StreamSupport.stream(new MembershipIterator(query, parameterSource, membershipMapper), false);
584    }
585
586    public Instant getLastUpdated(final String txId, final FedoraId subjectId) {
587        final MapSqlParameterSource parameterSource = new MapSqlParameterSource();
588
589        parameterSource.addValue(NO_END_TIME_PARAM, NO_END_TIMESTAMP);
590        final String lastUpdatedQuery;
591        if (subjectId.isMemento()) {
592            lastUpdatedQuery = SELECT_LAST_UPDATED_MEMENTO;
593            parameterSource.addValue(SUBJECT_ID_PARAM, subjectId.getBaseId());
594            parameterSource.addValue(MEMENTO_TIME_PARAM, formatInstant(subjectId.getMementoInstant()));
595        } else if (txId == null) {
596            lastUpdatedQuery = SELECT_LAST_UPDATED;
597            parameterSource.addValue(SUBJECT_ID_PARAM, subjectId.getFullId());
598        } else {
599            lastUpdatedQuery = SELECT_LAST_UPDATED_IN_TX;
600            parameterSource.addValue(SUBJECT_ID_PARAM, subjectId.getFullId());
601            parameterSource.addValue(TX_ID_PARAM, txId);
602            parameterSource.addValue(DELETE_OP_PARAM, DELETE_OPERATION);
603        }
604
605        return jdbcTemplate.queryForObject(lastUpdatedQuery, parameterSource, Instant.class);
606    }
607
608    /**
609     * Perform a commit of operations stored in the specified transaction
610     * @param txId transaction id
611     */
612    @Transactional
613    public void commitTransaction(final String txId) {
614        final Map<String, String> parameterSource = Map.of(TX_ID_PARAM, txId,
615                ADD_OP_PARAM, ADD_OPERATION,
616                DELETE_OP_PARAM, DELETE_OPERATION,
617                FORCE_PARAM, FORCE_FLAG);
618
619        jdbcTemplate.update(COMMIT_DELETES, parameterSource);
620        final int ends = jdbcTemplate.update(COMMIT_ENDS_MAP.get(this.dbPlatform), parameterSource);
621        final int adds = jdbcTemplate.update(COMMIT_ADDS, parameterSource);
622        final int cleaned = jdbcTemplate.update(DELETE_TRANSACTION, parameterSource);
623
624        log.debug("Completed commit, {} ended, {} adds, {} operations", ends, adds, cleaned);
625    }
626
627    /**
628     * Delete all entries related to a transaction
629     * @param txId transaction id
630     */
631    public void deleteTransaction(final String txId) {
632        final Map<String, String> parameterSource = Map.of(TX_ID_PARAM, txId);
633        jdbcTemplate.update(DELETE_TRANSACTION, parameterSource);
634    }
635
636    /**
637     * Format an instant to a timestamp without milliseconds, due to precision
638     * issues with memento datetimes.
639     * @param instant
640     * @return
641     */
642    private Timestamp formatInstant(final Instant instant) {
643        final var timestamp = Timestamp.from(instant);
644        timestamp.setNanos(0);
645        return timestamp;
646    }
647
648    /**
649     * Clear all entries from the index
650     */
651    @Transactional
652    public void clearIndex() {
653        jdbcTemplate.update(TRUNCATE_MEMBERSHIP, Map.of());
654        jdbcTemplate.update(TRUNCATE_MEMBERSHIP_TX, Map.of());
655    }
656
657    /**
658     * Log all membership entries, for debugging usage only
659     */
660    public void logMembership() {
661        log.info("source_id, subject_id, property, object_id, start_time, end_time, last_updated");
662        jdbcTemplate.query(SELECT_ALL_MEMBERSHIP, new RowCallbackHandler() {
663            @Override
664            public void processRow(final ResultSet rs) throws SQLException {
665                log.info("{}, {}, {}, {}, {}, {}, {}", rs.getString("source_id"), rs.getString("subject_id"),
666                        rs.getString("property"), rs.getString("object_id"), rs.getTimestamp("start_time"),
667                        rs.getTimestamp("end_time"), rs.getTimestamp("last_updated"));
668            }
669        });
670    }
671
672    /**
673     * Log all membership operations, for debugging usage only
674     */
675    public void logOperations() {
676        log.info("source_id, subject_id, property, object_id, start_time, end_time,"
677                + " last_updated, tx_id, operation, force_flag");
678        jdbcTemplate.query(SELECT_ALL_OPERATIONS, new RowCallbackHandler() {
679            @Override
680            public void processRow(final ResultSet rs) throws SQLException {
681                log.info("{}, {}, {}, {}, {}, {}, {}, {}, {}, {}",
682                        rs.getString("source_id"), rs.getString("subject_id"), rs.getString("property"),
683                        rs.getString("object_id"), rs.getTimestamp("start_time"), rs.getTimestamp("end_time"),
684                        rs.getTimestamp("last_updated"), rs.getString("tx_id"), rs.getString("operation"),
685                        rs.getString("force_flag"));
686            }
687        });
688    }
689
690    /**
691     * Set the JDBC datastore.
692     * @param dataSource the dataStore.
693     */
694    public void setDataSource(final DataSource dataSource) {
695        this.dataSource = dataSource;
696    }
697
698    /**
699     * Get the JDBC datastore.
700     * @return the dataStore.
701     */
702    public DataSource getDataSource() {
703        return dataSource;
704    }
705
706    /**
707     * Private class to back a stream with a paged DB query.
708     *
709     * If this needs to be run in parallel we will have to override trySplit() and determine a good method to split on.
710     */
711    private class MembershipIterator extends Spliterators.AbstractSpliterator<Triple> {
712        final Queue<Triple> children = new ConcurrentLinkedQueue<>();
713        int numOffsets = 0;
714        final String queryToUse;
715        final MapSqlParameterSource parameterSource;
716        final RowMapper<Triple> rowMapper;
717
718        public MembershipIterator(final String query, final MapSqlParameterSource parameters,
719                                  final RowMapper<Triple> mapper) {
720            super(Long.MAX_VALUE, Spliterator.ORDERED);
721            queryToUse = query;
722            parameterSource = parameters;
723            rowMapper = mapper;
724            parameterSource.addValue(ADD_OP_PARAM, ADD_OPERATION);
725            parameterSource.addValue(DELETE_OP_PARAM, DELETE_OPERATION);
726            parameterSource.addValue(LIMIT_PARAM, MEMBERSHIP_LIMIT);
727        }
728
729        @Override
730        public boolean tryAdvance(final Consumer<? super Triple> action) {
731            try {
732                action.accept(children.remove());
733            } catch (final NoSuchElementException e) {
734                parameterSource.addValue(OFFSET_PARAM, numOffsets * MEMBERSHIP_LIMIT);
735                numOffsets += 1;
736                children.addAll(jdbcTemplate.query(queryToUse, parameterSource, rowMapper));
737                if (children.size() == 0) {
738                    // no more elements.
739                    return false;
740                }
741                action.accept(children.remove());
742            }
743            return true;
744        }
745    }
746}