Source code

001/*
002 * Licensed to DuraSpace under one or more contributor license agreements.
003 * See the NOTICE file distributed with this work for additional information
004 * regarding copyright ownership.
005 *
006 * DuraSpace licenses this file to you under the Apache License,
007 * Version 2.0 (the "License"); you may not use this file except in
008 * compliance with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.fcrepo.kernel.impl.services;
019
020import static org.slf4j.LoggerFactory.getLogger;
021
022import java.sql.ResultSet;
023import java.sql.SQLException;
024import java.sql.Timestamp;
025import java.time.Instant;
026import java.util.Map;
027import java.util.NoSuchElementException;
028import java.util.Queue;
029import java.util.Spliterator;
030import java.util.Spliterators;
031import java.util.concurrent.ConcurrentLinkedQueue;
032import java.util.function.Consumer;
033import java.util.stream.Stream;
034import java.util.stream.StreamSupport;
035
036import javax.annotation.PostConstruct;
037import javax.inject.Inject;
038import javax.sql.DataSource;
039import javax.transaction.Transactional;
040
041import org.fcrepo.common.db.DbPlatform;
042import org.fcrepo.kernel.api.identifiers.FedoraId;
043
044import org.apache.jena.graph.Node;
045import org.apache.jena.graph.NodeFactory;
046import org.apache.jena.graph.Triple;
047import org.slf4j.Logger;
048import org.springframework.jdbc.core.RowCallbackHandler;
049import org.springframework.jdbc.core.RowMapper;
050import org.springframework.jdbc.core.namedparam.MapSqlParameterSource;
051import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate;
052import org.springframework.stereotype.Component;
053
054/**
055 * Manager for the membership index
056 *
057 * @author bbpennel
058 */
059@Component
060public class MembershipIndexManager {
061    private static final Logger log = getLogger(MembershipIndexManager.class);
062
063    private static final Timestamp NO_END_TIMESTAMP = Timestamp.from(MembershipServiceImpl.NO_END_INSTANT);
064    private static final Timestamp NO_START_TIMESTAMP = Timestamp.from(Instant.parse("1000-01-01T00:00:00.000Z"));
065
066    private static final String ADD_OPERATION = "add";
067    private static final String DELETE_OPERATION = "delete";
068    private static final String FORCE_FLAG = "force";
069
070    private static final String TX_ID_PARAM = "txId";
071    private static final String SUBJECT_ID_PARAM = "subjectId";
072    private static final String NO_END_TIME_PARAM = "noEndTime";
073    private static final String ADD_OP_PARAM = "addOp";
074    private static final String DELETE_OP_PARAM = "deleteOp";
075    private static final String MEMENTO_TIME_PARAM = "mementoTime";
076    private static final String PROPERTY_PARAM = "property";
077    private static final String TARGET_ID_PARAM = "targetId";
078    private static final String SOURCE_ID_PARAM = "sourceId";
079    private static final String PROXY_ID_PARAM = "proxyId";
080    private static final String START_TIME_PARAM = "startTime";
081    private static final String END_TIME_PARAM = "endTime";
082    private static final String LAST_UPDATED_PARAM = "lastUpdated";
083    private static final String OPERATION_PARAM = "operation";
084    private static final String FORCE_PARAM = "forceFlag";
085    private static final String LIMIT_PARAM = "limit";
086    private static final String OFFSET_PARAM = "offSet";
087
088    private static final String SELECT_ALL_MEMBERSHIP = "SELECT * FROM membership";
089
090    private static final String SELECT_ALL_OPERATIONS = "SELECT * FROM membership_tx_operations";
091
092    private static final String SELECT_MEMBERSHIP_IN_TX =
093            "SELECT property, object_id" +
094            " FROM membership m" +
095            " WHERE subject_id = :subjectId" +
096                " AND end_time = :noEndTime" +
097                " AND NOT EXISTS (" +
098                    " SELECT 1" +
099                    " FROM membership_tx_operations mto" +
100                    " WHERE mto.subject_id = :subjectId" +
101                        " AND mto.source_id = m.source_id" +
102                        " AND mto.object_id = m.object_id" +
103                        " AND mto.tx_id = :txId" +
104                        " AND mto.operation = :deleteOp)" +
105            " UNION" +
106            " SELECT property, object_id" +
107            " FROM membership_tx_operations" +
108            " WHERE subject_id = :subjectId" +
109                " AND tx_id = :txId" +
110                " AND end_time = :noEndTime" +
111                " AND operation = :addOp" +
112            " ORDER BY property, object_id" +
113            " LIMIT :limit OFFSET :offSet";
114
115    private static final String SELECT_MEMBERSHIP_MEMENTO_IN_TX =
116            "SELECT property, object_id" +
117            " FROM membership m" +
118            " WHERE m.subject_id = :subjectId" +
119                " AND m.start_time <= :mementoTime" +
120                " AND m.end_time > :mementoTime" +
121                " AND NOT EXISTS (" +
122                    " SELECT 1" +
123                    " FROM membership_tx_operations mto" +
124                    " WHERE mto.subject_id = :subjectId" +
125                        " AND mto.source_id = m.source_id" +
126                        " AND mto.property = m.property" +
127                        " AND mto.object_id = m.object_id" +
128                        " AND mto.end_time <= :mementoTime" +
129                        " AND mto.tx_id = :txId" +
130                        " AND mto.operation = :deleteOp)" +
131            " UNION" +
132            " SELECT property, object_id" +
133            " FROM membership_tx_operations" +
134            " WHERE subject_id = :subjectId" +
135                " AND tx_id = :txId" +
136                " AND start_time <= :mementoTime" +
137                " AND end_time > :mementoTime" +
138                " AND operation = :addOp" +
139            " ORDER BY property, object_id" +
140            " LIMIT :limit OFFSET :offSet";
141
142    private static final String SELECT_LAST_UPDATED =
143            "SELECT max(last_updated) as last_updated" +
144            " FROM membership" +
145            " WHERE subject_id = :subjectId";
146
147    // For mementos, use the start_time instead of last_updated as the
148    // end_time reflects when the next version starts
149    private static final String SELECT_LAST_UPDATED_MEMENTO =
150            "SELECT max(start_time)" +
151            " FROM membership" +
152            " WHERE subject_id = :subjectId" +
153                " AND start_time <= :mementoTime" +
154                " AND end_time > :mementoTime";
155
156    private static final String SELECT_LAST_UPDATED_IN_TX =
157            "SELECT max(combined.updated) as last_updated" +
158            " FROM (" +
159                " SELECT max(last_updated) as updated" +
160                " FROM membership m" +
161                " WHERE subject_id = :subjectId" +
162                    " AND NOT EXISTS (" +
163                        " SELECT 1" +
164                        " FROM membership_tx_operations mto" +
165                        " WHERE mto.subject_id = :subjectId" +
166                            " AND mto.source_id = m.source_id" +
167                            " AND mto.object_id = m.object_id" +
168                            " AND mto.tx_id = :txId" +
169                            " AND mto.operation = :deleteOp)" +
170                " UNION" +
171                " SELECT max(last_updated) as updated" +
172                " FROM membership_tx_operations" +
173                " WHERE subject_id = :subjectId" +
174                    " AND tx_id = :txId" +
175            ") combined";
176
177    private static final String INSERT_MEMBERSHIP_IN_TX =
178            "INSERT INTO membership_tx_operations (subject_id, property, object_id, source_id," +
179                    " proxy_id, start_time, end_time, last_updated, tx_id, operation)" +
180            " VALUES (:subjectId, :property, :targetId, :sourceId," +
181                    " :proxyId, :startTime, :endTime, :lastUpdated, :txId, :operation)";
182
183    private static final String END_EXISTING_MEMBERSHIP =
184            "INSERT INTO membership_tx_operations (subject_id, property, object_id, source_id," +
185                    " proxy_id, start_time, end_time, last_updated, tx_id, operation)" +
186            " SELECT m.subject_id, m.property, m.object_id, m.source_id, m.proxy_id, m.start_time," +
187                    " :endTime, :endTime, :txId, :deleteOp" +
188            " FROM membership m" +
189            " WHERE m.source_id = :sourceId" +
190                " AND m.proxy_id = :proxyId" +
191                " AND m.end_time = :noEndTime";
192
193    private static final String CLEAR_FOR_PROXY_IN_TX =
194            "DELETE FROM membership_tx_operations" +
195            " WHERE source_id = :sourceId" +
196                " AND tx_id = :txId" +
197                " AND proxy_id = :proxyId" +
198                " AND force_flag IS NULL";
199
200    private static final String CLEAR_ALL_ADDED_FOR_SOURCE_IN_TX =
201            "DELETE FROM membership_tx_operations" +
202            " WHERE source_id = :sourceId" +
203                " AND tx_id = :txId" +
204                " AND operation = :addOp";
205
206    // Add "delete" entries for all existing membership from the given source, if not already deleted
207    private static final String END_EXISTING_FOR_SOURCE =
208            "INSERT INTO membership_tx_operations (subject_id, property, object_id, source_id," +
209                    " proxy_id, start_time, end_time, last_updated, tx_id, operation)" +
210            " SELECT subject_id, property, object_id, source_id," +
211                    " proxy_id, start_time, :endTime, :endTime, :txId, :deleteOp" +
212            " FROM membership m" +
213            " WHERE source_id = :sourceId" +
214                " AND end_time = :noEndTime" +
215                " AND NOT EXISTS (" +
216                    " SELECT TRUE" +
217                    " FROM membership_tx_operations mtx" +
218                    " WHERE mtx.subject_id = m.subject_id" +
219                        " AND mtx.property = m.property" +
220                        " AND mtx.object_id = m.object_id" +
221                        " AND mtx.source_id = m.source_id" +
222                        " AND mtx.proxy_id = m.proxy_id" +
223                        " AND mtx.operation = :deleteOp" +
224                    ")";
225
226    private static final String DELETE_EXISTING_FOR_SOURCE_AFTER =
227            "INSERT INTO membership_tx_operations(subject_id, property, object_id, source_id," +
228                    " proxy_id, start_time, end_time, last_updated, tx_id, operation, force_flag)" +
229            " SELECT subject_id, property, object_id, source_id, proxy_id, start_time, end_time," +
230                    " last_updated, :txId, :deleteOp, :forceFlag" +
231            " FROM membership m" +
232            " WHERE m.source_id = :sourceId" +
233                " AND (m.start_time >= :startTime" +
234                " OR m.end_time >= :startTime)";
235
236    private static final String DELETE_EXISTING_FOR_PROXY_AFTER =
237            "INSERT INTO membership_tx_operations(subject_id, property, object_id, source_id," +
238                    " proxy_id, start_time, end_time, last_updated, tx_id, operation, force_flag)" +
239            " SELECT subject_id, property, object_id, source_id, proxy_id, start_time, end_time," +
240                    " last_updated, :txId, :deleteOp, :forceFlag" +
241            " FROM membership m" +
242            " WHERE m.proxy_id = :proxyId" +
243                " AND (m.start_time >= :startTime" +
244                " OR m.end_time >= :startTime)";
245
246    private static final String PURGE_ALL_REFERENCES_MEMBERSHIP =
247            "DELETE from membership" +
248            " where source_id = :targetId" +
249                " OR subject_id = :targetId" +
250                " OR object_id = :targetId";
251
252    private static final String PURGE_ALL_REFERENCES_TRANSACTION =
253            "DELETE from membership_tx_operations" +
254            " WHERE tx_id = :txId" +
255                " AND (source_id = :targetId" +
256                " OR subject_id = :targetId" +
257                " OR object_id = :targetId)";
258
259    private static final String COMMIT_DELETES =
260            "DELETE from membership" +
261            " WHERE EXISTS (" +
262                " SELECT TRUE" +
263                " FROM membership_tx_operations mto" +
264                " WHERE mto.tx_id = :txId" +
265                    " AND mto.operation = :deleteOp" +
266                    " AND mto.force_flag = :forceFlag" +
267                    " AND membership.source_id = mto.source_id" +
268                    " AND membership.proxy_id = mto.proxy_id" +
269                    " AND membership.subject_id = mto.subject_id" +
270                    " AND membership.property = mto.property" +
271                    " AND membership.object_id = mto.object_id" +
272                " )";
273
274    private static final String COMMIT_ENDS_H2 =
275            "UPDATE membership m" +
276            " SET end_time = (" +
277                " SELECT mto.end_time" +
278                " FROM membership_tx_operations mto" +
279                " WHERE mto.tx_id = :txId" +
280                    " AND m.source_id = mto.source_id" +
281                    " AND m.proxy_id = mto.proxy_id" +
282                    " AND m.subject_id = mto.subject_id" +
283                    " AND m.property = mto.property" +
284                    " AND m.object_id = mto.object_id" +
285                    " AND mto.operation = :deleteOp" +
286                " )," +
287                " last_updated = (" +
288                    " SELECT mto.end_time" +
289                    " FROM membership_tx_operations mto" +
290                    " WHERE mto.tx_id = :txId" +
291                        " AND m.source_id = mto.source_id" +
292                        " AND m.proxy_id = mto.proxy_id" +
293                        " AND m.subject_id = mto.subject_id" +
294                        " AND m.property = mto.property" +
295                        " AND m.object_id = mto.object_id" +
296                        " AND mto.operation = :deleteOp" +
297                    " )" +
298            " WHERE EXISTS (" +
299                "SELECT TRUE" +
300                " FROM membership_tx_operations mto" +
301                " WHERE mto.tx_id = :txId" +
302                    " AND mto.operation = :deleteOp" +
303                    " AND m.source_id = mto.source_id" +
304                    " AND m.proxy_id = mto.proxy_id" +
305                    " AND m.subject_id = mto.subject_id" +
306                    " AND m.property = mto.property" +
307                    " AND m.object_id = mto.object_id" +
308                " )";
309
310    private static final String COMMIT_ENDS_POSTGRES =
311            "UPDATE membership" +
312            " SET end_time = mto.end_time, last_updated = mto.end_time" +
313            " FROM membership_tx_operations mto" +
314            " WHERE mto.tx_id = :txId" +
315                " AND mto.operation = :deleteOp" +
316                " AND membership.source_id = mto.source_id" +
317                " AND membership.proxy_id = mto.proxy_id" +
318                " AND membership.subject_id = mto.subject_id" +
319                " AND membership.property = mto.property" +
320                " AND membership.object_id = mto.object_id";
321
322    private static final String COMMIT_ENDS_MYSQL =
323            "UPDATE membership m" +
324            " INNER JOIN membership_tx_operations mto ON" +
325                " m.source_id = mto.source_id" +
326                " AND m.proxy_id = mto.proxy_id" +
327                " AND m.subject_id = mto.subject_id" +
328                " AND m.property = mto.property" +
329                " AND m.object_id = mto.object_id" +
330            " SET m.end_time = mto.end_time, m.last_updated = mto.end_time" +
331            " WHERE mto.tx_id = :txId" +
332                " AND mto.operation = :deleteOp";
333
334    private static final Map<DbPlatform, String> COMMIT_ENDS_MAP = Map.of(
335            DbPlatform.MYSQL, COMMIT_ENDS_MYSQL,
336            DbPlatform.MARIADB, COMMIT_ENDS_MYSQL,
337            DbPlatform.POSTGRESQL, COMMIT_ENDS_POSTGRES,
338            DbPlatform.H2, COMMIT_ENDS_H2
339    );
340
341    // Transfer all "add" operations from tx to committed membership, unless the entry already exists
342    private static final String COMMIT_ADDS =
343            "INSERT INTO membership" +
344            " (subject_id, property, object_id, source_id, proxy_id, start_time, end_time, last_updated)" +
345            " SELECT subject_id, property, object_id, source_id, proxy_id, start_time, end_time, last_updated" +
346            " FROM membership_tx_operations mto" +
347            " WHERE mto.tx_id = :txId" +
348                " AND mto.operation = :addOp" +
349                " AND NOT EXISTS (" +
350                    " SELECT TRUE" +
351                    " FROM membership m" +
352                    " WHERE m.source_id = mto.source_id" +
353                        " AND m.proxy_id = mto.proxy_id" +
354                        " AND m.subject_id = mto.subject_id" +
355                        " AND m.property = mto.property" +
356                        " AND m.object_id = mto.object_id" +
357                        " AND m.start_time = mto.start_time" +
358                        " AND m.end_time = mto.end_time" +
359                " )";
360
361    private static final String DELETE_TRANSACTION =
362            "DELETE FROM membership_tx_operations" +
363            " WHERE tx_id = :txId";
364
365    private static final String TRUNCATE_MEMBERSHIP = "TRUNCATE TABLE membership";
366
367    private static final String TRUNCATE_MEMBERSHIP_TX = "TRUNCATE TABLE membership_tx_operations";
368
369    @Inject
370    private DataSource dataSource;
371
372    private NamedParameterJdbcTemplate jdbcTemplate;
373
374    private DbPlatform dbPlatform;
375
376    private static final int MEMBERSHIP_LIMIT = 50000;
377
378    @PostConstruct
379    public void setUp() {
380        jdbcTemplate = new NamedParameterJdbcTemplate(getDataSource());
381        dbPlatform = DbPlatform.fromDataSource(dataSource);
382    }
383
384    /**
385     * End a membership from the child of a Direct/IndirectContainer, setting an end time if committed,
386     * or clearing from the current tx if it was newly added.
387     *
388     * @param txId transaction id
389     * @param sourceId ID of the direct/indirect container whose membership should be ended
390     * @param proxyId ID of the proxy producing this membership, when applicable
391     * @param endTime the time the resource was deleted, generally its last modified
392     */
393    @Transactional
394    public void endMembershipFromChild(final String txId, final FedoraId sourceId, final FedoraId proxyId,
395            final Instant endTime) {
396        final MapSqlParameterSource parameterSource = new MapSqlParameterSource();
397        parameterSource.addValue(TX_ID_PARAM, txId);
398        parameterSource.addValue(SOURCE_ID_PARAM, sourceId.getFullId());
399        parameterSource.addValue(PROXY_ID_PARAM, proxyId.getFullId());
400
401        final int affected = jdbcTemplate.update(CLEAR_FOR_PROXY_IN_TX, parameterSource);
402
403        // If no rows were deleted, then assume we need to delete permanent entry
404        if (affected == 0) {
405            final MapSqlParameterSource parameterSource2 = new MapSqlParameterSource();
406            parameterSource2.addValue(TX_ID_PARAM, txId);
407            parameterSource2.addValue(SOURCE_ID_PARAM, sourceId.getFullId());
408            parameterSource2.addValue(PROXY_ID_PARAM, proxyId.getFullId());
409            parameterSource2.addValue(END_TIME_PARAM, formatInstant(endTime));
410            parameterSource2.addValue(NO_END_TIME_PARAM, NO_END_TIMESTAMP);
411            parameterSource2.addValue(DELETE_OP_PARAM, DELETE_OPERATION);
412            jdbcTemplate.update(END_EXISTING_MEMBERSHIP, parameterSource2);
413        }
414    }
415
416    @Transactional
417    public void deleteMembershipForProxyAfter(final String txId, final FedoraId sourceId, final FedoraId proxyId,
418            final Instant afterTime) {
419        // Clear all membership added in this transaction
420        final var parameterSource =  Map.of(
421                TX_ID_PARAM, txId,
422                SOURCE_ID_PARAM, sourceId.getFullId(),
423                PROXY_ID_PARAM, proxyId.getFullId(),
424                OPERATION_PARAM, ADD_OPERATION);
425
426        jdbcTemplate.update(CLEAR_FOR_PROXY_IN_TX, parameterSource);
427
428        final var afterTimestamp = afterTime == null ? NO_START_TIMESTAMP : formatInstant(afterTime);
429
430        // Delete all existing membership entries that start after or end after the given timestamp
431        final Map<String, Object> parameterSource2 = Map.of(
432                TX_ID_PARAM, txId,
433                PROXY_ID_PARAM, proxyId.getFullId(),
434                START_TIME_PARAM, afterTimestamp,
435                FORCE_PARAM, FORCE_FLAG,
436                DELETE_OP_PARAM, DELETE_OPERATION);
437        jdbcTemplate.update(DELETE_EXISTING_FOR_PROXY_AFTER, parameterSource2);
438    }
439
440    /**
441     * End all membership properties resulting from the specified source container
442     * @param txId transaction id
443     * @param sourceId ID of the direct/indirect container whose membership should be ended
444     * @param endTime the time the resource was deleted, generally its last modified
445     */
446    @Transactional
447    public void endMembershipForSource(final String txId, final FedoraId sourceId, final Instant endTime) {
448        final Map<String, Object> parameterSource = Map.of(
449                TX_ID_PARAM, txId,
450                SOURCE_ID_PARAM, sourceId.getFullId(),
451                ADD_OP_PARAM, ADD_OPERATION);
452
453        jdbcTemplate.update(CLEAR_ALL_ADDED_FOR_SOURCE_IN_TX, parameterSource);
454
455        final Map<String, Object> parameterSource2 = Map.of(
456                TX_ID_PARAM, txId,
457                SOURCE_ID_PARAM, sourceId.getFullId(),
458                END_TIME_PARAM, formatInstant(endTime),
459                NO_END_TIME_PARAM, NO_END_TIMESTAMP,
460                DELETE_OP_PARAM, DELETE_OPERATION);
461        jdbcTemplate.update(END_EXISTING_FOR_SOURCE, parameterSource2);
462    }
463
464    /**
465     * Delete membership entries that are active at or after the given timestamp for the specified source
466     * @param txId transaction id
467     * @param sourceId ID of the direct/indirect container
468     * @param afterTime time at or after which membership should be removed
469     */
470    @Transactional
471    public void deleteMembershipForSourceAfter(final String txId, final FedoraId sourceId, final Instant afterTime) {
472        // Clear all membership added in this transaction
473        final Map<String, Object> parameterSource = Map.of(
474                TX_ID_PARAM, txId,
475                SOURCE_ID_PARAM, sourceId.getFullId(),
476                ADD_OP_PARAM, ADD_OPERATION);
477
478        jdbcTemplate.update(CLEAR_ALL_ADDED_FOR_SOURCE_IN_TX, parameterSource);
479
480        final var afterTimestamp = afterTime == null ? NO_START_TIMESTAMP : formatInstant(afterTime);
481
482        // Delete all existing membership entries that start after or end after the given timestamp
483        final Map<String, Object> parameterSource2 = Map.of(
484                TX_ID_PARAM, txId,
485                SOURCE_ID_PARAM, sourceId.getFullId(),
486                START_TIME_PARAM, afterTimestamp,
487                FORCE_PARAM, FORCE_FLAG,
488                DELETE_OP_PARAM, DELETE_OPERATION);
489        jdbcTemplate.update(DELETE_EXISTING_FOR_SOURCE_AFTER, parameterSource2);
490    }
491
492    /**
493     * Clean up any references to the target id, in transactions and outside
494     * @param txId transaction id
495     * @param targetId identifier of the resource to cleanup membership references for
496     */
497    @Transactional
498    public void deleteMembershipReferences(final String txId, final FedoraId targetId) {
499        final Map<String, Object> parameterSource = Map.of(
500                TARGET_ID_PARAM, targetId.getFullId(),
501                TX_ID_PARAM, txId);
502
503        jdbcTemplate.update(PURGE_ALL_REFERENCES_TRANSACTION, parameterSource);
504        jdbcTemplate.update(PURGE_ALL_REFERENCES_MEMBERSHIP, parameterSource);
505    }
506
507    /**
508     * Add new membership property to the index, clearing any delete
509     * operations for the property if necessary.
510     * @param txId transaction id
511     * @param sourceId ID of the direct/indirect container which produced the membership
512     * @param proxyId ID of the proxy producing this membership, when applicable
513     * @param membership membership triple
514     * @param startTime time the membership triple was added
515     */
516    @Transactional
517    public void addMembership(final String txId, final FedoraId sourceId, final FedoraId proxyId,
518            final Triple membership, final Instant startTime) {
519        if (membership == null) {
520            return;
521        }
522        addMembership(txId, sourceId, proxyId, membership, startTime, null);
523    }
524
525    /**
526     * Add new membership property to the index
527     * @param txId transaction id
528     * @param sourceId ID of the direct/indirect container which produced the membership
529     * @param proxyId ID of the proxy producing this membership, when applicable
530     * @param membership membership triple
531     * @param startTime time the membership triple was added
532     * @param endTime time the membership triple ends, or never if not provided
533     */
534    @Transactional
535    public void addMembership(final String txId, final FedoraId sourceId, final FedoraId proxyId,
536            final Triple membership, final Instant startTime, final Instant endTime) {
537        final Timestamp endTimestamp;
538        final Timestamp lastUpdated;
539        final Timestamp startTimestamp = formatInstant(startTime);
540        if (endTime == null) {
541            endTimestamp = NO_END_TIMESTAMP;
542            lastUpdated = startTimestamp;
543        } else {
544            endTimestamp = formatInstant(endTime);
545            lastUpdated = endTimestamp;
546        }
547        // Add the new membership operation
548        final MapSqlParameterSource parameterSource = new MapSqlParameterSource();
549        parameterSource.addValue(SUBJECT_ID_PARAM, membership.getSubject().getURI());
550        parameterSource.addValue(PROPERTY_PARAM, membership.getPredicate().getURI());
551        parameterSource.addValue(TARGET_ID_PARAM, membership.getObject().getURI());
552        parameterSource.addValue(SOURCE_ID_PARAM, sourceId.getFullId());
553        parameterSource.addValue(PROXY_ID_PARAM, proxyId.getFullId());
554        parameterSource.addValue(START_TIME_PARAM, startTimestamp);
555        parameterSource.addValue(END_TIME_PARAM, endTimestamp);
556        parameterSource.addValue(LAST_UPDATED_PARAM, lastUpdated);
557        parameterSource.addValue(TX_ID_PARAM, txId);
558        parameterSource.addValue(OPERATION_PARAM, ADD_OPERATION);
559
560        jdbcTemplate.update(INSERT_MEMBERSHIP_IN_TX, parameterSource);
561    }
562
563    /**
564     * Get a stream of membership triples with
565     * @param txId transaction from which membership will be retrieved, or null for no transaction
566     * @param subjectId ID of the subject
567     * @return Stream of membership triples
568     */
569    public Stream<Triple> getMembership(final String txId, final FedoraId subjectId) {
570        final Node subjectNode = NodeFactory.createURI(subjectId.getBaseId());
571
572        final RowMapper<Triple> membershipMapper = (rs, rowNum) ->
573                Triple.create(subjectNode,
574                        NodeFactory.createURI(rs.getString("property")),
575                        NodeFactory.createURI(rs.getString("object_id")));
576
577        final MapSqlParameterSource parameterSource = new MapSqlParameterSource();
578        parameterSource.addValue(TX_ID_PARAM, txId);
579
580        final String query;
581        if (subjectId.isMemento()) {
582            parameterSource.addValue(SUBJECT_ID_PARAM, subjectId.getBaseId());
583            parameterSource.addValue(MEMENTO_TIME_PARAM, formatInstant(subjectId.getMementoInstant()));
584            query = SELECT_MEMBERSHIP_MEMENTO_IN_TX;
585        } else {
586            parameterSource.addValue(SUBJECT_ID_PARAM, subjectId.getFullId());
587            parameterSource.addValue(NO_END_TIME_PARAM, NO_END_TIMESTAMP);
588            query = SELECT_MEMBERSHIP_IN_TX;
589        }
590
591        return StreamSupport.stream(new MembershipIterator(query, parameterSource, membershipMapper), false);
592    }
593
594    public Instant getLastUpdated(final String txId, final FedoraId subjectId) {
595        final MapSqlParameterSource parameterSource = new MapSqlParameterSource();
596
597        parameterSource.addValue(NO_END_TIME_PARAM, NO_END_TIMESTAMP);
598        final String lastUpdatedQuery;
599        if (subjectId.isMemento()) {
600            lastUpdatedQuery = SELECT_LAST_UPDATED_MEMENTO;
601            parameterSource.addValue(SUBJECT_ID_PARAM, subjectId.getBaseId());
602            parameterSource.addValue(MEMENTO_TIME_PARAM, formatInstant(subjectId.getMementoInstant()));
603        } else if (txId == null) {
604            lastUpdatedQuery = SELECT_LAST_UPDATED;
605            parameterSource.addValue(SUBJECT_ID_PARAM, subjectId.getFullId());
606        } else {
607            lastUpdatedQuery = SELECT_LAST_UPDATED_IN_TX;
608            parameterSource.addValue(SUBJECT_ID_PARAM, subjectId.getFullId());
609            parameterSource.addValue(TX_ID_PARAM, txId);
610            parameterSource.addValue(DELETE_OP_PARAM, DELETE_OPERATION);
611        }
612
613        return jdbcTemplate.queryForObject(lastUpdatedQuery, parameterSource, Instant.class);
614    }
615
616    /**
617     * Perform a commit of operations stored in the specified transaction
618     * @param txId transaction id
619     */
620    @Transactional
621    public void commitTransaction(final String txId) {
622        final Map<String, String> parameterSource = Map.of(TX_ID_PARAM, txId,
623                ADD_OP_PARAM, ADD_OPERATION,
624                DELETE_OP_PARAM, DELETE_OPERATION,
625                FORCE_PARAM, FORCE_FLAG);
626
627        jdbcTemplate.update(COMMIT_DELETES, parameterSource);
628        final int ends = jdbcTemplate.update(COMMIT_ENDS_MAP.get(this.dbPlatform), parameterSource);
629        final int adds = jdbcTemplate.update(COMMIT_ADDS, parameterSource);
630        final int cleaned = jdbcTemplate.update(DELETE_TRANSACTION, parameterSource);
631
632        log.debug("Completed commit, {} ended, {} adds, {} operations", ends, adds, cleaned);
633    }
634
635    /**
636     * Delete all entries related to a transaction
637     * @param txId transaction id
638     */
639    public void deleteTransaction(final String txId) {
640        final Map<String, String> parameterSource = Map.of(TX_ID_PARAM, txId);
641        jdbcTemplate.update(DELETE_TRANSACTION, parameterSource);
642    }
643
644    /**
645     * Format an instant to a timestamp without milliseconds, due to precision
646     * issues with memento datetimes.
647     * @param instant
648     * @return
649     */
650    private Timestamp formatInstant(final Instant instant) {
651        final var timestamp = Timestamp.from(instant);
652        timestamp.setNanos(0);
653        return timestamp;
654    }
655
656    /**
657     * Clear all entries from the index
658     */
659    @Transactional
660    public void clearIndex() {
661        jdbcTemplate.update(TRUNCATE_MEMBERSHIP, Map.of());
662        jdbcTemplate.update(TRUNCATE_MEMBERSHIP_TX, Map.of());
663    }
664
665    /**
666     * Log all membership entries, for debugging usage only
667     */
668    public void logMembership() {
669        log.info("source_id, proxy_id, subject_id, property, object_id, start_time, end_time, last_updated");
670        jdbcTemplate.query(SELECT_ALL_MEMBERSHIP, new RowCallbackHandler() {
671            @Override
672            public void processRow(final ResultSet rs) throws SQLException {
673                log.info("{}, {}, {}, {}, {}, {}, {}, {}",
674                        rs.getString("source_id"), rs.getString("proxy_id"), rs.getString("subject_id"),
675                        rs.getString("property"), rs.getString("object_id"), rs.getTimestamp("start_time"),
676                        rs.getTimestamp("end_time"), rs.getTimestamp("last_updated"));
677            }
678        });
679    }
680
681    /**
682     * Log all membership operations, for debugging usage only
683     */
684    public void logOperations() {
685        log.info("source_id, proxy_id, subject_id, property, object_id, start_time, end_time,"
686                + " last_updated, tx_id, operation, force_flag");
687        jdbcTemplate.query(SELECT_ALL_OPERATIONS, new RowCallbackHandler() {
688            @Override
689            public void processRow(final ResultSet rs) throws SQLException {
690                log.info("{}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}",
691                        rs.getString("source_id"), rs.getString("proxy_id"), rs.getString("subject_id"),
692                        rs.getString("property"), rs.getString("object_id"), rs.getTimestamp("start_time"),
693                        rs.getTimestamp("end_time"), rs.getTimestamp("last_updated"), rs.getString("tx_id"),
694                        rs.getString("operation"), rs.getString("force_flag"));
695            }
696        });
697    }
698
699    /**
700     * Set the JDBC datastore.
701     * @param dataSource the dataStore.
702     */
703    public void setDataSource(final DataSource dataSource) {
704        this.dataSource = dataSource;
705    }
706
707    /**
708     * Get the JDBC datastore.
709     * @return the dataStore.
710     */
711    public DataSource getDataSource() {
712        return dataSource;
713    }
714
715    /**
716     * Private class to back a stream with a paged DB query.
717     *
718     * If this needs to be run in parallel we will have to override trySplit() and determine a good method to split on.
719     */
720    private class MembershipIterator extends Spliterators.AbstractSpliterator<Triple> {
721        final Queue<Triple> children = new ConcurrentLinkedQueue<>();
722        int numOffsets = 0;
723        final String queryToUse;
724        final MapSqlParameterSource parameterSource;
725        final RowMapper<Triple> rowMapper;
726
727        public MembershipIterator(final String query, final MapSqlParameterSource parameters,
728                                  final RowMapper<Triple> mapper) {
729            super(Long.MAX_VALUE, Spliterator.ORDERED);
730            queryToUse = query;
731            parameterSource = parameters;
732            rowMapper = mapper;
733            parameterSource.addValue(ADD_OP_PARAM, ADD_OPERATION);
734            parameterSource.addValue(DELETE_OP_PARAM, DELETE_OPERATION);
735            parameterSource.addValue(LIMIT_PARAM, MEMBERSHIP_LIMIT);
736        }
737
738        @Override
739        public boolean tryAdvance(final Consumer<? super Triple> action) {
740            try {
741                action.accept(children.remove());
742            } catch (final NoSuchElementException e) {
743                parameterSource.addValue(OFFSET_PARAM, numOffsets * MEMBERSHIP_LIMIT);
744                numOffsets += 1;
745                children.addAll(jdbcTemplate.query(queryToUse, parameterSource, rowMapper));
746                if (children.size() == 0) {
747                    // no more elements.
748                    return false;
749                }
750                action.accept(children.remove());
751            }
752            return true;
753        }
754    }
755}