001/*
002 * Licensed to DuraSpace under one or more contributor license agreements.
003 * See the NOTICE file distributed with this work for additional information
004 * regarding copyright ownership.
005 *
006 * DuraSpace licenses this file to you under the Apache License,
007 * Version 2.0 (the "License"); you may not use this file except in
008 * compliance with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.fcrepo.kernel.impl.services;
019
020import static org.slf4j.LoggerFactory.getLogger;
021
022import java.sql.ResultSet;
023import java.sql.SQLException;
024import java.sql.Timestamp;
025import java.time.Instant;
026import java.util.Map;
027import java.util.NoSuchElementException;
028import java.util.Queue;
029import java.util.Spliterator;
030import java.util.Spliterators;
031import java.util.concurrent.ConcurrentLinkedQueue;
032import java.util.function.Consumer;
033import java.util.stream.Stream;
034import java.util.stream.StreamSupport;
035
036import javax.annotation.PostConstruct;
037import javax.inject.Inject;
038import javax.sql.DataSource;
039
040import org.fcrepo.common.db.DbPlatform;
041import org.fcrepo.kernel.api.Transaction;
042import org.fcrepo.kernel.api.identifiers.FedoraId;
043
044import org.apache.jena.graph.Node;
045import org.apache.jena.graph.NodeFactory;
046import org.apache.jena.graph.Triple;
047import org.slf4j.Logger;
048import org.springframework.jdbc.core.RowCallbackHandler;
049import org.springframework.jdbc.core.RowMapper;
050import org.springframework.jdbc.core.namedparam.MapSqlParameterSource;
051import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate;
052import org.springframework.stereotype.Component;
053import org.springframework.transaction.annotation.Propagation;
054import org.springframework.transaction.annotation.Transactional;
055
056/**
057 * Manager for the membership index
058 *
059 * @author bbpennel
060 */
061@Component
062public class MembershipIndexManager {
063    private static final Logger log = getLogger(MembershipIndexManager.class);
064
065    private static final Timestamp NO_END_TIMESTAMP = Timestamp.from(MembershipServiceImpl.NO_END_INSTANT);
066    private static final Timestamp NO_START_TIMESTAMP = Timestamp.from(Instant.parse("1000-01-01T00:00:00.000Z"));
067
068    private static final String ADD_OPERATION = "add";
069    private static final String DELETE_OPERATION = "delete";
070    private static final String FORCE_FLAG = "force";
071
072    private static final String TX_ID_PARAM = "txId";
073    private static final String SUBJECT_ID_PARAM = "subjectId";
074    private static final String NO_END_TIME_PARAM = "noEndTime";
075    private static final String ADD_OP_PARAM = "addOp";
076    private static final String DELETE_OP_PARAM = "deleteOp";
077    private static final String MEMENTO_TIME_PARAM = "mementoTime";
078    private static final String PROPERTY_PARAM = "property";
079    private static final String TARGET_ID_PARAM = "targetId";
080    private static final String SOURCE_ID_PARAM = "sourceId";
081    private static final String PROXY_ID_PARAM = "proxyId";
082    private static final String START_TIME_PARAM = "startTime";
083    private static final String END_TIME_PARAM = "endTime";
084    private static final String LAST_UPDATED_PARAM = "lastUpdated";
085    private static final String OPERATION_PARAM = "operation";
086    private static final String FORCE_PARAM = "forceFlag";
087    private static final String LIMIT_PARAM = "limit";
088    private static final String OFFSET_PARAM = "offSet";
089
090    private static final String SELECT_ALL_MEMBERSHIP = "SELECT * FROM membership";
091
092    private static final String SELECT_ALL_OPERATIONS = "SELECT * FROM membership_tx_operations";
093
094    private static final String SELECT_MEMBERSHIP_IN_TX =
095            "SELECT property, object_id" +
096            " FROM membership m" +
097            " WHERE subject_id = :subjectId" +
098                " AND end_time = :noEndTime" +
099                " AND NOT EXISTS (" +
100                    " SELECT 1" +
101                    " FROM membership_tx_operations mto" +
102                    " WHERE mto.subject_id = :subjectId" +
103                        " AND mto.source_id = m.source_id" +
104                        " AND mto.object_id = m.object_id" +
105                        " AND mto.tx_id = :txId" +
106                        " AND mto.operation = :deleteOp)" +
107            " UNION" +
108            " SELECT property, object_id" +
109            " FROM membership_tx_operations" +
110            " WHERE subject_id = :subjectId" +
111                " AND tx_id = :txId" +
112                " AND end_time = :noEndTime" +
113                " AND operation = :addOp" +
114            " ORDER BY property, object_id" +
115            " LIMIT :limit OFFSET :offSet";
116
117    private static final String DIRECT_SELECT_MEMBERSHIP =
118            "SELECT property, object_id" +
119            " FROM membership" +
120            " WHERE subject_id = :subjectId" +
121                " AND end_time = :noEndTime" +
122            " ORDER BY property, object_id" +
123            " LIMIT :limit OFFSET :offSet";
124
125    private static final String SELECT_MEMBERSHIP_MEMENTO_IN_TX =
126            "SELECT property, object_id" +
127            " FROM membership m" +
128            " WHERE m.subject_id = :subjectId" +
129                " AND m.start_time <= :mementoTime" +
130                " AND m.end_time > :mementoTime" +
131                " AND NOT EXISTS (" +
132                    " SELECT 1" +
133                    " FROM membership_tx_operations mto" +
134                    " WHERE mto.subject_id = :subjectId" +
135                        " AND mto.source_id = m.source_id" +
136                        " AND mto.property = m.property" +
137                        " AND mto.object_id = m.object_id" +
138                        " AND mto.end_time <= :mementoTime" +
139                        " AND mto.tx_id = :txId" +
140                        " AND mto.operation = :deleteOp)" +
141            " UNION" +
142            " SELECT property, object_id" +
143            " FROM membership_tx_operations" +
144            " WHERE subject_id = :subjectId" +
145                " AND tx_id = :txId" +
146                " AND start_time <= :mementoTime" +
147                " AND end_time > :mementoTime" +
148                " AND operation = :addOp" +
149            " ORDER BY property, object_id" +
150            " LIMIT :limit OFFSET :offSet";
151
152    private static final String DIRECT_SELECT_MEMBERSHIP_MEMENTO =
153            "SELECT property, object_id" +
154            " FROM membership" +
155            " WHERE subject_id = :subjectId" +
156                " AND start_time <= :mementoTime" +
157                " AND end_time > :mementoTime" +
158            " ORDER BY property, object_id" +
159            " LIMIT :limit OFFSET :offSet";
160
161    private static final String SELECT_LAST_UPDATED =
162            "SELECT max(last_updated) as last_updated" +
163            " FROM membership" +
164            " WHERE subject_id = :subjectId";
165
166    // For mementos, use the start_time instead of last_updated as the
167    // end_time reflects when the next version starts
168    private static final String SELECT_LAST_UPDATED_MEMENTO =
169            "SELECT max(start_time)" +
170            " FROM membership" +
171            " WHERE subject_id = :subjectId" +
172                " AND start_time <= :mementoTime" +
173                " AND end_time > :mementoTime";
174
175    private static final String SELECT_LAST_UPDATED_IN_TX =
176            "SELECT max(combined.updated) as last_updated" +
177            " FROM (" +
178                " SELECT max(last_updated) as updated" +
179                " FROM membership m" +
180                " WHERE subject_id = :subjectId" +
181                    " AND NOT EXISTS (" +
182                        " SELECT 1" +
183                        " FROM membership_tx_operations mto" +
184                        " WHERE mto.subject_id = :subjectId" +
185                            " AND mto.source_id = m.source_id" +
186                            " AND mto.object_id = m.object_id" +
187                            " AND mto.tx_id = :txId" +
188                            " AND mto.operation = :deleteOp)" +
189                " UNION" +
190                " SELECT max(last_updated) as updated" +
191                " FROM membership_tx_operations" +
192                " WHERE subject_id = :subjectId" +
193                    " AND tx_id = :txId" +
194            ") combined";
195
196    private static final String INSERT_MEMBERSHIP_IN_TX =
197            "INSERT INTO membership_tx_operations (subject_id, property, object_id, source_id," +
198                    " proxy_id, start_time, end_time, last_updated, tx_id, operation)" +
199            " VALUES (:subjectId, :property, :targetId, :sourceId," +
200                    " :proxyId, :startTime, :endTime, :lastUpdated, :txId, :operation)";
201
202    private static final String DIRECT_INSERT_MEMBERSHIP =
203            "INSERT INTO membership (subject_id, property, object_id, source_id," +
204                    " proxy_id, start_time, end_time, last_updated)" +
205            " VALUES (:subjectId, :property, :targetId, :sourceId," +
206                    " :proxyId, :startTime, :endTime, :lastUpdated)";
207
208    private static final String END_EXISTING_MEMBERSHIP =
209            "INSERT INTO membership_tx_operations (subject_id, property, object_id, source_id," +
210                    " proxy_id, start_time, end_time, last_updated, tx_id, operation)" +
211            " SELECT m.subject_id, m.property, m.object_id, m.source_id, m.proxy_id, m.start_time," +
212                    " :endTime, :endTime, :txId, :deleteOp" +
213            " FROM membership m" +
214            " WHERE m.source_id = :sourceId" +
215                " AND m.proxy_id = :proxyId" +
216                " AND m.end_time = :noEndTime";
217
218    private static final String DIRECT_END_EXISTING_MEMBERSHIP =
219            "UPDATE membership SET end_time = :endTime, last_updated = :endTime" +
220            " WHERE source_id = :sourceId" +
221                " AND proxy_id = :proxyId" +
222                " AND end_time = :noEndTime";
223
224    private static final String CLEAR_FOR_PROXY_IN_TX =
225            "DELETE FROM membership_tx_operations" +
226            " WHERE source_id = :sourceId" +
227                " AND tx_id = :txId" +
228                " AND proxy_id = :proxyId" +
229                " AND force_flag IS NULL";
230
231    private static final String CLEAR_ALL_ADDED_FOR_SOURCE_IN_TX =
232            "DELETE FROM membership_tx_operations" +
233            " WHERE source_id = :sourceId" +
234                " AND tx_id = :txId" +
235                " AND operation = :addOp";
236
237    // Add "delete" entries for all existing membership from the given source, if not already deleted
238    private static final String END_EXISTING_FOR_SOURCE =
239            "INSERT INTO membership_tx_operations (subject_id, property, object_id, source_id," +
240                    " proxy_id, start_time, end_time, last_updated, tx_id, operation)" +
241            " SELECT subject_id, property, object_id, source_id," +
242                    " proxy_id, start_time, :endTime, :endTime, :txId, :deleteOp" +
243            " FROM membership m" +
244            " WHERE source_id = :sourceId" +
245                " AND end_time = :noEndTime" +
246                " AND NOT EXISTS (" +
247                    " SELECT TRUE" +
248                    " FROM membership_tx_operations mtx" +
249                    " WHERE mtx.subject_id = m.subject_id" +
250                        " AND mtx.property = m.property" +
251                        " AND mtx.object_id = m.object_id" +
252                        " AND mtx.source_id = m.source_id" +
253                        " AND mtx.proxy_id = m.proxy_id" +
254                        " AND mtx.operation = :deleteOp" +
255                    ")";
256
257    private static final String DIRECT_END_EXISTING_FOR_SOURCE =
258            "UPDATE membership SET end_time = :endTime, last_updated = :endTime" +
259                    " WHERE source_id = :sourceId" +
260                    " AND end_time = :noEndTime";
261
262    private static final String DELETE_EXISTING_FOR_SOURCE_AFTER =
263            "INSERT INTO membership_tx_operations(subject_id, property, object_id, source_id," +
264                    " proxy_id, start_time, end_time, last_updated, tx_id, operation, force_flag)" +
265            " SELECT subject_id, property, object_id, source_id, proxy_id, start_time, end_time," +
266                    " last_updated, :txId, :deleteOp, :forceFlag" +
267            " FROM membership m" +
268            " WHERE m.source_id = :sourceId" +
269                " AND (m.start_time >= :startTime" +
270                " OR m.end_time >= :startTime)";
271
272    private static final String DIRECT_DELETE_EXISTING_FOR_SOURCE_AFTER =
273            "DELETE FROM membership" +
274                    " WHERE source_id = :sourceId" +
275                    " AND (start_time >= :startTime" +
276                    " OR end_time >= :startTime)";
277
278    private static final String DELETE_EXISTING_FOR_PROXY_AFTER =
279            "INSERT INTO membership_tx_operations(subject_id, property, object_id, source_id," +
280                    " proxy_id, start_time, end_time, last_updated, tx_id, operation, force_flag)" +
281            " SELECT subject_id, property, object_id, source_id, proxy_id, start_time, end_time," +
282                    " last_updated, :txId, :deleteOp, :forceFlag" +
283            " FROM membership m" +
284            " WHERE m.proxy_id = :proxyId" +
285                " AND (m.start_time >= :startTime" +
286                " OR m.end_time >= :startTime)";
287
288    private static final String DIRECT_DELETE_EXISTING_FOR_PROXY_AFTER =
289            "UPDATE membership SET end_time = :endTime, last_updated = :endTime" +
290            " WHERE proxy_id = :proxyId" +
291                " AND (start_time >= :endTime" +
292                " OR end_time >= :endTime)";
293
294    private static final String PURGE_ALL_REFERENCES_MEMBERSHIP =
295            "DELETE from membership" +
296            " where source_id = :targetId" +
297                " OR subject_id = :targetId" +
298                " OR object_id = :targetId";
299
300    private static final String PURGE_ALL_REFERENCES_TRANSACTION =
301            "DELETE from membership_tx_operations" +
302            " WHERE tx_id = :txId" +
303                " AND (source_id = :targetId" +
304                " OR subject_id = :targetId" +
305                " OR object_id = :targetId)";
306
307    private static final String COMMIT_DELETES =
308            "DELETE from membership" +
309            " WHERE EXISTS (" +
310                " SELECT TRUE" +
311                " FROM membership_tx_operations mto" +
312                " WHERE mto.tx_id = :txId" +
313                    " AND mto.operation = :deleteOp" +
314                    " AND mto.force_flag = :forceFlag" +
315                    " AND membership.source_id = mto.source_id" +
316                    " AND membership.proxy_id = mto.proxy_id" +
317                    " AND membership.subject_id = mto.subject_id" +
318                    " AND membership.property = mto.property" +
319                    " AND membership.object_id = mto.object_id" +
320                " )";
321
322    private static final String COMMIT_ENDS_H2 =
323            "UPDATE membership m" +
324            " SET end_time = (" +
325                " SELECT mto.end_time" +
326                " FROM membership_tx_operations mto" +
327                " WHERE mto.tx_id = :txId" +
328                    " AND m.source_id = mto.source_id" +
329                    " AND m.proxy_id = mto.proxy_id" +
330                    " AND m.subject_id = mto.subject_id" +
331                    " AND m.property = mto.property" +
332                    " AND m.object_id = mto.object_id" +
333                    " AND mto.operation = :deleteOp" +
334                " )," +
335                " last_updated = (" +
336                    " SELECT mto.end_time" +
337                    " FROM membership_tx_operations mto" +
338                    " WHERE mto.tx_id = :txId" +
339                        " AND m.source_id = mto.source_id" +
340                        " AND m.proxy_id = mto.proxy_id" +
341                        " AND m.subject_id = mto.subject_id" +
342                        " AND m.property = mto.property" +
343                        " AND m.object_id = mto.object_id" +
344                        " AND mto.operation = :deleteOp" +
345                    " )" +
346            " WHERE EXISTS (" +
347                "SELECT TRUE" +
348                " FROM membership_tx_operations mto" +
349                " WHERE mto.tx_id = :txId" +
350                    " AND mto.operation = :deleteOp" +
351                    " AND m.source_id = mto.source_id" +
352                    " AND m.proxy_id = mto.proxy_id" +
353                    " AND m.subject_id = mto.subject_id" +
354                    " AND m.property = mto.property" +
355                    " AND m.object_id = mto.object_id" +
356                " )";
357
358    private static final String COMMIT_ENDS_POSTGRES =
359            "UPDATE membership" +
360            " SET end_time = mto.end_time, last_updated = mto.end_time" +
361            " FROM membership_tx_operations mto" +
362            " WHERE mto.tx_id = :txId" +
363                " AND mto.operation = :deleteOp" +
364                " AND membership.source_id = mto.source_id" +
365                " AND membership.proxy_id = mto.proxy_id" +
366                " AND membership.subject_id = mto.subject_id" +
367                " AND membership.property = mto.property" +
368                " AND membership.object_id = mto.object_id";
369
370    private static final String COMMIT_ENDS_MYSQL =
371            "UPDATE membership m" +
372            " INNER JOIN membership_tx_operations mto ON" +
373                " m.source_id = mto.source_id" +
374                " AND m.proxy_id = mto.proxy_id" +
375                " AND m.subject_id = mto.subject_id" +
376                " AND m.property = mto.property" +
377                " AND m.object_id = mto.object_id" +
378            " SET m.end_time = mto.end_time, m.last_updated = mto.end_time" +
379            " WHERE mto.tx_id = :txId" +
380                " AND mto.operation = :deleteOp";
381
382    private static final Map<DbPlatform, String> COMMIT_ENDS_MAP = Map.of(
383            DbPlatform.MYSQL, COMMIT_ENDS_MYSQL,
384            DbPlatform.MARIADB, COMMIT_ENDS_MYSQL,
385            DbPlatform.POSTGRESQL, COMMIT_ENDS_POSTGRES,
386            DbPlatform.H2, COMMIT_ENDS_H2
387    );
388
389    // Transfer all "add" operations from tx to committed membership, unless the entry already exists
390    private static final String COMMIT_ADDS =
391            "INSERT INTO membership" +
392            " (subject_id, property, object_id, source_id, proxy_id, start_time, end_time, last_updated)" +
393            " SELECT subject_id, property, object_id, source_id, proxy_id, start_time, end_time, last_updated" +
394            " FROM membership_tx_operations mto" +
395            " WHERE mto.tx_id = :txId" +
396                " AND mto.operation = :addOp" +
397                " AND NOT EXISTS (" +
398                    " SELECT TRUE" +
399                    " FROM membership m" +
400                    " WHERE m.source_id = mto.source_id" +
401                        " AND m.proxy_id = mto.proxy_id" +
402                        " AND m.subject_id = mto.subject_id" +
403                        " AND m.property = mto.property" +
404                        " AND m.object_id = mto.object_id" +
405                        " AND m.start_time = mto.start_time" +
406                        " AND m.end_time = mto.end_time" +
407                " )";
408
409    private static final String DELETE_TRANSACTION =
410            "DELETE FROM membership_tx_operations" +
411            " WHERE tx_id = :txId";
412
413    private static final String TRUNCATE_MEMBERSHIP = "TRUNCATE TABLE membership";
414
415    private static final String TRUNCATE_MEMBERSHIP_TX = "TRUNCATE TABLE membership_tx_operations";
416
417    @Inject
418    private DataSource dataSource;
419
420    private NamedParameterJdbcTemplate jdbcTemplate;
421
422    private DbPlatform dbPlatform;
423
424    private static final int MEMBERSHIP_LIMIT = 50000;
425
426    @PostConstruct
427    public void setUp() {
428        jdbcTemplate = new NamedParameterJdbcTemplate(getDataSource());
429        dbPlatform = DbPlatform.fromDataSource(dataSource);
430    }
431
432    /**
433     * End a membership from the child of a Direct/IndirectContainer, setting an end time if committed,
434     * or clearing from the current tx if it was newly added.
435     *
436     * @param tx transaction
437     * @param sourceId ID of the direct/indirect container whose membership should be ended
438     * @param proxyId ID of the proxy producing this membership, when applicable
439     * @param endTime the time the resource was deleted, generally its last modified
440     */
441    public void endMembershipFromChild(final Transaction tx, final FedoraId sourceId, final FedoraId proxyId,
442            final Instant endTime) {
443        tx.doInTx(() -> {
444            if (!tx.isShortLived()) {
445                final MapSqlParameterSource parameterSource = new MapSqlParameterSource();
446                parameterSource.addValue(TX_ID_PARAM, tx.getId());
447                parameterSource.addValue(SOURCE_ID_PARAM, sourceId.getFullId());
448                parameterSource.addValue(PROXY_ID_PARAM, proxyId.getFullId());
449
450                final int affected = jdbcTemplate.update(CLEAR_FOR_PROXY_IN_TX, parameterSource);
451
452                // If no rows were deleted, then assume we need to delete permanent entry
453                if (affected == 0) {
454                    final MapSqlParameterSource parameterSource2 = new MapSqlParameterSource();
455                    parameterSource2.addValue(TX_ID_PARAM, tx.getId());
456                    parameterSource2.addValue(SOURCE_ID_PARAM, sourceId.getFullId());
457                    parameterSource2.addValue(PROXY_ID_PARAM, proxyId.getFullId());
458                    parameterSource2.addValue(END_TIME_PARAM, formatInstant(endTime));
459                    parameterSource2.addValue(NO_END_TIME_PARAM, NO_END_TIMESTAMP);
460                    parameterSource2.addValue(DELETE_OP_PARAM, DELETE_OPERATION);
461                    jdbcTemplate.update(END_EXISTING_MEMBERSHIP, parameterSource2);
462                }
463            } else {
464                final MapSqlParameterSource parameterSource = new MapSqlParameterSource();
465                parameterSource.addValue(SOURCE_ID_PARAM, sourceId.getFullId());
466                parameterSource.addValue(PROXY_ID_PARAM, proxyId.getFullId());
467                parameterSource.addValue(END_TIME_PARAM, formatInstant(endTime));
468                parameterSource.addValue(NO_END_TIME_PARAM, NO_END_TIMESTAMP);
469                jdbcTemplate.update(DIRECT_END_EXISTING_MEMBERSHIP, parameterSource);
470            }
471        });
472    }
473
474    public void deleteMembershipForProxyAfter(final Transaction tx,
475                                              final FedoraId sourceId,
476                                              final FedoraId proxyId,
477                                              final Instant afterTime) {
478        tx.doInTx(() -> {
479            final var afterTimestamp = afterTime == null ? NO_START_TIMESTAMP : formatInstant(afterTime);
480
481            if (!tx.isShortLived()) {
482                // Clear all membership added in this transaction
483                final var parameterSource = Map.of(
484                        TX_ID_PARAM, tx.getId(),
485                        SOURCE_ID_PARAM, sourceId.getFullId(),
486                        PROXY_ID_PARAM, proxyId.getFullId(),
487                        OPERATION_PARAM, ADD_OPERATION);
488
489                jdbcTemplate.update(CLEAR_FOR_PROXY_IN_TX, parameterSource);
490
491                // Delete all existing membership entries that start after or end after the given timestamp
492                final Map<String, Object> parameterSource2 = Map.of(
493                        TX_ID_PARAM, tx.getId(),
494                        PROXY_ID_PARAM, proxyId.getFullId(),
495                        START_TIME_PARAM, afterTimestamp,
496                        FORCE_PARAM, FORCE_FLAG,
497                        DELETE_OP_PARAM, DELETE_OPERATION);
498                jdbcTemplate.update(DELETE_EXISTING_FOR_PROXY_AFTER, parameterSource2);
499            } else {
500                final Map<String, Object> parameterSource = Map.of(
501                        PROXY_ID_PARAM, proxyId.getFullId(),
502                        END_TIME_PARAM, afterTimestamp);
503                jdbcTemplate.update(DIRECT_DELETE_EXISTING_FOR_PROXY_AFTER, parameterSource);
504            }
505        });
506    }
507
508    /**
509     * End all membership properties resulting from the specified source container
510     * @param tx transaction
511     * @param sourceId ID of the direct/indirect container whose membership should be ended
512     * @param endTime the time the resource was deleted, generally its last modified
513     */
514    public void endMembershipForSource(final Transaction tx, final FedoraId sourceId, final Instant endTime) {
515        tx.doInTx(() -> {
516            if (!tx.isShortLived()) {
517                final Map<String, Object> parameterSource = Map.of(
518                        TX_ID_PARAM, tx.getId(),
519                        SOURCE_ID_PARAM, sourceId.getFullId(),
520                        ADD_OP_PARAM, ADD_OPERATION);
521
522                jdbcTemplate.update(CLEAR_ALL_ADDED_FOR_SOURCE_IN_TX, parameterSource);
523
524                final Map<String, Object> parameterSource2 = Map.of(
525                        TX_ID_PARAM, tx.getId(),
526                        SOURCE_ID_PARAM, sourceId.getFullId(),
527                        END_TIME_PARAM, formatInstant(endTime),
528                        NO_END_TIME_PARAM, NO_END_TIMESTAMP,
529                        DELETE_OP_PARAM, DELETE_OPERATION);
530                jdbcTemplate.update(END_EXISTING_FOR_SOURCE, parameterSource2);
531            } else {
532                final Map<String, Object> parameterSource = Map.of(
533                        SOURCE_ID_PARAM, sourceId.getFullId(),
534                        END_TIME_PARAM, formatInstant(endTime),
535                        NO_END_TIME_PARAM, NO_END_TIMESTAMP);
536                jdbcTemplate.update(DIRECT_END_EXISTING_FOR_SOURCE, parameterSource);
537            }
538        });
539    }
540
541    /**
542     * Delete membership entries that are active at or after the given timestamp for the specified source
543     * @param tx transaction
544     * @param sourceId ID of the direct/indirect container
545     * @param afterTime time at or after which membership should be removed
546     */
547    public void deleteMembershipForSourceAfter(final Transaction tx, final FedoraId sourceId, final Instant afterTime) {
548        tx.doInTx(() -> {
549            final var afterTimestamp = afterTime == null ? NO_START_TIMESTAMP : formatInstant(afterTime);
550
551            if (!tx.isShortLived()) {
552                // Clear all membership added in this transaction
553                final Map<String, Object> parameterSource = Map.of(
554                        TX_ID_PARAM, tx.getId(),
555                        SOURCE_ID_PARAM, sourceId.getFullId(),
556                        ADD_OP_PARAM, ADD_OPERATION);
557
558                jdbcTemplate.update(CLEAR_ALL_ADDED_FOR_SOURCE_IN_TX, parameterSource);
559
560                // Delete all existing membership entries that start after or end after the given timestamp
561                final Map<String, Object> parameterSource2 = Map.of(
562                        TX_ID_PARAM, tx.getId(),
563                        SOURCE_ID_PARAM, sourceId.getFullId(),
564                        START_TIME_PARAM, afterTimestamp,
565                        FORCE_PARAM, FORCE_FLAG,
566                        DELETE_OP_PARAM, DELETE_OPERATION);
567                jdbcTemplate.update(DELETE_EXISTING_FOR_SOURCE_AFTER, parameterSource2);
568            } else {
569                final Map<String, Object> parameterSource = Map.of(
570                        SOURCE_ID_PARAM, sourceId.getFullId(),
571                        START_TIME_PARAM, afterTimestamp);
572                jdbcTemplate.update(DIRECT_DELETE_EXISTING_FOR_SOURCE_AFTER, parameterSource);
573            }
574        });
575    }
576
577    /**
578     * Clean up any references to the target id, in transactions and outside
579     * @param txId transaction id
580     * @param targetId identifier of the resource to cleanup membership references for
581     */
582    public void deleteMembershipReferences(final String txId, final FedoraId targetId) {
583        final Map<String, Object> parameterSource = Map.of(
584                TARGET_ID_PARAM, targetId.getFullId(),
585                TX_ID_PARAM, txId);
586
587        jdbcTemplate.update(PURGE_ALL_REFERENCES_TRANSACTION, parameterSource);
588        jdbcTemplate.update(PURGE_ALL_REFERENCES_MEMBERSHIP, parameterSource);
589    }
590
591    /**
592     * Add new membership property to the index, clearing any delete
593     * operations for the property if necessary.
594     * @param tx transaction
595     * @param sourceId ID of the direct/indirect container which produced the membership
596     * @param proxyId ID of the proxy producing this membership, when applicable
597     * @param membership membership triple
598     * @param startTime time the membership triple was added
599     */
600    public void addMembership(final Transaction tx, final FedoraId sourceId, final FedoraId proxyId,
601            final Triple membership, final Instant startTime) {
602        if (membership == null) {
603            return;
604        }
605        addMembership(tx, sourceId, proxyId, membership, startTime, null);
606    }
607
608    /**
609     * Add new membership property to the index
610     * @param tx transaction
611     * @param sourceId ID of the direct/indirect container which produced the membership
612     * @param proxyId ID of the proxy producing this membership, when applicable
613     * @param membership membership triple
614     * @param startTime time the membership triple was added
615     * @param endTime time the membership triple ends, or never if not provided
616     */
617    public void addMembership(final Transaction tx, final FedoraId sourceId, final FedoraId proxyId,
618            final Triple membership, final Instant startTime, final Instant endTime) {
619        tx.doInTx(() -> {
620            final Timestamp endTimestamp;
621            final Timestamp lastUpdated;
622            final Timestamp startTimestamp = formatInstant(startTime);
623            if (endTime == null) {
624                endTimestamp = NO_END_TIMESTAMP;
625                lastUpdated = startTimestamp;
626            } else {
627                endTimestamp = formatInstant(endTime);
628                lastUpdated = endTimestamp;
629            }
630            // Add the new membership operation
631            final MapSqlParameterSource parameterSource = new MapSqlParameterSource();
632            parameterSource.addValue(SUBJECT_ID_PARAM, membership.getSubject().getURI());
633            parameterSource.addValue(PROPERTY_PARAM, membership.getPredicate().getURI());
634            parameterSource.addValue(TARGET_ID_PARAM, membership.getObject().getURI());
635            parameterSource.addValue(SOURCE_ID_PARAM, sourceId.getFullId());
636            parameterSource.addValue(PROXY_ID_PARAM, proxyId.getFullId());
637            parameterSource.addValue(START_TIME_PARAM, startTimestamp);
638            parameterSource.addValue(END_TIME_PARAM, endTimestamp);
639            parameterSource.addValue(LAST_UPDATED_PARAM, lastUpdated);
640
641            if (!tx.isShortLived()) {
642                parameterSource.addValue(TX_ID_PARAM, tx.getId());
643                parameterSource.addValue(OPERATION_PARAM, ADD_OPERATION);
644                jdbcTemplate.update(INSERT_MEMBERSHIP_IN_TX, parameterSource);
645            } else {
646                jdbcTemplate.update(DIRECT_INSERT_MEMBERSHIP, parameterSource);
647            }
648        });
649    }
650
651    /**
652     * Get a stream of membership triples with
653     * @param tx transaction from which membership will be retrieved, or null for no transaction
654     * @param subjectId ID of the subject
655     * @return Stream of membership triples
656     */
657    public Stream<Triple> getMembership(final Transaction tx, final FedoraId subjectId) {
658        final Node subjectNode = NodeFactory.createURI(subjectId.getBaseId());
659
660        final RowMapper<Triple> membershipMapper = (rs, rowNum) ->
661                Triple.create(subjectNode,
662                        NodeFactory.createURI(rs.getString("property")),
663                        NodeFactory.createURI(rs.getString("object_id")));
664
665        final MapSqlParameterSource parameterSource = new MapSqlParameterSource();
666        final String query;
667
668        if (subjectId.isMemento()) {
669            parameterSource.addValue(SUBJECT_ID_PARAM, subjectId.getBaseId());
670            parameterSource.addValue(MEMENTO_TIME_PARAM, formatInstant(subjectId.getMementoInstant()));
671        } else {
672            parameterSource.addValue(SUBJECT_ID_PARAM, subjectId.getFullId());
673            parameterSource.addValue(NO_END_TIME_PARAM, NO_END_TIMESTAMP);
674        }
675
676        if (tx.isOpenLongRunning()) {
677            parameterSource.addValue(TX_ID_PARAM, tx.getId());
678
679            if (subjectId.isMemento()) {
680                query = SELECT_MEMBERSHIP_MEMENTO_IN_TX;
681            } else {
682                query = SELECT_MEMBERSHIP_IN_TX;
683            }
684        } else {
685            if (subjectId.isMemento()) {
686                query = DIRECT_SELECT_MEMBERSHIP_MEMENTO;
687            } else {
688                query = DIRECT_SELECT_MEMBERSHIP;
689            }
690        }
691
692        return StreamSupport.stream(new MembershipIterator(query, parameterSource, membershipMapper), false);
693    }
694
695    public Instant getLastUpdated(final Transaction transaction, final FedoraId subjectId) {
696        final MapSqlParameterSource parameterSource = new MapSqlParameterSource();
697
698        parameterSource.addValue(NO_END_TIME_PARAM, NO_END_TIMESTAMP);
699        final String lastUpdatedQuery;
700        if (subjectId.isMemento()) {
701            lastUpdatedQuery = SELECT_LAST_UPDATED_MEMENTO;
702            parameterSource.addValue(SUBJECT_ID_PARAM, subjectId.getBaseId());
703            parameterSource.addValue(MEMENTO_TIME_PARAM, formatInstant(subjectId.getMementoInstant()));
704        } else if (transaction.isOpenLongRunning()) {
705            lastUpdatedQuery = SELECT_LAST_UPDATED_IN_TX;
706            parameterSource.addValue(SUBJECT_ID_PARAM, subjectId.getFullId());
707            parameterSource.addValue(TX_ID_PARAM, transaction.getId());
708            parameterSource.addValue(DELETE_OP_PARAM, DELETE_OPERATION);
709        } else {
710            lastUpdatedQuery = SELECT_LAST_UPDATED;
711            parameterSource.addValue(SUBJECT_ID_PARAM, subjectId.getFullId());
712        }
713
714        final var updated = jdbcTemplate.queryForObject(lastUpdatedQuery, parameterSource, Timestamp.class);
715        if (updated != null) {
716            return updated.toInstant();
717        }
718        return null;
719    }
720
721    /**
722     * Perform a commit of operations stored in the specified transaction
723     * @param tx transaction
724     */
725    public void commitTransaction(final Transaction tx) {
726        if (!tx.isShortLived()) {
727            tx.ensureCommitting();
728            final Map<String, String> parameterSource = Map.of(
729                    TX_ID_PARAM, tx.getId(),
730                    ADD_OP_PARAM, ADD_OPERATION,
731                    DELETE_OP_PARAM, DELETE_OPERATION,
732                    FORCE_PARAM, FORCE_FLAG);
733
734            jdbcTemplate.update(COMMIT_DELETES, parameterSource);
735            final int ends = jdbcTemplate.update(COMMIT_ENDS_MAP.get(this.dbPlatform), parameterSource);
736            final int adds = jdbcTemplate.update(COMMIT_ADDS, parameterSource);
737            final int cleaned = jdbcTemplate.update(DELETE_TRANSACTION, parameterSource);
738
739            log.debug("Completed commit, {} ended, {} adds, {} operations", ends, adds, cleaned);
740        }
741    }
742
743    /**
744     * Delete all entries related to a transaction
745     * @param tx transaction
746     */
747    @Transactional(propagation = Propagation.NOT_SUPPORTED)
748    public void deleteTransaction(final Transaction tx) {
749        if (!tx.isShortLived()) {
750            final Map<String, String> parameterSource = Map.of(TX_ID_PARAM, tx.getId());
751            jdbcTemplate.update(DELETE_TRANSACTION, parameterSource);
752        }
753    }
754
755    /**
756     * Format an instant to a timestamp without milliseconds, due to precision
757     * issues with memento datetimes.
758     * @param instant the instant
759     * @return a Timestamp
760     */
761    private Timestamp formatInstant(final Instant instant) {
762        final var timestamp = Timestamp.from(instant);
763        timestamp.setNanos(0);
764        return timestamp;
765    }
766
767    /**
768     * Clear all entries from the index
769     */
770    public void clearIndex() {
771        jdbcTemplate.update(TRUNCATE_MEMBERSHIP, Map.of());
772        jdbcTemplate.update(TRUNCATE_MEMBERSHIP_TX, Map.of());
773    }
774
775    /**
776     * Log all membership entries, for debugging usage only
777     */
778    protected void logMembership() {
779        log.info("source_id, proxy_id, subject_id, property, object_id, start_time, end_time, last_updated");
780        jdbcTemplate.query(SELECT_ALL_MEMBERSHIP, new RowCallbackHandler() {
781            @Override
782            public void processRow(final ResultSet rs) throws SQLException {
783                log.info("{}, {}, {}, {}, {}, {}, {}, {}",
784                        rs.getString("source_id"), rs.getString("proxy_id"), rs.getString("subject_id"),
785                        rs.getString("property"), rs.getString("object_id"), rs.getTimestamp("start_time"),
786                        rs.getTimestamp("end_time"), rs.getTimestamp("last_updated"));
787            }
788        });
789    }
790
791    /**
792     * Log all membership operations, for debugging usage only
793     */
794    protected void logOperations() {
795        log.info("source_id, proxy_id, subject_id, property, object_id, start_time, end_time,"
796                + " last_updated, tx_id, operation, force_flag");
797        jdbcTemplate.query(SELECT_ALL_OPERATIONS, new RowCallbackHandler() {
798            @Override
799            public void processRow(final ResultSet rs) throws SQLException {
800                log.info("{}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}",
801                        rs.getString("source_id"), rs.getString("proxy_id"), rs.getString("subject_id"),
802                        rs.getString("property"), rs.getString("object_id"), rs.getTimestamp("start_time"),
803                        rs.getTimestamp("end_time"), rs.getTimestamp("last_updated"), rs.getString("tx_id"),
804                        rs.getString("operation"), rs.getString("force_flag"));
805            }
806        });
807    }
808
809    /**
810     * Set the JDBC datastore.
811     * @param dataSource the dataStore.
812     */
813    public void setDataSource(final DataSource dataSource) {
814        this.dataSource = dataSource;
815    }
816
817    /**
818     * Get the JDBC datastore.
819     * @return the dataStore.
820     */
821    public DataSource getDataSource() {
822        return dataSource;
823    }
824
825    /**
826     * Private class to back a stream with a paged DB query.
827     *
828     * If this needs to be run in parallel we will have to override trySplit() and determine a good method to split on.
829     */
830    private class MembershipIterator extends Spliterators.AbstractSpliterator<Triple> {
831        final Queue<Triple> children = new ConcurrentLinkedQueue<>();
832        int numOffsets = 0;
833        final String queryToUse;
834        final MapSqlParameterSource parameterSource;
835        final RowMapper<Triple> rowMapper;
836
837        public MembershipIterator(final String query, final MapSqlParameterSource parameters,
838                                  final RowMapper<Triple> mapper) {
839            super(Long.MAX_VALUE, Spliterator.ORDERED);
840            queryToUse = query;
841            parameterSource = parameters;
842            rowMapper = mapper;
843            parameterSource.addValue(ADD_OP_PARAM, ADD_OPERATION);
844            parameterSource.addValue(DELETE_OP_PARAM, DELETE_OPERATION);
845            parameterSource.addValue(LIMIT_PARAM, MEMBERSHIP_LIMIT);
846        }
847
848        @Override
849        public boolean tryAdvance(final Consumer<? super Triple> action) {
850            try {
851                action.accept(children.remove());
852            } catch (final NoSuchElementException e) {
853                parameterSource.addValue(OFFSET_PARAM, numOffsets * MEMBERSHIP_LIMIT);
854                numOffsets += 1;
855                children.addAll(jdbcTemplate.query(queryToUse, parameterSource, rowMapper));
856                if (children.size() == 0) {
857                    // no more elements.
858                    return false;
859                }
860                action.accept(children.remove());
861            }
862            return true;
863        }
864    }
865}