001/* 002 * Licensed to DuraSpace under one or more contributor license agreements. 003 * See the NOTICE file distributed with this work for additional information 004 * regarding copyright ownership. 005 * 006 * DuraSpace licenses this file to you under the Apache License, 007 * Version 2.0 (the "License"); you may not use this file except in 008 * compliance with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.fcrepo.kernel.impl; 019 020import com.google.common.base.Preconditions; 021import org.fcrepo.common.db.DbPlatform; 022import org.fcrepo.kernel.api.ContainmentIndex; 023import org.fcrepo.kernel.api.exception.RepositoryRuntimeException; 024import org.fcrepo.kernel.api.identifiers.FedoraId; 025import org.slf4j.Logger; 026import org.springframework.core.io.DefaultResourceLoader; 027import org.springframework.dao.EmptyResultDataAccessException; 028import org.springframework.jdbc.core.namedparam.MapSqlParameterSource; 029import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; 030import org.springframework.jdbc.datasource.init.DatabasePopulatorUtils; 031import org.springframework.jdbc.datasource.init.ResourceDatabasePopulator; 032import org.springframework.stereotype.Component; 033import org.springframework.transaction.annotation.Transactional; 034 035import javax.annotation.Nonnull; 036import javax.annotation.PostConstruct; 037import javax.inject.Inject; 038import javax.sql.DataSource; 039 040import java.sql.Timestamp; 041import java.time.Instant; 042import java.time.temporal.ChronoUnit; 043import java.util.Collections; 044import java.util.List; 045import java.util.Map; 046import java.util.NoSuchElementException; 047import java.util.Queue; 048import java.util.Spliterator; 049import java.util.Spliterators; 050import java.util.concurrent.ConcurrentLinkedQueue; 051import java.util.function.Consumer; 052import java.util.stream.Stream; 053import java.util.stream.StreamSupport; 054 055import static org.fcrepo.kernel.api.FedoraTypes.FEDORA_ID_PREFIX; 056import static org.slf4j.LoggerFactory.getLogger; 057 058/** 059 * @author peichman 060 * @author whikloj 061 * @since 6.0.0 062 */ 063@Component("containmentIndexImpl") 064public class ContainmentIndexImpl implements ContainmentIndex { 065 066 private static final Logger LOGGER = getLogger(ContainmentIndexImpl.class); 067 068 private int containsLimit = 50000; 069 070 @Inject 071 private DataSource dataSource; 072 073 private NamedParameterJdbcTemplate jdbcTemplate; 074 075 private DbPlatform dbPlatform; 076 077 public static final String RESOURCES_TABLE = "containment"; 078 079 private static final String TRANSACTION_OPERATIONS_TABLE = "containment_transactions"; 080 081 public static final String FEDORA_ID_COLUMN = "fedora_id"; 082 083 private static final String PARENT_COLUMN = "parent"; 084 085 private static final String TRANSACTION_ID_COLUMN = "transaction_id"; 086 087 private static final String OPERATION_COLUMN = "operation"; 088 089 private static final String START_TIME_COLUMN = "start_time"; 090 091 private static final String END_TIME_COLUMN = "end_time"; 092 093 private static final String UPDATED_COLUMN = "updated"; 094 095 /* 096 * Select children of a resource that are not marked as deleted. 097 */ 098 private static final String SELECT_CHILDREN = "SELECT " + FEDORA_ID_COLUMN + 099 " FROM " + RESOURCES_TABLE + " WHERE " + PARENT_COLUMN + " = :parent AND " + END_TIME_COLUMN + " IS NULL" + 100 " ORDER BY " + FEDORA_ID_COLUMN + " LIMIT :containsLimit OFFSET :offSet"; 101 102 /* 103 * Select children of a memento of a resource. 104 */ 105 private static final String SELECT_CHILDREN_OF_MEMENTO = "SELECT " + FEDORA_ID_COLUMN + 106 " FROM " + RESOURCES_TABLE + " WHERE " + PARENT_COLUMN + " = :parent AND " + START_TIME_COLUMN + 107 " <= :asOfTime AND (" + END_TIME_COLUMN + " > :asOfTime OR " + END_TIME_COLUMN + " IS NULL) ORDER BY " + 108 FEDORA_ID_COLUMN + " LIMIT :containsLimit OFFSET :offSet"; 109 110 /* 111 * Select children of a parent from resources table and from the transaction table with an 'add' operation, 112 * but exclude any records that also exist in the transaction table with a 'delete' or 'purge' operation. 113 */ 114 private static final String SELECT_CHILDREN_IN_TRANSACTION = "SELECT x." + FEDORA_ID_COLUMN + " FROM" + 115 " (SELECT " + FEDORA_ID_COLUMN + " FROM " + RESOURCES_TABLE + " WHERE " + PARENT_COLUMN + " = :parent" + 116 " AND " + END_TIME_COLUMN + " IS NULL " + 117 " UNION SELECT " + FEDORA_ID_COLUMN + " FROM " + TRANSACTION_OPERATIONS_TABLE + 118 " WHERE " + PARENT_COLUMN + " = :parent AND " + TRANSACTION_ID_COLUMN + " = :transactionId" + 119 " AND " + OPERATION_COLUMN + " = 'add') x" + 120 " WHERE NOT EXISTS " + 121 " (SELECT 1 FROM " + TRANSACTION_OPERATIONS_TABLE + 122 " WHERE " + PARENT_COLUMN + " = :parent AND " + FEDORA_ID_COLUMN + " = x." + FEDORA_ID_COLUMN + 123 " AND " + TRANSACTION_ID_COLUMN + " = :transactionId AND " + OPERATION_COLUMN + " IN ('delete', 'purge'))" + 124 " ORDER BY x." + FEDORA_ID_COLUMN + " LIMIT :containsLimit OFFSET :offSet"; 125 126 /* 127 * Select all children of a resource that are marked for deletion. 128 */ 129 private static final String SELECT_DELETED_CHILDREN = "SELECT " + FEDORA_ID_COLUMN + 130 " FROM " + RESOURCES_TABLE + " WHERE " + PARENT_COLUMN + " = :parent AND " + END_TIME_COLUMN + 131 " IS NOT NULL ORDER BY " + FEDORA_ID_COLUMN + " LIMIT :containsLimit OFFSET :offSet"; 132 133 /* 134 * Select children of a resource plus children 'delete'd in the non-committed transaction, but excluding any 135 * 'add'ed in the non-committed transaction. 136 */ 137 private static final String SELECT_DELETED_CHILDREN_IN_TRANSACTION = "SELECT x." + FEDORA_ID_COLUMN + 138 " FROM (SELECT " + FEDORA_ID_COLUMN + " FROM " + RESOURCES_TABLE + 139 " WHERE " + PARENT_COLUMN + " = :parent AND " + END_TIME_COLUMN + " IS NOT NULL UNION" + 140 " SELECT " + FEDORA_ID_COLUMN + " FROM " + TRANSACTION_OPERATIONS_TABLE + " WHERE " + 141 PARENT_COLUMN + " = :parent AND " + TRANSACTION_ID_COLUMN + " = :transactionId AND " + 142 OPERATION_COLUMN + " = 'delete') x" + 143 " WHERE NOT EXISTS " + 144 "(SELECT 1 FROM " + TRANSACTION_OPERATIONS_TABLE + " WHERE " + PARENT_COLUMN + " = :parent AND " + 145 FEDORA_ID_COLUMN + " = x." + FEDORA_ID_COLUMN + " AND " + TRANSACTION_ID_COLUMN + " = :transactionId AND " + 146 OPERATION_COLUMN + " = 'add') ORDER BY x." + FEDORA_ID_COLUMN + " LIMIT :containsLimit OFFSET :offSet"; 147 148 /* 149 * Insert a parent child relationship to the transaction operation table. 150 */ 151 private static final String INSERT_CHILD_IN_TRANSACTION = "INSERT INTO " + TRANSACTION_OPERATIONS_TABLE + 152 " ( " + PARENT_COLUMN + ", " + FEDORA_ID_COLUMN + ", " + START_TIME_COLUMN + ", " + END_TIME_COLUMN + ", " + 153 TRANSACTION_ID_COLUMN + ", " + OPERATION_COLUMN + " ) VALUES (:parent, :child, :startTime, :endTime, " + 154 ":transactionId, 'add')"; 155 156 /* 157 * Remove an insert row from the transaction operation table for this parent child relationship. 158 */ 159 private static final String UNDO_INSERT_CHILD_IN_TRANSACTION = "DELETE FROM " + TRANSACTION_OPERATIONS_TABLE + 160 " WHERE " + PARENT_COLUMN + " = :parent AND " + FEDORA_ID_COLUMN + " = :child AND " + TRANSACTION_ID_COLUMN 161 + " = :transactionId AND " + OPERATION_COLUMN + " = 'add'"; 162 163 /* 164 * Add a parent child relationship deletion to the transaction operation table. 165 */ 166 private static final String DELETE_CHILD_IN_TRANSACTION = "INSERT INTO " + TRANSACTION_OPERATIONS_TABLE + 167 " ( " + PARENT_COLUMN + ", " + FEDORA_ID_COLUMN + ", " + END_TIME_COLUMN + ", " + TRANSACTION_ID_COLUMN + 168 ", " + OPERATION_COLUMN + " ) VALUES (:parent, :child, :endTime, :transactionId, 'delete')"; 169 170 /* 171 * Add a parent child relationship purge to the transaction operation table. 172 */ 173 private static final String PURGE_CHILD_IN_TRANSACTION = "INSERT INTO " + TRANSACTION_OPERATIONS_TABLE + 174 " ( " + PARENT_COLUMN + ", " + FEDORA_ID_COLUMN + ", " + TRANSACTION_ID_COLUMN + ", " + OPERATION_COLUMN + 175 " ) VALUES (:parent, :child, :transactionId, 'purge')"; 176 177 /* 178 * Remove a mark as deleted row from the transaction operation table for this child relationship (no parent). 179 */ 180 private static final String UNDO_DELETE_CHILD_IN_TRANSACTION_NO_PARENT = "DELETE FROM " + 181 TRANSACTION_OPERATIONS_TABLE + " WHERE " + FEDORA_ID_COLUMN + " = :child AND " + TRANSACTION_ID_COLUMN 182 + " = :transactionId AND " + OPERATION_COLUMN + " = 'delete'"; 183 184 /* 185 * Remove a purge row from the transaction operation table for this parent child relationship. 186 */ 187 private static final String UNDO_PURGE_CHILD_IN_TRANSACTION = "DELETE FROM " + TRANSACTION_OPERATIONS_TABLE + 188 " WHERE " + PARENT_COLUMN + " = :parent AND " + FEDORA_ID_COLUMN + " = :child AND " + TRANSACTION_ID_COLUMN 189 + " = :transactionId AND " + OPERATION_COLUMN + " = 'purge'"; 190 191 /* 192 * Is this parent child relationship being added in this transaction? 193 */ 194 private static final String IS_CHILD_ADDED_IN_TRANSACTION = "SELECT TRUE FROM " + TRANSACTION_OPERATIONS_TABLE + 195 " WHERE " + FEDORA_ID_COLUMN + " = :child AND " + PARENT_COLUMN + " = :parent" + 196 " AND " + TRANSACTION_ID_COLUMN + " = :transactionId AND " + OPERATION_COLUMN + " = 'add'"; 197 198 /* 199 * Is this child's relationship being marked for deletion in this transaction (no parent)? 200 */ 201 private static final String IS_CHILD_DELETED_IN_TRANSACTION_NO_PARENT = "SELECT TRUE FROM " + 202 TRANSACTION_OPERATIONS_TABLE + " WHERE " + FEDORA_ID_COLUMN + " = :child " + 203 " AND " + TRANSACTION_ID_COLUMN + " = :transactionId AND " + OPERATION_COLUMN + " = 'delete'"; 204 205 /* 206 * Is this parent child relationship being purged in this transaction? 207 */ 208 private static final String IS_CHILD_PURGED_IN_TRANSACTION = "SELECT TRUE FROM " + TRANSACTION_OPERATIONS_TABLE + 209 " WHERE " + FEDORA_ID_COLUMN + " = :child AND " + PARENT_COLUMN + " = :parent" + 210 " AND " + TRANSACTION_ID_COLUMN + " = :transactionId AND " + OPERATION_COLUMN + " = 'purge'"; 211 212 /* 213 * Delete all rows from the transaction operation table for this transaction. 214 */ 215 private static final String DELETE_ENTIRE_TRANSACTION = "DELETE FROM " + TRANSACTION_OPERATIONS_TABLE + " WHERE " + 216 TRANSACTION_ID_COLUMN + " = :transactionId"; 217 218 /* 219 * Add to the main table all rows from the transaction operation table marked 'add' for this transaction. 220 */ 221 private static final String COMMIT_ADD_RECORDS = "INSERT INTO " + RESOURCES_TABLE + " ( " + FEDORA_ID_COLUMN + ", " 222 + PARENT_COLUMN + ", " + START_TIME_COLUMN + ", " + END_TIME_COLUMN + " ) SELECT " + FEDORA_ID_COLUMN + 223 ", " + PARENT_COLUMN + ", " + START_TIME_COLUMN + ", " + END_TIME_COLUMN + " FROM " + 224 TRANSACTION_OPERATIONS_TABLE + " WHERE " + TRANSACTION_ID_COLUMN + " = :transactionId AND " + 225 OPERATION_COLUMN + " = 'add'"; 226 227 /* 228 * Add an end time to the rows in the main table that match all rows from transaction operation table marked 229 * 'delete' for this transaction. 230 */ 231 private static final String COMMIT_DELETE_RECORDS_H2 = "UPDATE " + RESOURCES_TABLE + 232 " r SET r." + END_TIME_COLUMN + " = ( SELECT t." + END_TIME_COLUMN + " FROM " + 233 TRANSACTION_OPERATIONS_TABLE + " t " + 234 " WHERE t." + FEDORA_ID_COLUMN + " = r." + FEDORA_ID_COLUMN + " AND t." + TRANSACTION_ID_COLUMN + 235 " = :transactionId AND t." + OPERATION_COLUMN + 236 " = 'delete' AND t." + PARENT_COLUMN + " = r." + PARENT_COLUMN + " AND r." + 237 END_TIME_COLUMN + " IS NULL)" + 238 " WHERE EXISTS (SELECT * FROM " + TRANSACTION_OPERATIONS_TABLE + " t WHERE t." + FEDORA_ID_COLUMN + 239 " = r." + FEDORA_ID_COLUMN + " AND t." + TRANSACTION_ID_COLUMN + " = :transactionId AND t." + 240 OPERATION_COLUMN + " = 'delete' AND t." + PARENT_COLUMN + " = r." + PARENT_COLUMN + " AND r." + 241 END_TIME_COLUMN + " IS NULL)"; 242 243 private static final String COMMIT_DELETE_RECORDS_MYSQL = "UPDATE " + RESOURCES_TABLE + 244 " r INNER JOIN " + TRANSACTION_OPERATIONS_TABLE + " t ON t." + FEDORA_ID_COLUMN + " = r." + 245 FEDORA_ID_COLUMN + " SET r." + END_TIME_COLUMN + " = t." + END_TIME_COLUMN + 246 " WHERE t." + PARENT_COLUMN + " = r." + 247 PARENT_COLUMN + " AND t." + TRANSACTION_ID_COLUMN + " = :transactionId AND t." + OPERATION_COLUMN + 248 " = 'delete' AND r." + END_TIME_COLUMN + " IS NULL"; 249 250 private static final String COMMIT_DELETE_RECORDS_POSTGRES = "UPDATE " + RESOURCES_TABLE + " SET " + 251 END_TIME_COLUMN + " = t." + END_TIME_COLUMN + " FROM " + TRANSACTION_OPERATIONS_TABLE + " t WHERE t." + 252 FEDORA_ID_COLUMN + " = " + RESOURCES_TABLE + "." + FEDORA_ID_COLUMN + " AND t." + PARENT_COLUMN + 253 " = " + RESOURCES_TABLE + "." + PARENT_COLUMN + " AND t." + TRANSACTION_ID_COLUMN + 254 " = :transactionId AND t." + OPERATION_COLUMN + " = 'delete' AND " + RESOURCES_TABLE + "." + 255 END_TIME_COLUMN + " IS NULL"; 256 257 private Map<DbPlatform, String> COMMIT_DELETE_RECORDS = Map.of( 258 DbPlatform.H2, COMMIT_DELETE_RECORDS_H2, 259 DbPlatform.MARIADB, COMMIT_DELETE_RECORDS_MYSQL, 260 DbPlatform.MYSQL, COMMIT_DELETE_RECORDS_MYSQL, 261 DbPlatform.POSTGRESQL, COMMIT_DELETE_RECORDS_POSTGRES 262 ); 263 264 /* 265 * Remove from the main table all rows from transaction operation table marked 'purge' for this transaction. 266 */ 267 private static final String COMMIT_PURGE_RECORDS = "DELETE FROM " + RESOURCES_TABLE + " WHERE " + 268 "EXISTS (SELECT * FROM " + TRANSACTION_OPERATIONS_TABLE + " t WHERE t." + 269 TRANSACTION_ID_COLUMN + " = :transactionId AND t." + OPERATION_COLUMN + " = 'purge' AND" + 270 " t." + FEDORA_ID_COLUMN + " = " + RESOURCES_TABLE + "." + FEDORA_ID_COLUMN + 271 " AND t." + PARENT_COLUMN + " = " + RESOURCES_TABLE + "." + PARENT_COLUMN + ")"; 272 273 /* 274 * Query if a resource exists in the main table and is not deleted. 275 */ 276 private static final String RESOURCE_EXISTS = "SELECT " + FEDORA_ID_COLUMN + " FROM " + RESOURCES_TABLE + 277 " WHERE " + FEDORA_ID_COLUMN + " = :child AND " + END_TIME_COLUMN + " IS NULL"; 278 279 /* 280 * Resource exists as a record in the transaction operations table with an 'add' operation and not also 281 * exists as a 'delete' operation. 282 */ 283 private static final String RESOURCE_EXISTS_IN_TRANSACTION = "SELECT x." + FEDORA_ID_COLUMN + " FROM" + 284 " (SELECT " + FEDORA_ID_COLUMN + " FROM " + RESOURCES_TABLE + " WHERE " + FEDORA_ID_COLUMN + " = :child" + 285 " AND " + END_TIME_COLUMN + " IS NULL UNION SELECT " + FEDORA_ID_COLUMN + " FROM " + 286 TRANSACTION_OPERATIONS_TABLE + " WHERE " + FEDORA_ID_COLUMN + " = :child AND " + TRANSACTION_ID_COLUMN + 287 " = :transactionId" + " AND " + OPERATION_COLUMN + " = 'add') x WHERE NOT EXISTS " + 288 " (SELECT 1 FROM " + TRANSACTION_OPERATIONS_TABLE + 289 " WHERE " + FEDORA_ID_COLUMN + " = :child AND " + TRANSACTION_ID_COLUMN + " = :transactionId" + 290 " AND " + OPERATION_COLUMN + " IN ('delete', 'purge'))"; 291 292 /* 293 * Query if a resource exists in the main table even if it is deleted. 294 */ 295 private static final String RESOURCE_OR_TOMBSTONE_EXISTS = "SELECT " + FEDORA_ID_COLUMN + " FROM " + 296 RESOURCES_TABLE + " WHERE " + FEDORA_ID_COLUMN + " = :child"; 297 298 /* 299 * Resource exists as a record in the main table even if deleted or in the transaction operations table with an 300 * 'add' operation and not also exists as a 'delete' operation. 301 */ 302 private static final String RESOURCE_OR_TOMBSTONE_EXISTS_IN_TRANSACTION = "SELECT x." + FEDORA_ID_COLUMN + " FROM" + 303 " (SELECT " + FEDORA_ID_COLUMN + " FROM " + RESOURCES_TABLE + " WHERE " + FEDORA_ID_COLUMN + " = :child" + 304 " UNION SELECT " + FEDORA_ID_COLUMN + " FROM " + 305 TRANSACTION_OPERATIONS_TABLE + " WHERE " + FEDORA_ID_COLUMN + " = :child AND " + TRANSACTION_ID_COLUMN + 306 " = :transactionId" + " AND " + OPERATION_COLUMN + " = 'add') x WHERE NOT EXISTS " + 307 " (SELECT 1 FROM " + TRANSACTION_OPERATIONS_TABLE + 308 " WHERE " + FEDORA_ID_COLUMN + " = :child AND " + TRANSACTION_ID_COLUMN + " = :transactionId" + 309 " AND " + OPERATION_COLUMN + " IN ('delete', 'purge'))"; 310 311 312 /* 313 * Get the parent ID for this resource from the main table if not deleted. 314 */ 315 private static final String PARENT_EXISTS = "SELECT " + PARENT_COLUMN + " FROM " + RESOURCES_TABLE + 316 " WHERE " + FEDORA_ID_COLUMN + " = :child AND " + END_TIME_COLUMN + " IS NULL"; 317 318 /* 319 * Get the parent ID for this resource from the operations table for an 'add' operation in this transaction, but 320 * exclude any 'delete' operations for this resource in this transaction. 321 */ 322 private static final String PARENT_EXISTS_IN_TRANSACTION = "SELECT x." + PARENT_COLUMN + " FROM" + 323 " (SELECT " + PARENT_COLUMN + " FROM " + RESOURCES_TABLE + " WHERE " + FEDORA_ID_COLUMN + " = :child" + 324 " AND " + END_TIME_COLUMN + " IS NULL" + 325 " UNION SELECT " + PARENT_COLUMN + " FROM " + TRANSACTION_OPERATIONS_TABLE + 326 " WHERE " + FEDORA_ID_COLUMN + " = :child AND " + TRANSACTION_ID_COLUMN + " = :transactionId" + 327 " AND " + OPERATION_COLUMN + " = 'add') x" + 328 " WHERE NOT EXISTS " + 329 " (SELECT 1 FROM " + TRANSACTION_OPERATIONS_TABLE + 330 " WHERE " + FEDORA_ID_COLUMN + " = :child AND " + TRANSACTION_ID_COLUMN + " = :transactionId" + 331 " AND " + OPERATION_COLUMN + " = 'delete')"; 332 333 /* 334 * Get the parent ID for this resource from the main table if deleted. 335 */ 336 private static final String PARENT_EXISTS_DELETED = "SELECT " + PARENT_COLUMN + " FROM " + RESOURCES_TABLE + 337 " WHERE " + FEDORA_ID_COLUMN + " = :child AND " + END_TIME_COLUMN + " IS NOT NULL"; 338 339 /* 340 * Get the parent ID for this resource from main table and the operations table for a 'delete' operation in this 341 * transaction, excluding any 'add' operations for this resource in this transaction. 342 */ 343 private static final String PARENT_EXISTS_DELETED_IN_TRANSACTION = "SELECT x." + PARENT_COLUMN + " FROM" + 344 " (SELECT " + PARENT_COLUMN + " FROM " + RESOURCES_TABLE + " WHERE " + FEDORA_ID_COLUMN + " = :child" + 345 " AND " + END_TIME_COLUMN + " IS NOT NULL UNION SELECT " + PARENT_COLUMN + " FROM " + 346 TRANSACTION_OPERATIONS_TABLE + " WHERE " + FEDORA_ID_COLUMN + " = :child AND " + TRANSACTION_ID_COLUMN + 347 " = :transactionId AND " + OPERATION_COLUMN + " = 'delete') x WHERE NOT EXISTS " + 348 " (SELECT 1 FROM " + TRANSACTION_OPERATIONS_TABLE + " WHERE " + FEDORA_ID_COLUMN + " = :child AND " + 349 TRANSACTION_ID_COLUMN + " = :transactionId AND " + OPERATION_COLUMN + " = 'add')"; 350 351 /* 352 * Does this resource exist in the transaction operation table for an 'add' record. 353 */ 354 private static final String IS_CHILD_ADDED_IN_TRANSACTION_NO_PARENT = "SELECT TRUE FROM " + 355 TRANSACTION_OPERATIONS_TABLE + " WHERE " + FEDORA_ID_COLUMN + " = :child AND " + 356 TRANSACTION_ID_COLUMN + " = :transactionId AND " + OPERATION_COLUMN + " = 'add'"; 357 358 /* 359 * Delete a row from the transaction operation table with this resource and 'add' operation, no parent required. 360 */ 361 private static final String UNDO_INSERT_CHILD_IN_TRANSACTION_NO_PARENT = "DELETE FROM " + 362 TRANSACTION_OPERATIONS_TABLE + " WHERE " + FEDORA_ID_COLUMN + " = :child AND " + TRANSACTION_ID_COLUMN 363 + " = :transactionId AND " + OPERATION_COLUMN + " = 'add'"; 364 365 private static final String TRUNCATE_TABLE = "TRUNCATE TABLE "; 366 367 /* 368 * Any record tracked in the containment index is either active or a tombstone. Either way it exists for the 369 * purpose of finding ghost nodes. 370 */ 371 private static final String SELECT_ID_LIKE = "SELECT " + FEDORA_ID_COLUMN + " FROM " + RESOURCES_TABLE + " WHERE " + 372 FEDORA_ID_COLUMN + " LIKE :resourceId"; 373 374 private static final String SELECT_ID_LIKE_IN_TRANSACTION = "SELECT x." + FEDORA_ID_COLUMN + " FROM (SELECT " + 375 FEDORA_ID_COLUMN + " FROM " + RESOURCES_TABLE + " WHERE " + FEDORA_ID_COLUMN + " LIKE :resourceId" + 376 " UNION SELECT " + FEDORA_ID_COLUMN + " FROM " + TRANSACTION_OPERATIONS_TABLE + " WHERE " + 377 FEDORA_ID_COLUMN + " LIKE :resourceId AND " + TRANSACTION_ID_COLUMN + " = :transactionId AND " + 378 OPERATION_COLUMN + " = 'add') x WHERE NOT EXISTS (SELECT 1 FROM " + TRANSACTION_OPERATIONS_TABLE + 379 " WHERE " + FEDORA_ID_COLUMN + " LIKE :resourceId AND " + TRANSACTION_ID_COLUMN + " = :transactionId AND " + 380 OPERATION_COLUMN + " = 'delete')"; 381 382 private static final Map<DbPlatform, String> DDL_MAP = Map.of( 383 DbPlatform.MYSQL, "sql/mysql-containment.sql", 384 DbPlatform.H2, "sql/default-containment.sql", 385 DbPlatform.POSTGRESQL, "sql/postgres-containment.sql", 386 DbPlatform.MARIADB, "sql/default-containment.sql" 387 ); 388 389 private static final String SELECT_LAST_UPDATED = "SELECT " + UPDATED_COLUMN + " FROM " + RESOURCES_TABLE + 390 " WHERE " + FEDORA_ID_COLUMN + " = :resourceId"; 391 392 private static final String UPDATE_LAST_UPDATED = "UPDATE " + RESOURCES_TABLE + " SET " + UPDATED_COLUMN + 393 " = :updated WHERE " + FEDORA_ID_COLUMN + " = :resourceId"; 394 395 private static final String SELECT_LAST_UPDATED_IN_TX = "SELECT MAX(x.updated)" + 396 " FROM (SELECT " + UPDATED_COLUMN + " as updated FROM " + RESOURCES_TABLE + " WHERE " + 397 FEDORA_ID_COLUMN + " = :resourceId UNION SELECT " + START_TIME_COLUMN + 398 " as updated FROM " + TRANSACTION_OPERATIONS_TABLE + " WHERE " + PARENT_COLUMN + " = :resourceId AND " + 399 OPERATION_COLUMN + " = 'add' AND " + TRANSACTION_ID_COLUMN + " = :transactionId UNION SELECT " + 400 END_TIME_COLUMN + " as updated FROM " + TRANSACTION_OPERATIONS_TABLE + " WHERE " + PARENT_COLUMN + 401 " = :resourceId AND " + OPERATION_COLUMN + " = 'delete' AND " + TRANSACTION_ID_COLUMN + 402 " = :transactionId UNION SELECT " + END_TIME_COLUMN + 403 " as updated FROM " + TRANSACTION_OPERATIONS_TABLE + " WHERE " + PARENT_COLUMN + " = :resourceId AND " + 404 OPERATION_COLUMN + " = 'add' AND " + TRANSACTION_ID_COLUMN + " = :transactionId) x"; 405 406 private static final String GET_UPDATED_RESOURCES = "SELECT DISTINCT " + PARENT_COLUMN + " FROM " + 407 TRANSACTION_OPERATIONS_TABLE + " WHERE " + TRANSACTION_ID_COLUMN + " = :transactionId AND " + 408 OPERATION_COLUMN + " in ('add', 'delete')"; 409 410 /** 411 * Connect to the database 412 */ 413 @PostConstruct 414 private void setup() { 415 jdbcTemplate = getNamedParameterJdbcTemplate(); 416 417 dbPlatform = DbPlatform.fromDataSource(dataSource); 418 419 Preconditions.checkArgument(DDL_MAP.containsKey(dbPlatform), 420 "Missing DDL mapping for %s", dbPlatform); 421 422 final var ddl = DDL_MAP.get(dbPlatform); 423 LOGGER.info("Applying ddl: {}", ddl); 424 DatabasePopulatorUtils.execute( 425 new ResourceDatabasePopulator(new DefaultResourceLoader().getResource("classpath:" + ddl)), 426 dataSource); 427 } 428 429 private NamedParameterJdbcTemplate getNamedParameterJdbcTemplate() { 430 return new NamedParameterJdbcTemplate(getDataSource()); 431 } 432 433 void setContainsLimit(final int limit) { 434 containsLimit = limit; 435 } 436 437 @Override 438 public Stream<String> getContains(final String txId, final FedoraId fedoraId) { 439 final String resourceId = fedoraId.isMemento() ? fedoraId.getBaseId() : fedoraId.getFullId(); 440 final Instant asOfTime = fedoraId.isMemento() ? fedoraId.getMementoInstant() : null; 441 final MapSqlParameterSource parameterSource = new MapSqlParameterSource(); 442 parameterSource.addValue("parent", resourceId); 443 444 LOGGER.debug("getContains for {} in transaction {} and instant {}", resourceId, txId, asOfTime); 445 446 final String query; 447 if (asOfTime == null) { 448 if (txId != null) { 449 // we are in a transaction 450 parameterSource.addValue("transactionId", txId); 451 query = SELECT_CHILDREN_IN_TRANSACTION; 452 } else { 453 // not in a transaction 454 query = SELECT_CHILDREN; 455 } 456 } else { 457 parameterSource.addValue("asOfTime", formatInstant(asOfTime)); 458 query = SELECT_CHILDREN_OF_MEMENTO; 459 } 460 461 return StreamSupport.stream(new ContainmentIterator(query, parameterSource), false); 462 } 463 464 @Override 465 public Stream<String> getContainsDeleted(final String txId, final FedoraId fedoraId) { 466 final String resourceId = fedoraId.getFullId(); 467 final MapSqlParameterSource parameterSource = new MapSqlParameterSource(); 468 parameterSource.addValue("parent", resourceId); 469 470 final String query; 471 if (txId != null) { 472 // we are in a transaction 473 parameterSource.addValue("transactionId", txId); 474 query = SELECT_DELETED_CHILDREN_IN_TRANSACTION; 475 } else { 476 // not in a transaction 477 query = SELECT_DELETED_CHILDREN; 478 } 479 LOGGER.debug("getContainsDeleted for {} in transaction {}", resourceId, txId); 480 return StreamSupport.stream(new ContainmentIterator(query, parameterSource), false); 481 } 482 483 @Override 484 public String getContainedBy(final String txId, final FedoraId resource) { 485 final String resourceID = resource.getFullId(); 486 final MapSqlParameterSource parameterSource = new MapSqlParameterSource(); 487 parameterSource.addValue("child", resourceID); 488 final List<String> parentID; 489 if (txId != null) { 490 parameterSource.addValue("transactionId", txId); 491 parentID = jdbcTemplate.queryForList(PARENT_EXISTS_IN_TRANSACTION, parameterSource, String.class); 492 } else { 493 parentID = jdbcTemplate.queryForList(PARENT_EXISTS, parameterSource, String.class); 494 } 495 return parentID.stream().findFirst().orElse(null); 496 } 497 498 @Override 499 public void addContainedBy(@Nonnull final String txId, final FedoraId parent, final FedoraId child) { 500 addContainedBy(txId, parent, child, Instant.now(), null); 501 } 502 503 @Override 504 public void addContainedBy(@Nonnull final String txId, final FedoraId parent, final FedoraId child, 505 final Instant startTime, final Instant endTime) { 506 final String parentID = parent.getFullId(); 507 final String childID = child.getFullId(); 508 final MapSqlParameterSource parameterSource = new MapSqlParameterSource(); 509 510 LOGGER.debug("Adding: parent: {}, child: {}, in txn: {}, start time {}, end time {}", parentID, childID, txId, 511 formatInstant(startTime), formatInstant(endTime)); 512 513 parameterSource.addValue("parent", parentID); 514 parameterSource.addValue("child", childID); 515 parameterSource.addValue("transactionId", txId); 516 parameterSource.addValue("startTime", formatInstant(startTime)); 517 parameterSource.addValue("endTime", formatInstant(endTime)); 518 final boolean purgedInTxn = !jdbcTemplate.queryForList(IS_CHILD_PURGED_IN_TRANSACTION, parameterSource) 519 .isEmpty(); 520 if (purgedInTxn) { 521 // We purged it, but are re-adding it so remove the purge operation. 522 jdbcTemplate.update(UNDO_PURGE_CHILD_IN_TRANSACTION, parameterSource); 523 } 524 jdbcTemplate.update(INSERT_CHILD_IN_TRANSACTION, parameterSource); 525 } 526 527 @Override 528 public void removeContainedBy(@Nonnull final String txId, final FedoraId parent, final FedoraId child) { 529 final String parentID = parent.getFullId(); 530 final String childID = child.getFullId(); 531 final MapSqlParameterSource parameterSource = new MapSqlParameterSource(); 532 parameterSource.addValue("parent", parentID); 533 parameterSource.addValue("child", childID); 534 parameterSource.addValue("transactionId", txId); 535 parameterSource.addValue("endTime", formatInstant(Instant.now())); 536 final boolean addedInTxn = !jdbcTemplate.queryForList(IS_CHILD_ADDED_IN_TRANSACTION, parameterSource) 537 .isEmpty(); 538 if (addedInTxn) { 539 jdbcTemplate.update(UNDO_INSERT_CHILD_IN_TRANSACTION, parameterSource); 540 } else { 541 jdbcTemplate.update(DELETE_CHILD_IN_TRANSACTION, parameterSource); 542 } 543 } 544 545 @Override 546 public void removeResource(@Nonnull final String txId, final FedoraId resource) { 547 final String resourceID = resource.getFullId(); 548 final MapSqlParameterSource parameterSource = new MapSqlParameterSource(); 549 parameterSource.addValue("child", resourceID); 550 parameterSource.addValue("transactionId", txId); 551 final boolean addedInTxn = !jdbcTemplate.queryForList(IS_CHILD_ADDED_IN_TRANSACTION_NO_PARENT, 552 parameterSource).isEmpty(); 553 if (addedInTxn) { 554 jdbcTemplate.update(UNDO_INSERT_CHILD_IN_TRANSACTION_NO_PARENT, parameterSource); 555 } else { 556 final String parent = getContainedBy(txId, resource); 557 if (parent != null) { 558 LOGGER.debug("Marking containment relationship between parent ({}) and child ({}) deleted", parent, 559 resourceID); 560 parameterSource.addValue("parent", parent); 561 parameterSource.addValue("endTime", formatInstant(Instant.now())); 562 jdbcTemplate.update(DELETE_CHILD_IN_TRANSACTION, parameterSource); 563 } 564 } 565 } 566 567 @Override 568 public void purgeResource(@Nonnull final String txId, final FedoraId resource) { 569 final String resourceID = resource.getFullId(); 570 final MapSqlParameterSource parameterSource = new MapSqlParameterSource(); 571 parameterSource.addValue("child", resourceID); 572 parameterSource.addValue("transactionId", txId); 573 final String parent = getContainedByDeleted(txId, resource); 574 final boolean deletedInTxn = !jdbcTemplate.queryForList(IS_CHILD_DELETED_IN_TRANSACTION_NO_PARENT, 575 parameterSource).isEmpty(); 576 if (deletedInTxn) { 577 jdbcTemplate.update(UNDO_DELETE_CHILD_IN_TRANSACTION_NO_PARENT, parameterSource); 578 } 579 if (parent != null) { 580 LOGGER.debug("Removing containment relationship between parent ({}) and child ({})", parent, resourceID); 581 parameterSource.addValue("parent", parent); 582 jdbcTemplate.update(PURGE_CHILD_IN_TRANSACTION, parameterSource); 583 } 584 } 585 586 /** 587 * Find parent for a resource using a deleted containment relationship. 588 * @param txId the transaction id. 589 * @param resource the child resource id. 590 * @return the parent id. 591 */ 592 private String getContainedByDeleted(final String txId, final FedoraId resource) { 593 final String resourceID = resource.getFullId(); 594 final MapSqlParameterSource parameterSource = new MapSqlParameterSource(); 595 parameterSource.addValue("child", resourceID); 596 final List<String> parentID; 597 if (txId != null) { 598 parameterSource.addValue("transactionId", txId); 599 parentID = jdbcTemplate.queryForList(PARENT_EXISTS_DELETED_IN_TRANSACTION, parameterSource, String.class); 600 } else { 601 parentID = jdbcTemplate.queryForList(PARENT_EXISTS_DELETED, parameterSource, String.class); 602 } 603 return parentID.stream().findFirst().orElse(null); 604 } 605 606 @Transactional 607 @Override 608 public void commitTransaction(final String txId) { 609 if (txId != null) { 610 try { 611 final MapSqlParameterSource parameterSource = new MapSqlParameterSource(); 612 parameterSource.addValue("transactionId", txId); 613 final List<String> changedParents = jdbcTemplate.queryForList(GET_UPDATED_RESOURCES, parameterSource, 614 String.class); 615 jdbcTemplate.update(COMMIT_PURGE_RECORDS, parameterSource); 616 jdbcTemplate.update(COMMIT_DELETE_RECORDS.get(dbPlatform), parameterSource); 617 jdbcTemplate.update(COMMIT_ADD_RECORDS, parameterSource); 618 for (final var parent : changedParents) { 619 final var updated = jdbcTemplate.queryForObject(SELECT_LAST_UPDATED_IN_TX, 620 Map.of("resourceId", parent, "transactionId", txId), Instant.class); 621 if (updated != null) { 622 jdbcTemplate.update(UPDATE_LAST_UPDATED, 623 Map.of("resourceId", parent, "updated", formatInstant(updated))); 624 } 625 } 626 jdbcTemplate.update(DELETE_ENTIRE_TRANSACTION, parameterSource); 627 } catch (final Exception e) { 628 LOGGER.warn("Unable to commit containment index transaction {}: {}", txId, e.getMessage()); 629 throw new RepositoryRuntimeException("Unable to commit containment index transaction", e); 630 } 631 } 632 } 633 634 @Override 635 public void rollbackTransaction(final String txId) { 636 if (txId != null) { 637 final MapSqlParameterSource parameterSource = new MapSqlParameterSource(); 638 parameterSource.addValue("transactionId", txId); 639 jdbcTemplate.update(DELETE_ENTIRE_TRANSACTION, parameterSource); 640 } 641 } 642 643 @Override 644 public boolean resourceExists(final String txId, final FedoraId fedoraId, final boolean includeDeleted) { 645 // Get the containing ID because fcr:metadata will not exist here but MUST exist if the containing resource does 646 final String resourceId = fedoraId.getBaseId(); 647 LOGGER.debug("Checking if {} exists in transaction {}", resourceId, txId); 648 if (fedoraId.isRepositoryRoot()) { 649 // Root always exists. 650 return true; 651 } 652 final MapSqlParameterSource parameterSource = new MapSqlParameterSource(); 653 parameterSource.addValue("child", resourceId); 654 final String queryToUse; 655 if (txId != null) { 656 queryToUse = includeDeleted ? RESOURCE_OR_TOMBSTONE_EXISTS_IN_TRANSACTION : 657 RESOURCE_EXISTS_IN_TRANSACTION; 658 parameterSource.addValue("transactionId", txId); 659 } else { 660 queryToUse = includeDeleted ? RESOURCE_OR_TOMBSTONE_EXISTS : 661 RESOURCE_EXISTS; 662 } 663 return !jdbcTemplate.queryForList(queryToUse, parameterSource, String.class).isEmpty(); 664 } 665 666 @Override 667 public FedoraId getContainerIdByPath(final String txId, final FedoraId fedoraId, final boolean checkDeleted) { 668 if (fedoraId.isRepositoryRoot()) { 669 // If we are root then we are the top. 670 return fedoraId; 671 } 672 final String parent = getContainedBy(txId, fedoraId); 673 if (parent != null) { 674 return FedoraId.create(parent); 675 } 676 String fullId = fedoraId.getFullId(); 677 while (fullId.contains("/")) { 678 fullId = fedoraId.getResourceId().substring(0, fullId.lastIndexOf("/")); 679 if (fullId.equals(FEDORA_ID_PREFIX)) { 680 return FedoraId.getRepositoryRootId(); 681 } 682 final FedoraId testID = FedoraId.create(fullId); 683 if (resourceExists(txId, testID, checkDeleted)) { 684 return testID; 685 } 686 } 687 return FedoraId.getRepositoryRootId(); 688 } 689 690 @Transactional 691 @Override 692 public void reset() { 693 try { 694 jdbcTemplate.update(TRUNCATE_TABLE + RESOURCES_TABLE, Collections.emptyMap()); 695 jdbcTemplate.update(TRUNCATE_TABLE + TRANSACTION_OPERATIONS_TABLE, Collections.emptyMap()); 696 } catch (final Exception e) { 697 throw new RepositoryRuntimeException("Failed to truncate containment tables", e); 698 } 699 } 700 701 @Override 702 public boolean hasResourcesStartingWith(final String txId, final FedoraId fedoraId) { 703 final MapSqlParameterSource parameterSource = new MapSqlParameterSource(); 704 parameterSource.addValue("resourceId", fedoraId.getFullId() + "/%"); 705 final boolean matchingIds; 706 if (txId != null) { 707 parameterSource.addValue("transactionId", txId); 708 matchingIds = !jdbcTemplate.queryForList(SELECT_ID_LIKE_IN_TRANSACTION, parameterSource, String.class) 709 .isEmpty(); 710 } else { 711 matchingIds = !jdbcTemplate.queryForList(SELECT_ID_LIKE, parameterSource, String.class).isEmpty(); 712 } 713 return matchingIds; 714 } 715 716 @Override 717 public Instant containmentLastUpdated(final String txId, final FedoraId fedoraId) { 718 final MapSqlParameterSource parameterSource = new MapSqlParameterSource(); 719 parameterSource.addValue("resourceId", fedoraId.getFullId()); 720 final String queryToUse; 721 if (txId == null) { 722 queryToUse = SELECT_LAST_UPDATED; 723 } else { 724 parameterSource.addValue("transactionId", txId); 725 queryToUse = SELECT_LAST_UPDATED_IN_TX; 726 } 727 try { 728 return jdbcTemplate.queryForObject(queryToUse, parameterSource, Instant.class); 729 } catch (final EmptyResultDataAccessException e) { 730 return null; 731 } 732 } 733 734 /** 735 * Get the data source backing this containment index 736 * @return data source 737 */ 738 public DataSource getDataSource() { 739 return dataSource; 740 } 741 742 /** 743 * Set the data source backing this containment index 744 * @param dataSource data source 745 */ 746 public void setDataSource(final DataSource dataSource) { 747 this.dataSource = dataSource; 748 } 749 750 /** 751 * Format an instant to a timestamp without milliseconds, due to precision 752 * issues with memento datetimes. 753 * @param instant the instant to format. 754 * @return the datetime timestamp 755 */ 756 private Timestamp formatInstant(final Instant instant) { 757 if (instant == null) { 758 return null; 759 } 760 return Timestamp.from(instant.truncatedTo(ChronoUnit.SECONDS)); 761 } 762 763 /** 764 * Private class to back a stream with a paged DB query. 765 * 766 * If this needs to be run in parallel we will have to override trySplit() and determine a good method to split on. 767 */ 768 private class ContainmentIterator extends Spliterators.AbstractSpliterator<String> { 769 final Queue<String> children = new ConcurrentLinkedQueue<>(); 770 int numOffsets = 0; 771 final String queryToUse; 772 final MapSqlParameterSource parameterSource; 773 774 public ContainmentIterator(final String query, final MapSqlParameterSource parameters) { 775 super(Long.MAX_VALUE, Spliterator.ORDERED); 776 queryToUse = query; 777 parameterSource = parameters; 778 parameterSource.addValue("containsLimit", containsLimit); 779 } 780 781 @Override 782 public boolean tryAdvance(final Consumer<? super String> action) { 783 try { 784 action.accept(children.remove()); 785 } catch (final NoSuchElementException e) { 786 parameterSource.addValue("offSet", numOffsets * containsLimit); 787 numOffsets += 1; 788 children.addAll(jdbcTemplate.queryForList(queryToUse, parameterSource, String.class)); 789 if (children.size() == 0) { 790 // no more elements. 791 return false; 792 } 793 action.accept(children.remove()); 794 } 795 return true; 796 } 797 } 798}