Source code

001/*
002 * The contents of this file are subject to the license and copyright detailed
003 * in the LICENSE and NOTICE files at the root of the source tree.
004 */
005package org.duraspace.bagit;
006
007import static java.util.function.Function.identity;
008import static org.assertj.core.api.Assertions.assertThat;
009import static org.duraspace.bagit.BagConfig.ACCESS_KEY;
010import static org.duraspace.bagit.BagConfig.BAGGING_DATE_KEY;
011import static org.duraspace.bagit.BagConfig.BAG_SIZE_KEY;
012import static org.duraspace.bagit.BagConfig.CONTACT_EMAIL_KEY;
013import static org.duraspace.bagit.BagConfig.CONTACT_NAME_KEY;
014import static org.duraspace.bagit.BagConfig.CONTACT_PHONE_KEY;
015import static org.duraspace.bagit.BagConfig.EXTERNAL_DESCRIPTION_KEY;
016import static org.duraspace.bagit.BagConfig.ORGANIZATION_ADDRESS_KEY;
017import static org.duraspace.bagit.BagConfig.PAYLOAD_OXUM_KEY;
018import static org.duraspace.bagit.BagConfig.SOURCE_ORGANIZATION_KEY;
019import static org.duraspace.bagit.BagConfig.TITLE_KEY;
020import static org.duraspace.bagit.BagProfileConstants.ACCEPT_BAGIT_VERSION;
021import static org.duraspace.bagit.BagProfileConstants.ACCEPT_SERIALIZATION;
022import static org.duraspace.bagit.BagProfileConstants.BAGIT_PROFILE_IDENTIFIER;
023import static org.duraspace.bagit.BagProfileConstants.BAGIT_PROFILE_INFO;
024import static org.duraspace.bagit.BagProfileConstants.BAGIT_PROFILE_VERSION;
025import static org.duraspace.bagit.BagProfileConstants.BAG_INFO;
026import static org.duraspace.bagit.BagProfileConstants.MANIFESTS_REQUIRED;
027import static org.duraspace.bagit.BagProfileConstants.PROFILE_VERSION;
028import static org.duraspace.bagit.BagProfileConstants.TAG_FILES_REQUIRED;
029import static org.duraspace.bagit.BagProfileConstants.TAG_MANIFESTS_REQUIRED;
030import static org.junit.Assert.assertEquals;
031import static org.junit.Assert.assertFalse;
032import static org.junit.Assert.assertNotNull;
033import static org.junit.Assert.assertTrue;
034
035import java.io.IOException;
036import java.net.URISyntaxException;
037import java.net.URL;
038import java.nio.file.Files;
039import java.nio.file.Path;
040import java.nio.file.Paths;
041import java.util.Arrays;
042import java.util.Collection;
043import java.util.Collections;
044import java.util.List;
045import java.util.Map;
046import java.util.Objects;
047import java.util.Set;
048import java.util.stream.Collectors;
049
050import gov.loc.repository.bagit.domain.Bag;
051import gov.loc.repository.bagit.domain.FetchItem;
052import gov.loc.repository.bagit.domain.Manifest;
053import gov.loc.repository.bagit.domain.Version;
054import gov.loc.repository.bagit.hash.StandardSupportedAlgorithms;
055import org.junit.Assert;
056import org.junit.Before;
057import org.junit.Test;
058import org.slf4j.Logger;
059import org.slf4j.LoggerFactory;
060
061/**
062 * @author escowles
063 * @since 2016-12-13
064 */
065public class BagProfileTest {
066
067    private final String testValue = "test-value";
068    private final String defaultBag = "bag";
069
070    // profile locations
071    private final String defaultProfilePath = "profiles/profile.json";
072    private final String extraTagsPath = "profiles/profileWithExtraTags.json";
073    private final String invalidPath = "profiles/invalidProfile.json";
074    private final String invalidSerializationPath = "profiles/invalidProfileSerializationError.json";
075
076    // config locations
077    private final String bagitConfig = "configs/bagit-config.yml";
078    private final String bagitConfigBadAccess = "configs/bagit-config-bad-access.yml";
079    private final String bagitConfigMissingAccess = "configs/bagit-config-missing-access.yml";
080    private final String bagitConfigNoAptrust = "configs/bagit-config-no-aptrust.yml";
081
082    private final Version defaultVersion = new Version(1, 0);
083    private Path targetDir;
084
085    private final Logger logger = LoggerFactory.getLogger(BagProfileTest.class);
086
087    @Before
088    public void setup() throws URISyntaxException {
089        final URL url = this.getClass().getClassLoader().getResource("sample");
090        targetDir = Paths.get(Objects.requireNonNull(url).toURI());
091    }
092
093    @Test
094    public void testBuiltInProfiles() {
095        // validate that the string passed in the constructor matches the string used in the switch statement
096        for (BagProfile.BuiltIn value : BagProfile.BuiltIn.values()) {
097            assertEquals(value, BagProfile.BuiltIn.from(value.getIdentifier()));
098        }
099    }
100
101    @Test(expected = IllegalArgumentException.class)
102    public void testBuiltInNotSupported() {
103        BagProfile.BuiltIn.from("test");
104    }
105
106    @Test
107    public void testBasicProfileFromFile() throws Exception {
108        final BagProfile profile = new BagProfile(Files.newInputStream(resolveResourcePath(defaultProfilePath)));
109
110        final String md5 = BagItDigest.MD5.bagitName();
111        final String sha1 = BagItDigest.SHA1.bagitName();
112        final String sha256 = BagItDigest.SHA256.bagitName();
113        final String sha512 = BagItDigest.SHA512.bagitName();
114        assertTrue(profile.getPayloadDigestAlgorithms().contains(md5));
115        assertTrue(profile.getPayloadDigestAlgorithms().contains(sha1));
116        assertTrue(profile.getPayloadDigestAlgorithms().contains(sha256));
117        assertTrue(profile.getPayloadDigestAlgorithms().contains(sha512));
118
119        assertFalse(profile.getTagDigestAlgorithms().contains(md5));
120        assertTrue(profile.getTagDigestAlgorithms().contains(sha1));
121        assertTrue(profile.getTagDigestAlgorithms().contains(sha256));
122        assertTrue(profile.getTagDigestAlgorithms().contains(sha512));
123
124        assertTrue(profile.getMetadataFields().get(SOURCE_ORGANIZATION_KEY).isRequired());
125        assertFalse(profile.getMetadataFields().get(SOURCE_ORGANIZATION_KEY).isRepeatable());
126        assertEquals(profile.getMetadataFields().get(SOURCE_ORGANIZATION_KEY).getDescription(),
127                     SOURCE_ORGANIZATION_KEY);
128        assertTrue(profile.getMetadataFields().get(ORGANIZATION_ADDRESS_KEY).isRequired());
129        assertTrue(profile.getMetadataFields().get(ORGANIZATION_ADDRESS_KEY).isRepeatable());
130        assertTrue(profile.getMetadataFields().get(CONTACT_NAME_KEY).isRequired());
131        assertTrue(profile.getMetadataFields().get(CONTACT_PHONE_KEY).isRequired());
132        assertTrue(profile.getMetadataFields().get(BAG_SIZE_KEY).isRequired());
133        assertTrue(profile.getMetadataFields().get(BAGGING_DATE_KEY).isRequired());
134        assertTrue(profile.getMetadataFields().get(PAYLOAD_OXUM_KEY).isRequired());
135        assertFalse(profile.getMetadataFields().get(CONTACT_EMAIL_KEY).isRequired());
136
137        assertTrue(profile.getMetadataFields().get(BAGIT_PROFILE_IDENTIFIER).isRepeatable());
138        assertFalse(profile.getMetadataFields().get(BAGIT_PROFILE_IDENTIFIER).isRequired());
139        assertFalse(profile.getMetadataFields().get(BAGIT_PROFILE_IDENTIFIER).isRecommended());
140        assertEquals(profile.getMetadataFields().get(BAGIT_PROFILE_IDENTIFIER).getDescription(), "No description");
141
142        assertTrue(profile.getSectionNames().stream().allMatch(t -> t.equalsIgnoreCase(BAG_INFO)));
143
144        assertFalse(profile.isAllowFetch());
145        assertEquals(BagProfile.Serialization.OPTIONAL, profile.getSerialization());
146        assertTrue(profile.getAcceptedBagItVersions().contains("0.97"));
147        assertTrue(profile.getAcceptedBagItVersions().contains("1.0"));
148        assertTrue(profile.getAcceptedSerializations().contains("application/tar"));
149        assertTrue(profile.getTagFilesAllowed().contains("*"));
150        assertTrue(profile.getTagFilesRequired().isEmpty());
151        assertTrue(profile.getAllowedTagAlgorithms().isEmpty());
152        assertTrue(profile.getAllowedPayloadAlgorithms().isEmpty());
153    }
154
155    @Test
156    public void testLoadsEmptyMap() throws URISyntaxException, IOException {
157        final String profilePath = "profiles/profileNoBagInfo.json";
158        final BagProfile profile = new BagProfile(Files.newInputStream(resolveResourcePath(profilePath)));
159        final Map<String, ProfileFieldRule> bagInfoFields = profile.getMetadataFields(BAG_INFO);
160        assertNotNull(bagInfoFields);
161        assertTrue(bagInfoFields.isEmpty());
162    }
163
164    @Test
165    public void testExtendedProfile() throws Exception {
166        final String aptrustInfo = "APTrust-Info";
167        final BagProfile profile = new BagProfile(Files.newInputStream(resolveResourcePath(extraTagsPath)));
168
169        assertTrue(profile.getSectionNames().stream().anyMatch(t -> t.equalsIgnoreCase(BAG_INFO)));
170        assertTrue(profile.getSectionNames().stream().anyMatch(t -> t.equalsIgnoreCase(aptrustInfo)));
171        assertTrue(profile.getSectionNames().stream().noneMatch(t -> t.equalsIgnoreCase("Wrong-Tags")));
172        assertTrue(profile.getMetadataFields(aptrustInfo).containsKey(TITLE_KEY));
173        assertTrue(profile.getMetadataFields(aptrustInfo).containsKey(ACCESS_KEY));
174        assertTrue(profile.getMetadataFields(aptrustInfo).get(ACCESS_KEY).getValues().contains("Consortia"));
175        assertTrue(profile.getMetadataFields(aptrustInfo).get(ACCESS_KEY).getValues().contains("Institution"));
176        assertTrue(profile.getMetadataFields(aptrustInfo).get(ACCESS_KEY).getValues().contains("Restricted"));
177    }
178
179
180    @Test
181    public void testGoodConfig() throws Exception {
182        final BagConfig config = new BagConfig(resolveResourcePath(bagitConfig).toFile());
183        final Map<String, Map<String, String>> configAsMap =
184            config.getTagFiles().stream()
185                  .collect(Collectors.toMap(identity(), config::getFieldsForTagFile));
186        final BagProfile profile = new BagProfile(Files.newInputStream(resolveResourcePath(extraTagsPath)));
187        profile.validateConfig(config);
188        profile.validateTagFiles(configAsMap);
189    }
190
191    @Test(expected = RuntimeException.class)
192    public void testBadAccessValue() throws Exception {
193        final BagConfig config = new BagConfig(resolveResourcePath(bagitConfigBadAccess).toFile());
194        final BagProfile profile = new BagProfile(Files.newInputStream(resolveResourcePath(extraTagsPath)));
195        profile.validateConfig(config);
196    }
197
198    @Test(expected = RuntimeException.class)
199    public void testMissingAccessValue() throws Exception {
200        final BagConfig config = new BagConfig(resolveResourcePath(bagitConfigMissingAccess).toFile());
201        final BagProfile profile = new BagProfile(Files.newInputStream(resolveResourcePath(extraTagsPath)));
202        profile.validateConfig(config);
203    }
204
205    @Test
206    public void testMissingSectionNotNeeded() throws Exception {
207        final BagConfig config = new BagConfig(resolveResourcePath(bagitConfigNoAptrust).toFile());
208        final BagProfile profile = new BagProfile(Files.newInputStream(resolveResourcePath(defaultProfilePath)));
209        profile.validateConfig(config);
210    }
211
212    @Test(expected = RuntimeException.class)
213    public void testMissingSectionRequired() throws Exception {
214        final BagConfig config = new BagConfig(resolveResourcePath(bagitConfigNoAptrust).toFile());
215        final BagProfile profile = new BagProfile(Files.newInputStream(resolveResourcePath(extraTagsPath)));
216        profile.validateConfig(config);
217    }
218
219    @Test
220    public void testAllProfilesPassValidation() throws IOException {
221        final Path profiles = Paths.get("src/main/resources/profiles");
222
223        Files.list(profiles).forEach(path -> {
224            String profileIdentifier = path.getFileName().toString();
225            profileIdentifier = profileIdentifier.substring(0, profileIdentifier.indexOf("."));
226            logger.debug("Validating {}", profileIdentifier);
227            BagProfile profile = null;
228            try {
229                profile = new BagProfile(BagProfile.BuiltIn.from(profileIdentifier));
230            } catch (IOException e) {
231                Assert.fail(e.getMessage());
232            }
233
234            validateProfile(Objects.requireNonNull(profile));
235        });
236
237    }
238
239    @Test
240    public void testInvalidBagProfile() throws IOException, URISyntaxException {
241        final BagProfile profile = new BagProfile(Files.newInputStream(resolveResourcePath(invalidPath)));
242        try {
243            assertThat(profile.getIdentifier()).isEmpty();
244            validateProfile(profile);
245            Assert.fail("Should throw an exception");
246        } catch (RuntimeException e) {
247            final String message = e.getMessage();
248            // check that the error message contains each failed section
249            Assert.assertTrue(message.contains(BAGIT_PROFILE_INFO));
250            Assert.assertTrue(message.contains(BAGIT_PROFILE_IDENTIFIER));
251            Assert.assertTrue(message.contains(ACCEPT_SERIALIZATION));
252            Assert.assertTrue(message.contains(MANIFESTS_REQUIRED));
253            Assert.assertTrue(message.contains(TAG_MANIFESTS_REQUIRED));
254            Assert.assertTrue(message.contains(TAG_FILES_REQUIRED));
255            Assert.assertTrue(message.contains(ACCEPT_BAGIT_VERSION));
256        }
257    }
258
259    @Test
260    public void testInvalidBagProfileSerializationTypo() throws IOException, URISyntaxException {
261        final BagProfile profile = new BagProfile(Files.newInputStream(resolveResourcePath(invalidSerializationPath)));
262        try {
263            validateProfile(profile);
264            Assert.fail("Should throw an exception");
265        } catch (RuntimeException e) {
266            final String message = e.getMessage();
267            // check that the serialization field failed to parse
268            Assert.assertTrue(message.contains("Unknown Serialization"));
269        }
270    }
271
272    /**
273     * Validates this {@link BagProfile} according to the BagIt Profiles specification found at
274     * https://bagit-profiles.github.io/bagit-profiles-specification/
275     *
276     * This checks the following fields:
277     *
278     * BagIt-Profile-Info
279     * Existence of the Source-Organization, External-Description, Version, BagIt-Profile-Identifier, and
280     * BagIt-Profile-Version fields
281     *
282     * Serialization
283     * Is equal to one of "forbidden", "required", or "optional"
284     *
285     * Accept-Serialization
286     * If serialization has a value of required or optional, at least one value is needed.
287     *
288     * Manifests-Allowed
289     * If specified, the {@link BagProfile#getPayloadDigestAlgorithms()} must be a subset of
290     * {@link BagProfile#getAllowedPayloadAlgorithms()}
291     *
292     * Tag-Manifests-Allowed
293     * If specified, the {@link BagProfile#getTagDigestAlgorithms()} must be a subset of
294     * {@link BagProfile#getAllowedTagAlgorithms()}
295     *
296     * Tag-Files-Allowed
297     * If specified, the {@link BagProfile#getTagFilesRequired()} must be a subset of
298     * {@link BagProfile#getTagFilesAllowed()}. If not specified, all tags must match the '*' glob
299     *
300     * Accept-BagIt-Version
301     * At least one version is required
302     */
303    private void validateProfile(final BagProfile profile) {
304        final StringBuilder errors = new StringBuilder();
305
306        // Bag-Profile-Info
307        final List<String> expectedInfoFields = Arrays.asList(SOURCE_ORGANIZATION_KEY,
308                                                              EXTERNAL_DESCRIPTION_KEY,
309                                                              PROFILE_VERSION,
310                                                              BAGIT_PROFILE_IDENTIFIER,
311                                                              BAGIT_PROFILE_VERSION);
312        final Map<String, String> bagInfo = profile.getProfileMetadata();
313        for (final String expected : expectedInfoFields) {
314            if (!bagInfo.containsKey(expected)) {
315                if (errors.length() == 0) {
316                    errors.append("Error(s) in BagIt-Profile-Info:\n");
317                }
318                errors.append("  * Missing key ").append(expected).append("\n");
319            }
320        }
321
322        // Serialization / Accept-Serialization
323        final BagProfile.Serialization serialization = profile.getSerialization();
324        if (serialization == BagProfile.Serialization.REQUIRED || serialization == BagProfile.Serialization.OPTIONAL) {
325            if (profile.getAcceptedSerializations().isEmpty()) {
326                errors.append("Serialization value of ").append(serialization)
327                      .append(" requires at least one value in the Accept-Serialization field!\n");
328            }
329        } else if(serialization == BagProfile.Serialization.UNKNOWN) {
330            errors.append("Unknown Serialization value ").append(serialization)
331                  .append(". Allowed values are forbidden, required, or optional.\n");
332        }
333
334        // Manifests-Allowed / Manifests-Required
335        final Set<String> allowedPayloadAlgorithms = profile.getAllowedPayloadAlgorithms();
336        final Set<String> payloadDigestAlgorithms = profile.getPayloadDigestAlgorithms();
337        if (!(allowedPayloadAlgorithms.isEmpty() || isSubset(payloadDigestAlgorithms, allowedPayloadAlgorithms))) {
338            errors.append("Manifests-Required must be a subset of Manifests-Allowed!\n");
339        }
340
341        // Tag-Manifests-Allowed / Tag-Manifests-Required
342        final Set<String> allowedTagAlgorithms = profile.getAllowedTagAlgorithms();
343        final Set<String> tagDigestAlgorithms = profile.getTagDigestAlgorithms();
344        if (!(allowedTagAlgorithms.isEmpty() || isSubset(tagDigestAlgorithms, allowedTagAlgorithms))) {
345            errors.append("Tag-Manifests-Required must be a subset of Tag-Manifests-Allowed!\n");
346        }
347
348        // Tag-Files-Allowed / Tag-Files-Required
349        final Set<String> tagFilesAllowed = profile.getTagFilesAllowed();
350        final Set<String> tagFilesRequired = profile.getTagFilesRequired();
351        if (!(tagFilesAllowed.isEmpty() || isSubset(tagFilesRequired, tagFilesAllowed))) {
352            errors.append("Tag-Files-Required must be a subset of Tag-Files-Allowed!\n");
353        }
354
355        if (profile.getAcceptedBagItVersions().isEmpty()) {
356            errors.append("Accept-BagIt-Version requires at least one entry!");
357        }
358
359        if (errors.length() > 0) {
360            errors.insert(0, "Bag Profile json does not conform to BagIt Profiles specification! " +
361                             "The following errors occurred:\n");
362            throw new RuntimeException(errors.toString());
363        }
364    }
365
366    /**
367     * Check to see if a collection (labelled as {@code subCollection}) is a subset of the {@code superCollection}
368     *
369     * @param subCollection   the sub collection to iterate against and check if elements are contained within
370     *                        {@code superCollection}
371     * @param superCollection the super collection containing all the elements
372     * @param <T>             the type of each collection
373     * @return true if all elements of {@code subCollection} are contained within {@code superCollection}
374     */
375    private <T> boolean isSubset(final Collection<T> subCollection, final Collection<T> superCollection) {
376        for (T t : subCollection) {
377            if (!superCollection.contains(t)) {
378                return false;
379            }
380        }
381
382        return true;
383    }
384
385    @Test
386    public void testValidateBag() throws IOException, URISyntaxException {
387        final Bag bag = new Bag();
388        bag.setVersion(defaultVersion);
389        bag.setRootDir(targetDir.resolve(defaultBag));
390        final BagProfile bagProfile = new BagProfile(Files.newInputStream(resolveResourcePath(defaultProfilePath)));
391
392        putRequiredBagInfo(bag, bagProfile);
393        putRequiredManifests(bag.getTagManifests(), bagProfile.getTagDigestAlgorithms());
394        putRequiredManifests(bag.getPayLoadManifests(), bagProfile.getPayloadDigestAlgorithms());
395        putRequiredTags(bag, bagProfile);
396
397        bagProfile.validateBag(bag);
398    }
399
400    @Test
401    public void testValidateBagFailure() throws IOException {
402        final Long fetchLength = 0L;
403        final Path fetchFile = Paths.get("data/fetch.txt");
404        final URL fetchUrl = new URL("http://localhost/data/fetch.txt");
405
406        final Bag bag = new Bag();
407        bag.setItemsToFetch(Collections.singletonList(new FetchItem(fetchUrl, fetchLength, fetchFile)));
408        bag.setVersion(new Version(0, 0));
409        bag.setRootDir(targetDir.resolve(defaultBag));
410        final BagProfile bagProfile = new BagProfile(BagProfile.BuiltIn.APTRUST);
411
412        putRequiredBagInfo(bag, bagProfile);
413        putRequiredManifests(bag.getPayLoadManifests(), bagProfile.getPayloadDigestAlgorithms());
414        putRequiredTags(bag, bagProfile);
415
416        try {
417            bagProfile.validateBag(bag);
418            Assert.fail("Validation did not throw an exception");
419        } catch (RuntimeException e) {
420            final String message = e.getMessage();
421            Assert.assertTrue(message.contains("Profile does not allow a fetch.txt"));
422            Assert.assertTrue(message.contains("No tag manifest"));
423            Assert.assertTrue(message.contains("Required tag file \"aptrust-info.txt\" does not exist"));
424            Assert.assertTrue(message.contains("Could not read info from \"aptrust-info.txt\""));
425            Assert.assertTrue(message.contains("BagIt version incompatible"));
426
427            Assert.assertFalse(message.contains("Missing tag manifest algorithm"));
428        }
429    }
430
431    /**
432     * Add required tag files to a Bag from a BagProfile
433     *
434     * @param bag the Bag
435     * @param bagProfile the BagProfile defining the required files
436     */
437    private void putRequiredTags(final Bag bag, final BagProfile bagProfile) {
438        final List<String> tagManifestExpected = Arrays.asList("manifest-sha1.txt", "bag-info.txt", "bagit.txt");
439
440        // Always populate with the files we expect to see
441        for (String expected : tagManifestExpected) {
442            final Path required = Paths.get(expected);
443            for (Manifest manifest : bag.getTagManifests()) {
444                manifest.getFileToChecksumMap().put(required, testValue);
445            }
446        }
447
448        for (String requiredTag : bagProfile.getTagFilesRequired()) {
449            final Path requiredPath = Paths.get(requiredTag);
450            for (Manifest manifest : bag.getTagManifests()) {
451                manifest.getFileToChecksumMap().put(requiredPath, testValue);
452            }
453        }
454    }
455
456    /**
457     *
458     * @param manifests the manifests to add algorithms to
459     * @param algorithms the algorithms to add
460     */
461    private void putRequiredManifests(final Set<Manifest> manifests, final Set<String> algorithms) {
462        for (String algorithm : algorithms) {
463            manifests.add(new Manifest(StandardSupportedAlgorithms.valueOf(algorithm.toUpperCase())));
464        }
465    }
466
467    /**
468     *
469     * @param bag the Bag to set info fields for
470     * @param profile the BagProfile defining the required info fields
471     */
472    private void putRequiredBagInfo(final Bag bag, final BagProfile profile) {
473        final Map<String, ProfileFieldRule> bagInfoMeta = profile.getMetadataFields(BAG_INFO);
474        bag.getMetadata().add(BAGIT_PROFILE_IDENTIFIER, profile.getIdentifier());
475        for (Map.Entry<String, ProfileFieldRule> entry : bagInfoMeta.entrySet()) {
476            if (entry.getValue().isRequired())  {
477                bag.getMetadata().add(entry.getKey(), testValue);
478            }
479        }
480    }
481
482    private Path resolveResourcePath(final String resource) throws URISyntaxException {
483        final URL url = this.getClass().getClassLoader().getResource(resource);
484        return Paths.get(Objects.requireNonNull(url).toURI());
485    }
486
487}