Source code

001/*
002 * The contents of this file are subject to the license and copyright detailed
003 * in the LICENSE and NOTICE files at the root of the source tree.
004 */
005package org.duraspace.bagit;
006
007import static org.duraspace.bagit.BagConfig.ACCESS_KEY;
008import static org.duraspace.bagit.BagConfig.BAGGING_DATE_KEY;
009import static org.duraspace.bagit.BagConfig.BAG_SIZE_KEY;
010import static org.duraspace.bagit.BagConfig.CONTACT_EMAIL_KEY;
011import static org.duraspace.bagit.BagConfig.CONTACT_NAME_KEY;
012import static org.duraspace.bagit.BagConfig.CONTACT_PHONE_KEY;
013import static org.duraspace.bagit.BagConfig.EXTERNAL_DESCRIPTION_KEY;
014import static org.duraspace.bagit.BagConfig.ORGANIZATION_ADDRESS_KEY;
015import static org.duraspace.bagit.BagConfig.PAYLOAD_OXUM_KEY;
016import static org.duraspace.bagit.BagConfig.SOURCE_ORGANIZATION_KEY;
017import static org.duraspace.bagit.BagConfig.TITLE_KEY;
018import static org.duraspace.bagit.BagProfileConstants.ACCEPT_BAGIT_VERSION;
019import static org.duraspace.bagit.BagProfileConstants.ACCEPT_SERIALIZATION;
020import static org.duraspace.bagit.BagProfileConstants.BAGIT_PROFILE_IDENTIFIER;
021import static org.duraspace.bagit.BagProfileConstants.BAGIT_PROFILE_INFO;
022import static org.duraspace.bagit.BagProfileConstants.BAGIT_PROFILE_VERSION;
023import static org.duraspace.bagit.BagProfileConstants.BAG_INFO;
024import static org.duraspace.bagit.BagProfileConstants.MANIFESTS_REQUIRED;
025import static org.duraspace.bagit.BagProfileConstants.PROFILE_VERSION;
026import static org.duraspace.bagit.BagProfileConstants.TAG_FILES_REQUIRED;
027import static org.duraspace.bagit.BagProfileConstants.TAG_MANIFESTS_REQUIRED;
028import static org.junit.Assert.assertEquals;
029import static org.junit.Assert.assertFalse;
030import static org.junit.Assert.assertTrue;
031
032import java.io.IOException;
033import java.net.URISyntaxException;
034import java.net.URL;
035import java.nio.file.Files;
036import java.nio.file.Path;
037import java.nio.file.Paths;
038import java.util.Arrays;
039import java.util.Collection;
040import java.util.Collections;
041import java.util.List;
042import java.util.Map;
043import java.util.Objects;
044import java.util.Set;
045
046import gov.loc.repository.bagit.domain.Bag;
047import gov.loc.repository.bagit.domain.FetchItem;
048import gov.loc.repository.bagit.domain.Manifest;
049import gov.loc.repository.bagit.domain.Version;
050import gov.loc.repository.bagit.hash.StandardSupportedAlgorithms;
051import org.junit.Assert;
052import org.junit.Before;
053import org.junit.Test;
054import org.slf4j.Logger;
055import org.slf4j.LoggerFactory;
056
057/**
058 * @author escowles
059 * @since 2016-12-13
060 */
061public class BagProfileTest {
062
063    private final String testValue = "test-value";
064    private final String defaultBag = "bag";
065
066    // profile locations
067    private final String defaultProfilePath = "profiles/profile.json";
068    private final String extraTagsPath = "profiles/profileWithExtraTags.json";
069    private final String invalidPath = "profiles/invalidProfile.json";
070    private final String invalidSerializationPath = "profiles/invalidProfileSerializationError.json";
071
072    // config locations
073    private final String bagitConfig = "configs/bagit-config.yml";
074    private final String bagitConfigBadAccess = "configs/bagit-config-bad-access.yml";
075    private final String bagitConfigMissingAccess = "configs/bagit-config-missing-access.yml";
076    private final String bagitConfigNoAptrust = "configs/bagit-config-no-aptrust.yml";
077
078    private final Version defaultVersion = new Version(1, 0);
079    private Path targetDir;
080
081    private final Logger logger = LoggerFactory.getLogger(BagProfileTest.class);
082
083    @Before
084    public void setup() throws URISyntaxException {
085        final URL url = this.getClass().getClassLoader().getResource("sample");
086        targetDir = Paths.get(Objects.requireNonNull(url).toURI());
087    }
088
089    @Test
090    public void testBasicProfileFromFile() throws Exception {
091        final BagProfile profile = new BagProfile(Files.newInputStream(resolveResourcePath(defaultProfilePath)));
092
093        final String md5 = BagItDigest.MD5.bagitName();
094        final String sha1 = BagItDigest.SHA1.bagitName();
095        final String sha256 = BagItDigest.SHA256.bagitName();
096        final String sha512 = BagItDigest.SHA512.bagitName();
097        assertTrue(profile.getPayloadDigestAlgorithms().contains(md5));
098        assertTrue(profile.getPayloadDigestAlgorithms().contains(sha1));
099        assertTrue(profile.getPayloadDigestAlgorithms().contains(sha256));
100        assertTrue(profile.getPayloadDigestAlgorithms().contains(sha512));
101
102        assertFalse(profile.getTagDigestAlgorithms().contains(md5));
103        assertTrue(profile.getTagDigestAlgorithms().contains(sha1));
104        assertTrue(profile.getTagDigestAlgorithms().contains(sha256));
105        assertTrue(profile.getTagDigestAlgorithms().contains(sha512));
106
107        assertTrue(profile.getMetadataFields().get(SOURCE_ORGANIZATION_KEY).isRequired());
108        assertTrue(profile.getMetadataFields().get(ORGANIZATION_ADDRESS_KEY).isRequired());
109        assertTrue(profile.getMetadataFields().get(CONTACT_NAME_KEY).isRequired());
110        assertTrue(profile.getMetadataFields().get(CONTACT_PHONE_KEY).isRequired());
111        assertTrue(profile.getMetadataFields().get(BAG_SIZE_KEY).isRequired());
112        assertTrue(profile.getMetadataFields().get(BAGGING_DATE_KEY).isRequired());
113        assertTrue(profile.getMetadataFields().get(PAYLOAD_OXUM_KEY).isRequired());
114        assertFalse(profile.getMetadataFields().get(CONTACT_EMAIL_KEY).isRequired());
115
116        assertTrue(profile.getSectionNames().stream().allMatch(t -> t.equalsIgnoreCase(BAG_INFO)));
117
118        assertFalse(profile.isAllowFetch());
119        assertEquals(BagProfile.Serialization.OPTIONAL, profile.getSerialization());
120        assertTrue(profile.getAcceptedBagItVersions().contains("0.97"));
121        assertTrue(profile.getAcceptedBagItVersions().contains("1.0"));
122        assertTrue(profile.getAcceptedSerializations().contains("application/tar"));
123        assertTrue(profile.getTagFilesAllowed().contains("*"));
124        assertTrue(profile.getTagFilesRequired().isEmpty());
125        assertTrue(profile.getAllowedTagAlgorithms().isEmpty());
126        assertTrue(profile.getAllowedPayloadAlgorithms().isEmpty());
127    }
128
129    @Test
130    public void testExtendedProfile() throws Exception {
131        final String aptrustInfo = "APTrust-Info";
132        final BagProfile profile = new BagProfile(Files.newInputStream(resolveResourcePath(extraTagsPath)));
133
134        assertTrue(profile.getSectionNames().stream().anyMatch(t -> t.equalsIgnoreCase(BAG_INFO)));
135        assertTrue(profile.getSectionNames().stream().anyMatch(t -> t.equals(aptrustInfo)));
136        assertTrue(profile.getSectionNames().stream().noneMatch(t -> t.equals("Wrong-Tags")));
137        assertTrue(profile.getMetadataFields(aptrustInfo).containsKey(TITLE_KEY));
138        assertTrue(profile.getMetadataFields(aptrustInfo).containsKey(ACCESS_KEY));
139        assertTrue(profile.getMetadataFields(aptrustInfo).get(ACCESS_KEY).getValues().contains("Consortia"));
140        assertTrue(profile.getMetadataFields(aptrustInfo).get(ACCESS_KEY).getValues().contains("Institution"));
141        assertTrue(profile.getMetadataFields(aptrustInfo).get(ACCESS_KEY).getValues().contains("Restricted"));
142
143    }
144
145
146    @Test
147    public void testGoodConfig() throws Exception {
148        final BagConfig config = new BagConfig(resolveResourcePath(bagitConfig).toFile());
149        final BagProfile profile = new BagProfile(Files.newInputStream(resolveResourcePath(extraTagsPath)));
150        profile.validateConfig(config);
151    }
152
153    @Test(expected = RuntimeException.class)
154    public void testBadAccessValue() throws Exception {
155        final BagConfig config = new BagConfig(resolveResourcePath(bagitConfigBadAccess).toFile());
156        final BagProfile profile = new BagProfile(Files.newInputStream(resolveResourcePath(extraTagsPath)));
157        profile.validateConfig(config);
158    }
159
160    @Test(expected = RuntimeException.class)
161    public void testMissingAccessValue() throws Exception {
162        final BagConfig config = new BagConfig(resolveResourcePath(bagitConfigMissingAccess).toFile());
163        final BagProfile profile = new BagProfile(Files.newInputStream(resolveResourcePath(extraTagsPath)));
164        profile.validateConfig(config);
165    }
166
167    @Test
168    public void testMissingSectionNotNeeded() throws Exception {
169        final BagConfig config = new BagConfig(resolveResourcePath(bagitConfigNoAptrust).toFile());
170        final BagProfile profile = new BagProfile(Files.newInputStream(resolveResourcePath(defaultProfilePath)));
171        profile.validateConfig(config);
172    }
173
174    @Test(expected = RuntimeException.class)
175    public void testMissingSectionRequired() throws Exception {
176        final BagConfig config = new BagConfig(resolveResourcePath(bagitConfigNoAptrust).toFile());
177        final BagProfile profile = new BagProfile(Files.newInputStream(resolveResourcePath(extraTagsPath)));
178        profile.validateConfig(config);
179    }
180
181    @Test
182    public void testAllProfilesPassValidation() throws IOException {
183        final Path profiles = Paths.get("src/main/resources/profiles");
184
185        Files.list(profiles).forEach(path -> {
186            String profileIdentifier = path.getFileName().toString();
187            profileIdentifier = profileIdentifier.substring(0, profileIdentifier.indexOf("."));
188            logger.debug("Validating {}", profileIdentifier);
189            BagProfile profile = null;
190            try {
191                profile = new BagProfile(BagProfile.BuiltIn.from(profileIdentifier));
192            } catch (IOException e) {
193                Assert.fail(e.getMessage());
194            }
195
196            validateProfile(Objects.requireNonNull(profile));
197        });
198
199    }
200
201    @Test
202    public void testInvalidBagProfile() throws IOException, URISyntaxException {
203        final BagProfile profile = new BagProfile(Files.newInputStream(resolveResourcePath(invalidPath)));
204        try {
205            validateProfile(profile);
206            Assert.fail("Should throw an exception");
207        } catch (RuntimeException e) {
208            final String message = e.getMessage();
209            // check that the error message contains each failed section
210            Assert.assertTrue(message.contains(BAGIT_PROFILE_INFO));
211            Assert.assertTrue(message.contains(BAGIT_PROFILE_IDENTIFIER));
212            Assert.assertTrue(message.contains(ACCEPT_SERIALIZATION));
213            Assert.assertTrue(message.contains(MANIFESTS_REQUIRED));
214            Assert.assertTrue(message.contains(TAG_MANIFESTS_REQUIRED));
215            Assert.assertTrue(message.contains(TAG_FILES_REQUIRED));
216            Assert.assertTrue(message.contains(ACCEPT_BAGIT_VERSION));
217        }
218    }
219
220    @Test
221    public void testInvalidBagProfileSerializationTypo() throws IOException, URISyntaxException {
222        final BagProfile profile = new BagProfile(Files.newInputStream(resolveResourcePath(invalidSerializationPath)));
223        try {
224            validateProfile(profile);
225            Assert.fail("Should throw an exception");
226        } catch (RuntimeException e) {
227            final String message = e.getMessage();
228            // check that the serialization field failed to parse
229            Assert.assertTrue(message.contains("Unknown Serialization"));
230        }
231    }
232
233    /**
234     * Validates this {@link BagProfile} according to the BagIt Profiles specification found at
235     * https://bagit-profiles.github.io/bagit-profiles-specification/
236     *
237     * This checks the following fields:
238     *
239     * BagIt-Profile-Info
240     * Existence of the Source-Organization, External-Description, Version, BagIt-Profile-Identifier, and
241     * BagIt-Profile-Version fields
242     *
243     * Serialization
244     * Is equal to one of "forbidden", "required", or "optional"
245     *
246     * Accept-Serialization
247     * If serialization has a value of required or optional, at least one value is needed.
248     *
249     * Manifests-Allowed
250     * If specified, the {@link BagProfile#getPayloadDigestAlgorithms()} must be a subset of
251     * {@link BagProfile#getAllowedPayloadAlgorithms()}
252     *
253     * Tag-Manifests-Allowed
254     * If specified, the {@link BagProfile#getTagDigestAlgorithms()} must be a subset of
255     * {@link BagProfile#getAllowedTagAlgorithms()}
256     *
257     * Tag-Files-Allowed
258     * If specified, the {@link BagProfile#getTagFilesRequired()} must be a subset of
259     * {@link BagProfile#getTagFilesAllowed()}. If not specified, all tags must match the '*' glob
260     *
261     * Accept-BagIt-Version
262     * At least one version is required
263     */
264    private void validateProfile(final BagProfile profile) {
265        final StringBuilder errors = new StringBuilder();
266
267        // Bag-Profile-Info
268        final List<String> expectedInfoFields = Arrays.asList(SOURCE_ORGANIZATION_KEY,
269                                                              EXTERNAL_DESCRIPTION_KEY,
270                                                              PROFILE_VERSION,
271                                                              BAGIT_PROFILE_IDENTIFIER,
272                                                              BAGIT_PROFILE_VERSION);
273        final Map<String, String> bagInfo = profile.getProfileMetadata();
274        for (final String expected : expectedInfoFields) {
275            if (!bagInfo.containsKey(expected)) {
276                if (errors.length() == 0) {
277                    errors.append("Error(s) in BagIt-Profile-Info:\n");
278                }
279                errors.append("  * Missing key ").append(expected).append("\n");
280            }
281        }
282
283        // Serialization / Accept-Serialization
284        final BagProfile.Serialization serialization = profile.getSerialization();
285        if (serialization == BagProfile.Serialization.REQUIRED || serialization == BagProfile.Serialization.OPTIONAL) {
286            if (profile.getAcceptedSerializations().isEmpty()) {
287                errors.append("Serialization value of ").append(serialization)
288                      .append(" requires at least one value in the Accept-Serialization field!\n");
289            }
290        } else if(serialization == BagProfile.Serialization.UNKNOWN) {
291            errors.append("Unknown Serialization value ").append(serialization)
292                  .append(". Allowed values are forbidden, required, or optional.\n");
293        }
294
295        // Manifests-Allowed / Manifests-Required
296        final Set<String> allowedPayloadAlgorithms = profile.getAllowedPayloadAlgorithms();
297        final Set<String> payloadDigestAlgorithms = profile.getPayloadDigestAlgorithms();
298        if (!(allowedPayloadAlgorithms.isEmpty() || isSubset(payloadDigestAlgorithms, allowedPayloadAlgorithms))) {
299            errors.append("Manifests-Required must be a subset of Manifests-Allowed!\n");
300        }
301
302        // Tag-Manifests-Allowed / Tag-Manifests-Required
303        final Set<String> allowedTagAlgorithms = profile.getAllowedTagAlgorithms();
304        final Set<String> tagDigestAlgorithms = profile.getTagDigestAlgorithms();
305        if (!(allowedTagAlgorithms.isEmpty() || isSubset(tagDigestAlgorithms, allowedTagAlgorithms))) {
306            errors.append("Tag-Manifests-Required must be a subset of Tag-Manifests-Allowed!\n");
307        }
308
309        // Tag-Files-Allowed / Tag-Files-Required
310        final Set<String> tagFilesAllowed = profile.getTagFilesAllowed();
311        final Set<String> tagFilesRequired = profile.getTagFilesRequired();
312        if (!(tagFilesAllowed.isEmpty() || isSubset(tagFilesRequired, tagFilesAllowed))) {
313            errors.append("Tag-Files-Required must be a subset of Tag-Files-Allowed!\n");
314        }
315
316        if (profile.getAcceptedBagItVersions().isEmpty()) {
317            errors.append("Accept-BagIt-Version requires at least one entry!");
318        }
319
320        if (errors.length() > 0) {
321            errors.insert(0, "Bag Profile json does not conform to BagIt Profiles specification! " +
322                             "The following errors occurred:\n");
323            throw new RuntimeException(errors.toString());
324        }
325    }
326
327    /**
328     * Check to see if a collection (labelled as {@code subCollection}) is a subset of the {@code superCollection}
329     *
330     * @param subCollection   the sub collection to iterate against and check if elements are contained within
331     *                        {@code superCollection}
332     * @param superCollection the super collection containing all the elements
333     * @param <T>             the type of each collection
334     * @return true if all elements of {@code subCollection} are contained within {@code superCollection}
335     */
336    private <T> boolean isSubset(final Collection<T> subCollection, final Collection<T> superCollection) {
337        for (T t : subCollection) {
338            if (!superCollection.contains(t)) {
339                return false;
340            }
341        }
342
343        return true;
344    }
345
346    @Test
347    public void testValidateBag() throws IOException, URISyntaxException {
348        final Bag bag = new Bag();
349        bag.setVersion(defaultVersion);
350        bag.setRootDir(targetDir.resolve(defaultBag));
351        final BagProfile bagProfile = new BagProfile(Files.newInputStream(resolveResourcePath(defaultProfilePath)));
352
353        putRequiredBagInfo(bag, bagProfile);
354        putRequiredManifests(bag.getTagManifests(), bagProfile.getTagDigestAlgorithms());
355        putRequiredManifests(bag.getPayLoadManifests(), bagProfile.getPayloadDigestAlgorithms());
356        putRequiredTags(bag, bagProfile);
357
358        bagProfile.validateBag(bag);
359    }
360
361    @Test
362    public void testValidateBagFailure() throws IOException {
363        final Long fetchLength = 0L;
364        final Path fetchFile = Paths.get("data/fetch.txt");
365        final URL fetchUrl = new URL("http://localhost/data/fetch.txt");
366
367        final Bag bag = new Bag();
368        bag.setItemsToFetch(Collections.singletonList(new FetchItem(fetchUrl, fetchLength, fetchFile)));
369        bag.setVersion(new Version(0, 0));
370        bag.setRootDir(targetDir.resolve(defaultBag));
371        final BagProfile bagProfile = new BagProfile(BagProfile.BuiltIn.APTRUST);
372
373        putRequiredBagInfo(bag, bagProfile);
374        putRequiredManifests(bag.getPayLoadManifests(), bagProfile.getPayloadDigestAlgorithms());
375        putRequiredTags(bag, bagProfile);
376
377        try {
378            bagProfile.validateBag(bag);
379            Assert.fail("Validation did not throw an exception");
380        } catch (RuntimeException e) {
381            final String message = e.getMessage();
382            Assert.assertTrue(message.contains("Profile does not allow a fetch.txt"));
383            Assert.assertTrue(message.contains("No tag manifest"));
384            Assert.assertTrue(message.contains("Required tag file \"aptrust-info.txt\" does not exist"));
385            Assert.assertTrue(message.contains("Could not read info from \"aptrust-info.txt\""));
386            Assert.assertTrue(message.contains("BagIt version incompatible"));
387
388            Assert.assertFalse(message.contains("Missing tag manifest algorithm"));
389        }
390    }
391
392    /**
393     * Add required tag files to a Bag from a BagProfile
394     *
395     * @param bag the Bag
396     * @param bagProfile the BagProfile defining the required files
397     */
398    private void putRequiredTags(final Bag bag, final BagProfile bagProfile) {
399        final List<String> tagManifestExpected = Arrays.asList("manifest-sha1.txt", "bag-info.txt", "bagit.txt");
400
401        // Always populate with the files we expect to see
402        for (String expected : tagManifestExpected) {
403            final Path required = Paths.get(expected);
404            for (Manifest manifest : bag.getTagManifests()) {
405                manifest.getFileToChecksumMap().put(required, testValue);
406            }
407        }
408
409        for (String requiredTag : bagProfile.getTagFilesRequired()) {
410            final Path requiredPath = Paths.get(requiredTag);
411            for (Manifest manifest : bag.getTagManifests()) {
412                manifest.getFileToChecksumMap().put(requiredPath, testValue);
413            }
414        }
415    }
416
417    /**
418     *
419     * @param manifests the manifests to add algorithms to
420     * @param algorithms the algorithms to add
421     */
422    private void putRequiredManifests(final Set<Manifest> manifests, final Set<String> algorithms) {
423        for (String algorithm : algorithms) {
424            manifests.add(new Manifest(StandardSupportedAlgorithms.valueOf(algorithm.toUpperCase())));
425        }
426    }
427
428    /**
429     *
430     * @param bag the Bag to set info fields for
431     * @param profile the BagProfile defining the required info fields
432     */
433    private void putRequiredBagInfo(final Bag bag, final BagProfile profile) {
434        final Map<String, ProfileFieldRule> bagInfoMeta = profile.getMetadataFields(BAG_INFO);
435        for (Map.Entry<String, ProfileFieldRule> entry : bagInfoMeta.entrySet()) {
436            if (entry.getValue().isRequired())  {
437                bag.getMetadata().add(entry.getKey(), testValue);
438            }
439        }
440    }
441
442    private Path resolveResourcePath(final String resource) throws URISyntaxException {
443        final URL url = this.getClass().getClassLoader().getResource(resource);
444        return Paths.get(Objects.requireNonNull(url).toURI());
445    }
446
447}