001/*
002 * Licensed to DuraSpace under one or more contributor license agreements.
003 * See the NOTICE file distributed with this work for additional information
004 * regarding copyright ownership.
005 *
006 * DuraSpace licenses this file to you under the Apache License,
007 * Version 2.0 (the "License"); you may not use this file except in
008 * compliance with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.fcrepo.integration.connector.file;
019
020import static java.lang.Integer.MAX_VALUE;
021import static java.lang.Integer.parseInt;
022import static java.lang.System.currentTimeMillis;
023import static java.lang.Thread.sleep;
024import static java.util.UUID.randomUUID;
025import static javax.ws.rs.core.MediaType.TEXT_PLAIN;
026import static javax.ws.rs.core.Response.Status.CREATED;
027import static javax.ws.rs.core.Response.Status.NO_CONTENT;
028import static javax.ws.rs.core.Response.Status.OK;
029import static org.apache.jena.graph.Node.ANY;
030import static org.apache.jena.graph.NodeFactory.createURI;
031import static org.apache.jena.vocabulary.RDF.type;
032import static org.junit.Assert.assertEquals;
033import static org.junit.Assert.assertFalse;
034import static org.junit.Assert.assertTrue;
035import static org.junit.Assert.fail;
036import static org.fcrepo.kernel.api.RdfLexicon.MODE_NAMESPACE;
037import static org.fcrepo.http.commons.test.util.TestHelpers.parseTriples;
038
039import java.io.File;
040import java.io.IOException;
041import java.net.URI;
042import java.text.ParseException;
043import java.text.SimpleDateFormat;
044import java.util.Locale;
045
046import org.apache.http.HttpResponse;
047import org.apache.http.annotation.NotThreadSafe;
048import org.apache.http.client.methods.CloseableHttpResponse;
049import org.apache.http.client.methods.HttpGet;
050import org.apache.http.client.methods.HttpHead;
051import org.apache.http.client.methods.HttpPatch;
052import org.apache.http.client.methods.HttpPost;
053import org.apache.http.client.methods.HttpPut;
054import org.apache.http.client.methods.HttpRequestBase;
055import org.apache.http.client.methods.HttpUriRequest;
056import org.apache.http.entity.ByteArrayEntity;
057import org.apache.http.entity.StringEntity;
058import org.apache.http.impl.client.CloseableHttpClient;
059import org.apache.http.impl.client.HttpClientBuilder;
060import org.apache.http.util.EntityUtils;
061import org.apache.jena.sparql.core.DatasetGraph;
062import org.apache.jena.graph.Node;
063
064import org.fcrepo.http.commons.test.util.CloseableDataset;
065
066import org.junit.Ignore;
067import org.junit.Test;
068import org.slf4j.Logger;
069import org.slf4j.LoggerFactory;
070
071/**
072 * Tests around using the fcrepo-connector-file
073 *
074 * @author awoods
075 * @author ajs6f
076 */
077public class FileConnectorIT {
078
079    private static final Logger logger = LoggerFactory.getLogger(FileConnectorIT.class);
080
081    private static final int SERVER_PORT = parseInt(System.getProperty("fcrepo.dynamic.test.port", "8080"));
082
083    private static final String HOSTNAME = "localhost";
084
085    private static final String PROTOCOL = "http";
086
087    private static final String serverAddress = PROTOCOL + "://" + HOSTNAME + ":" + SERVER_PORT + "/fcrepo/rest/";
088
089    private static CloseableHttpClient client = createClient();
090
091    private static CloseableHttpClient createClient() {
092        return HttpClientBuilder.create().setMaxConnPerRoute(MAX_VALUE).setMaxConnTotal(MAX_VALUE).build();
093    }
094
095    private static SimpleDateFormat headerFormat = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss zzz", Locale.US);
096
097    /**
098     * I should be able to link to content on a federated filesystem.
099     *
100     * @throws IOException thrown during this function
101     **/
102    @Test
103    public void testFederatedDatastream() throws IOException {
104        final String federationAddress = serverAddress + "files/FileSystem1/ds1";
105        final String linkingAddress;
106        try (final CloseableHttpResponse response = client.execute(new HttpPost(serverAddress))) {
107            EntityUtils.consume(response.getEntity());
108            linkingAddress = response.getFirstHeader("Location").getValue();
109        }
110
111        // link from the object to the content of the file on the federated filesystem
112        final HttpPatch patch = new HttpPatch(linkingAddress);
113        patch.addHeader("Content-Type", "application/sparql-update");
114        patch.setEntity(new ByteArrayEntity(
115                ("INSERT DATA { <> <http://some-vocabulary#hasExternalContent> " + "<" + federationAddress + "> . }")
116                .getBytes()));
117        assertEquals("Couldn't link to external datastream!", NO_CONTENT.getStatusCode(), getStatus(patch));
118    }
119
120    /**
121     * Given a directory at: test-FileSystem1/ /ds1 /ds2 /TestSubdir/ and a projection of test-objects as
122     * fedora:/files, then I should be able to retrieve an object from fedora:/files/FileSystem1 that lists a child
123     * object at fedora:/files/FileSystem1/TestSubdir and lists datastreams ds and ds2
124     *
125     * @throws IOException thrown during this function
126     */
127    @Test
128    public void testGetProjectedNode() throws IOException {
129        final HttpGet method = new HttpGet(serverAddress + "files/FileSystem1");
130        try (final CloseableDataset dataset = getDataset(client, method)) {
131            final DatasetGraph graph = dataset.asDatasetGraph();
132            final Node subjectURI = createURI(serverAddress + "files/FileSystem1");
133            assertTrue("Didn't find the first datastream! ", graph.contains(ANY,
134                    subjectURI, ANY, createURI(subjectURI + "/ds1")));
135            assertTrue("Didn't find the second datastream! ", graph.contains(ANY,
136                    subjectURI, ANY, createURI(subjectURI + "/ds2")));
137            assertTrue("Didn't find the first object! ", graph.contains(ANY,
138                    subjectURI, ANY, createURI(subjectURI + "/TestSubdir")));
139        }
140    }
141
142    /**
143     * When I make changes to a resource in a federated filesystem, the parent folder's Last-Modified header should be
144     * updated.
145     *
146     * @throws IOException thrown during this function
147     **/
148    @Test
149    public void testLastModifiedUpdatedAfterUpdates() throws IOException {
150        // create directory containing a file in filesystem
151        final File fed = new File("target/test-classes/test-objects");
152        final String id = randomUUID().toString();
153        final File dir = new File(fed, id);
154        final File child = new File(dir, "child");
155        final long timestamp1 = currentTimeMillis();
156        dir.mkdir();
157        child.mkdir();
158        // TODO this seems really brittle
159        try {
160            sleep(2000);
161        } catch (final InterruptedException e) {
162        }
163
164        // check Last-Modified header is current
165        final long lastmod1;
166        try (final CloseableHttpResponse resp1 = client.execute(new HttpHead(serverAddress + "files/" + id))) {
167            assertEquals(OK.getStatusCode(), getStatus(resp1));
168            lastmod1 = headerFormat.parse(resp1.getFirstHeader("Last-Modified").getValue()).getTime();
169            assertTrue((timestamp1 - lastmod1) < 1000); // because rounding
170
171            // remove the file and wait for the TTL to expire
172            final long timestamp2 = currentTimeMillis();
173            child.delete();
174            try {
175                sleep(2000);
176            } catch (final InterruptedException e) {
177            }
178
179            // check Last-Modified header is updated
180            try (final CloseableHttpResponse resp2 = client.execute(new HttpHead(serverAddress + "files/" + id))) {
181                assertEquals(OK.getStatusCode(), getStatus(resp2));
182                final long lastmod2 = headerFormat.parse(resp2.getFirstHeader("Last-Modified").getValue()).getTime();
183                assertTrue((timestamp2 - lastmod2) < 1000); // because rounding
184                assertFalse("Last-Modified headers should have changed", lastmod1 == lastmod2);
185            } catch (final ParseException e) {
186                fail();
187            }
188        } catch (final ParseException e) {
189            fail();
190        }
191    }
192
193    /**
194     * I should be able to copy objects from a federated filesystem to the repository.
195     **/
196    @Test
197    public void testCopyFromProjection() {
198        final String destination = serverAddress + "copy-" + randomUUID().toString() + "-ds1";
199        final String source = serverAddress + "files/FileSystem1/ds1";
200
201        // ensure the source is present
202        assertEquals(OK.getStatusCode(), getStatus(new HttpGet(source)));
203
204        // copy to repository
205        final HttpCopy request = new HttpCopy(source);
206        request.addHeader("Destination", destination);
207        assertEquals(CREATED.getStatusCode(), getStatus(request));
208
209        // repository copy should now exist
210        assertEquals(OK.getStatusCode(), getStatus(new HttpGet(destination)));
211        assertEquals(OK.getStatusCode(), getStatus(new HttpGet(source)));
212    }
213
214    /**
215     * I should be able to copy objects from the repository to a federated filesystem.
216     *
217     * @throws IOException exception thrown during this function
218     **/
219    @Ignore("Enabled once the FedoraFileSystemConnector becomes readable/writable")
220    public void testCopyToProjection() throws IOException {
221        // create object in the repository
222        final String pid = randomUUID().toString();
223        final HttpPut put = new HttpPut(serverAddress + pid + "/ds1");
224        put.setEntity(new StringEntity("abc123"));
225        put.setHeader("Content-Type", TEXT_PLAIN);
226        client.execute(put);
227
228        // copy to federated filesystem
229        final HttpCopy request = new HttpCopy(serverAddress + pid);
230        request.addHeader("Destination", serverAddress + "files/copy-" + pid);
231        assertEquals(CREATED.getStatusCode(), getStatus(request));
232
233        // federated copy should now exist
234        final HttpGet copyGet = new HttpGet(serverAddress + "files/copy-" + pid);
235        assertEquals(OK.getStatusCode(), getStatus(copyGet));
236
237        // repository copy should still exist
238        final HttpGet originalGet = new HttpGet(serverAddress + pid);
239        assertEquals(OK.getStatusCode(), getStatus(originalGet));
240    }
241
242    @Test
243    public void testGetRepositoryGraph() throws IOException {
244        try (final CloseableDataset dataset = getDataset(client, new HttpGet(serverAddress))) {
245            final DatasetGraph graph = dataset.asDatasetGraph();
246            logger.trace("Retrieved repository graph:\n" + graph);
247            assertFalse("Should not find the root type", graph.contains(ANY,
248                    ANY, type.asNode(), createURI(MODE_NAMESPACE + "root")));
249        }
250    }
251
252    /**
253     * I should be able to create two subdirectories of a non-existent parent directory.
254     *
255     * @throws IOException thrown during this function
256     **/
257    @Ignore("Enabled once the FedoraFileSystemConnector becomes readable/writable")
258    // TODO
259    public
260    void testBreakFederation() throws IOException {
261        final String id = randomUUID().toString();
262        testGetRepositoryGraph();
263        createObjectAndClose("files/a0/" + id + "b0");
264        createObjectAndClose("files/a0/" + id + "b1");
265        testGetRepositoryGraph();
266    }
267
268    /**
269     * I should be able to upload a file to a read/write federated filesystem.
270     *
271     * @throws IOException thrown during this function
272     **/
273    @Ignore("Enabled once the FedoraFileSystemConnector becomes readable/writable")
274    // TODO
275    public
276    void testUploadToProjection() throws IOException {
277        // upload file to federated filesystem using rest api
278        final String id = randomUUID().toString();
279        final String uploadLocation = serverAddress + "files/" + id + "/ds1";
280        final String uploadContent = "abc123";
281        logger.debug("Uploading to federated filesystem via rest api: " + uploadLocation);
282        // final HttpResponse response = createDatastream("files/" + pid, "ds1", uploadContent);
283        // final String actualLocation = response.getFirstHeader("Location").getValue();
284        // assertEquals("Wrong URI in Location header", uploadLocation, actualLocation);
285
286        // validate content
287        try (final CloseableHttpResponse getResponse = client.execute(new HttpGet(uploadLocation))) {
288            final String actualContent = EntityUtils.toString(getResponse.getEntity());
289            assertEquals(OK.getStatusCode(), getResponse.getStatusLine().getStatusCode());
290            assertEquals("Content doesn't match", actualContent, uploadContent);
291        }
292        // validate object profile
293        try (final CloseableHttpResponse objResponse = client.execute(new HttpGet(serverAddress + "files/" + id))) {
294            assertEquals(OK.getStatusCode(), objResponse.getStatusLine().getStatusCode());
295        }
296    }
297
298    /* Many of the following private methods are copied directly from
299     * org.fcrepo.integration.http.api.AbstractResourceIT.java. The reason we're not
300     * simply inheriting from that class and re-using these methods has to do with the
301     * complexities of building a working spring-based container along with the assumptions of
302     * certain dependency chains. Basically, it didn't work, and this is a compromise
303     * to simply make this integration test work. */
304
305    /**
306     * Executes an HTTP request and returns the status code of the response, closing the response.
307     *
308     * @param req the request to execute
309     * @return the HTTP status code of the response
310     */
311    private static int getStatus(final HttpUriRequest req) {
312        try (final CloseableHttpResponse response = client.execute(req)) {
313            final int result = getStatus(response);
314            if (!(result > 199) || !(result < 400)) {
315                logger.warn("Got status {}", result);
316                if (response.getEntity() != null) {
317                    logger.trace(EntityUtils.toString(response.getEntity()));
318                }
319            }
320            EntityUtils.consume(response.getEntity());
321            return result;
322        } catch (final IOException e) {
323            throw new RuntimeException(e);
324        }
325    }
326
327    private static int getStatus(final HttpResponse response) {
328        return response.getStatusLine().getStatusCode();
329    }
330
331    private CloseableHttpResponse createObject(final String pid) {
332        final HttpPost httpPost = new HttpPost(serverAddress + "/");
333        if (pid.length() > 0) {
334            httpPost.addHeader("Slug", pid);
335        }
336        try {
337            final CloseableHttpResponse response = client.execute(httpPost);
338            assertEquals(CREATED.getStatusCode(), getStatus(response));
339            return response;
340        } catch (final IOException e) {
341            throw new RuntimeException(e);
342        }
343    }
344
345    private void createObjectAndClose(final String pid) {
346        try {
347            createObject(pid).close();
348        } catch (final IOException e) {
349            throw new RuntimeException(e);
350        }
351    }
352
353    private CloseableDataset getDataset(final CloseableHttpClient client, final HttpUriRequest req)
354            throws IOException {
355        if (!req.containsHeader("Accept")) {
356            req.addHeader("Accept", "application/n-triples");
357        }
358        logger.debug("Retrieving RDF using mimeType: {}", req.getFirstHeader("Accept"));
359
360        try (final CloseableHttpResponse response = client.execute(req)) {
361            assertEquals(OK.getStatusCode(), response.getStatusLine().getStatusCode());
362            final CloseableDataset result = parseTriples(response.getEntity());
363            logger.trace("Retrieved RDF: {}", result);
364            return result;
365        }
366    }
367
368    @NotThreadSafe // HttpRequestBase is @NotThreadSafe
369    private class HttpCopy extends HttpRequestBase {
370
371        /**
372         * @throws IllegalArgumentException if the uri is invalid.
373         */
374        public HttpCopy(final String uri) {
375            super();
376            setURI(URI.create(uri));
377        }
378
379        @Override
380        public String getMethod() {
381            return "COPY";
382        }
383    }
384}