public/docs
1
0
Fork 0
mirror of https://github.com/sismics/docs.git synced 2025-12-13 17:56:20 +00:00
Code Issues Releases Wiki Activity

Delete files from storage when necessary,

Browse Source 
batch to cleanup storage for orphan files,
better Lucene directory reader management
This commit is contained in:
jendib
2013-08-18 00:53:01 +02:00
parent 00ed2e3c25
commit 6b5c1b2b51
10 changed files with 140 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
docs-web/src/main/java/com/sismics/docs/rest/resource/AppResource.java
View File

@@ -1,7 +1,9 @@
package com.sismics.docs.rest.resource;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.ResourceBundle;
import javax.ws.rs.GET;
@@ -19,11 +21,14 @@ import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import com.sismics.docs.core.dao.jpa.DocumentDao;
import com.sismics.docs.core.dao.jpa.FileDao;
import com.sismics.docs.core.dao.jpa.criteria.DocumentCriteria;
import com.sismics.docs.core.dao.jpa.dto.DocumentDto;
import com.sismics.docs.core.event.OcrFileAsyncEvent;
import com.sismics.docs.core.model.context.AppContext;
import com.sismics.docs.core.model.jpa.File;
import com.sismics.docs.core.util.ConfigUtil;
import com.sismics.docs.core.util.DirectoryUtil;
import com.sismics.docs.core.util.jpa.PaginatedList;
import com.sismics.docs.core.util.jpa.PaginatedLists;
import com.sismics.docs.core.util.jpa.SortCriteria;
@@ -177,12 +182,53 @@ public class AppResource extends BaseResource {
}
checkBaseFunction(BaseFunction.ADMIN);
JSONObject response = new JSONObject();
try {
AppContext.getInstance().getIndexingService().rebuildIndex();
} catch (Exception e) {
throw new ServerException("IndexingError", "Error rebuilding index", e);
}
JSONObject response = new JSONObject();
response.put("status", "ok");
return Response.ok().entity(response).build();
}
/**
* Destroy and rebuild Lucene index.
*
* @return Response
* @throws JSONException
*/
@POST
@Path("batch/clean_storage")
@Produces(MediaType.APPLICATION_JSON)
public Response batchCleanStorage() throws JSONException {
if (!authenticate()) {
throw new ForbiddenClientException();
}
checkBaseFunction(BaseFunction.ADMIN);
// Get all files
FileDao fileDao = new FileDao();
List<File> fileList = fileDao.findAll();
Map<String, File> fileMap = new HashMap<>();
for (File file : fileList) {
fileMap.put(file.getId(), file);
}
// Check if each stored file is valid
java.io.File[] storedFileList = DirectoryUtil.getStorageDirectory().listFiles();
for (java.io.File storedFile : storedFileList) {
String fileName = storedFile.getName();
if (fileName.endsWith("_thumb")) {
fileName = fileName.replace("_thumb", "");
}
if (!fileMap.containsKey(fileName)) {
storedFile.delete();
}
}
JSONObject response = new JSONObject();
response.put("status", "ok");
return Response.ok().entity(response).build();
}

6
docs-web/src/test/java/com/sismics/docs/rest/TestAppResource.java
View File

@@ -55,6 +55,12 @@ public class TestAppResource extends BaseJerseyTest {
appResource.addFilter(new CookieAuthenticationFilter(adminAuthenticationToken));
response = appResource.post(ClientResponse.class);
Assert.assertEquals(Status.OK, Status.fromStatusCode(response.getStatus()));
// Clean storage
appResource = resource().path("/app/batch/clean_storage");
appResource.addFilter(new CookieAuthenticationFilter(adminAuthenticationToken));
response = appResource.post(ClientResponse.class);
Assert.assertEquals(Status.OK, Status.fromStatusCode(response.getStatus()));
}
/**

9
docs-web/src/test/java/com/sismics/docs/rest/TestDocumentResource.java
View File

@@ -284,6 +284,15 @@ public class TestDocumentResource extends BaseJerseyTest {
json = response.getEntity(JSONObject.class);
Assert.assertEquals(document1Id, json.getString("id"));
// Search documents by query
documentResource = resource().path("/document/list");
documentResource.addFilter(new CookieAuthenticationFilter(document1Token));
getParams = new MultivaluedMapImpl();
getParams.putSingle("search", "super");
response = documentResource.queryParams(getParams).get(ClientResponse.class);
json = response.getEntity(JSONObject.class);
Assert.assertEquals(Status.OK, Status.fromStatusCode(response.getStatus()));
// Get a document
documentResource = resource().path("/document/" + document1Id);
documentResource.addFilter(new CookieAuthenticationFilter(document1Token));

8
docs-web/src/test/java/com/sismics/docs/rest/TestFileResource.java
View File

@@ -2,6 +2,7 @@ package com.sismics.docs.rest;
import java.io.BufferedInputStream;
import java.io.InputStream;
import java.nio.file.Paths;
import javax.ws.rs.core.MediaType;
@@ -12,6 +13,7 @@ import org.codehaus.jettison.json.JSONObject;
import org.junit.Test;
import com.google.common.io.ByteStreams;
import com.sismics.docs.core.util.DirectoryUtil;
import com.sismics.docs.rest.filter.CookieAuthenticationFilter;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.ClientResponse.Status;
@@ -145,6 +147,12 @@ public class TestFileResource extends BaseJerseyTest {
json = response.getEntity(JSONObject.class);
Assert.assertEquals("ok", json.getString("status"));
// Check that files are deleted from FS
java.io.File thumbnailFile = Paths.get(DirectoryUtil.getStorageDirectory().getPath(), file1Id + "_thumb").toFile();
java.io.File storedFile = Paths.get(DirectoryUtil.getStorageDirectory().getPath(), file1Id).toFile();
Assert.assertFalse(thumbnailFile.exists());
Assert.assertFalse(storedFile.exists());
// Get all files from a document
fileResource = resource().path("/file/list");
fileResource.addFilter(new CookieAuthenticationFilter(file1AuthenticationToken));