public/docs
1
0
Fork 0
mirror of https://github.com/sismics/docs.git synced 2025-12-14 02:06:25 +00:00
Code Issues Releases Wiki Activity

Store file size in DB (#704)

Browse Source
This commit is contained in:
Julien Kirch
2023-09-14 16:50:39 +02:00
committed by GitHub
parent eedf19ad9d
commit ab7ff25929
29 changed files with 484 additions and 163 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
docs-core/src/main/java/com/sismics/docs/core/dao/FileDao.java
View File

@@ -160,6 +160,7 @@ public class FileDao {
fileDb.setMimeType(file.getMimeType());
fileDb.setVersionId(file.getVersionId());
fileDb.setLatestVersion(file.isLatestVersion());
fileDb.setSize(file.getSize());
return file;
}
@@ -224,4 +225,12 @@ public class FileDao {
q.setParameter("versionId", versionId);
return q.getResultList();
}
public List<File> getFilesWithUnknownSize(int limit) {
EntityManager em = ThreadLocalContext.get().getEntityManager();
TypedQuery<File> q = em.createQuery("select f from File f where f.size = :size and f.deleteDate is null order by f.order asc", File.class);
q.setParameter("size", File.UNKNOWN_SIZE);
q.setMaxResults(limit);
return q.getResultList();
}
}

13
docs-core/src/main/java/com/sismics/docs/core/event/FileDeletedAsyncEvent.java
View File

@@ -13,6 +13,8 @@ public class FileDeletedAsyncEvent extends UserEvent {
*/
private String fileId;
private Long fileSize;
public String getFileId() {
return fileId;
}
@@ -21,10 +23,19 @@ public class FileDeletedAsyncEvent extends UserEvent {
this.fileId = fileId;
}
public Long getFileSize() {
return fileSize;
}
public void setFileSize(Long fileSize) {
this.fileSize = fileSize;
}
@Override
public String toString() {
return MoreObjects.toStringHelper(this)
.add("fileId", fileId)
.add("fileSize", fileSize)
.toString();
}
}
}

25
docs-core/src/main/java/com/sismics/docs/core/listener/async/FileDeletedAsyncListener.java
View File

@@ -2,8 +2,11 @@ package com.sismics.docs.core.listener.async;
import com.google.common.eventbus.AllowConcurrentEvents;
import com.google.common.eventbus.Subscribe;
import com.sismics.docs.core.dao.UserDao;
import com.sismics.docs.core.event.FileDeletedAsyncEvent;
import com.sismics.docs.core.model.context.AppContext;
import com.sismics.docs.core.model.jpa.File;
import com.sismics.docs.core.model.jpa.User;
import com.sismics.docs.core.util.FileUtil;
import com.sismics.docs.core.util.TransactionUtil;
import org.slf4j.Logger;
@@ -11,7 +14,7 @@ import org.slf4j.LoggerFactory;
/**
* Listener on file deleted.
*
*
* @author bgamard
*/
public class FileDeletedAsyncListener {
@@ -22,7 +25,7 @@ public class FileDeletedAsyncListener {
/**
* File deleted.
*
*
* @param event File deleted event
* @throws Exception e
*/
@@ -32,6 +35,24 @@ public class FileDeletedAsyncListener {
if (log.isInfoEnabled()) {
log.info("File deleted event: " + event.toString());
}
TransactionUtil.handle(() -> {
// Update the user quota
UserDao userDao = new UserDao();
User user = userDao.getById(event.getUserId());
if (user != null) {
Long fileSize = event.getFileSize();
if (fileSize.equals(File.UNKNOWN_SIZE)) {
// The file size was not in the database, in this case we need to get from the unencrypted size.
fileSize = FileUtil.getFileSize(event.getFileId(), user);
}
if (! fileSize.equals(File.UNKNOWN_SIZE)) {
user.setStorageCurrent(user.getStorageCurrent() - fileSize);
userDao.updateQuota(user);
}
}
});
// Delete the file from storage
FileUtil.delete(event.getFileId());

15
docs-core/src/main/java/com/sismics/docs/core/model/context/AppContext.java
View File

@@ -9,6 +9,7 @@ import com.sismics.docs.core.dao.UserDao;
import com.sismics.docs.core.listener.async.*;
import com.sismics.docs.core.model.jpa.User;
import com.sismics.docs.core.service.FileService;
import com.sismics.docs.core.service.FileSizeService;
import com.sismics.docs.core.service.InboxService;
import com.sismics.docs.core.util.PdfUtil;
import com.sismics.docs.core.util.indexing.IndexingHandler;
@@ -65,6 +66,11 @@ public class AppContext {
*/
private FileService fileService;
/**
* File size service.
*/
private FileSizeService fileSizeService;
/**
* Asynchronous executors.
*/
@@ -102,6 +108,11 @@ public class AppContext {
inboxService.startAsync();
inboxService.awaitRunning();
// Start file size service
fileSizeService = new FileSizeService();
fileSizeService.startAsync();
fileSizeService.awaitRunning();
// Register fonts
PdfUtil.registerFonts();
@@ -238,6 +249,10 @@ public class AppContext {
fileService.stopAsync();
}
if (fileSizeService != null) {
fileSizeService.stopAsync();
}
instance = null;
}
}

20
docs-core/src/main/java/com/sismics/docs/core/model/jpa/File.java
View File

@@ -88,6 +88,14 @@ public class File implements Loggable {
@Column(name = "FIL_LATESTVERSION_B", nullable = false)
private boolean latestVersion;
public static final Long UNKNOWN_SIZE = -1L;
/**
* Can be {@link File#UNKNOWN_SIZE} if the size has not been stored in the database when the file has been uploaded
*/
@Column(name = "FIL_SIZE_N", nullable = false)
private Long size;
/**
* Private key to decrypt the file.
* Not saved to database, of course.
@@ -204,6 +212,18 @@ public class File implements Loggable {
return this;
}
/**
* Can return {@link File#UNKNOWN_SIZE} if the file size is not stored in the database.
*/
public Long getSize() {
return size;
}
public File setSize(Long size) {
this.size = size;
return this;
}
@Override
public String toString() {
return MoreObjects.toStringHelper(this)

78
docs-core/src/main/java/com/sismics/docs/core/service/FileSizeService.java Normal file
View File

@@ -0,0 +1,78 @@
package com.sismics.docs.core.service;
import com.google.common.util.concurrent.AbstractScheduledService;
import com.sismics.docs.core.dao.FileDao;
import com.sismics.docs.core.dao.UserDao;
import com.sismics.docs.core.model.jpa.File;
import com.sismics.docs.core.model.jpa.User;
import com.sismics.docs.core.util.FileUtil;
import com.sismics.docs.core.util.TransactionUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.List;
import java.util.concurrent.TimeUnit;
/**
* Service that retrieve files sizes when they are not in the database.
*/
public class FileSizeService extends AbstractScheduledService {
/**
* Logger.
*/
private static final Logger log = LoggerFactory.getLogger(FileSizeService.class);
public FileSizeService() {
}
@Override
protected void startUp() {
log.info("File size service starting up");
}
@Override
protected void shutDown() {
log.info("File size service shutting down");
}
private static final int BATCH_SIZE = 30;
@Override
protected void runOneIteration() {
try {
TransactionUtil.handle(() -> {
FileDao fileDao = new FileDao();
List<File> files = fileDao.getFilesWithUnknownSize(BATCH_SIZE);
for(File file : files) {
processFile(file);
}
if(files.size() < BATCH_SIZE) {
log.info("No more file to process, stopping the service");
stopAsync();
}
});
} catch (Throwable e) {
log.error("Exception during file service iteration", e);
}
}
void processFile(File file) {
UserDao userDao = new UserDao();
User user = userDao.getById(file.getUserId());
if(user == null) {
return;
}
long fileSize = FileUtil.getFileSize(file.getId(), user);
if(fileSize != File.UNKNOWN_SIZE){
FileDao fileDao = new FileDao();
file.setSize(fileSize);
fileDao.update(file);
}
}
@Override
protected Scheduler scheduler() {
return Scheduler.newFixedDelaySchedule(0, 1, TimeUnit.MINUTES);
}
}

3
docs-core/src/main/java/com/sismics/docs/core/service/InboxService.java
View File

@@ -1,14 +1,11 @@
package com.sismics.docs.core.service;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.google.common.util.concurrent.AbstractScheduledService;
import com.sismics.docs.core.constant.ConfigType;
import com.sismics.docs.core.dao.TagDao;
import com.sismics.docs.core.dao.criteria.TagCriteria;
import com.sismics.docs.core.dao.dto.TagDto;
import com.sismics.docs.core.event.DocumentCreatedAsyncEvent;
import com.sismics.docs.core.model.jpa.Config;
import com.sismics.docs.core.model.jpa.Document;
import com.sismics.docs.core.model.jpa.Tag;
import com.sismics.docs.core.util.ConfigUtil;

33
docs-core/src/main/java/com/sismics/docs/core/util/FileUtil.java
View File

@@ -16,6 +16,9 @@ import com.sismics.util.Scalr;
import com.sismics.util.context.ThreadLocalContext;
import com.sismics.util.io.InputStreamReaderThread;
import com.sismics.util.mime.MimeTypeUtil;
import org.apache.commons.io.IOUtils;
import org.apache.commons.io.input.CountingInputStream;
import org.apache.commons.io.output.NullOutputStream;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -46,7 +49,7 @@ public class FileUtil {
/**
* File ID of files currently being processed.
*/
private static Set<String> processingFileSet = Collections.synchronizedSet(new HashSet<>());
private static final Set<String> processingFileSet = Collections.synchronizedSet(new HashSet<>());
/**
* Optical character recognition on an image.
@@ -149,6 +152,7 @@ public class FileUtil {
file.setName(StringUtils.abbreviate(name, 200));
file.setMimeType(mimeType);
file.setUserId(userId);
file.setSize(fileSize);
// Get files of this document
FileDao fileDao = new FileDao();
@@ -240,4 +244,31 @@ public class FileUtil {
public static boolean isProcessingFile(String fileId) {
return processingFileSet.contains(fileId);
}
/**
* Get the size of a file on disk.
*
* @param fileId the file id
* @param user the file owner
* @return the size or -1 if something went wrong
*/
public static long getFileSize(String fileId, User user) {
// To get the size we copy the decrypted content into a null output stream
// and count the copied byte size.
Path storedFile = DirectoryUtil.getStorageDirectory().resolve(fileId);
if (! Files.exists(storedFile)) {
log.debug("File does not exist " + fileId);
return File.UNKNOWN_SIZE;
}
try (InputStream fileInputStream = Files.newInputStream(storedFile);
InputStream inputStream = EncryptionUtil.decryptInputStream(fileInputStream, user.getPrivateKey());
CountingInputStream countingInputStream = new CountingInputStream(inputStream);
) {
IOUtils.copy(countingInputStream, NullOutputStream.NULL_OUTPUT_STREAM);
return countingInputStream.getByteCount();
} catch (Exception e) {
log.debug("Can't find size of file " + fileId, e);
return File.UNKNOWN_SIZE;
}
}
}

1
docs-core/src/main/java/com/sismics/docs/core/util/format/FormatHandlerUtil.java
View File

@@ -3,7 +3,6 @@ package com.sismics.docs.core.util.format;
import com.google.common.collect.Lists;
import com.sismics.util.ClasspathScanner;
import java.lang.reflect.InvocationTargetException;
import java.util.List;
/**

1
docs-core/src/main/java/com/sismics/docs/core/util/format/TextPlainFormatHandler.java
View File

@@ -1,6 +1,5 @@
package com.sismics.docs.core.util.format;
import com.google.common.base.Charsets;
import com.google.common.io.Closer;
import com.lowagie.text.*;
import com.lowagie.text.pdf.PdfWriter;

2
docs-core/src/main/resources/config.properties
View File

@@ -1 +1 @@
db.version=28
db.version=29

2
docs-core/src/main/resources/db/update/dbupdate-029-0.sql Normal file
View File

@@ -0,0 +1,2 @@
alter table T_FILE add column FIL_SIZE_N bigint not null default -1;
update T_CONFIG set CFG_VALUE_C = '29' where CFG_ID_C = 'DB_VERSION';

49
docs-core/src/test/java/com/sismics/BaseTest.java Normal file
View File

@@ -0,0 +1,49 @@
package com.sismics;
import java.io.InputStream;
import java.net.URL;
public abstract class BaseTest {
protected static final String FILE_CSV = "document.csv";
protected static final String FILE_DOCX = "document.docx";
protected static final String FILE_GIF = "image.gif";
protected static final String FILE_JPG = "apollo_portrait.jpg";
protected static final Long FILE_JPG_SIZE = 7_907L;
protected static final String FILE_JPG2 = "apollo_landscape.jpg";
protected static final String FILE_MP4 = "video.mp4";
protected static final String FILE_ODT = "document.odt";
protected static final String FILE_PDF = "udhr.pdf";
protected static final String FILE_PDF_ENCRYPTED = "udhr_encrypted.pdf";
protected static final String FILE_PDF_SCANNED = "scanned.pdf";
protected static final String FILE_PNG = "image.png";
protected static final String FILE_PPTX = "apache.pptx";
protected static final String FILE_TXT = "document.txt";
protected static final String FILE_WEBM = "video.webm";
protected static final String FILE_XLSX = "document.xlsx";
protected static final String FILE_ZIP = "document.zip";
protected static URL getResource(String fileName) {
return ClassLoader.getSystemResource("file/" + fileName);
}
protected static InputStream getSystemResourceAsStream(String fileName) {
return ClassLoader.getSystemResourceAsStream("file/" + fileName);
}
}

45
docs-core/src/test/java/com/sismics/docs/BaseTransactionalTest.java
View File

@@ -1,19 +1,34 @@
package com.sismics.docs;
import com.sismics.BaseTest;
import com.sismics.docs.core.dao.FileDao;
import com.sismics.docs.core.dao.UserDao;
import com.sismics.docs.core.model.jpa.File;
import com.sismics.docs.core.model.jpa.User;
import com.sismics.docs.core.util.DirectoryUtil;
import com.sismics.docs.core.util.EncryptionUtil;
import com.sismics.util.context.ThreadLocalContext;
import com.sismics.util.jpa.EMF;
import com.sismics.util.mime.MimeType;
import org.junit.After;
import org.junit.Before;
import jakarta.persistence.EntityManager;
import jakarta.persistence.EntityTransaction;
import javax.crypto.Cipher;
import javax.crypto.CipherInputStream;
import java.io.InputStream;
import java.nio.file.Files;
import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
/**
* Base class of tests with a transactional context.
*
* @author jtremeaux
*/
public abstract class BaseTransactionalTest {
public abstract class BaseTransactionalTest extends BaseTest {
@Before
public void setUp() throws Exception {
// Initialize the entity manager
@@ -27,4 +42,32 @@ public abstract class BaseTransactionalTest {
@After
public void tearDown() throws Exception {
}
protected User createUser(String userName) throws Exception {
UserDao userDao = new UserDao();
User user = new User();
user.setUsername(userName);
user.setPassword("12345678");
user.setEmail("toto@docs.com");
user.setRoleId("admin");
user.setStorageQuota(100_000L);
userDao.create(user, userName);
return user;
}
protected File createFile(User user, long fileSize) throws Exception {
FileDao fileDao = new FileDao();
try(InputStream inputStream = getSystemResourceAsStream(FILE_JPG)) {
File file = new File();
file.setId("apollo_portrait");
file.setUserId(user.getId());
file.setVersion(0);
file.setMimeType(MimeType.IMAGE_JPEG);
file.setSize(fileSize);
String fileId = fileDao.create(file, user.getId());
Cipher cipher = EncryptionUtil.getEncryptionCipher(user.getPrivateKey());
Files.copy(new CipherInputStream(inputStream, cipher), DirectoryUtil.getStorageDirectory().resolve(fileId), REPLACE_EXISTING);
return file;
}
}
}

14
docs-core/src/test/java/com/sismics/docs/core/dao/jpa/TestJpa.java
View File

@@ -18,22 +18,16 @@ public class TestJpa extends BaseTransactionalTest {
public void testJpa() throws Exception {
// Create a user
UserDao userDao = new UserDao();
User user = new User();
user.setUsername("username");
user.setPassword("12345678");
user.setEmail("toto@docs.com");
user.setRoleId("admin");
user.setStorageQuota(10L);
String id = userDao.create(user, "me");
User user = createUser("testJpa");
TransactionUtil.commit();
// Search a user by his ID
user = userDao.getById(id);
user = userDao.getById(user.getId());
Assert.assertNotNull(user);
Assert.assertEquals("toto@docs.com", user.getEmail());
// Authenticate using the database
Assert.assertNotNull(new InternalAuthenticationHandler().authenticate("username", "12345678"));
Assert.assertNotNull(new InternalAuthenticationHandler().authenticate("testJpa", "12345678"));
}
}

52
docs-core/src/test/java/com/sismics/docs/core/listener/async/FileDeletedAsyncListenerTest.java Normal file
View File

@@ -0,0 +1,52 @@
package com.sismics.docs.core.listener.async;
import com.sismics.docs.BaseTransactionalTest;
import com.sismics.docs.core.dao.UserDao;
import com.sismics.docs.core.event.FileDeletedAsyncEvent;
import com.sismics.docs.core.model.jpa.File;
import com.sismics.docs.core.model.jpa.User;
import com.sismics.docs.core.util.TransactionUtil;
import org.junit.Assert;
import org.junit.Test;
public class FileDeletedAsyncListenerTest extends BaseTransactionalTest {
@Test
public void updateQuotaSizeKnown() throws Exception {
User user = createUser("updateQuotaSizeKnown");
File file = createFile(user, FILE_JPG_SIZE);
UserDao userDao = new UserDao();
user = userDao.getById(user.getId());
user.setStorageCurrent(10_000L);
userDao.updateQuota(user);
FileDeletedAsyncListener fileDeletedAsyncListener = new FileDeletedAsyncListener();
TransactionUtil.commit();
FileDeletedAsyncEvent event = new FileDeletedAsyncEvent();
event.setFileSize(FILE_JPG_SIZE);
event.setFileId(file.getId());
event.setUserId(user.getId());
fileDeletedAsyncListener.on(event);
Assert.assertEquals(userDao.getById(user.getId()).getStorageCurrent(), Long.valueOf(10_000 - FILE_JPG_SIZE));
}
@Test
public void updateQuotaSizeUnknown() throws Exception {
User user = createUser("updateQuotaSizeUnknown");
File file = createFile(user, File.UNKNOWN_SIZE);
UserDao userDao = new UserDao();
user = userDao.getById(user.getId());
user.setStorageCurrent(10_000L);
userDao.updateQuota(user);
FileDeletedAsyncListener fileDeletedAsyncListener = new FileDeletedAsyncListener();
TransactionUtil.commit();
FileDeletedAsyncEvent event = new FileDeletedAsyncEvent();
event.setFileSize(FILE_JPG_SIZE);
event.setFileId(file.getId());
event.setUserId(user.getId());
fileDeletedAsyncListener.on(event);
Assert.assertEquals(userDao.getById(user.getId()).getStorageCurrent(), Long.valueOf(10_000 - FILE_JPG_SIZE));
}
}

22
docs-core/src/test/java/com/sismics/docs/core/service/TestFileSizeService.java Normal file
View File

@@ -0,0 +1,22 @@
package com.sismics.docs.core.service;
import com.sismics.docs.BaseTransactionalTest;
import com.sismics.docs.core.dao.FileDao;
import com.sismics.docs.core.model.jpa.File;
import com.sismics.docs.core.model.jpa.User;
import org.junit.Assert;
import org.junit.Test;
public class TestFileSizeService extends BaseTransactionalTest {
@Test
public void processFileTest() throws Exception {
User user = createUser("processFileTest");
FileDao fileDao = new FileDao();
File file = createFile(user, File.UNKNOWN_SIZE);
FileSizeService fileSizeService = new FileSizeService();
fileSizeService.processFile(file);
Assert.assertEquals(fileDao.getFile(file.getId()).getSize(), Long.valueOf(FILE_JPG_SIZE));
}
}

11
docs-core/src/test/java/com/sismics/docs/core/util/TestEncryptUtil.java
View File

@@ -2,6 +2,7 @@ package com.sismics.docs.core.util;
import com.google.common.base.Strings;
import com.google.common.io.ByteStreams;
import com.sismics.BaseTest;
import org.junit.Assert;
import org.junit.Test;
@@ -14,7 +15,7 @@ import java.io.InputStream;
*
* @author bgamard
*/
public class TestEncryptUtil {
public class TestEncryptUtil extends BaseTest {
@Test
public void generatePrivateKeyTest() {
String key = EncryptionUtil.generatePrivateKey();
@@ -31,9 +32,9 @@ public class TestEncryptUtil {
// NOP
}
Cipher cipher = EncryptionUtil.getEncryptionCipher("OnceUponATime");
InputStream inputStream = new CipherInputStream(this.getClass().getResourceAsStream("/file/udhr.pdf"), cipher);
InputStream inputStream = new CipherInputStream(getSystemResourceAsStream(FILE_PDF), cipher);
byte[] encryptedData = ByteStreams.toByteArray(inputStream);
byte[] assertData = ByteStreams.toByteArray(this.getClass().getResourceAsStream("/file/udhr_encrypted.pdf"));
byte[] assertData = ByteStreams.toByteArray(getSystemResourceAsStream(FILE_PDF_ENCRYPTED));
Assert.assertEquals(encryptedData.length, assertData.length);
}
@@ -41,9 +42,9 @@ public class TestEncryptUtil {
@Test
public void decryptStreamTest() throws Exception {
InputStream inputStream = EncryptionUtil.decryptInputStream(
this.getClass().getResourceAsStream("/file/udhr_encrypted.pdf"), "OnceUponATime");
getSystemResourceAsStream(FILE_PDF_ENCRYPTED), "OnceUponATime");
byte[] encryptedData = ByteStreams.toByteArray(inputStream);
byte[] assertData = ByteStreams.toByteArray(this.getClass().getResourceAsStream("/file/udhr.pdf"));
byte[] assertData = ByteStreams.toByteArray(getSystemResourceAsStream(FILE_PDF));
Assert.assertEquals(encryptedData.length, assertData.length);
}

35
docs-core/src/test/java/com/sismics/docs/core/util/TestFileUtil.java
View File

@@ -2,6 +2,7 @@ package com.sismics.docs.core.util;
import com.google.common.collect.Lists;
import com.google.common.io.Resources;
import com.sismics.BaseTest;
import com.sismics.docs.core.dao.dto.DocumentDto;
import com.sismics.docs.core.model.jpa.File;
import com.sismics.docs.core.util.format.*;
@@ -23,11 +24,11 @@ import java.util.Date;
*
* @author bgamard
*/
public class TestFileUtil {
public class TestFileUtil extends BaseTest {
@Test
public void extractContentOpenDocumentTextTest() throws Exception {
Path path = Paths.get(ClassLoader.getSystemResource("file/document.odt").toURI());
FormatHandler formatHandler = FormatHandlerUtil.find(MimeTypeUtil.guessMimeType(path, "document.odt"));
Path path = Paths.get(getResource(FILE_ODT).toURI());
FormatHandler formatHandler = FormatHandlerUtil.find(MimeTypeUtil.guessMimeType(path, FILE_ODT));
Assert.assertNotNull(formatHandler);
Assert.assertTrue(formatHandler instanceof OdtFormatHandler);
String content = formatHandler.extractContent("eng", path);
@@ -36,8 +37,8 @@ public class TestFileUtil {
@Test
public void extractContentOfficeDocumentTest() throws Exception {
Path path = Paths.get(ClassLoader.getSystemResource("file/document.docx").toURI());
FormatHandler formatHandler = FormatHandlerUtil.find(MimeTypeUtil.guessMimeType(path, "document.docx"));
Path path = Paths.get(getResource(FILE_DOCX).toURI());
FormatHandler formatHandler = FormatHandlerUtil.find(MimeTypeUtil.guessMimeType(path, FILE_DOCX));
Assert.assertNotNull(formatHandler);
Assert.assertTrue(formatHandler instanceof DocxFormatHandler);
String content = formatHandler.extractContent("eng", path);
@@ -46,8 +47,8 @@ public class TestFileUtil {
@Test
public void extractContentPowerpointTest() throws Exception {
Path path = Paths.get(ClassLoader.getSystemResource("file/apache.pptx").toURI());
FormatHandler formatHandler = FormatHandlerUtil.find(MimeTypeUtil.guessMimeType(path, "apache.pptx"));
Path path = Paths.get(getResource(FILE_PPTX).toURI());
FormatHandler formatHandler = FormatHandlerUtil.find(MimeTypeUtil.guessMimeType(path, FILE_PPTX));
Assert.assertNotNull(formatHandler);
Assert.assertTrue(formatHandler instanceof PptxFormatHandler);
String content = formatHandler.extractContent("eng", path);
@@ -56,8 +57,8 @@ public class TestFileUtil {
@Test
public void extractContentPdf() throws Exception {
Path path = Paths.get(ClassLoader.getSystemResource("file/udhr.pdf").toURI());
FormatHandler formatHandler = FormatHandlerUtil.find(MimeTypeUtil.guessMimeType(path, "udhr.pdf"));
Path path = Paths.get(getResource(FILE_PDF).toURI());
FormatHandler formatHandler = FormatHandlerUtil.find(MimeTypeUtil.guessMimeType(path, FILE_PDF));
Assert.assertNotNull(formatHandler);
Assert.assertTrue(formatHandler instanceof PdfFormatHandler);
String content = formatHandler.extractContent("eng", path);
@@ -66,8 +67,8 @@ public class TestFileUtil {
@Test
public void extractContentScannedPdf() throws Exception {
Path path = Paths.get(ClassLoader.getSystemResource("file/scanned.pdf").toURI());
FormatHandler formatHandler = FormatHandlerUtil.find(MimeTypeUtil.guessMimeType(path, "scanned.pdf"));
Path path = Paths.get(getResource("scanned.pdf").toURI());
FormatHandler formatHandler = FormatHandlerUtil.find(MimeTypeUtil.guessMimeType(path, FILE_PDF_SCANNED));
Assert.assertNotNull(formatHandler);
Assert.assertTrue(formatHandler instanceof PdfFormatHandler);
String content = formatHandler.extractContent("eng", path);
@@ -76,12 +77,12 @@ public class TestFileUtil {
@Test
public void convertToPdfTest() throws Exception {
try (InputStream inputStream0 = Resources.getResource("file/apollo_landscape.jpg").openStream();
InputStream inputStream1 = Resources.getResource("file/apollo_portrait.jpg").openStream();
InputStream inputStream2 = Resources.getResource("file/udhr_encrypted.pdf").openStream();
InputStream inputStream3 = Resources.getResource("file/document.docx").openStream();
InputStream inputStream4 = Resources.getResource("file/document.odt").openStream();
InputStream inputStream5 = Resources.getResource("file/apache.pptx").openStream()) {
try (InputStream inputStream0 = getSystemResourceAsStream(FILE_JPG2);
InputStream inputStream1 = getSystemResourceAsStream(FILE_JPG);
InputStream inputStream2 = getSystemResourceAsStream(FILE_PDF_ENCRYPTED);
InputStream inputStream3 = getSystemResourceAsStream(FILE_DOCX);
InputStream inputStream4 = getSystemResourceAsStream(FILE_ODT);
InputStream inputStream5 = getSystemResourceAsStream(FILE_PPTX)) {
// Document
DocumentDto documentDto = new DocumentDto();
documentDto.setTitle("My super document 1");

55
docs-core/src/test/java/com/sismics/util/TestMimeTypeUtil.java
View File

@@ -1,5 +1,6 @@
package com.sismics.util;
import com.sismics.BaseTest;
import com.sismics.util.mime.MimeType;
import com.sismics.util.mime.MimeTypeUtil;
import org.junit.Assert;
@@ -13,59 +14,59 @@ import java.nio.file.Paths;
*
* @author bgamard
*/
public class TestMimeTypeUtil {
public class TestMimeTypeUtil extends BaseTest {
@Test
public void test() throws Exception {
// Detect ODT files
Path path = Paths.get(ClassLoader.getSystemResource("file/document.odt").toURI());
Assert.assertEquals(MimeType.OPEN_DOCUMENT_TEXT, MimeTypeUtil.guessMimeType(path, "document.odt"));
Path path = Paths.get(getResource(FILE_ODT).toURI());
Assert.assertEquals(MimeType.OPEN_DOCUMENT_TEXT, MimeTypeUtil.guessMimeType(path, FILE_ODT));
// Detect DOCX files
path = Paths.get(ClassLoader.getSystemResource("file/document.docx").toURI());
Assert.assertEquals(MimeType.OFFICE_DOCUMENT, MimeTypeUtil.guessMimeType(path, "document.odt"));
path = Paths.get(getResource(FILE_DOCX).toURI());
Assert.assertEquals(MimeType.OFFICE_DOCUMENT, MimeTypeUtil.guessMimeType(path, FILE_ODT));
// Detect PPTX files
path = Paths.get(ClassLoader.getSystemResource("file/apache.pptx").toURI());
Assert.assertEquals(MimeType.OFFICE_PRESENTATION, MimeTypeUtil.guessMimeType(path, "apache.pptx"));
path = Paths.get(getResource(FILE_PPTX).toURI());
Assert.assertEquals(MimeType.OFFICE_PRESENTATION, MimeTypeUtil.guessMimeType(path, FILE_PPTX));
// Detect XLSX files
path = Paths.get(ClassLoader.getSystemResource("file/document.xlsx").toURI());
Assert.assertEquals(MimeType.OFFICE_SHEET, MimeTypeUtil.guessMimeType(path, "document.xlsx"));
path = Paths.get(getResource(FILE_XLSX).toURI());
Assert.assertEquals(MimeType.OFFICE_SHEET, MimeTypeUtil.guessMimeType(path, FILE_XLSX));
// Detect TXT files
path = Paths.get(ClassLoader.getSystemResource("file/document.txt").toURI());
Assert.assertEquals(MimeType.TEXT_PLAIN, MimeTypeUtil.guessMimeType(path, "document.txt"));
path = Paths.get(getResource(FILE_TXT).toURI());
Assert.assertEquals(MimeType.TEXT_PLAIN, MimeTypeUtil.guessMimeType(path, FILE_TXT));
// Detect CSV files
path = Paths.get(ClassLoader.getSystemResource("file/document.csv").toURI());
Assert.assertEquals(MimeType.TEXT_CSV, MimeTypeUtil.guessMimeType(path, "document.csv"));
path = Paths.get(getResource(FILE_CSV).toURI());
Assert.assertEquals(MimeType.TEXT_CSV, MimeTypeUtil.guessMimeType(path, FILE_CSV));
// Detect PDF files
path = Paths.get(ClassLoader.getSystemResource("file/udhr.pdf").toURI());
Assert.assertEquals(MimeType.APPLICATION_PDF, MimeTypeUtil.guessMimeType(path, "udhr.pdf"));
path = Paths.get(getResource(FILE_PDF).toURI());
Assert.assertEquals(MimeType.APPLICATION_PDF, MimeTypeUtil.guessMimeType(path, FILE_PDF));
// Detect JPEG files
path = Paths.get(ClassLoader.getSystemResource("file/apollo_portrait.jpg").toURI());
Assert.assertEquals(MimeType.IMAGE_JPEG, MimeTypeUtil.guessMimeType(path, "apollo_portrait.jpg"));
path = Paths.get(getResource(FILE_JPG).toURI());
Assert.assertEquals(MimeType.IMAGE_JPEG, MimeTypeUtil.guessMimeType(path, FILE_JPG));
// Detect GIF files
path = Paths.get(ClassLoader.getSystemResource("file/image.gif").toURI());
Assert.assertEquals(MimeType.IMAGE_GIF, MimeTypeUtil.guessMimeType(path, "image.gif"));
path = Paths.get(getResource(FILE_GIF).toURI());
Assert.assertEquals(MimeType.IMAGE_GIF, MimeTypeUtil.guessMimeType(path, FILE_GIF));
// Detect PNG files
path = Paths.get(ClassLoader.getSystemResource("file/image.png").toURI());
Assert.assertEquals(MimeType.IMAGE_PNG, MimeTypeUtil.guessMimeType(path, "image.png"));
path = Paths.get(getResource(FILE_PNG).toURI());
Assert.assertEquals(MimeType.IMAGE_PNG, MimeTypeUtil.guessMimeType(path, FILE_PNG));
// Detect ZIP files
path = Paths.get(ClassLoader.getSystemResource("file/document.zip").toURI());
Assert.assertEquals(MimeType.APPLICATION_ZIP, MimeTypeUtil.guessMimeType(path, "document.zip"));
path = Paths.get(getResource(FILE_ZIP).toURI());
Assert.assertEquals(MimeType.APPLICATION_ZIP, MimeTypeUtil.guessMimeType(path, FILE_ZIP));
// Detect WEBM files
path = Paths.get(ClassLoader.getSystemResource("file/video.webm").toURI());
Assert.assertEquals(MimeType.VIDEO_WEBM, MimeTypeUtil.guessMimeType(path, "video.webm"));
path = Paths.get(getResource(FILE_WEBM).toURI());
Assert.assertEquals(MimeType.VIDEO_WEBM, MimeTypeUtil.guessMimeType(path, FILE_WEBM));
// Detect MP4 files
path = Paths.get(ClassLoader.getSystemResource("file/video.mp4").toURI());
Assert.assertEquals(MimeType.VIDEO_MP4, MimeTypeUtil.guessMimeType(path, "video.mp4"));
path = Paths.get(getResource(FILE_MP4).toURI());
Assert.assertEquals(MimeType.VIDEO_MP4, MimeTypeUtil.guessMimeType(path, FILE_MP4));
}
}

5
docs-core/src/test/java/com/sismics/util/format/TestPdfFormatHandler.java
View File

@@ -1,5 +1,6 @@
package com.sismics.util.format;
import com.sismics.BaseTest;
import com.sismics.docs.core.util.format.PdfFormatHandler;
import org.junit.Assert;
import org.junit.Test;
@@ -11,14 +12,14 @@ import java.nio.file.Paths;
*
* @author bgamard
*/
public class TestPdfFormatHandler {
public class TestPdfFormatHandler extends BaseTest {
/**
* Test related to https://github.com/sismics/docs/issues/373.
*/
@Test
public void testIssue373() throws Exception {
PdfFormatHandler formatHandler = new PdfFormatHandler();
String content = formatHandler.extractContent("deu", Paths.get(ClassLoader.getSystemResource("file/issue373.pdf").toURI()));
String content = formatHandler.extractContent("deu", Paths.get(getResource("issue373.pdf").toURI()));
Assert.assertTrue(content.contains("Aufrechterhaltung"));
Assert.assertTrue(content.contains("Außentemperatur"));
Assert.assertTrue(content.contains("Grundumsatzmessungen"));