public/docs
1
0
Fork 0
mirror of https://github.com/sismics/docs.git synced 2025-12-13 17:56:20 +00:00
Code Issues Releases Wiki Activity

Delete files from storage when necessary,

Browse Source 
batch to cleanup storage for orphan files,
better Lucene directory reader management
This commit is contained in:
jendib
2013-08-18 00:53:01 +02:00
parent 00ed2e3c25
commit 6b5c1b2b51
10 changed files with 140 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
docs-core/src/main/java/com/sismics/docs/core/dao/lucene/LuceneDao.java
View File

@@ -10,8 +10,6 @@ import java.util.Set;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.queries.TermsFilter;
@@ -23,8 +21,6 @@ import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.util.Version;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.sismics.docs.core.model.context.AppContext;
import com.sismics.docs.core.model.jpa.Document;
@@ -38,11 +34,6 @@ import com.sismics.docs.core.util.LuceneUtil.LuceneRunnable;
* @author bgamard
*/
public class LuceneDao {
/**
* Logger.
*/
private static final Logger log = LoggerFactory.getLogger(LuceneDao.class);
/**
* Destroy and rebuild index.
*
@@ -178,17 +169,12 @@ public class LuceneDao {
TermsFilter userFilter = new TermsFilter(terms);
// Search
Set<String> documentIdList = new HashSet<String>();
if (!DirectoryReader.indexExists(AppContext.getInstance().getLuceneDirectory())) {
log.warn("Lucene directory not yet initialized");
return documentIdList;
}
IndexReader reader = DirectoryReader.open(AppContext.getInstance().getLuceneDirectory());
IndexSearcher searcher = new IndexSearcher(reader);
IndexSearcher searcher = new IndexSearcher(AppContext.getInstance().getIndexingService().getDirectoryReader());
TopDocs topDocs = searcher.search(query, userFilter, Integer.MAX_VALUE);
ScoreDoc[] docs = topDocs.scoreDocs;
// Extract document IDs
Set<String> documentIdList = new HashSet<String>();
for (int i = 0; i < docs.length; i++) {
org.apache.lucene.document.Document document = searcher.doc(docs[i].doc);
String type = document.get("type");
@@ -201,8 +187,6 @@ public class LuceneDao {
documentIdList.add(documentId);
}
reader.close();
return documentIdList;
}

18
docs-core/src/main/java/com/sismics/docs/core/listener/async/FileDeletedAsyncListener.java
View File

@@ -1,11 +1,15 @@
package com.sismics.docs.core.listener.async;
import java.nio.file.Paths;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.eventbus.Subscribe;
import com.sismics.docs.core.dao.lucene.LuceneDao;
import com.sismics.docs.core.event.FileDeletedAsyncEvent;
import com.sismics.docs.core.model.jpa.File;
import com.sismics.docs.core.util.DirectoryUtil;
/**
* Listener on file deleted.
@@ -30,10 +34,20 @@ public class FileDeletedAsyncListener {
log.info("File deleted event: " + fileDeletedAsyncEvent.toString());
}
// TODO Delete the file from storage
// Delete the file from storage
File file = fileDeletedAsyncEvent.getFile();
java.io.File thumbnailFile = Paths.get(DirectoryUtil.getStorageDirectory().getPath(), file.getId() + "_thumb").toFile();
java.io.File storedFile = Paths.get(DirectoryUtil.getStorageDirectory().getPath(), file.getId()).toFile();
if (thumbnailFile.exists()) {
thumbnailFile.delete();
}
if (storedFile.exists()) {
storedFile.delete();
}
// Update Lucene index
LuceneDao luceneDao = new LuceneDao();
luceneDao.deleteDocument(fileDeletedAsyncEvent.getFile().getId());
luceneDao.deleteDocument(file.getId());
}
}

22
docs-core/src/main/java/com/sismics/docs/core/model/context/AppContext.java
View File

@@ -7,8 +7,6 @@ import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import org.apache.lucene.store.Directory;
import com.google.common.eventbus.AsyncEventBus;
import com.google.common.eventbus.EventBus;
import com.sismics.docs.core.constant.ConfigType;
@@ -51,11 +49,6 @@ public class AppContext {
*/
private IndexingService indexingService;
/**
* Lucene directory.
*/
private Directory luceneDirectory;
/**
* Asynchronous executors.
*/
@@ -71,8 +64,6 @@ public class AppContext {
Config luceneStorageConfig = configDao.getById(ConfigType.LUCENE_DIRECTORY_STORAGE);
indexingService = new IndexingService(luceneStorageConfig != null ? luceneStorageConfig.getValue() : null);
indexingService.startAndWait();
luceneDirectory = indexingService.getDirectory();
}
/**
@@ -165,20 +156,11 @@ public class AppContext {
}
/**
* Getter of feedService.
* Getter of indexingService.
*
* @return feedService
* @return indexingService
*/
public IndexingService getIndexingService() {
return indexingService;
}
/**
* Getter of- luceneDirectory.
*
* @return the luceneDirectory
*/
public Directory getLuceneDirectory() {
return luceneDirectory;
}
}

50
docs-core/src/main/java/com/sismics/docs/core/service/IndexingService.java
View File

@@ -4,6 +4,7 @@ import java.io.File;
import java.io.IOException;
import java.util.concurrent.TimeUnit;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.store.SimpleFSDirectory;
@@ -34,6 +35,11 @@ public class IndexingService extends AbstractScheduledService {
*/
private Directory directory;
/**
* Directory reader.
*/
private DirectoryReader directoryReader;
/**
* Lucene storage config.
*/
@@ -62,10 +68,16 @@ public class IndexingService extends AbstractScheduledService {
@Override
protected void shutDown() {
Directory luceneIndex = AppContext.getInstance().getLuceneDirectory();
if (luceneIndex != null) {
if (directoryReader != null) {
try {
luceneIndex.close();
directoryReader.close();
} catch (IOException e) {
log.error("Error closing the index reader", e);
}
}
if (directory != null) {
try {
directory.close();
} catch (IOException e) {
log.error("Error closing Lucene index", e);
}
@@ -105,4 +117,36 @@ public class IndexingService extends AbstractScheduledService {
public Directory getDirectory() {
return directory;
}
/**
* Returns a valid directory reader.
* Take care of reopening the reader if the index has changed
* and closing the previous one.
*
* @return the directoryReader
*/
public DirectoryReader getDirectoryReader() {
if (directoryReader == null) {
if (!DirectoryReader.indexExists(directory)) {
log.info("Lucene directory not yet created");
return null;
}
try {
directoryReader = DirectoryReader.open(directory);
} catch (IOException e) {
log.error("Error creating the directory reader", e);
}
} else {
try {
DirectoryReader newReader = DirectoryReader.openIfChanged(directoryReader);
if (newReader != null) {
directoryReader.close();
directoryReader = newReader;
}
} catch (IOException e) {
log.error("Error while reopening the directory reader", e);
}
}
return directoryReader;
}
}

2
docs-core/src/main/java/com/sismics/docs/core/util/LuceneUtil.java
View File

@@ -37,7 +37,7 @@ public class LuceneUtil {
config.setMergeScheduler(new SerialMergeScheduler());
// Creating index writer
Directory directory = AppContext.getInstance().getLuceneDirectory();
Directory directory = AppContext.getInstance().getIndexingService().getDirectory();
IndexWriter indexWriter = null;
try {
indexWriter = new IndexWriter(directory, config);