Skip to content

V0.12.18 fix perf log #19

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 11 commits into
base: v0.12
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
fix(cmis:path): on folder metadata
perf(threadpool): use a 10 thread executor on start
perf(memory): do not load content on memory at startup
  • Loading branch information
johanlelan-mgdis committed Apr 16, 2021
commit a84e96dc51388e9b62b3503304c77e9bdedf00f2
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@
import org.apache.chemistry.opencmis.commons.spi.BindingsObjectFactory;
import org.apache.chemistry.opencmis.inmemory.FilterParser;
import org.apache.chemistry.opencmis.inmemory.storedobj.api.Document;
import org.apache.chemistry.opencmis.inmemory.storedobj.api.Fileable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand All @@ -48,14 +47,12 @@ public DocumentImpl() { // visibility should be package


public ContentStream getContent() {
return fContent;
return this.fContent;
}


public void setContent(ContentStream content) {
fContent = content;
this.fContent = content;
}


public void fillProperties(Map<String, PropertyData<?>> properties, BindingsObjectFactory objFactory,
List<String> requestedIds) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,9 @@ public String getPath() {
}
if (this.getParentId() != null && this.getStore() != null) {
FolderImpl parent = (FolderImpl) this.getStore().getObjectById(this.getParentId());
if (parent == null) {
return null;
}
String path = "/" + this.getName();
if (!parent.getPath().equals("/")) {
path = parent.getPath() + path;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.AbstractMap.SimpleImmutableEntry;

import javax.xml.stream.XMLStreamReader;

Expand All @@ -35,11 +36,13 @@
import org.apache.chemistry.opencmis.inmemory.storedobj.api.Document;
import org.apache.chemistry.opencmis.inmemory.storedobj.api.Fileable;
import org.apache.chemistry.opencmis.inmemory.storedobj.api.Folder;
import org.apache.chemistry.opencmis.inmemory.storedobj.api.ObjectStore;
import org.apache.chemistry.opencmis.inmemory.storedobj.api.Relationship;
import org.apache.chemistry.opencmis.inmemory.storedobj.api.StoredObject;
import org.apache.chemistry.opencmis.inmemory.storedobj.impl.DocumentImpl;
import org.apache.chemistry.opencmis.inmemory.storedobj.impl.FilingImpl;
import org.apache.chemistry.opencmis.inmemory.storedobj.impl.FolderImpl;
import org.apache.chemistry.opencmis.inmemory.storedobj.impl.ObjectStoreImpl;
import org.apache.chemistry.opencmis.server.support.TypeManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand Down Expand Up @@ -204,7 +207,7 @@ public InputStream getStream(File file) {
// stream = new FileInputStream(file);
stream = org.apache.commons.io.FileUtils.openInputStream(file);

LOG.info("Get stream content from " + file.getAbsolutePath());
LOG.debug("Get stream content from " + file.getAbsolutePath());
// stream = new BufferedInputStream(new FileInputStream(file),
// BUFFER_SIZE);

Expand All @@ -228,7 +231,7 @@ public int writeContent(File newFile, InputStream stream) {
int length = (int) newFile.length();
// Files.copy(stream, Paths.get(newFile.getAbsolutePath()),
// StandardCopyOption.REPLACE_EXISTING);
LOG.info("Write content in " + newFile.getAbsolutePath());
LOG.debug("Write content in " + newFile.getAbsolutePath());
return length;
} catch (IOException e) {
throw new CmisStorageException("Could not write content in "
Expand Down Expand Up @@ -299,7 +302,7 @@ public void writeCMISToDisc(File newFile, StoredObject so) {
metadataFile), json.toString());
// out.print(json);
// out.flush();
LOG.info("Writing metadata in " + metadataFile);
LOG.debug("Writing metadata in " + metadataFile);
} catch (IOException e) {
throw new CmisStorageException("Could not write metadata: "
+ e.getMessage(), e);
Expand All @@ -309,17 +312,17 @@ public void writeCMISToDisc(File newFile, StoredObject so) {
/**
* Read the metadata from disc.
*/
public StoredObject readCMISFromDisk(File metadataFile) {
public SimpleImmutableEntry<Boolean, StoredObject> readCMISFromDisk(File metadataFile, ObjectStore store) {

if (root == null)
return null;

if (!metadataFile.exists()) {
// check if a XXXX.cmis.xml file exists at the same position
if (new File(metadataFile.getAbsolutePath().replace(FilePersistenceLoader.SUFFIXE_METADATA, "") + "/" + FilePersistenceLoader.SHADOW_FOLDER).exists())
return readCMISFromDisk(new File(metadataFile.getAbsolutePath().replace(FilePersistenceLoader.SUFFIXE_METADATA, "") + "/cmis.xml"));
return readCMISFromDisk(new File(metadataFile.getAbsolutePath().replace(FilePersistenceLoader.SUFFIXE_METADATA, "") + "/cmis.xml"), store);
else if (new File(metadataFile.getAbsolutePath().replace(FilePersistenceLoader.SUFFIXE_METADATA, "") + FilePersistenceLoader.SHADOW_EXT).exists())
return readCMISFromDisk(new File(metadataFile.getAbsolutePath().replace(FilePersistenceLoader.SUFFIXE_METADATA, "") + ".cmis.xml"));
return readCMISFromDisk(new File(metadataFile.getAbsolutePath().replace(FilePersistenceLoader.SUFFIXE_METADATA, "") + ".cmis.xml"), store);
else
return null;
}
Expand All @@ -338,6 +341,7 @@ else if (new File(metadataFile.getAbsolutePath().replace(FilePersistenceLoader.S
if (storedObjectStr.equals("")) {
return null;
}
Boolean toBeSaved = false;
StoredObject result = null;
LOG.debug(storedObjectStr);

Expand Down Expand Up @@ -392,10 +396,10 @@ else if (new File(metadataFile.getAbsolutePath().replace(FilePersistenceLoader.S
IOUtils.closeQuietly(stream);
}
} else if (metadataFile.getName().endsWith(FilePersistenceLoader.SUFFIXE_METADATA)) {
result = new StoredObjectJsonSerializer().deserialize(storedObjectStr, typeManager);
return new StoredObjectJsonSerializer().deserialize(storedObjectStr, typeManager, store);
}

return result;
return new SimpleImmutableEntry<Boolean, StoredObject>(toBeSaved, result);
}

@SuppressWarnings("unchecked")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,10 @@
import java.io.FilenameFilter;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.AbstractMap.SimpleImmutableEntry;


import org.apache.chemistry.opencmis.commons.PropertyIds;
import org.apache.chemistry.opencmis.commons.data.PropertyData;
Expand All @@ -19,6 +23,8 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.sun.corba.se.impl.orbutil.closure.Future;

public class FilePersistenceLoader {

private static final Logger LOG = LoggerFactory
Expand Down Expand Up @@ -57,109 +63,120 @@ public boolean accept(File dir, String name) {
private static void loadFolder(String repositoryId, ObjectStore store,
File folder, String folderId, FilenameFilter filenameFilter,
PersistenceManager persistenceManager) {
LOG.info("Scanning " + folder.getAbsolutePath());
LOG.debug("Scanning " + folder.getAbsolutePath());

ExecutorService exec = Executors.newFixedThreadPool(10);

// iterate through children
for (File child : folder.listFiles(filenameFilter)) {
// skip hidden files
// skip hidden files
if (child.isHidden()) continue;

LOG.debug("Loading file " + child.getAbsolutePath());

boolean toBeSaved = false;

StoredObject so = null;
if (child.isDirectory()) {
so = new FolderImpl(child.getName(), folderId);
so.setName(child.getName());
so.setId(persistenceManager.generateId());
File metadataFile = new File(child.getAbsolutePath()
+ SUFFIXE_METADATA);

String relativePath = child.getAbsolutePath().replace(persistenceManager.getRootPath(), "");

FolderImpl meta = (FolderImpl) persistenceManager
.readCMISFromDisk(metadataFile);
if (meta != null) {
// if metadata file exists then meta is the CMIS Object
so = meta;
} else {
LOG.warn("Missing metadata or malformed file for " + child.getAbsolutePath());
so.setProperties(new LinkedHashMap<String, PropertyData<?>>());
so.setTypeId(BaseTypeId.CMIS_FOLDER.value());
so.setName(child.getName());
toBeSaved = true;
}
Map<String, PropertyData<?>> properties = so.getProperties();
if (!properties.containsKey(PropertyIds.PATH)
|| !properties.get(PropertyIds.PATH).getFirstValue().equals(relativePath)) {
LOG.warn("Fixing cmis:path for " + relativePath);
properties.put(PropertyIds.PATH, new PropertyStringImpl(PropertyIds.PATH, relativePath));
//toBeSaved = true;
}
Folder fso = (Folder) so;
if (fso.getTypeId() == null) fso.setTypeId("cmis:folder");
if (!folderId.equals(fso.getParentId())) {
LOG.warn("Fixing folder.parentId : setting " + folderId + " into cmis:parentId");
fso.setParentId(folderId);
//toBeSaved = true;
}
so.setRepositoryId(repositoryId);
so.setStore(store);
} else {
so = new DocumentImpl();
File metadataFile = new File(child.getAbsolutePath()
+ SUFFIXE_METADATA);
so.setId(persistenceManager.generateId());
exec.submit(new Runnable() {
public void run() {
LOG.debug("Loading file " + child.getAbsolutePath());

boolean toBeSaved = false;

StoredObject so = null;
if (child.isDirectory()) {
so = new FolderImpl(child.getName(), folderId);
so.setName(child.getName());
so.setId(persistenceManager.generateId());
so.setRepositoryId(repositoryId);
so.setStore(store);
File metadataFile = new File(child.getAbsolutePath()
+ SUFFIXE_METADATA);

String relativePath = child.getAbsolutePath().replace(persistenceManager.getRootPath(), "");

SimpleImmutableEntry<Boolean, StoredObject> unmarshalled = persistenceManager
.readCMISFromDisk(metadataFile, store);
if (unmarshalled != null) {
FolderImpl meta = (FolderImpl) unmarshalled.getValue();
toBeSaved = unmarshalled.getKey();
// if metadata file exists then meta is the CMIS Object
so = meta;
so.setRepositoryId(repositoryId);
so.setStore(store);
} else {
LOG.warn("Missing metadata or malformed file for " + child.getAbsolutePath());
so.setProperties(new LinkedHashMap<String, PropertyData<?>>());
so.setTypeId(BaseTypeId.CMIS_FOLDER.value());
so.setName(child.getName());
toBeSaved = true;
}
Map<String, PropertyData<?>> properties = so.getProperties();
if (!properties.containsKey(PropertyIds.PATH)
|| !properties.get(PropertyIds.PATH).getFirstValue().equals(relativePath)) {
LOG.warn("Fixing cmis:path for " + relativePath);
properties.put(PropertyIds.PATH, new PropertyStringImpl(PropertyIds.PATH, relativePath));
toBeSaved = true;
}
Folder fso = (Folder) so;
if (fso.getTypeId() == null) fso.setTypeId("cmis:folder");
if (!folderId.equals(fso.getParentId())) {
LOG.warn("Fixing folder.parentId : setting " + folderId + " into cmis:parentId");
fso.setParentId(folderId);
toBeSaved = true;
}
} else {
so = new DocumentImpl();
File metadataFile = new File(child.getAbsolutePath()
+ SUFFIXE_METADATA);
so.setId(persistenceManager.generateId());
so.setRepositoryId(repositoryId);
so.setStore(store);

DocumentImpl meta = (DocumentImpl) persistenceManager
.readCMISFromDisk(metadataFile);

if (meta != null) {
// if metadata file exists then meta is the CMIS Object
so = meta;
} else {
LOG.warn("Missing metadata or malformed file for " + child.getAbsolutePath());
so.setProperties(new LinkedHashMap<String, PropertyData<?>>());
so.setName(child.getName());
so.setTypeId(BaseTypeId.CMIS_DOCUMENT.value());
toBeSaved = true;
}
DocumentImpl dso = (DocumentImpl) so;
if(dso.getTypeId() == null) dso.setTypeId("cmis:document");
if (dso.getParentIds() == null ||
dso.getParentIds().size() == 0 ||
!dso.getParentIds().contains(folderId)) {
LOG.warn("Fixing document.parentIds : setting " + folderId + " into cmis:parentIds");
/*
* This is a bug fixing patch, in order to fix the orphean documents by adding
* their current physical directory to the list of ParentIds.
* but we can't know if the orphean situation comes from a 'move' or a 'addToFolder' bug
* therefore it is impossible to state if we have to set the current directory
* to a unique parentId or just add the missing parentId
*
* By default it will be : set the current directory to a unique parentId
*
*/
dso.getParentIds().clear();
dso.getParentIds().add(0, folderId);
toBeSaved = true;
SimpleImmutableEntry<Boolean, StoredObject> unmarshalled = persistenceManager
.readCMISFromDisk(metadataFile, store);

if (unmarshalled != null) {
DocumentImpl meta = (DocumentImpl) unmarshalled.getValue();
toBeSaved = unmarshalled.getKey();
// if metadata file exists then meta is the CMIS Object
so = meta;
} else {
LOG.warn("Missing metadata or malformed file for " + child.getAbsolutePath());
so.setProperties(new LinkedHashMap<String, PropertyData<?>>());
so.setName(child.getName());
so.setTypeId(BaseTypeId.CMIS_DOCUMENT.value());
toBeSaved = true;
}
DocumentImpl dso = (DocumentImpl) so;
if(dso.getTypeId() == null) dso.setTypeId("cmis:document");
if (dso.getParentIds() == null ||
dso.getParentIds().size() == 0 ||
!dso.getParentIds().contains(folderId)) {
LOG.warn("Fixing document.parentIds : setting " + folderId + " into cmis:parentIds");
/*
* This is a bug fixing patch, in order to fix the orphan documents by adding
* their current physical directory to the list of ParentIds.
* but we can't know if the orphan situation comes from a 'move' or a 'addToFolder' bug
* therefore it is impossible to state if we have to set the current directory
* to a unique parentId or just add the missing parentId
*
* By default it will be : set the current directory to a unique parentId
*
*/
dso.getParentIds().clear();
dso.getParentIds().add(0, folderId);
toBeSaved = true;
}
}
// save metadata to disk
if (toBeSaved) {
persistenceManager.writeCMISToDisc(child, so);
}
// save object in memory
store.storeObject(so, false);
// invoke recursiveness if needed
if (child.isDirectory()) {
// recursive loading
loadFolder(repositoryId, store, child, so.getId(), filenameFilter, persistenceManager);
}
}
// read contentStream
((DocumentImpl) so).setContent(persistenceManager.readContent(child, true));
so.setRepositoryId(repositoryId);
so.setStore(store);
}
// save metadata to disk
if (toBeSaved) {
persistenceManager.writeCMISToDisc(child, so);
}
// save object in memory
store.storeObject(so, false);
// invoke recursiveness if needed
if (child.isDirectory()) {
// recursive loading
loadFolder(repositoryId, store, child, so.getId(), filenameFilter, persistenceManager);
}
});
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,11 @@
import java.io.IOException;
import java.io.InputStream;
import java.util.Map;
import java.util.AbstractMap.SimpleImmutableEntry;

import org.apache.chemistry.opencmis.commons.data.ContentStream;
import org.apache.chemistry.opencmis.inmemory.storedobj.api.Folder;
import org.apache.chemistry.opencmis.inmemory.storedobj.api.ObjectStore;
import org.apache.chemistry.opencmis.inmemory.storedobj.api.StoredObject;

public class InMemoryPersistence extends PersistenceManager {
Expand Down Expand Up @@ -71,7 +73,7 @@ public void writeCMISToDisc(File newFile, StoredObject so) {
}


public StoredObject readCMISFromDisk(File metadataFile) {
public SimpleImmutableEntry<Boolean, StoredObject> readCMISFromDisk(File metadataFile, ObjectStore store) {
// TODO Auto-generated method stub
return null;
}
Expand Down
Loading