Materials: External Module Support

The final PR for the external module feature that allows materials to
be stored in an external datastore, webservice, etc.

This includes the final material manager classes, and the UI support in
the form of commands and preference pages.
This commit is contained in:
David Carter
2025-04-28 16:11:02 -04:00
committed by Chris Hennes
parent 1bf21f85bd
commit ffb7ab779b
23 changed files with 1336 additions and 16 deletions

View File

@@ -33,6 +33,10 @@
#include "MaterialManagerLocal.h"
#include "ModelManagerLocal.h"
#include "PropertyMaterial.h"
#if defined(BUILD_MATERIAL_EXTERNAL)
#include "ModelManagerExternal.h"
#include "MaterialManagerExternal.h"
#endif
#include "Array2DPy.h"
#include "Array3DPy.h"
@@ -107,6 +111,10 @@ PyMOD_INIT_FUNC(Materials)
Materials::MaterialManagerLocal ::init();
Materials::Model ::init();
Materials::ModelManager ::init();
#if defined(BUILD_MATERIAL_EXTERNAL)
Materials::MaterialManagerExternal ::init();
Materials::ModelManagerExternal ::init();
#endif
Materials::ModelManagerLocal ::init();
Materials::ModelUUIDs ::init();

View File

@@ -22,6 +22,13 @@ include_directories(
SYSTEM
${YAML_CPP_INCLUDE_DIR}
)
if (BUILD_MATERIAL_EXTERNAL)
include_directories(
${CMAKE_SOURCE_DIR}/src/3rdParty/lru-cache/include
)
endif(BUILD_MATERIAL_EXTERNAL)
link_directories(${YAML_CPP_LIBRARY_DIR})
if(BUILD_MATERIAL_EXTERNAL)
@@ -70,6 +77,11 @@ SET(MaterialsAPI_Files
MaterialAPI/MaterialManagerExternal.py
)
SET(MaterialsAPI_Files
MaterialAPI/__init__.py
MaterialAPI/MaterialManagerExternal.py
)
SET(Python_SRCS
Exceptions.h
Array2D.pyi
@@ -145,6 +157,8 @@ if(BUILD_MATERIAL_EXTERNAL)
list(APPEND Materials_SRCS
ExternalManager.cpp
ExternalManager.h
MaterialManagerExternal.cpp
MaterialManagerExternal.h
ModelManagerExternal.cpp
ModelManagerExternal.h
)

View File

@@ -188,7 +188,6 @@ ExternalManager::libraryFromObject(const Py::Object& entry)
if (!pyName.isNone()) {
libraryName = QString::fromStdString(pyName.as_string());
}
QString icon;
if (!pyIcon.isNone()) {
icon = QString::fromStdString(pyIcon.as_string());

View File

@@ -56,6 +56,11 @@ MaterialLibrary::MaterialLibrary(const QString& libraryName,
, _local(false)
{}
MaterialLibrary::MaterialLibrary(const Library& library)
: Library(library)
, _local(false)
{}
bool MaterialLibrary::isLocal() const
{
return _local;

View File

@@ -57,6 +57,7 @@ public:
const QString& dir,
const QString& icon,
bool readOnly = true);
MaterialLibrary(const Library& library);
MaterialLibrary(const MaterialLibrary&) = delete;
~MaterialLibrary() override = default;

View File

@@ -35,6 +35,9 @@
#include "MaterialConfigLoader.h"
#include "MaterialLoader.h"
#include "MaterialManager.h"
#if defined(BUILD_MATERIAL_EXTERNAL)
#include "MaterialManagerExternal.h"
#endif
#include "MaterialManagerLocal.h"
#include "ModelManager.h"
#include "ModelUuids.h"
@@ -49,14 +52,25 @@ using namespace Materials;
TYPESYSTEM_SOURCE(Materials::MaterialManager, Base::BaseClass)
QMutex MaterialManager::_mutex;
bool MaterialManager::_useExternal = false;
MaterialManager* MaterialManager::_manager = nullptr;
std::unique_ptr<MaterialManagerLocal> MaterialManager::_localManager;
#if defined(BUILD_MATERIAL_EXTERNAL)
std::unique_ptr<MaterialManagerExternal> MaterialManager::_externalManager;
#endif
MaterialManager::MaterialManager()
{}
{
_hGrp = App::GetApplication().GetParameterGroupByPath(
"User parameter:BaseApp/Preferences/Mod/Material/ExternalInterface");
_useExternal = _hGrp->GetBool("UseExternal", false);
_hGrp->Attach(this);
}
MaterialManager::~MaterialManager()
{}
{
_hGrp->Detach(this);
}
MaterialManager& MaterialManager::getManager()
{
@@ -77,6 +91,22 @@ void MaterialManager::initManagers()
if (!_localManager) {
_localManager = std::make_unique<MaterialManagerLocal>();
}
#if defined(BUILD_MATERIAL_EXTERNAL)
if (!_externalManager) {
_externalManager = std::make_unique<MaterialManagerExternal>();
}
#endif
}
void MaterialManager::OnChange(ParameterGrp::SubjectType& rCaller, ParameterGrp::MessageType Reason)
{
const ParameterGrp& rGrp = static_cast<ParameterGrp&>(rCaller);
if (strcmp(Reason, "UseExternal") == 0) {
Base::Console().Log("Use external changed\n");
_useExternal = rGrp.GetBool("UseExternal", false);
// _dbManager->refresh();
}
}
void MaterialManager::cleanup()
@@ -84,6 +114,11 @@ void MaterialManager::cleanup()
if (_localManager) {
_localManager->cleanup();
}
#if defined(BUILD_MATERIAL_EXTERNAL)
if (_externalManager) {
_externalManager->cleanup();
}
#endif
}
void MaterialManager::refresh()
@@ -183,22 +218,52 @@ QString MaterialManager::defaultMaterialUUID()
std::shared_ptr<std::list<std::shared_ptr<MaterialLibrary>>> MaterialManager::getLibraries()
{
auto libraries = std::make_shared<std::list<std::shared_ptr<MaterialLibrary>>>();
// External libraries take precedence over local libraries
auto libMap = std::map<QString, std::shared_ptr<MaterialLibrary>>();
#if defined(BUILD_MATERIAL_EXTERNAL)
if (_useExternal) {
auto remoteLibraries = _externalManager->getLibraries();
for (auto& remote : *remoteLibraries) {
libMap.try_emplace(remote->getName(), remote);
}
}
#endif
auto localLibraries = _localManager->getLibraries();
for (auto& local : *localLibraries) {
libraries->push_back(local);
libMap.try_emplace(local->getName(), local);
}
// Consolidate into a single list
auto libraries = std::make_shared<std::list<std::shared_ptr<MaterialLibrary>>>();
for (auto libEntry : libMap) {
libraries->push_back(libEntry.second);
}
return libraries;
}
std::shared_ptr<std::list<std::shared_ptr<MaterialLibrary>>> MaterialManager::getLocalLibraries()
std::shared_ptr<std::list<std::shared_ptr<MaterialLibrary>>>
MaterialManager::getLocalLibraries()
{
return _localManager->getLibraries();
}
std::shared_ptr<MaterialLibrary> MaterialManager::getLibrary(const QString& name) const
{
#if defined(BUILD_MATERIAL_EXTERNAL)
if (_useExternal) {
try
{
auto lib = _externalManager->getLibrary(name);
if (lib) {
return lib;
}
}
catch (const LibraryNotFound& e) {
}
}
#endif
// We really want to return the local library if not found, such as for User folder models
return _localManager->getLibrary(name);
}
@@ -233,6 +298,18 @@ void MaterialManager::removeLibrary(const QString& libraryName)
std::shared_ptr<std::vector<std::tuple<QString, QString, QString>>>
MaterialManager::libraryMaterials(const QString& libraryName)
{
#if defined(BUILD_MATERIAL_EXTERNAL)
if (_useExternal) {
try {
auto materials = _externalManager->libraryMaterials(libraryName);
if (materials) {
return materials;
}
}
catch (const LibraryNotFound& e) {
}
}
#endif
return _localManager->libraryMaterials(libraryName);
}
@@ -241,13 +318,42 @@ MaterialManager::libraryMaterials(const QString& libraryName,
const std::shared_ptr<MaterialFilter>& filter,
const MaterialFilterOptions& options)
{
#if defined(BUILD_MATERIAL_EXTERNAL)
if (_useExternal) {
try {
auto materials = _externalManager->libraryMaterials(libraryName, filter, options);
if (materials) {
return materials;
}
}
catch (const LibraryNotFound& e) {
}
}
#endif
return _localManager->libraryMaterials(libraryName, filter, options);
}
#if defined(BUILD_MATERIAL_EXTERNAL)
bool MaterialManager::isLocalLibrary(const QString& libraryName)
{
if (_useExternal) {
try {
auto lib = _externalManager->getLibrary(libraryName);
if (lib) {
return false;
}
}
catch (const LibraryNotFound& e) {
}
}
return true;
}
#else
bool MaterialManager::isLocalLibrary(const QString& /*libraryName*/)
{
return true;
}
#endif
//=====
//
@@ -346,6 +452,15 @@ MaterialManager::getLocalMaterials() const
std::shared_ptr<Material> MaterialManager::getMaterial(const QString& uuid) const
{
#if defined(BUILD_MATERIAL_EXTERNAL)
if (_useExternal) {
auto material = _externalManager->getMaterial(uuid);
if (material) {
return material;
}
}
#endif
// We really want to return the local material if not found, such as for User folder models
return _localManager->getMaterial(uuid);
}
@@ -443,3 +558,55 @@ void MaterialManager::dereference(std::shared_ptr<Material> material) const
{
_localManager->dereference(material);
}
#if defined(BUILD_MATERIAL_EXTERNAL)
void MaterialManager::migrateToExternal(const std::shared_ptr<Materials::MaterialLibrary>& library)
{
_externalManager->createLibrary(library->getName(),
library->getIconPath(),
library->isReadOnly());
auto materials = _localManager->libraryMaterials(library->getName());
for (auto& tuple : *materials) {
auto uuid = std::get<0>(tuple);
auto path = std::get<1>(tuple);
auto name = std::get<2>(tuple);
Base::Console().Log("\t('%s', '%s', '%s')\n",
uuid.toStdString().c_str(),
path.toStdString().c_str(),
name.toStdString().c_str());
auto material = _localManager->getMaterial(uuid);
if (!material->isOldFormat()) {
_externalManager->migrateMaterial(library->getName(), path, material);
}
}
}
void MaterialManager::validateMigration(const std::shared_ptr<Materials::MaterialLibrary>& library)
{
auto materials = _localManager->libraryMaterials(library->getName());
for (auto& tuple : *materials) {
auto uuid = std::get<0>(tuple);
auto path = std::get<1>(tuple);
auto name = std::get<2>(tuple);
Base::Console().Log("\t('%s', '%s', '%s')\n",
uuid.toStdString().c_str(),
path.toStdString().c_str(),
name.toStdString().c_str());
auto material = _localManager->getMaterial(uuid);
if (!material->isOldFormat()) {
auto externalMaterial = _externalManager->getMaterial(uuid);
material->validate(externalMaterial);
}
}
}
// Cache stats
double MaterialManager::materialHitRate()
{
initManagers();
return _externalManager->materialHitRate();
}
#endif

View File

@@ -51,7 +51,7 @@ class MaterialManagerLocal;
class MaterialFilter;
class MaterialFilterOptions;
class MaterialsExport MaterialManager: public Base::BaseClass
class MaterialsExport MaterialManager: public Base::BaseClass, ParameterGrp::ObserverType
{
TYPESYSTEM_HEADER_WITH_OVERRIDE();
@@ -136,14 +136,31 @@ public:
void dereference(std::shared_ptr<Material> material) const;
void dereference() const;
/// Observer message from the ParameterGrp
void OnChange(ParameterGrp::SubjectType& rCaller, ParameterGrp::MessageType Reason) override;
#if defined(BUILD_MATERIAL_EXTERNAL)
void migrateToExternal(const std::shared_ptr<Materials::MaterialLibrary>& library);
void validateMigration(const std::shared_ptr<Materials::MaterialLibrary>& library);
// Cache functions
static double materialHitRate();
#endif
private:
MaterialManager();
static void initManagers();
static MaterialManager* _manager;
#if defined(BUILD_MATERIAL_EXTERNAL)
static std::unique_ptr<MaterialManagerExternal> _externalManager;
#endif
static std::unique_ptr<MaterialManagerLocal> _localManager;
static QMutex _mutex;
static bool _useExternal;
ParameterGrp::handle _hGrp;
};
} // namespace Materials

View File

@@ -0,0 +1,210 @@
/***************************************************************************
* Copyright (c) 2023 David Carter <dcarter@david.carter.ca> *
* *
* This file is part of FreeCAD. *
* *
* FreeCAD is free software: you can redistribute it and/or modify it *
* under the terms of the GNU Lesser General Public License as *
* published by the Free Software Foundation, either version 2.1 of the *
* License, or (at your option) any later version. *
* *
* FreeCAD is distributed in the hope that it will be useful, but *
* WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU *
* Lesser General Public License for more details. *
* *
* You should have received a copy of the GNU Lesser General Public *
* License along with FreeCAD. If not, see *
* <https://www.gnu.org/licenses/>. *
* *
**************************************************************************/
#include "PreCompiled.h"
#ifndef _PreComp_
#endif
#include <QMutexLocker>
#include <App/Application.h>
#include "Exceptions.h"
#include "ExternalManager.h"
#include "MaterialLibrary.h"
#include "MaterialManagerExternal.h"
using namespace Materials;
/* TRANSLATOR Material::Materials */
QMutex MaterialManagerExternal::_mutex;
LRU::Cache<std::string, std::shared_ptr<Material>>
MaterialManagerExternal::_cache(DEFAULT_CACHE_SIZE);
TYPESYSTEM_SOURCE(Materials::MaterialManagerExternal, Base::BaseClass)
MaterialManagerExternal::MaterialManagerExternal()
{
initCache();
}
void MaterialManagerExternal::initCache()
{
QMutexLocker locker(&_mutex);
ParameterGrp::handle hGrp = App::GetApplication().GetParameterGroupByPath(
"User parameter:BaseApp/Preferences/Mod/Material/ExternalInterface");
auto cacheSize = hGrp->GetInt("MaterialCacheSize", DEFAULT_CACHE_SIZE);
_cache.capacity(cacheSize);
_cache.monitor();
}
void MaterialManagerExternal::cleanup()
{}
void MaterialManagerExternal::refresh()
{
resetCache();
}
//=====
//
// Library management
//
//=====
std::shared_ptr<std::list<std::shared_ptr<MaterialLibrary>>> MaterialManagerExternal::getLibraries()
{
auto libraryList = std::make_shared<std::list<std::shared_ptr<MaterialLibrary>>>();
try {
auto externalLibraries = ExternalManager::getManager()->libraries();
for (auto& entry : *externalLibraries) {
auto library = std::make_shared<MaterialLibrary>(*entry);
libraryList->push_back(library);
}
}
catch (const LibraryNotFound& e) {
}
catch (const ConnectionError& e) {
}
return libraryList;
}
std::shared_ptr<std::list<std::shared_ptr<MaterialLibrary>>>
MaterialManagerExternal::getMaterialLibraries()
{
auto libraryList = std::make_shared<std::list<std::shared_ptr<MaterialLibrary>>>();
try {
auto externalLibraries = ExternalManager::getManager()->materialLibraries();
for (auto& entry : *externalLibraries) {
auto library = std::make_shared<MaterialLibrary>(*entry);
libraryList->push_back(library);
}
}
catch (const LibraryNotFound& e) {
}
catch (const ConnectionError& e) {
}
return libraryList;
}
std::shared_ptr<MaterialLibrary> MaterialManagerExternal::getLibrary(const QString& name) const
{
try {
auto lib = ExternalManager::getManager()->getLibrary(name);
auto library = std::make_shared<MaterialLibrary>(*lib);
return library;
}
catch (const LibraryNotFound& e) {
throw LibraryNotFound(e);
}
catch (const ConnectionError& e) {
throw LibraryNotFound(e.what());
}
}
void MaterialManagerExternal::createLibrary(const QString& libraryName,
const QString& icon,
bool readOnly)
{
ExternalManager::getManager()->createLibrary(libraryName, icon, readOnly);
}
std::shared_ptr<std::vector<std::tuple<QString, QString, QString>>>
MaterialManagerExternal::libraryMaterials(const QString& libraryName)
{
return ExternalManager::getManager()->libraryMaterials(libraryName);
}
std::shared_ptr<std::vector<std::tuple<QString, QString, QString>>>
MaterialManagerExternal::libraryMaterials(const QString& libraryName,
const std::shared_ptr<MaterialFilter>& filter,
const MaterialFilterOptions& options)
{
return ExternalManager::getManager()->libraryMaterials(libraryName, filter, options);
}
//=====
//
// Material management
//
//=====
std::shared_ptr<Material> MaterialManagerExternal::getMaterial(const QString& uuid) const
{
if (_cache.contains(uuid.toStdString())) {
return _cache.lookup(uuid.toStdString());
}
try {
auto material = ExternalManager::getManager()->getMaterial(uuid);
_cache.emplace(uuid.toStdString(), material);
return material;
}
catch (const MaterialNotFound& e) {
_cache.emplace(uuid.toStdString(), nullptr);
return nullptr;
}
catch (const ConnectionError& e) {
_cache.emplace(uuid.toStdString(), nullptr);
return nullptr;
}
}
void MaterialManagerExternal::addMaterial(const QString& libraryName,
const QString& path,
const std::shared_ptr<Material>& material)
{
_cache.erase(material->getUUID().toStdString());
ExternalManager::getManager()->addMaterial(libraryName, path, material);
}
void MaterialManagerExternal::migrateMaterial(const QString& libraryName,
const QString& path,
const std::shared_ptr<Material>& material)
{
_cache.erase(material->getUUID().toStdString());
ExternalManager::getManager()->migrateMaterial(libraryName, path, material);
}
//=====
//
// Cache management
//
//=====
void MaterialManagerExternal::resetCache()
{
_cache.clear();
}
double MaterialManagerExternal::materialHitRate()
{
auto hitRate = _cache.stats().hit_rate();
if (std::isnan(hitRate)) {
return 0;
}
return hitRate;
}

View File

@@ -0,0 +1,102 @@
/***************************************************************************
* Copyright (c) 2024 David Carter <dcarter@david.carter.ca> *
* *
* This file is part of FreeCAD. *
* *
* FreeCAD is free software: you can redistribute it and/or modify it *
* under the terms of the GNU Lesser General Public License as *
* published by the Free Software Foundation, either version 2.1 of the *
* License, or (at your option) any later version. *
* *
* FreeCAD is distributed in the hope that it will be useful, but *
* WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU *
* Lesser General Public License for more details. *
* *
* You should have received a copy of the GNU Lesser General Public *
* License along with FreeCAD. If not, see *
* <https://www.gnu.org/licenses/>. *
* *
**************************************************************************/
#ifndef MATERIAL_MATERIALMANAGEREXTERNAl_H
#define MATERIAL_MATERIALMANAGEREXTERNAl_H
#include <memory>
#include <lru/lru.hpp>
#include <Mod/Material/MaterialGlobal.h>
#include <QMutex>
#include "FolderTree.h"
#include "Materials.h"
class QMutex;
namespace App
{
class Material;
}
namespace Materials
{
class MaterialLibrary;
class MaterialLibraryExternal;
class MaterialFilter;
class MaterialFilterOptions;
class MaterialsExport MaterialManagerExternal: public Base::BaseClass
{
TYPESYSTEM_HEADER_WITH_OVERRIDE();
public:
MaterialManagerExternal();
~MaterialManagerExternal() override = default;
static void cleanup();
void refresh();
static const int DEFAULT_CACHE_SIZE = 100;
// Library management
std::shared_ptr<std::list<std::shared_ptr<MaterialLibrary>>> getLibraries();
std::shared_ptr<std::list<std::shared_ptr<MaterialLibrary>>> getMaterialLibraries();
std::shared_ptr<MaterialLibrary> getLibrary(const QString& name) const;
void createLibrary(const QString& libraryName, const QString& icon, bool readOnly = true);
std::shared_ptr<std::vector<std::tuple<QString, QString, QString>>>
libraryMaterials(const QString& libraryName);
std::shared_ptr<std::vector<std::tuple<QString, QString, QString>>>
libraryMaterials(const QString& libraryName,
const std::shared_ptr<MaterialFilter>& filter,
const MaterialFilterOptions& options);
// Folder management
// Material management
std::shared_ptr<Material> getMaterial(const QString& uuid) const;
void addMaterial(const QString& libraryName,
const QString& path,
const std::shared_ptr<Material>& material);
void migrateMaterial(const QString& libraryName,
const QString& path,
const std::shared_ptr<Material>& material);
// Cache functions
void resetCache();
double materialHitRate();
private:
static void initCache();
static QMutex _mutex;
// Older platforms (Ubuntu 20.04) can't use QString as the index
// due to a lack of a move constructor
static LRU::Cache<std::string, std::shared_ptr<Material>> _cache;
};
} // namespace Materials
#endif // MATERIAL_MATERIALMANAGEREXTERNAl_H

View File

@@ -551,6 +551,7 @@ PyObject* MaterialPy::setAppearanceValue(PyObject* args)
Py_INCREF(Py_None);
return Py_None;
}
PyObject* MaterialPy::setValue(PyObject* args)
{
char* name;
@@ -595,7 +596,8 @@ PyObject* MaterialPy::setValue(PyObject* args)
Py_Return;
}
PyErr_SetString(PyExc_TypeError, "Either a string, a list, or an array are expected");
PyErr_SetString(PyExc_TypeError,
"Either a string, a list, or an array are expected");
return nullptr;
}
@@ -669,4 +671,4 @@ PyObject* MaterialPy::mapping_subscript(PyObject* self, PyObject* key)
{
Py::Dict dict = static_cast<MaterialPy*>(self)->getProperties();
return Py::new_reference_to(dict.getItem(Py::Object(key)));
}
}

View File

@@ -112,7 +112,7 @@ public:
// The precision is based on the value from the original materials editor
static const int PRECISION = 6;
void validate(const MaterialValue& other) const;
protected:

View File

@@ -46,7 +46,7 @@ class MaterialsExport ModelLibrary: public Library,
public:
ModelLibrary();
ModelLibrary(const Library& other);
ModelLibrary(const Library& library);
ModelLibrary(const QString& libraryName,
const QString& dir,
const QString& icon,

View File

@@ -121,7 +121,7 @@ void ModelManagerLocal::createLibrary(const QString& libraryName,
void ModelManagerLocal::renameLibrary(const QString& libraryName, const QString& newName)
{
for (auto& library : *_libraryList) {
if (library->getName() == libraryName) {
if (library->isName(libraryName)) {
library->setName(newName);
return;
}
@@ -133,7 +133,7 @@ void ModelManagerLocal::renameLibrary(const QString& libraryName, const QString&
void ModelManagerLocal::changeIcon(const QString& libraryName, const QString& icon)
{
for (auto& library : *_libraryList) {
if (library->getName() == libraryName) {
if (library->isName(libraryName)) {
library->setIconPath(icon);
return;
}
@@ -145,7 +145,7 @@ void ModelManagerLocal::changeIcon(const QString& libraryName, const QString& ic
void ModelManagerLocal::removeLibrary(const QString& libraryName)
{
for (auto& library : *_libraryList) {
if (library->getName() == libraryName) {
if (library->isName(libraryName)) {
_libraryList->remove(library);
// At this point we should rebuild the model map
@@ -163,7 +163,7 @@ ModelManagerLocal::libraryModels(const QString& libraryName)
for (auto& it : *_modelMap) {
// This is needed to resolve cyclic dependencies
if (it.second->getLibrary()->getName() == libraryName) {
if (it.second->getLibrary()->isName(libraryName)) {
models->push_back(
std::tuple<QString, QString, QString>(it.first, it.second->getDirectory(), it.second->getName()));
}
@@ -217,7 +217,7 @@ std::shared_ptr<Model> ModelManagerLocal::getModelByPath(const QString& path,
std::shared_ptr<ModelLibrary> ModelManagerLocal::getLibrary(const QString& name) const
{
for (auto& library : *_libraryList) {
if (library->getName() == name) {
if (library->isName(name)) {
return library;
}
}