Skip to content

Commit ace3cc9

Browse files
ganikganik
authored
Merge pull request microsoft#60 from ganik/ganik/dotnet2
Use dotnetcore2 package
2 parents bec566c + 062d55a commit ace3cc9

File tree

11 files changed

+217
-222
lines changed

11 files changed

+217
-222
lines changed

‎build.cmd

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -256,14 +256,20 @@ if %PythonVersion% == 3.6 (
256256
)
257257

258258
echo Placing binaries in libs dir for wheel packaging
259-
echo dummy > excludedfileslist.txt
260-
echo .exe >> excludedfileslist.txt
261-
if "%DebugBuild%" == "False" (
262-
echo .pdb >> excludedfileslist.txt
263-
echo .ipdb >> excludedfileslist.txt
259+
copy "%BuildOutputDir%%Configuration%\DotNetBridge.dll" "%__currentScriptDir%src\python\nimbusml\internal\libs\"
260+
copy "%BuildOutputDir%%Configuration%\pybridge.pyd" "%__currentScriptDir%src\python\nimbusml\internal\libs\"
261+
262+
if %PythonVersion% == 2.7 (
263+
copy "%BuildOutputDir%%Configuration%\Platform\win-x64\publish\*.dll" "%__currentScriptDir%src\python\nimbusml\internal\libs\"
264+
)
265+
else (
266+
for /F "tokens=*" %%A in (build/libs_win.txt) do copy "%BuildOutputDir%%Configuration%\Platform\win-x64\publish\%%A" "%__currentScriptDir%src\python\nimbusml\internal\libs\"
267+
)
268+
269+
if "%DebugBuild%" == "True" (
270+
copy "%BuildOutputDir%%Configuration%\DotNetBridge.pdb" "%__currentScriptDir%src\python\nimbusml\internal\libs\"
271+
copy "%BuildOutputDir%%Configuration%\pybridge.pdb" "%__currentScriptDir%src\python\nimbusml\internal\libs\"
264272
)
265-
xcopy /E /I /exclude:excludedfileslist.txt "%BuildOutputDir%%Configuration%" "%__currentScriptDir%src\python\nimbusml\internal\libs"
266-
del excludedfileslist.txt
267273

268274
call "%PythonExe%" -m pip install --upgrade "wheel>=0.31.0"
269275
cd "%__currentScriptDir%src\python"

‎build.sh

Lines changed: 29 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -171,14 +171,37 @@ then
171171
touch "${__currentScriptDir}/src/python/nimbusml/internal/libs/__init__.py"
172172

173173
echo "Placing binaries in libs dir for wheel packaging ... "
174-
mv "${BuildOutputDir}/${__configuration}"/Platform "${__currentScriptDir}/src/python/nimbusml/internal/libs/Platform"
175-
mv "${BuildOutputDir}/${__configuration}"/*.* "${__currentScriptDir}/src/python/nimbusml/internal/libs/"
176-
find "${__currentScriptDir}/src/python/nimbusml/internal/libs/" \( -name "dummy*" -o -name "*.exe" \) -print | xargs rm
177-
if [[ ! $__configuration = Dbg* ]]
174+
cp "${BuildOutputDir}/${__configuration}"/DotNetBridge.dll "${__currentScriptDir}/src/python/nimbusml/internal/libs/"
175+
cp "${BuildOutputDir}/${__configuration}"/pybridge.so "${__currentScriptDir}/src/python/nimbusml/internal/libs/"
176+
177+
if [ ${PythonVersion} = 2.7 ]
178178
then
179-
find "${__currentScriptDir}/src/python/nimbusml/internal/libs/" \( -name "*.pdb" -o -name "*.ipdb" \) -print | xargs rm
179+
cp "${BuildOutputDir}/${__configuration}/Platform/${PublishDir}"/publish/*.dll "${__currentScriptDir}/src/python/nimbusml/internal/libs/"
180+
cp "${BuildOutputDir}/${__configuration}/Platform/${PublishDir}"/publish/System.Native.a "${__currentScriptDir}/src/python/nimbusml/internal/libs/"
181+
cp "${BuildOutputDir}/${__configuration}/Platform/${PublishDir}"/publish/createdump "${__currentScriptDir}/src/python/nimbusml/internal/libs/" || :
182+
cp "${BuildOutputDir}/${__configuration}/Platform/${PublishDir}"/publish/sosdocsunix.txt "${__currentScriptDir}/src/python/nimbusml/internal/libs/"
183+
ext=*.so
184+
if [ "$(uname -s)" = "Darwin" ]
185+
then
186+
ext=*.dylib
187+
fi
188+
cp "${BuildOutputDir}/${__configuration}/Platform/${PublishDir}"/publish/${ext} "${__currentScriptDir}/src/python/nimbusml/internal/libs/"
189+
else
190+
libs_txt=libs_linux.txt
191+
if [ "$(uname -s)" = "Darwin" ]
192+
then
193+
libs_txt=libs_mac.txt
194+
fi
195+
cat build/${libs_txt} | while read i; do
196+
cp "${BuildOutputDir}/${__configuration}/Platform/${PublishDir}"/publish/$i "${__currentScriptDir}/src/python/nimbusml/internal/libs/"
197+
done
180198
fi
181-
199+
200+
if [[ $__configuration = Dbg* ]]
201+
then
202+
cp "${BuildOutputDir}/${__configuration}"/DotNetBridge.pdb "${__currentScriptDir}/src/python/nimbusml/internal/libs/"
203+
fi
204+
182205
"${PythonExe}" -m pip install --upgrade "wheel>=0.31.0"
183206
cd "${__currentScriptDir}/src/python"
184207

‎build/libs_linux.txt

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
Google.Protobuf.dll
2+
Newtonsoft.Json.dll
3+
libCpuMathNative.so
4+
libFactorizationMachineNative.so
5+
libFastTreeNative.so
6+
libLdaNative.so
7+
libMklImports.so
8+
libSymSgdNative.so
9+
lib_lightgbm.so
10+
libtensorflow.so
11+
libtensorflow_framework.so
12+
Microsoft.ML.*

‎build/libs_mac.txt

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
Google.Protobuf.dll
2+
Newtonsoft.Json.dll
3+
libCpuMathNative.dylib
4+
libFactorizationMachineNative.dylib
5+
libFastTreeNative.dylib
6+
libLdaNative.dylib
7+
libMklImports.dylib
8+
libSymSgdNative.dylib
9+
lib_lightgbm.dylib
10+
libtensorflow.dylib
11+
libtensorflow_framework.dylib
12+
Microsoft.ML.*

‎build/libs_win.txt

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
Google.Protobuf.dll
2+
Newtonsoft.Json.dll
3+
CpuMathNative.dll
4+
FactorizationMachineNative.dll
5+
FastTreeNative.dll
6+
LdaNative.dll
7+
lib_lightgbm.dll
8+
MklImports.dll
9+
SymSgdNative.dll
10+
tensorflow.dll
11+
Microsoft.ML.*

‎src/NativeBridge/UnixInterface.h

Lines changed: 11 additions & 70 deletions
Original file line numberDiff line numberDiff line change
@@ -12,22 +12,17 @@
1212
#include <unistd.h>
1313
#include <vector>
1414

15-
#define NATIVE_FOLDER "/Linux/"
16-
#define AUTOLOAD_FOLDER "/AutoLoad/"
17-
#define PLATFORM_FOLDER "/Platform/"
18-
#define PUBLISH_FOLDER "/publish/"
1915
#define NATIVE_BRIDGE_LIB "/pybridge.so"
2016
#ifdef __APPLE__
21-
#define CORECLR_LIB "libcoreclr.dylib"
17+
#define CORECLR_LIB "/libcoreclr.dylib"
2218
#else
23-
#define CORECLR_LIB "libcoreclr.so"
19+
#define CORECLR_LIB "/libcoreclr.so"
2420
#endif
2521

2622
#define CORECLR_INIT "coreclr_initialize"
2723
#define CORECLR_DELEGATE "coreclr_create_delegate"
2824
#define CORECLR_SHUTDOWN "coreclr_shutdown"
2925

30-
#define DOTNETBRIDGE_DLL "DotNetBridge.dll"
3126
#define DOTNETBRIDGE "DotNetBridge"
3227
#define DOTNETBRIDGE_FQDN "Microsoft.MachineLearning.DotNetBridge.Bridge"
3328

@@ -149,18 +144,15 @@ class UnixMlNetInterface
149144
{
150145
}
151146

152-
FNGETTER EnsureGetter(const char *path, const char *coreclrpath)
147+
FNGETTER EnsureGetter(const char *nimbuslibspath, const char *coreclrpath)
153148
{
154149
if (_getter != nullptr)
155150
return _getter;
156151

157-
std::string dir(path);
152+
std::string libsroot(nimbuslibspath);
158153
std::string coreclrdir(coreclrpath);
159154

160-
std::string dll(dir);
161-
dll.append(W(DOTNETBRIDGE_DLL));
162-
163-
ICLRRuntimeHost2* host = EnsureClrHost(dir.c_str(), coreclrdir.c_str());
155+
ICLRRuntimeHost2* host = EnsureClrHost(libsroot.c_str(), coreclrdir.c_str());
164156
if (host == nullptr)
165157
return nullptr;
166158

@@ -254,58 +246,19 @@ class UnixMlNetInterface
254246
closedir(dir);
255247
}
256248

257-
const char* GetDistribution()
258-
{
259-
#ifdef __APPLE__
260-
return "osx-x64";
261-
#else
262-
return "linux-x64";
263-
#endif
264-
}
265-
266-
ICLRRuntimeHost2* EnsureClrHost(const char * dirRoot, const char * coreclrDirRoot)
249+
ICLRRuntimeHost2* EnsureClrHost(const char * libsRoot, const char * coreclrDirRoot)
267250
{
268251
if (_host != nullptr)
269252
return _host;
270253

271-
// Set up paths.
272-
std::string dirNative(dirRoot);
273-
dirNative.append(NATIVE_FOLDER);
274-
275-
std::string dirClr(coreclrDirRoot);
276-
277-
const char* distribution = GetDistribution();
278-
if (distribution == nullptr)
279-
throw std::runtime_error("Found unsupported platform when looking for Core CLR libs. The supported Linux distributions include Redhat (CentOS) and Ubuntu.");
280-
281-
dirClr.append(PLATFORM_FOLDER);
282-
dirClr.append(distribution);
283-
dirClr.append(PUBLISH_FOLDER);
284-
285-
// REVIEW: now the assemblies in AutoLoad are added to the TPA list.
286-
// This is a workaround to circumvent this CoreCLR issue: https://github.com/dotnet/coreclr/issues/5837
287-
// This bug is fixed but not published yet. When a newer version of CoreCLR is available, we should
288-
// 1. Remove the assemblies in AutoLoad from TPA.
289-
// 2. Modify AppDomainProxy (in ML.NET Core) so that all assemblies are resolved using events.
290-
std::string dirAutoLoad(dirRoot);
291-
dirAutoLoad.append(AUTOLOAD_FOLDER);
292-
293-
std::string appPath(dirRoot);
294-
std::string appNiPath(dirRoot);
295-
appNiPath.append(":").append(dirClr);
296-
297-
std::string nativeDllSearchDirs(dirNative);
298-
nativeDllSearchDirs.append(":").append(appNiPath);
299-
300254
std::string tpaList;
301-
AddDllsToList(dirRoot, tpaList);
302-
AddDllsToList(dirClr.c_str(), tpaList);
303-
AddDllsToList(dirAutoLoad.c_str(), tpaList);
255+
AddDllsToList(libsRoot, tpaList);
256+
AddDllsToList(coreclrDirRoot, tpaList);
304257

305258
// Start the CoreCLR.
306-
HMODULE hmodCore = EnsureCoreClrModule(dirClr.c_str());
259+
HMODULE hmodCore = EnsureCoreClrModule(coreclrDirRoot);
307260

308-
ICLRRuntimeHost2 *host = new ICLRRuntimeHost2(hmodCore, dirRoot);
261+
ICLRRuntimeHost2 *host = new ICLRRuntimeHost2(hmodCore, libsRoot);
309262
HRESULT hr;
310263
// App domain flags are not used by UnixCoreConsole.
311264
DWORD appDomainFlags = 0;
@@ -322,28 +275,16 @@ class UnixMlNetInterface
322275
// APP_PATHS
323276
// - The list of paths which will be probed by the assembly loader
324277
//
325-
// APP_NI_PATHS
326-
// - The list of additional paths that the assembly loader will probe for ngen images
327-
//
328-
// NATIVE_DLL_SEARCH_DIRECTORIES
329-
// - The list of paths that will be probed for native DLLs called by PInvoke
330-
//
331278
const char *property_keys[] = {
332279
W("TRUSTED_PLATFORM_ASSEMBLIES"),
333280
W("APP_PATHS"),
334-
W("APP_NI_PATHS"),
335-
W("NATIVE_DLL_SEARCH_DIRECTORIES"),
336281
W("AppDomainCompatSwitch"),
337282
};
338283
const char *property_values[] = {
339284
// TRUSTED_PLATFORM_ASSEMBLIES
340285
tpaList.c_str(),
341286
// APP_PATHS
342-
appPath.c_str(),
343-
// APP_NI_PATHS
344-
appNiPath.c_str(),
345-
// NATIVE_DLL_SEARCH_DIRECTORIES
346-
nativeDllSearchDirs.c_str(),
287+
libsRoot,
347288
// AppDomainCompatSwitch
348289
W("UseLatestBehaviorWhenTFMNotSpecified")
349290
};

‎src/NativeBridge/WinInterface.h

Lines changed: 12 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -117,7 +117,7 @@ class WinMlNetInterface
117117
LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR
118118
// | LOAD_LIBRARY_SEARCH_SYSTEM32
119119
| LOAD_LIBRARY_SEARCH_DEFAULT_DIRS
120-
);
120+
);
121121
SetDllDirectoryW(nullptr);
122122
if (!hmodCore)
123123
{
@@ -161,34 +161,18 @@ class WinMlNetInterface
161161
FindClose(findHandle);
162162
}
163163

164-
ICLRRuntimeHost2* EnsureClrHost(const wchar_t * dirRoot, const wchar_t * coreclrDirRoot)
164+
ICLRRuntimeHost2* EnsureClrHost(const wchar_t * libsRoot, const wchar_t * coreclrDirRoot)
165165
{
166166
if (_host != nullptr)
167167
return _host;
168168

169169
// Set up paths.
170-
std::wstring dirNative(dirRoot);
171-
dirNative.append(L"Win\\");
172-
173-
std::wstring dirClr(coreclrDirRoot);
174-
dirClr.append(L"Platform\\win-x64\\publish\\");
175-
176-
std::wstring dirAutoLoad(dirRoot);
177-
dirAutoLoad.append(L"AutoLoad\\");
178-
179-
std::wstring appPath(dirRoot);
180-
std::wstring appNiPath(dirRoot);
181-
appNiPath.append(W(";")).append(dirClr);
182-
std::wstring nativeDllSearchDirs(dirNative);
183-
nativeDllSearchDirs.append(W(";")).append(appNiPath);
184-
185170
std::wstring tpaList;
186-
AddDllsToList(dirRoot, tpaList);
187-
AddDllsToList(dirClr.c_str(), tpaList);
188-
AddDllsToList(dirAutoLoad.c_str(), tpaList);
171+
AddDllsToList(libsRoot, tpaList);
172+
AddDllsToList(coreclrDirRoot, tpaList);
189173

190174
// Start the CoreCLR.
191-
HMODULE hmodCore = EnsureCoreClrModule(dirClr.c_str());
175+
HMODULE hmodCore = EnsureCoreClrModule(coreclrDirRoot);
192176

193177
FnGetCLRRuntimeHost pfnGetCLRRuntimeHost =
194178
(FnGetCLRRuntimeHost)::GetProcAddress(hmodCore, "GetCLRRuntimeHost");
@@ -235,28 +219,16 @@ class WinMlNetInterface
235219
// APP_PATHS
236220
// - The list of paths which will be probed by the assembly loader
237221
//
238-
// APP_NI_PATHS
239-
// - The list of additional paths that the assembly loader will probe for ngen images
240-
//
241-
// NATIVE_DLL_SEARCH_DIRECTORIES
242-
// - The list of paths that will be probed for native DLLs called by PInvoke
243-
//
244222
const wchar_t *property_keys[] = {
245223
W("TRUSTED_PLATFORM_ASSEMBLIES"),
246224
W("APP_PATHS"),
247-
W("APP_NI_PATHS"),
248-
W("NATIVE_DLL_SEARCH_DIRECTORIES"),
249225
W("AppDomainCompatSwitch"),
250226
};
251227
const wchar_t *property_values[] = {
252228
// TRUSTED_PLATFORM_ASSEMBLIES
253229
tpaList.c_str(),
254230
// APP_PATHS
255-
appPath.c_str(),
256-
// APP_NI_PATHS
257-
appNiPath.c_str(),
258-
// NATIVE_DLL_SEARCH_DIRECTORIES
259-
nativeDllSearchDirs.c_str(),
231+
libsRoot,
260232
// AppDomainCompatSwitch
261233
W("UseLatestBehaviorWhenTFMNotSpecified")
262234
};
@@ -295,13 +267,13 @@ class WinMlNetInterface
295267
}
296268

297269
public:
298-
FNGETTER EnsureGetter(const char *path, const char *coreclrpath)
270+
FNGETTER EnsureGetter(const char *nimbuslibspath, const char *coreclrpath)
299271
{
300272
if (_getter != nullptr)
301273
return _getter;
302274

303-
std::wstring dir = Utf8ToUtf16le(path);
304-
ConvertToWinPath(dir);
275+
std::wstring libsdir = Utf8ToUtf16le(nimbuslibspath);
276+
ConvertToWinPath(libsdir);
305277

306278
std::wstring coreclrdir;
307279
if (strlen(coreclrpath) != 0)
@@ -311,16 +283,16 @@ class WinMlNetInterface
311283
}
312284
else
313285
{
314-
coreclrdir = dir;
286+
coreclrdir = libsdir;
315287
}
316288

317-
ICLRRuntimeHost2* host = EnsureClrHost(dir.c_str(), coreclrdir.c_str());
289+
ICLRRuntimeHost2* host = EnsureClrHost(libsdir.c_str(), coreclrdir.c_str());
318290
if (host == nullptr)
319291
return nullptr;
320292

321293
// CoreCLR currently requires using environment variables to set most CLR flags.
322294
// cf. https://github.com/dotnet/coreclr/blob/master/Documentation/project-docs/clr-configuration-knobs.md
323-
if(_wputenv(W("COMPlus_gcAllowVeryLargeObjects=1")) == -1)
295+
if (_wputenv(W("COMPlus_gcAllowVeryLargeObjects=1")) == -1)
324296
return nullptr;
325297

326298
INT_PTR getter;

0 commit comments

Comments
 (0)