Skip to content

Commit e56d2be

Browse files
kreebenkreeben
committed
proofed for 4.6.1 (cli is back)
1 parent 8dd4349 commit e56d2be

17 files changed

+412
-14
lines changed

‎README.md

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,10 @@
22

33
Resin is a vector space model implementation, a modern type information retrieval system and a document based search engine library with fast fuzzy and prefix querying and with customizable tokenizers and scoring (tf-idf included, other schemes supported).
44

5+
## .net version
6+
7+
4.6.1
8+
59
## Documentation
610

711
### A document.

‎Resin.sln

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution
1515
todo.txt = todo.txt
1616
EndProjectSection
1717
EndProject
18+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Cli", "src\Resin.Cli\Cli.csproj", "{61294CF9-8B5F-44F1-8A65-743728A04AE4}"
19+
EndProject
1820
Global
1921
GlobalSection(SolutionConfigurationPlatforms) = preSolution
2022
Debug|Any CPU = Debug|Any CPU
@@ -29,6 +31,10 @@ Global
2931
{FF84114A-C58C-4E65-B25F-DB49AB98A4F7}.Debug|Any CPU.Build.0 = Debug|Any CPU
3032
{FF84114A-C58C-4E65-B25F-DB49AB98A4F7}.Release|Any CPU.ActiveCfg = Release|Any CPU
3133
{FF84114A-C58C-4E65-B25F-DB49AB98A4F7}.Release|Any CPU.Build.0 = Release|Any CPU
34+
{61294CF9-8B5F-44F1-8A65-743728A04AE4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
35+
{61294CF9-8B5F-44F1-8A65-743728A04AE4}.Debug|Any CPU.Build.0 = Debug|Any CPU
36+
{61294CF9-8B5F-44F1-8A65-743728A04AE4}.Release|Any CPU.ActiveCfg = Release|Any CPU
37+
{61294CF9-8B5F-44F1-8A65-743728A04AE4}.Release|Any CPU.Build.0 = Release|Any CPU
3238
EndGlobalSection
3339
GlobalSection(SolutionProperties) = preSolution
3440
HideSolutionNode = FALSE

‎rn-write.bat

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
call rn write --file c:\temp\0wikipedia.json --dir d:\resin\wikipedia --skip 0 --take 2000000
2+
rem call rn write --file c:\temp\0wikipedia.json --dir d:\resin\wikipedia --skip 1000000 --take 1000000
3+
rem call rn write --file c:\temp\0wikipedia.json --dir d:\resin\wikipedia --skip 0 --take 2000000
4+
rem call rn write --file c:\temp\1wikipedia.json --dir d:\resin\wikipedia --skip 0 --take 2000000
5+
rem call rn write --file c:\temp\2wikipedia.json --dir d:\resin\wikipedia --skip 0 --take 2000000
6+
rem call rn write --file c:\temp\3wikipedia.json --dir d:\resin\wikipedia --skip 0 --take 2000000

‎rn.bat

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
src\Resin.Cli\bin\Release\rn.exe %*

‎rnw-write.bat

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
rem call rnw.bat d:\wikipedia\latest-all.json c:\temp\0wikipedia.json 0 2000000
2+
call rnw.bat d:\wikipedia\latest-all.json c:\temp\1wikipedia.json 2000000 4000000
3+
call rnw.bat d:\wikipedia\latest-all.json c:\temp\2wikipedia.json 4000000 8000000
4+
call rnw.bat d:\wikipedia\latest-all.json c:\temp\3wikipedia.json 8000000 10000000

‎rnw.bat

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
src\Resin.WikipediaJsonParser\bin\Release\rnw.exe %*

‎src/Resin.Cli/App.config

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
<?xml version="1.0" encoding="utf-8"?>
2+
<configuration>
3+
<configSections>
4+
<section name="log4net" type="log4net.Config.Log4NetConfigurationSectionHandler, log4net" />
5+
</configSections>
6+
<appSettings>
7+
<add key="sir.endpoint" value="http:\\localhost:1234/" />
8+
</appSettings>
9+
<startup>
10+
<supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.6.1" />
11+
</startup>
12+
<log4net>
13+
<appender name="Console" type="log4net.Appender.ConsoleAppender">
14+
<layout type="log4net.Layout.PatternLayout">
15+
<conversionPattern value="%date %-5level: %message%newline" />
16+
</layout>
17+
</appender>
18+
<appender name="RollingFileAppender" type="log4net.Appender.RollingFileAppender">
19+
<file value="resin-cli.log" />
20+
<appendToFile value="true" />
21+
<rollingStyle value="Size" />
22+
<maxSizeRollBackups value="10" />
23+
<maximumFileSize value="50MB" />
24+
<staticLogFileName value="true" />
25+
<layout type="log4net.Layout.PatternLayout">
26+
<conversionPattern value="%date %-5level[%logger]: %message%newline" />
27+
</layout>
28+
</appender>
29+
<appender name="TermFileAppender" type="log4net.Appender.RollingFileAppender">
30+
<file value="terms.log" />
31+
<appendToFile value="true" />
32+
<rollingStyle value="Size" />
33+
<maxSizeRollBackups value="10" />
34+
<maximumFileSize value="10000MB" />
35+
<staticLogFileName value="true" />
36+
<layout type="log4net.Layout.PatternLayout">
37+
<conversionPattern value="%message%newline" />
38+
</layout>
39+
</appender>
40+
<root>
41+
<level value="DEBUG" />
42+
<appender-ref ref="Console" />
43+
<appender-ref ref="RollingFileAppender" />
44+
</root>
45+
</log4net>
46+
<runtime>
47+
<assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
48+
<dependentAssembly>
49+
<assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
50+
<bindingRedirect oldVersion="0.0.0.0-4.5.0.0" newVersion="4.5.0.0" />
51+
</dependentAssembly>
52+
</assemblyBinding>
53+
</runtime>
54+
</configuration>

‎src/Resin.Cli/Cli.csproj

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
<?xml version="1.0" encoding="utf-8"?>
2+
<Project ToolsVersion="12.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
3+
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
4+
<PropertyGroup>
5+
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
6+
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
7+
<ProjectGuid>{61294CF9-8B5F-44F1-8A65-743728A04AE4}</ProjectGuid>
8+
<OutputType>Exe</OutputType>
9+
<AppDesignerFolder>Properties</AppDesignerFolder>
10+
<RootNamespace>Resin.Cli</RootNamespace>
11+
<AssemblyName>rn</AssemblyName>
12+
<TargetFrameworkVersion>v4.6.1</TargetFrameworkVersion>
13+
<FileAlignment>512</FileAlignment>
14+
<TargetFrameworkProfile />
15+
<SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\</SolutionDir>
16+
<RestorePackages>true</RestorePackages>
17+
</PropertyGroup>
18+
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
19+
<PlatformTarget>AnyCPU</PlatformTarget>
20+
<DebugSymbols>true</DebugSymbols>
21+
<DebugType>full</DebugType>
22+
<Optimize>false</Optimize>
23+
<OutputPath>bin\Debug\</OutputPath>
24+
<DefineConstants>DEBUG;TRACE</DefineConstants>
25+
<ErrorReport>prompt</ErrorReport>
26+
<WarningLevel>4</WarningLevel>
27+
<Prefer32Bit>false</Prefer32Bit>
28+
</PropertyGroup>
29+
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
30+
<PlatformTarget>AnyCPU</PlatformTarget>
31+
<DebugType>pdbonly</DebugType>
32+
<Optimize>true</Optimize>
33+
<OutputPath>bin\Release\</OutputPath>
34+
<DefineConstants>TRACE</DefineConstants>
35+
<ErrorReport>prompt</ErrorReport>
36+
<WarningLevel>4</WarningLevel>
37+
<Prefer32Bit>false</Prefer32Bit>
38+
</PropertyGroup>
39+
<PropertyGroup>
40+
<StartupObject>Resin.Cli.Program</StartupObject>
41+
</PropertyGroup>
42+
<ItemGroup>
43+
<Reference Include="log4net, Version=2.0.8.0, Culture=neutral, PublicKeyToken=669e0ddf0bb1aa2a, processorArchitecture=MSIL">
44+
<HintPath>..\..\packages\log4net.2.0.8\lib\net45-full\log4net.dll</HintPath>
45+
<Private>True</Private>
46+
</Reference>
47+
<Reference Include="Newtonsoft.Json, Version=4.5.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
48+
<SpecificVersion>False</SpecificVersion>
49+
<HintPath>..\..\lib\resin\Newtonsoft.Json.dll</HintPath>
50+
</Reference>
51+
<Reference Include="Sir.Client">
52+
<HintPath>..\..\lib\sir.client\Sir.Client.dll</HintPath>
53+
</Reference>
54+
<Reference Include="System" />
55+
<Reference Include="System.Configuration" />
56+
<Reference Include="System.Core" />
57+
<Reference Include="Microsoft.CSharp" />
58+
<Reference Include="System.Xml" />
59+
</ItemGroup>
60+
<ItemGroup>
61+
<Compile Include="Program.cs" />
62+
<Compile Include="Properties\AssemblyInfo.cs" />
63+
</ItemGroup>
64+
<ItemGroup>
65+
<None Include="App.config" />
66+
<None Include="packages.config" />
67+
</ItemGroup>
68+
<ItemGroup>
69+
<ProjectReference Include="..\Resin\Resin.csproj">
70+
<Project>{2be53bd7-0454-4289-af1b-e1781c983ec3}</Project>
71+
<Name>Resin</Name>
72+
</ProjectReference>
73+
</ItemGroup>
74+
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
75+
<Import Project="$(SolutionDir)\.nuget\NuGet.targets" Condition="Exists('$(SolutionDir)\.nuget\NuGet.targets')" />
76+
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
77+
<PropertyGroup>
78+
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
79+
</PropertyGroup>
80+
<Error Condition="!Exists('$(SolutionDir)\.nuget\NuGet.targets')" Text="$([System.String]::Format('$(ErrorText)', '$(SolutionDir)\.nuget\NuGet.targets'))" />
81+
</Target>
82+
<PropertyGroup>
83+
<PostBuildEvent>xcopy "$(TargetDir)*.*" "$(SolutionDir)bin" /y</PostBuildEvent>
84+
</PropertyGroup>
85+
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
86+
Other similar extension points exist, see Microsoft.Common.targets.
87+
<Target Name="BeforeBuild">
88+
</Target>
89+
<Target Name="AfterBuild">
90+
</Target>
91+
-->
92+
</Project>

‎src/Resin.Cli/Program.cs

Lines changed: 189 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,189 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.Configuration;
4+
using System.Diagnostics;
5+
using System.Globalization;
6+
using System.IO;
7+
using System.Linq;
8+
using log4net.Config;
9+
using Resin.Analysis;
10+
using Resin.Querying;
11+
using Sir.Client;
12+
13+
//TODO: move to own repo.
14+
namespace Resin.Cli
15+
{
16+
class Program
17+
{
18+
//inproc:
19+
// query --dir D:\resin\wikipedia -q "label:porn~" -p 0 -s 10
20+
// write --file c:\temp\0wikipedia.json --dir d:\resin\wikipedia --skip 0 --take 10000
21+
// delete --ids "Q1476435" --dir d:\resin\wikipedia
22+
static void Main(string[] args)
23+
{
24+
XmlConfigurator.Configure();
25+
26+
if (args[0].ToLower() == "write")
27+
{
28+
if (Array.IndexOf(args, "--file") == -1)
29+
{
30+
Console.WriteLine("I need a file.");
31+
return;
32+
}
33+
Write(args);
34+
}
35+
else if (args[0].ToLower() == "query")
36+
{
37+
if (Array.IndexOf(args, "-q") == -1)
38+
{
39+
Console.WriteLine("I need a query.");
40+
return;
41+
}
42+
Query(args);
43+
}
44+
else
45+
{
46+
Console.WriteLine("usage:");
47+
Console.WriteLine("rn.exe write --file source.json --dir c:\\target_dir");
48+
Console.WriteLine("rn.exe query --dir c:\\my_index -q field:value");
49+
}
50+
}
51+
52+
static void Query(string[] args)
53+
{
54+
string dir = null;
55+
string indexName = null;
56+
57+
if (Array.IndexOf(args, "--dir") > 0) dir = args[Array.IndexOf(args, "--dir") + 1];
58+
if (Array.IndexOf(args, "--name") > 0) indexName = args[Array.IndexOf(args, "--name") + 1];
59+
60+
var inproc = !string.IsNullOrWhiteSpace(dir);
61+
62+
var q = args[Array.IndexOf(args, "-q") + 1];
63+
var page = 0;
64+
var size = 10;
65+
var url = ConfigurationManager.AppSettings.Get("sir.endpoint");
66+
Result result;
67+
68+
if (Array.IndexOf(args, "-p") > 0) page = int.Parse(args[Array.IndexOf(args, "-p") + 1]);
69+
if (Array.IndexOf(args, "-s") > 0) size = int.Parse(args[Array.IndexOf(args, "-s") + 1]);
70+
if (Array.IndexOf(args, "--url") > 0) url = args[Array.IndexOf(args, "--url") + 1];
71+
72+
73+
74+
if (inproc)
75+
{
76+
var timer = new Stopwatch();
77+
timer.Start();
78+
using (var s = new Searcher(dir, new QueryParser(new Analyzer()), new Tfidf()))
79+
{
80+
result = s.Search(q, page, size);
81+
82+
Console.WriteLine(timer.Elapsed);
83+
84+
timer.Stop();
85+
86+
var docs = result.Docs;
87+
88+
PrintHeaders();
89+
90+
foreach (var doc in docs)
91+
{
92+
Print(doc);
93+
}
94+
95+
Console.WriteLine("\r\n{0} results of {1} in {2}", (page + 1) * size, result.Total, timer.Elapsed);
96+
}
97+
}
98+
else
99+
{
100+
var timer = new Stopwatch();
101+
timer.Start();
102+
using (var s = new SearchClient(indexName, url))
103+
{
104+
result = s.Search(q, page, size);
105+
var docs = result.Docs.ToList();
106+
107+
timer.Stop();
108+
109+
PrintHeaders();
110+
111+
foreach (var doc in docs)
112+
{
113+
Print(doc);
114+
}
115+
116+
Console.WriteLine("\r\n{0} results of {1} in {2}", (page + 1) * size, result.Total, timer.Elapsed);
117+
}
118+
}
119+
120+
}
121+
122+
private static void PrintHeaders()
123+
{
124+
Console.WriteLine();
125+
126+
Console.WriteLine(string.Join(string.Empty,
127+
"docid".PadRight(10),
128+
"score".PadRight(10),
129+
"label".PadRight(70),
130+
"description"
131+
));
132+
Console.WriteLine();
133+
}
134+
135+
private static void Print(Document doc)
136+
{
137+
Console.WriteLine(string.Join(string.Empty,
138+
doc.Fields["_id"].ToString(CultureInfo.InvariantCulture).PadRight(10),
139+
doc.Fields["__score"].ToString(CultureInfo.InvariantCulture).PadRight(10).Substring(0, 9).PadRight(10),
140+
(doc.Fields["label"] ?? string.Empty).Substring(0, Math.Min(69, (doc.Fields["label"] ?? string.Empty).Length)).PadRight(70),
141+
(doc.Fields["description"] ?? string.Empty).Substring(0, Math.Min(30, (doc.Fields["description"] ?? string.Empty).Length))
142+
));
143+
}
144+
145+
static void Write(string[] args)
146+
{
147+
var take = 1000;
148+
149+
if (Array.IndexOf(args, "--take") > 0) take = int.Parse(args[Array.IndexOf(args, "--take") + 1]);
150+
151+
var fileName = args[Array.IndexOf(args, "--file") + 1];
152+
string dir = null;
153+
string indexName = null;
154+
155+
if (Array.IndexOf(args, "--dir") > 0) dir = args[Array.IndexOf(args, "--dir") + 1];
156+
if (Array.IndexOf(args, "--name") > 0) indexName = args[Array.IndexOf(args, "--name") + 1];
157+
158+
var url = ConfigurationManager.AppSettings.Get("sir.endpoint");
159+
var inproc = !string.IsNullOrWhiteSpace(dir);
160+
161+
Console.WriteLine("writing...");
162+
163+
var docs = new List<Dictionary<string, string>>();
164+
165+
var writeTimer = new Stopwatch();
166+
writeTimer.Start();
167+
168+
if (inproc)
169+
{
170+
if (!Directory.Exists(dir)) Directory.CreateDirectory(dir);
171+
using (var writer = new StreamWriteOperation(dir, new Analyzer(), fileName, take))
172+
{
173+
writer.Write();
174+
}
175+
}
176+
else
177+
{
178+
Console.WriteLine("Executing HTTP POST");
179+
180+
using (var client = new WriterClient(indexName, url))
181+
{
182+
client.Write(docs);
183+
}
184+
}
185+
186+
Console.WriteLine("write operation took {0}", writeTimer.Elapsed);
187+
}
188+
}
189+
}

0 commit comments

Comments
 (0)