Skip to content

Commit ce8b03a

Browse files
obrakmannobrakmann
committed
Merge pull request OpenRA#10530 from pchote/remove-external-hashes
Index filesystem contents by filename instead of hash.
2 parents 01cd6c9 + 6085de7 commit ce8b03a

File tree

22 files changed

+139
-333
lines changed

22 files changed

+139
-333
lines changed

‎OpenRA.Game/ContentInstaller.cs

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,11 +15,14 @@ namespace OpenRA
1515
// Referenced from ModMetadata, so needs to be in OpenRA.Game :(
1616
public class ContentInstaller : IGlobalModData
1717
{
18+
public enum FilenameCase { Input, ForceLower, ForceUpper }
19+
1820
public readonly string[] TestFiles = { };
1921
public readonly string[] DiskTestFiles = { };
2022
public readonly string PackageToExtractFromCD = null;
2123
public readonly bool OverwriteFiles = true;
2224

25+
public readonly FilenameCase OutputFilenameCase = FilenameCase.ForceLower;
2326
public readonly Dictionary<string, string[]> CopyFilesFromCD = new Dictionary<string, string[]>();
2427
public readonly Dictionary<string, string[]> ExtractFilesFromCD = new Dictionary<string, string[]>();
2528

‎OpenRA.Game/FileSystem/BagFile.cs

Lines changed: 6 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -21,19 +21,15 @@ namespace OpenRA.FileSystem
2121
{
2222
public sealed class BagFile : IReadOnlyPackage
2323
{
24-
static readonly uint[] Nothing = { };
25-
2624
readonly string bagFilename;
2725
readonly Stream s;
2826
readonly int bagFilePriority;
29-
readonly Dictionary<uint, IdxEntry> index;
30-
readonly FileSystem context;
27+
readonly Dictionary<string, IdxEntry> index;
3128

3229
public BagFile(FileSystem context, string filename, int priority)
3330
{
3431
bagFilename = filename;
3532
bagFilePriority = priority;
36-
this.context = context;
3733

3834
// A bag file is always accompanied with an .idx counterpart
3935
// For example: audio.bag requires the audio.idx file
@@ -44,7 +40,7 @@ public BagFile(FileSystem context, string filename, int priority)
4440
using (var indexStream = context.Open(indexFilename))
4541
entries = new IdxReader(indexStream).Entries;
4642

47-
index = entries.ToDictionaryWithConflictLog(x => x.Hash,
43+
index = entries.ToDictionaryWithConflictLog(x => x.Filename,
4844
"{0} (bag format)".F(filename),
4945
null, x => "(offs={0}, len={1})".F(x.Offset, x.Length));
5046

@@ -54,10 +50,10 @@ public BagFile(FileSystem context, string filename, int priority)
5450
public int Priority { get { return 1000 + bagFilePriority; } }
5551
public string Name { get { return bagFilename; } }
5652

57-
public Stream GetContent(uint hash)
53+
public Stream GetContent(string filename)
5854
{
5955
IdxEntry entry;
60-
if (!index.TryGetValue(hash, out entry))
56+
if (!index.TryGetValue(filename, out entry))
6157
return null;
6258

6359
s.Seek(entry.Offset, SeekOrigin.Begin);
@@ -120,59 +116,14 @@ public Stream GetContent(uint hash)
120116
return mergedStream;
121117
}
122118

123-
uint? FindMatchingHash(string filename)
124-
{
125-
var hash = IdxEntry.HashFilename(filename, PackageHashType.CRC32);
126-
if (index.ContainsKey(hash))
127-
return hash;
128-
129-
// Maybe we were given a raw hash?
130-
uint raw;
131-
if (!uint.TryParse(filename, NumberStyles.AllowHexSpecifier, CultureInfo.InvariantCulture, out raw))
132-
return null;
133-
134-
if ("{0:X}".F(raw) == filename && index.ContainsKey(raw))
135-
return raw;
136-
137-
return null;
138-
}
139-
140-
public Stream GetContent(string filename)
141-
{
142-
var hash = FindMatchingHash(filename);
143-
return hash.HasValue ? GetContent(hash.Value) : null;
144-
}
145-
146119
public bool Exists(string filename)
147120
{
148-
return FindMatchingHash(filename).HasValue;
149-
}
150-
151-
public IEnumerable<uint> ClassicHashes()
152-
{
153-
return Nothing;
154-
}
155-
156-
public IEnumerable<uint> CrcHashes()
157-
{
158-
return index.Keys;
121+
return index.ContainsKey(filename);
159122
}
160123

161124
public IEnumerable<string> AllFileNames()
162125
{
163-
var lookup = new Dictionary<uint, string>();
164-
if (context.Exists("global mix database.dat"))
165-
{
166-
var db = new XccGlobalDatabase(context.Open("global mix database.dat"));
167-
foreach (var e in db.Entries)
168-
{
169-
var hash = IdxEntry.HashFilename(e, PackageHashType.CRC32);
170-
if (!lookup.ContainsKey(hash))
171-
lookup.Add(hash, e);
172-
}
173-
}
174-
175-
return index.Keys.Select(k => lookup.ContainsKey(k) ? lookup[k] : "{0:X}".F(k));
126+
return index.Keys;
176127
}
177128

178129
public void Dispose()

‎OpenRA.Game/FileSystem/BigFile.cs

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -96,16 +96,6 @@ public bool Exists(string filename)
9696
return entries.ContainsKey(filename);
9797
}
9898

99-
public IEnumerable<uint> ClassicHashes()
100-
{
101-
return entries.Keys.Select(filename => PackageEntry.HashFilename(filename, PackageHashType.Classic));
102-
}
103-
104-
public IEnumerable<uint> CrcHashes()
105-
{
106-
return Enumerable.Empty<uint>();
107-
}
108-
10999
public IEnumerable<string> AllFileNames()
110100
{
111101
return entries.Keys;

‎OpenRA.Game/FileSystem/D2kSoundResources.cs

Lines changed: 20 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -10,18 +10,29 @@
1010

1111
using System.Collections.Generic;
1212
using System.IO;
13+
using OpenRA.Primitives;
1314

1415
namespace OpenRA.FileSystem
1516
{
1617
public sealed class D2kSoundResources : IReadOnlyPackage
1718
{
19+
struct Entry
20+
{
21+
public readonly uint Offset;
22+
public readonly uint Length;
23+
24+
public Entry(uint offset, uint length)
25+
{
26+
Offset = offset;
27+
Length = length;
28+
}
29+
}
30+
1831
readonly Stream s;
1932

2033
readonly string filename;
21-
readonly List<string> filenames;
2234
readonly int priority;
23-
24-
readonly Dictionary<uint, PackageEntry> index = new Dictionary<uint, PackageEntry>();
35+
readonly Dictionary<string, Entry> index = new Dictionary<string, Entry>();
2536

2637
public D2kSoundResources(FileSystem context, string filename, int priority)
2738
{
@@ -31,20 +42,13 @@ public D2kSoundResources(FileSystem context, string filename, int priority)
3142
s = context.Open(filename);
3243
try
3344
{
34-
filenames = new List<string>();
35-
3645
var headerLength = s.ReadUInt32();
3746
while (s.Position < headerLength + 4)
3847
{
3948
var name = s.ReadASCIIZ();
4049
var offset = s.ReadUInt32();
4150
var length = s.ReadUInt32();
42-
43-
var hash = PackageEntry.HashFilename(name, PackageHashType.Classic);
44-
if (!index.ContainsKey(hash))
45-
index.Add(hash, new PackageEntry(hash, offset, length));
46-
47-
filenames.Add(name);
51+
index.Add(name, new Entry(offset, length));
4852
}
4953
}
5054
catch
@@ -54,45 +58,30 @@ public D2kSoundResources(FileSystem context, string filename, int priority)
5458
}
5559
}
5660

57-
public Stream GetContent(uint hash)
61+
public Stream GetContent(string filename)
5862
{
59-
PackageEntry e;
60-
if (!index.TryGetValue(hash, out e))
63+
Entry e;
64+
if (!index.TryGetValue(filename, out e))
6165
return null;
6266

6367
s.Seek(e.Offset, SeekOrigin.Begin);
6468
return new MemoryStream(s.ReadBytes((int)e.Length));
6569
}
6670

67-
public Stream GetContent(string filename)
68-
{
69-
return GetContent(PackageEntry.HashFilename(filename, PackageHashType.Classic));
70-
}
71-
7271
public bool Exists(string filename)
7372
{
74-
return index.ContainsKey(PackageEntry.HashFilename(filename, PackageHashType.Classic));
73+
return index.ContainsKey(filename);
7574
}
7675

7776
public IEnumerable<string> AllFileNames()
7877
{
79-
return filenames;
78+
return index.Keys;
8079
}
8180

8281
public string Name { get { return filename; } }
8382

8483
public int Priority { get { return 1000 + priority; } }
8584

86-
public IEnumerable<uint> ClassicHashes()
87-
{
88-
return index.Keys;
89-
}
90-
91-
public IEnumerable<uint> CrcHashes()
92-
{
93-
yield break;
94-
}
95-
9685
public void Dispose()
9786
{
9887
s.Dispose();

‎OpenRA.Game/FileSystem/FileSystem.cs

Lines changed: 14 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -19,14 +19,12 @@ namespace OpenRA.FileSystem
1919
{
2020
public class FileSystem
2121
{
22-
public readonly List<string> PackagePaths = new List<string>();
2322
public readonly List<IReadOnlyPackage> MountedPackages = new List<IReadOnlyPackage>();
2423

2524
static readonly Dictionary<string, Assembly> AssemblyCache = new Dictionary<string, Assembly>();
2625

2726
int order;
28-
Cache<uint, List<IReadOnlyPackage>> crcHashIndex = new Cache<uint, List<IReadOnlyPackage>>(_ => new List<IReadOnlyPackage>());
29-
Cache<uint, List<IReadOnlyPackage>> classicHashIndex = new Cache<uint, List<IReadOnlyPackage>>(_ => new List<IReadOnlyPackage>());
27+
Cache<string, List<IReadOnlyPackage>> fileIndex = new Cache<string, List<IReadOnlyPackage>>(_ => new List<IReadOnlyPackage>());
3028

3129
public IReadWritePackage CreatePackage(string filename, int order, Dictionary<string, byte[]> content)
3230
{
@@ -93,7 +91,6 @@ public void Mount(string name, string annotation = null)
9391

9492
name = Platform.ResolvePath(name);
9593

96-
PackagePaths.Add(name);
9794
Action a = () => MountInner(OpenPackage(name, annotation, order++));
9895

9996
if (optional)
@@ -107,27 +104,23 @@ void MountInner(IReadOnlyPackage package)
107104
{
108105
MountedPackages.Add(package);
109106

110-
foreach (var hash in package.ClassicHashes())
107+
foreach (var filename in package.AllFileNames())
111108
{
112-
var packageList = classicHashIndex[hash];
113-
if (!packageList.Contains(package))
114-
packageList.Add(package);
115-
}
116-
117-
foreach (var hash in package.CrcHashes())
118-
{
119-
var packageList = crcHashIndex[hash];
109+
var packageList = fileIndex[filename];
120110
if (!packageList.Contains(package))
121111
packageList.Add(package);
122112
}
123113
}
124114

125-
public bool Unmount(IReadOnlyPackage mount)
115+
public bool Unmount(IReadOnlyPackage package)
126116
{
127-
if (MountedPackages.Contains(mount))
128-
mount.Dispose();
117+
foreach (var packagesForFile in fileIndex.Values)
118+
packagesForFile.RemoveAll(p => p == package);
129119

130-
return MountedPackages.RemoveAll(f => f == mount) > 0;
120+
if (MountedPackages.Contains(package))
121+
package.Dispose();
122+
123+
return MountedPackages.RemoveAll(p => p == package) > 0;
131124
}
132125

133126
public void UnmountAll()
@@ -136,9 +129,7 @@ public void UnmountAll()
136129
package.Dispose();
137130

138131
MountedPackages.Clear();
139-
PackagePaths.Clear();
140-
classicHashIndex = new Cache<uint, List<IReadOnlyPackage>>(_ => new List<IReadOnlyPackage>());
141-
crcHashIndex = new Cache<uint, List<IReadOnlyPackage>>(_ => new List<IReadOnlyPackage>());
132+
fileIndex = new Cache<string, List<IReadOnlyPackage>>(_ => new List<IReadOnlyPackage>());
142133
}
143134

144135
public void LoadFromManifest(Manifest manifest)
@@ -151,10 +142,9 @@ public void LoadFromManifest(Manifest manifest)
151142
Mount(pkg.Key, pkg.Value);
152143
}
153144

154-
Stream GetFromCache(PackageHashType type, string filename)
145+
Stream GetFromCache(string filename)
155146
{
156-
var index = type == PackageHashType.CRC32 ? crcHashIndex : classicHashIndex;
157-
var package = index[PackageEntry.HashFilename(filename, type)]
147+
var package = fileIndex[filename]
158148
.Where(x => x.Exists(filename))
159149
.MinByOrDefault(x => x.Priority);
160150

@@ -191,11 +181,7 @@ public bool TryOpen(string name, out Stream s)
191181
// TODO: This disables caching for explicit package requests
192182
if (filename.IndexOfAny(new[] { '/', '\\' }) == -1 && !explicitPackage)
193183
{
194-
s = GetFromCache(PackageHashType.Classic, filename);
195-
if (s != null)
196-
return true;
197-
198-
s = GetFromCache(PackageHashType.CRC32, filename);
184+
s = GetFromCache(filename);
199185
if (s != null)
200186
return true;
201187
}

‎OpenRA.Game/FileSystem/Folder.cs

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -43,17 +43,6 @@ public Stream GetContent(string filename)
4343
catch { return null; }
4444
}
4545

46-
public IEnumerable<uint> ClassicHashes()
47-
{
48-
foreach (var filename in Directory.GetFiles(path, "*", SearchOption.TopDirectoryOnly))
49-
yield return PackageEntry.HashFilename(Path.GetFileName(filename), PackageHashType.Classic);
50-
}
51-
52-
public IEnumerable<uint> CrcHashes()
53-
{
54-
yield break;
55-
}
56-
5746
public IEnumerable<string> AllFileNames()
5847
{
5948
foreach (var filename in Directory.GetFiles(path, "*", SearchOption.TopDirectoryOnly))

‎OpenRA.Game/FileSystem/IPackage.cs

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,6 @@ public interface IReadOnlyPackage : IDisposable
1818
{
1919
Stream GetContent(string filename);
2020
bool Exists(string filename);
21-
IEnumerable<uint> ClassicHashes();
22-
IEnumerable<uint> CrcHashes();
2321
IEnumerable<string> AllFileNames();
2422
int Priority { get; }
2523
string Name { get; }

0 commit comments

Comments
 (0)