merge

BitFaster.Caching.Benchmarks/BitFaster.Caching.Benchmarks.csproj

@@ -2,6 +2,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
+    <LangVersion>latest</LangVersion>
     <TargetFrameworks>net6.0;net8.0</TargetFrameworks>
     <AllowUnsafeBlocks>True</AllowUnsafeBlocks>
     <!-- https://stackoverflow.com/a/59916801/131345 -->
@@ -41,4 +42,4 @@
         <DefineConstants>MacOS</DefineConstants>
     </PropertyGroup>
 
-</Project>
+</Project>

BitFaster.Caching.Benchmarks/Lfu/SketchFrequency.cs

@@ -23,7 +23,8 @@ public class SketchFrequency
 
         private CmSketchCore<int, DisableHardwareIntrinsics> blockStd;
         private CmSketchNoPin<int, DetectIsa> blockAvxNoPin;
-        private CmSketchCore<int, DetectIsa> blockAvx;
+        private CmSketchCore<int, DetectIsa> blockVector;
+        private CmSketchCore512<int, DetectIsa> blockVector512;
 
         [Params(32_768, 524_288, 8_388_608, 134_217_728)]
         public int Size { get; set; }
@@ -36,7 +37,8 @@ public void Setup()
 
             blockStd = new CmSketchCore<int, DisableHardwareIntrinsics>(Size, EqualityComparer<int>.Default);
             blockAvxNoPin = new CmSketchNoPin<int, DetectIsa>(Size, EqualityComparer<int>.Default);
-            blockAvx = new CmSketchCore<int, DetectIsa>(Size, EqualityComparer<int>.Default);
+            blockVector = new CmSketchCore<int, DetectIsa>(Size, EqualityComparer<int>.Default);
+            blockVector512 = new CmSketchCore512<int, DetectIsa>(Size, EqualityComparer<int>.Default);
         }
 
         [Benchmark(Baseline = true, OperationsPerInvoke = iterations)]
@@ -74,7 +76,7 @@ public int FrequencyBlockAvxNotPinned()
         {
             int count = 0;
             for (int i = 0; i < iterations; i++)
-                count += blockAvxNoPin.EstimateFrequency(i) > blockAvx.EstimateFrequency(i + 1) ? 1 : 0;
+                count += blockAvxNoPin.EstimateFrequency(i) > blockVector.EstimateFrequency(i + 1) ? 1 : 0;
 
             return count;
         }
@@ -84,7 +86,17 @@ public int FrequencyBlockAvxPinned()
         {
             int count = 0;
             for (int i = 0; i < iterations; i++)
-                count += blockAvx.EstimateFrequency(i) > blockAvx.EstimateFrequency(i + 1) ? 1 : 0;
+                count += blockVector.EstimateFrequency(i) > blockVector.EstimateFrequency(i + 1) ? 1 : 0;
+
+            return count;
+        }
+
+        [Benchmark(OperationsPerInvoke = iterations)]
+        public int FrequencyBlockAvxPinned512()
+        {
+            int count = 0;
+            for (int i = 0; i < iterations; i++)
+                count += blockVector512.EstimateFrequency(i) > blockVector.EstimateFrequency(i + 1) ? 1 : 0;
 
             return count;
         }

BitFaster.Caching.Benchmarks/Lfu/SketchIncrement.cs

@@ -12,7 +12,7 @@ namespace BitFaster.Caching.Benchmarks.Lfu
     [SimpleJob(RuntimeMoniker.Net90)]
     [MemoryDiagnoser(displayGenColumns: false)]
     [HideColumns("Job", "Median", "RatioSD", "Alloc Ratio")]
-    [ColumnChart(Title = "Sketch Increment ({JOB})")]
+    [ColumnChart(Title = "Sketch Increment ({JOB})", Colors = "#cd5c5c,#fa8072,#ffa07a")]
     public class SketchIncrement
     {
         const int iterations = 1_048_576;
@@ -23,6 +23,8 @@ public class SketchIncrement
         private CmSketchCore<int, DisableHardwareIntrinsics> blockStd;
         private CmSketchNoPin<int, DetectIsa> blockAvxNoPin;
         private CmSketchCore<int, DetectIsa> blockAvx;
+        private CmSketchCore512<int, DetectIsa> blockVector512;
+
 
         [Params(32_768, 524_288, 8_388_608, 134_217_728)]
         public int Size { get; set; }
@@ -36,6 +38,7 @@ public void Setup()
             blockStd = new CmSketchCore<int, DisableHardwareIntrinsics>(Size, EqualityComparer<int>.Default);
             blockAvxNoPin = new CmSketchNoPin<int, DetectIsa>(Size, EqualityComparer<int>.Default);
             blockAvx = new CmSketchCore<int, DetectIsa>(Size, EqualityComparer<int>.Default);
+            blockVector512 = new CmSketchCore512<int, DetectIsa>(Size, EqualityComparer<int>.Default);
         }
 
         [Benchmark(Baseline = true, OperationsPerInvoke = iterations)]
@@ -82,5 +85,14 @@ public void IncBlockAvxPinned()
                 blockAvx.Increment(i);
             }
         }
+
+        [Benchmark(OperationsPerInvoke = iterations)]
+        public void IncBlockAvxPinned512()
+        {
+            for (int i = 0; i < iterations; i++)
+            {
+                blockVector512.Increment(i);
+            }
+        }
     }
 }

BitFaster.Caching/BitFaster.Caching.csproj

@@ -29,8 +29,8 @@
     <IsTrimmable>true</IsTrimmable>
     <Nullable>enable</Nullable>
     <!--Package Validation-->
-    <EnablePackageValidation>false</EnablePackageValidation>
-    <PackageValidationBaselineVersion>2.5.0</PackageValidationBaselineVersion>    
+    <EnablePackageValidation>true</EnablePackageValidation>
+    <PackageValidationBaselineVersion>2.5.1</PackageValidationBaselineVersion>    
   </PropertyGroup>
 
   <PropertyGroup Condition="'$(GITHUB_ACTIONS)' == 'true'">

BitFaster.Caching/Lfu/CmSketchCore.cs

@@ -2,6 +2,7 @@
 using System.Collections.Generic;
 using System.Diagnostics.CodeAnalysis;
 using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
 
 
 #if !NETSTANDARD2_0
@@ -37,13 +38,14 @@ public unsafe class CmSketchCore<T, I>
         private const long OneMask = 0x1111111111111111L;
 
         private long[] table;
-#if NET6_0_OR_GREATER
-        private long* tableAddr;
-#endif
         private int sampleSize;
         private int blockMask;
         private int size;
 
+#if NET6_0_OR_GREATER
+        private long* tableAddr;
+#endif
+
         private readonly IEqualityComparer<T> comparer;
 
         /// <summary>
@@ -142,7 +144,7 @@ private void EnsureCapacity(long maximumSize)
 
 #if NET6_0_OR_GREATER
             I isa = default;
-            if (isa.IsAvx2Supported)
+            if (isa.IsAvx2Supported || isa.IsArm64Supported)
             {
                 // over alloc by 8 to give 64 bytes padding, tableAddr is then aligned to 64 bytes
                 const int pad = 8;
@@ -378,7 +380,7 @@ private unsafe void IncrementAvx(T value)
 #endif
 
 #if NET6_0_OR_GREATER
-        [MethodImpl(MethodImplOptions.AggressiveOptimization | MethodImplOptions.AggressiveInlining)]
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
         private unsafe void IncrementArm(T value)
         {
             int blockHash = Spread(comparer.GetHashCode(value));
@@ -389,7 +391,8 @@ private unsafe void IncrementArm(T value)
             Vector128<int> index = AdvSimd.And(AdvSimd.ShiftRightLogical(h, 1), Vector128.Create(0xf));
             Vector128<int> blockOffset = AdvSimd.Add(AdvSimd.Add(Vector128.Create(block), AdvSimd.And(h, Vector128.Create(1))), Vector128.Create(0, 2, 4, 6));
 
-            fixed (long* tablePtr = table)
+            long* tablePtr = tableAddr;
+            //fixed (long* tablePtr = table)
             {
                 int t0 = AdvSimd.Extract(blockOffset, 0);
                 int t1 = AdvSimd.Extract(blockOffset, 1);
@@ -429,7 +432,7 @@ private unsafe void IncrementArm(T value)
             }
         }
 
-        [MethodImpl(MethodImplOptions.AggressiveOptimization | MethodImplOptions.AggressiveInlining)]
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
         private unsafe int EstimateFrequencyArm(T value)
         {
             int blockHash = Spread(comparer.GetHashCode(value));
@@ -440,7 +443,8 @@ private unsafe int EstimateFrequencyArm(T value)
             Vector128<int> index = AdvSimd.And(AdvSimd.ShiftRightLogical(h, 1), Vector128.Create(0xf));
             Vector128<int> blockOffset = AdvSimd.Add(AdvSimd.Add(Vector128.Create(block), AdvSimd.And(h, Vector128.Create(1))), Vector128.Create(0, 2, 4, 6));
 
-            fixed (long* tablePtr = table)
+            long* tablePtr = tableAddr;
+            //fixed (long* tablePtr = table)
             {
                 Vector128<long> tableVectorA = Vector128.Create(AdvSimd.LoadVector64(tablePtr + AdvSimd.Extract(blockOffset, 0)), AdvSimd.LoadVector64(tablePtr + AdvSimd.Extract(blockOffset, 1)));
                 Vector128<long> tableVectorB = Vector128.Create(AdvSimd.LoadVector64(tablePtr + AdvSimd.Extract(blockOffset, 2)), AdvSimd.LoadVector64(tablePtr + AdvSimd.Extract(blockOffset, 3)));