From e6f3627e846cb3377f20d2eaaea04fefd0601fd0 Mon Sep 17 00:00:00 2001 From: slorello89 Date: Fri, 6 Oct 2023 07:41:51 -0400 Subject: [PATCH 01/36] starting vectors --- Redis.OM.sln | 58 ---------- src/Redis.OM/Modeling/RedisSchemaField.cs | 104 +++++++++++++++++- src/Redis.OM/Modeling/SearchFieldType.cs | 5 + .../Modeling/Vectors/DistanceMetric.cs | 48 ++++++++ .../Modeling/Vectors/VectorAlgorithm.cs | 43 ++++++++ .../Modeling/Vectors/VectorAttribute.cs | 89 +++++++++++++++ .../Modeling/Vectors/VectorJsonConverter.cs | 103 +++++++++++++++++ src/Redis.OM/Modeling/Vectors/VectorType.cs | 42 +++++++ .../Modeling/Vectors/VectorizerAttribute.cs | 38 +++++++ src/Redis.OM/RedisObjectHandler.cs | 82 ++++++++++++-- src/Redis.OM/stylecop.ruleset | 1 + .../VectorTests/ObjectWithVector.cs | 31 ++++++ .../VectorTests/SimpleVectorizer.cs | 22 ++++ .../VectorTests/VectorFunctionalTests.cs | 53 +++++++++ .../VectorTests/VectorTests.cs | 104 ++++++++++++++++++ 15 files changed, 754 insertions(+), 69 deletions(-) create mode 100644 src/Redis.OM/Modeling/Vectors/DistanceMetric.cs create mode 100644 src/Redis.OM/Modeling/Vectors/VectorAlgorithm.cs create mode 100644 src/Redis.OM/Modeling/Vectors/VectorAttribute.cs create mode 100644 src/Redis.OM/Modeling/Vectors/VectorJsonConverter.cs create mode 100644 src/Redis.OM/Modeling/Vectors/VectorType.cs create mode 100644 src/Redis.OM/Modeling/Vectors/VectorizerAttribute.cs create mode 100644 test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/ObjectWithVector.cs create mode 100644 test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/SimpleVectorizer.cs create mode 100644 test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorFunctionalTests.cs create mode 100644 test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorTests.cs diff --git a/Redis.OM.sln b/Redis.OM.sln index 9846fb54..9924b2ca 100644 --- a/Redis.OM.sln +++ b/Redis.OM.sln @@ -10,14 +10,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Redis.OM.POC", "src\Redis.O EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Redis.OM.Unit.Tests", "test\Redis.OM.Unit.Tests\Redis.OM.Unit.Tests.csproj", "{570BF479-BCF4-4D1B-A702-2234CA0A3E7D}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Redis.OM.Test.ConsoleApp", "test\Redis.OM.Test.ConsoleApp\Redis.OM.Test.ConsoleApp.csproj", "{FC7E5ED3-51AC-45E6-A178-6287C9227975}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Redis.OM.Analyzer", "src\Redis.OM.Analyzer\Redis.OM.Analyzer.csproj", "{44FAD9BB-C6DF-402C-BCE7-64E7C674F8D1}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Redis.OM.AspNetCore", "src\Redis.OM.AspNetCore\Redis.OM.AspNetCore.csproj", "{230ED77D-D625-43BC-94D6-6BDBACEA3EAF}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Redis.OM.Test.AspDotnetCore", "test\Redis.OM.Test.AspDotnetCore\Redis.OM.Test.AspDotnetCore.csproj", "{3F609AB2-1492-4EBE-9FF2-B47829307E9E}" -EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -64,54 +56,6 @@ Global {570BF479-BCF4-4D1B-A702-2234CA0A3E7D}.Release|x64.Build.0 = Release|Any CPU {570BF479-BCF4-4D1B-A702-2234CA0A3E7D}.Release|x86.ActiveCfg = Release|Any CPU {570BF479-BCF4-4D1B-A702-2234CA0A3E7D}.Release|x86.Build.0 = Release|Any CPU - {FC7E5ED3-51AC-45E6-A178-6287C9227975}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {FC7E5ED3-51AC-45E6-A178-6287C9227975}.Debug|Any CPU.Build.0 = Debug|Any CPU - {FC7E5ED3-51AC-45E6-A178-6287C9227975}.Debug|x64.ActiveCfg = Debug|Any CPU - {FC7E5ED3-51AC-45E6-A178-6287C9227975}.Debug|x64.Build.0 = Debug|Any CPU - {FC7E5ED3-51AC-45E6-A178-6287C9227975}.Debug|x86.ActiveCfg = Debug|Any CPU - {FC7E5ED3-51AC-45E6-A178-6287C9227975}.Debug|x86.Build.0 = Debug|Any CPU - {FC7E5ED3-51AC-45E6-A178-6287C9227975}.Release|Any CPU.ActiveCfg = Release|Any CPU - {FC7E5ED3-51AC-45E6-A178-6287C9227975}.Release|Any CPU.Build.0 = Release|Any CPU - {FC7E5ED3-51AC-45E6-A178-6287C9227975}.Release|x64.ActiveCfg = Release|Any CPU - {FC7E5ED3-51AC-45E6-A178-6287C9227975}.Release|x64.Build.0 = Release|Any CPU - {FC7E5ED3-51AC-45E6-A178-6287C9227975}.Release|x86.ActiveCfg = Release|Any CPU - {FC7E5ED3-51AC-45E6-A178-6287C9227975}.Release|x86.Build.0 = Release|Any CPU - {44FAD9BB-C6DF-402C-BCE7-64E7C674F8D1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {44FAD9BB-C6DF-402C-BCE7-64E7C674F8D1}.Debug|Any CPU.Build.0 = Debug|Any CPU - {44FAD9BB-C6DF-402C-BCE7-64E7C674F8D1}.Debug|x64.ActiveCfg = Debug|Any CPU - {44FAD9BB-C6DF-402C-BCE7-64E7C674F8D1}.Debug|x64.Build.0 = Debug|Any CPU - {44FAD9BB-C6DF-402C-BCE7-64E7C674F8D1}.Debug|x86.ActiveCfg = Debug|Any CPU - {44FAD9BB-C6DF-402C-BCE7-64E7C674F8D1}.Debug|x86.Build.0 = Debug|Any CPU - {44FAD9BB-C6DF-402C-BCE7-64E7C674F8D1}.Release|Any CPU.ActiveCfg = Release|Any CPU - {44FAD9BB-C6DF-402C-BCE7-64E7C674F8D1}.Release|Any CPU.Build.0 = Release|Any CPU - {44FAD9BB-C6DF-402C-BCE7-64E7C674F8D1}.Release|x64.ActiveCfg = Release|Any CPU - {44FAD9BB-C6DF-402C-BCE7-64E7C674F8D1}.Release|x64.Build.0 = Release|Any CPU - {44FAD9BB-C6DF-402C-BCE7-64E7C674F8D1}.Release|x86.ActiveCfg = Release|Any CPU - {44FAD9BB-C6DF-402C-BCE7-64E7C674F8D1}.Release|x86.Build.0 = Release|Any CPU - {230ED77D-D625-43BC-94D6-6BDBACEA3EAF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {230ED77D-D625-43BC-94D6-6BDBACEA3EAF}.Debug|Any CPU.Build.0 = Debug|Any CPU - {230ED77D-D625-43BC-94D6-6BDBACEA3EAF}.Debug|x64.ActiveCfg = Debug|Any CPU - {230ED77D-D625-43BC-94D6-6BDBACEA3EAF}.Debug|x64.Build.0 = Debug|Any CPU - {230ED77D-D625-43BC-94D6-6BDBACEA3EAF}.Debug|x86.ActiveCfg = Debug|Any CPU - {230ED77D-D625-43BC-94D6-6BDBACEA3EAF}.Debug|x86.Build.0 = Debug|Any CPU - {230ED77D-D625-43BC-94D6-6BDBACEA3EAF}.Release|Any CPU.ActiveCfg = Release|Any CPU - {230ED77D-D625-43BC-94D6-6BDBACEA3EAF}.Release|Any CPU.Build.0 = Release|Any CPU - {230ED77D-D625-43BC-94D6-6BDBACEA3EAF}.Release|x64.ActiveCfg = Release|Any CPU - {230ED77D-D625-43BC-94D6-6BDBACEA3EAF}.Release|x64.Build.0 = Release|Any CPU - {230ED77D-D625-43BC-94D6-6BDBACEA3EAF}.Release|x86.ActiveCfg = Release|Any CPU - {230ED77D-D625-43BC-94D6-6BDBACEA3EAF}.Release|x86.Build.0 = Release|Any CPU - {3F609AB2-1492-4EBE-9FF2-B47829307E9E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {3F609AB2-1492-4EBE-9FF2-B47829307E9E}.Debug|Any CPU.Build.0 = Debug|Any CPU - {3F609AB2-1492-4EBE-9FF2-B47829307E9E}.Debug|x64.ActiveCfg = Debug|Any CPU - {3F609AB2-1492-4EBE-9FF2-B47829307E9E}.Debug|x64.Build.0 = Debug|Any CPU - {3F609AB2-1492-4EBE-9FF2-B47829307E9E}.Debug|x86.ActiveCfg = Debug|Any CPU - {3F609AB2-1492-4EBE-9FF2-B47829307E9E}.Debug|x86.Build.0 = Debug|Any CPU - {3F609AB2-1492-4EBE-9FF2-B47829307E9E}.Release|Any CPU.ActiveCfg = Release|Any CPU - {3F609AB2-1492-4EBE-9FF2-B47829307E9E}.Release|Any CPU.Build.0 = Release|Any CPU - {3F609AB2-1492-4EBE-9FF2-B47829307E9E}.Release|x64.ActiveCfg = Release|Any CPU - {3F609AB2-1492-4EBE-9FF2-B47829307E9E}.Release|x64.Build.0 = Release|Any CPU - {3F609AB2-1492-4EBE-9FF2-B47829307E9E}.Release|x86.ActiveCfg = Release|Any CPU - {3F609AB2-1492-4EBE-9FF2-B47829307E9E}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -119,8 +63,6 @@ Global GlobalSection(NestedProjects) = preSolution {7994382C-28EF-4F55-9B6D-810D35247816} = {8D9ECCFF-E022-4B68-BB43-228CEA248DEC} {E3A31119-E4F1-4793-B5C2-ED2D51502B01} = {8D9ECCFF-E022-4B68-BB43-228CEA248DEC} - {44FAD9BB-C6DF-402C-BCE7-64E7C674F8D1} = {8D9ECCFF-E022-4B68-BB43-228CEA248DEC} - {230ED77D-D625-43BC-94D6-6BDBACEA3EAF} = {8D9ECCFF-E022-4B68-BB43-228CEA248DEC} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {E5752441-184B-4F17-BAD0-93823AC68607} diff --git a/src/Redis.OM/Modeling/RedisSchemaField.cs b/src/Redis.OM/Modeling/RedisSchemaField.cs index 38d44ee3..b1e09521 100644 --- a/src/Redis.OM/Modeling/RedisSchemaField.cs +++ b/src/Redis.OM/Modeling/RedisSchemaField.cs @@ -3,6 +3,7 @@ using System.Globalization; using System.Linq; using System.Reflection; +using System.Runtime.InteropServices.ComTypes; using System.Text.Json.Serialization; namespace Redis.OM.Modeling @@ -61,7 +62,15 @@ internal static string[] SerializeArgsJson(this PropertyInfo info, int remaining { var innerType = Nullable.GetUnderlyingType(info.PropertyType) ?? info.PropertyType; - if (IsComplexType(innerType)) + if (attr is VectorAttribute) + { + var pathPostfix = info.GetCustomAttributes().Any() ? ".Vector" : string.Empty; + ret.Add(!string.IsNullOrEmpty(attr.PropertyName) ? $"{pathPrefix}{attr.PropertyName}{pathPostfix}" : $"{pathPrefix}{info.Name}{pathPostfix}"); + ret.Add("AS"); + ret.Add(!string.IsNullOrEmpty(attr.PropertyName) ? $"{aliasPrefix}{attr.PropertyName}" : $"{aliasPrefix}{info.Name}"); + ret.AddRange(CommonSerialization(attr, innerType, info)); + } + else if (IsComplexType(innerType)) { if (cascadeDepth > 0) { @@ -93,12 +102,18 @@ internal static string[] SerializeArgsJson(this PropertyInfo info, int remaining internal static string[] SerializeArgs(this PropertyInfo info) { var attr = Attribute.GetCustomAttribute(info, typeof(SearchFieldAttribute)) as SearchFieldAttribute; - if (attr == null) + if (attr is null) { return Array.Empty(); } - var ret = new List { !string.IsNullOrEmpty(attr.PropertyName) ? attr.PropertyName : info.Name }; + var suffix = string.Empty; + if (attr.SearchFieldType == SearchFieldType.VECTOR && info.GetCustomAttributes().Any()) + { + suffix = ".Vector"; + } + + var ret = new List { !string.IsNullOrEmpty(attr.PropertyName) ? $"attr.PropertyName{suffix}" : $"{info.Name}{suffix}" }; var innerType = Nullable.GetUnderlyingType(info.PropertyType); ret.AddRange(CommonSerialization(attr, innerType ?? info.PropertyType, info)); return ret.ToArray(); @@ -187,6 +202,84 @@ private static string GetSearchFieldType(Type declaredType, SearchFieldAttribute private static bool IsEnumTypeFlags(Type type) => type.GetCustomAttributes(typeof(FlagsAttribute), false).Any(); + private static IEnumerable VectorSerialization(VectorAttribute vectorAttribute, Type declaredType, PropertyInfo propertyInfo) + { + var vectorizer = propertyInfo.GetCustomAttributes().FirstOrDefault(); + if (vectorizer is null) + { + if (vectorAttribute.Dim == default) + { + throw new ArgumentException("Could not determine dimension of the vector"); + } + + if (declaredType != typeof(double[]) && declaredType != typeof(float[])) + { + throw new ArgumentException("Could not determine the Vector Type"); + } + } + + yield return vectorAttribute.Algorithm.AsRedisString(); + yield return vectorAttribute.NumArgs.ToString(); + yield return "TYPE"; + if (vectorizer is not null) + { + yield return vectorizer.VectorType.AsRedisString(); + } + else if (declaredType == typeof(double[])) + { + yield return "FLOAT64"; + } + else if (declaredType == typeof(float[])) + { + yield return "FLOAT32"; + } + + yield return "DIM"; + yield return vectorizer is null ? vectorAttribute.Dim!.ToString() : vectorizer.Dim.ToString(); + yield return "DISTANCE_METRIC"; + yield return vectorAttribute.DistanceMetric.AsRedisString(); + if (vectorAttribute.InitialCapacity != default) + { + yield return "INITIAL_CAP"; + yield return vectorAttribute.InitialCapacity.ToString(); + } + + if (vectorAttribute.Algorithm == VectorAlgorithm.FLAT) + { + if (vectorAttribute.BlockSize != default) + { + yield return "BLOCK_SIZE"; + yield return vectorAttribute.BlockSize.ToString(); + } + } + else if (vectorAttribute.Algorithm == VectorAlgorithm.HNSW) + { + if (vectorAttribute.M != default) + { + yield return "M"; + yield return vectorAttribute.M.ToString(); + } + + if (vectorAttribute.EfConstructor != default) + { + yield return "EF_CONSTRUCTION"; + yield return vectorAttribute.EfConstructor.ToString(); + } + + if (vectorAttribute.EfRuntime != default) + { + yield return "EF_RUNTIME"; + yield return vectorAttribute.EfRuntime.ToString(); + } + + if (vectorAttribute.Epsilon != default) + { + yield return "EPSILON"; + yield return vectorAttribute.Epsilon.ToString(CultureInfo.InvariantCulture); + } + } + } + private static string[] CommonSerialization(SearchFieldAttribute attr, Type declaredType, PropertyInfo propertyInfo) { var searchFieldType = GetSearchFieldType(declaredType, attr, propertyInfo); @@ -230,6 +323,11 @@ private static string[] CommonSerialization(SearchFieldAttribute attr, Type decl } } + if (searchFieldType == "VECTOR" && attr is VectorAttribute vector) + { + ret.AddRange(VectorSerialization(vector, declaredType, propertyInfo)); + } + if (attr.Sortable || attr.Aggregatable) { ret.Add("SORTABLE"); diff --git a/src/Redis.OM/Modeling/SearchFieldType.cs b/src/Redis.OM/Modeling/SearchFieldType.cs index 8ab425ce..8c1b9853 100644 --- a/src/Redis.OM/Modeling/SearchFieldType.cs +++ b/src/Redis.OM/Modeling/SearchFieldType.cs @@ -29,5 +29,10 @@ internal enum SearchFieldType /// A generically indexed field - the library will figure out how to index. /// INDEXED = 4, + + /// + /// A vector index field. + /// + VECTOR = 5, } } diff --git a/src/Redis.OM/Modeling/Vectors/DistanceMetric.cs b/src/Redis.OM/Modeling/Vectors/DistanceMetric.cs new file mode 100644 index 00000000..af39609f --- /dev/null +++ b/src/Redis.OM/Modeling/Vectors/DistanceMetric.cs @@ -0,0 +1,48 @@ +using System; + +namespace Redis.OM.Modeling +{ + /// + /// The Vector distance metric to use. + /// + public enum DistanceMetric + { + /// + /// Euclidean distance. + /// + L2, + + /// + /// Inner Product. + /// + IP, + + /// + /// The Cosine distance. + /// + COSINE, + } + + /// + /// Quality of life extensions for distance metrics. + /// + internal static class DistanceMetricExtensions + { + /// + /// Gets the Distance metric as a Redis usable string. + /// + /// The distance Metric. + /// A Redis Usable string. + /// thrown if illegal ordinal encountered. + internal static string AsRedisString(this DistanceMetric distanceMetric) + { + return distanceMetric switch + { + DistanceMetric.L2 => "L2", + DistanceMetric.IP => "IP", + DistanceMetric.COSINE => "COSINE", + _ => throw new ArgumentOutOfRangeException(nameof(distanceMetric)) + }; + } + } +} \ No newline at end of file diff --git a/src/Redis.OM/Modeling/Vectors/VectorAlgorithm.cs b/src/Redis.OM/Modeling/Vectors/VectorAlgorithm.cs new file mode 100644 index 00000000..f4d8c9db --- /dev/null +++ b/src/Redis.OM/Modeling/Vectors/VectorAlgorithm.cs @@ -0,0 +1,43 @@ +using System; + +namespace Redis.OM.Modeling +{ + /// + /// The Vector Algorithm. + /// + public enum VectorAlgorithm + { + /// + /// Uses a brute force algorithm to find nearest neighbors. + /// + FLAT = 0, + + /// + /// Uses the Hierarchical Small World Algorithm to build an efficient graph structure to + /// retrieve approximate nearest neighbors + /// + HNSW = 1, + } + + /// + /// Quality of life functions for VectorAlgorithm enum. + /// + internal static class VectorAlgorithmExtensions + { + /// + /// Returns the algorithm as a Redis Serialized String. + /// + /// The algorithm to use. + /// The algorithm's name. + /// Thrown if an invalid Algorithm is passed. + internal static string AsRedisString(this VectorAlgorithm algorithm) + { + return algorithm switch + { + VectorAlgorithm.FLAT => "FLAT", + VectorAlgorithm.HNSW => "HNSW", + _ => throw new ArgumentOutOfRangeException(nameof(algorithm)) + }; + } + } +} \ No newline at end of file diff --git a/src/Redis.OM/Modeling/Vectors/VectorAttribute.cs b/src/Redis.OM/Modeling/Vectors/VectorAttribute.cs new file mode 100644 index 00000000..05d607be --- /dev/null +++ b/src/Redis.OM/Modeling/Vectors/VectorAttribute.cs @@ -0,0 +1,89 @@ +namespace Redis.OM.Modeling +{ + /// + /// An indexed vector embedding, can be whatever type you want so long as you provide a serializer. + /// + public class VectorAttribute : SearchFieldAttribute + { + /// + /// Gets or sets the vector storage algorithm to use. Defaults to Flat (which is brute force). + /// + public VectorAlgorithm Algorithm { get; set; } = VectorAlgorithm.FLAT; + + /// + /// Gets or sets the Vector dimension specified as a positive integer. Will be inferred from the vectorizer + /// if provided. + /// + public int Dim { get; set; } + + /// + /// Gets or sets the Supported distance metric. + /// + public DistanceMetric DistanceMetric { get; set; } = DistanceMetric.L2; + + /// + /// Gets or sets the Initial vector capacity in the index affecting memory allocation size of the index. + /// + public int InitialCapacity { get; set; } + + /// + /// Gets or sets Block size to hold BLOCK_SIZE amount of vectors in a contiguous array. This is useful when the + /// index is dynamic with respect to addition and deletion. Defaults to 1024. + /// + public int BlockSize { get; set; } + + /// + /// gets or sets the number of maximum allowed outgoing edges for each node in the graph in each layer. + /// On layer zero the maximal number of outgoing edges will be 2M. Default is 16. + /// + public int M { get; set; } + + /// + /// Gets or sets the number of maximum allowed potential outgoing edges candidates for each node in the graph, + /// during the graph building. Default is 200. + /// + public int EfConstructor { get; set; } + + /// + /// Gets or sets the number of maximum top candidates to hold during the KNN search. Higher values of + /// EfRuntime lead to more accurate results at the expense of a longer runtime. Default is 10. + /// + public int EfRuntime { get; set; } + + /// + /// Gets or sets Relative factor that sets the boundaries in which a range query may search for candidates. + /// That is, vector candidates whose distance from the query vector is radius*(1 + EPSILON) are potentially + /// scanned, allowing more extensive search and more accurate results (on the expense of runtime). Default is 0.01. + /// + public double Epsilon { get; set; } + + /// + internal override SearchFieldType SearchFieldType => SearchFieldType.VECTOR; + + /// + /// gets the number of arguments that will be produced by this attribute. + /// + internal int NumArgs + { + get + { + var numArgs = 6; + numArgs += InitialCapacity != default ? 2 : 0; + if (Algorithm == VectorAlgorithm.FLAT) + { + numArgs += BlockSize != default ? 2 : 0; + } + + if (Algorithm == VectorAlgorithm.HNSW) + { + numArgs += M != default ? 2 : 0; + numArgs += EfConstructor != default ? 2 : 0; + numArgs += EfRuntime != default ? 2 : 0; + numArgs += Epsilon != default ? 2 : 0; + } + + return numArgs; + } + } + } +} \ No newline at end of file diff --git a/src/Redis.OM/Modeling/Vectors/VectorJsonConverter.cs b/src/Redis.OM/Modeling/Vectors/VectorJsonConverter.cs new file mode 100644 index 00000000..4af8d4d2 --- /dev/null +++ b/src/Redis.OM/Modeling/Vectors/VectorJsonConverter.cs @@ -0,0 +1,103 @@ +using System; +using System.Linq; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Redis.OM.Modeling +{ + /// + /// Converts the provided object to a json vector. + /// + internal class VectorJsonConverter : JsonConverter + { + private readonly VectorizerAttribute _vectorizerAttribute; + + /// + /// Initializes a new instance of the class. + /// + /// the attribute that will be used for vectorization. + internal VectorJsonConverter(VectorizerAttribute attribute) + { + _vectorizerAttribute = attribute; + } + + /// + public override object? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + reader.Read(); + reader.Read(); + var res = JsonSerializer.Deserialize(reader.GetString() !, typeToConvert); + reader.Read(); + reader.Read(); // Vector + reader.Read(); // start array + for (var i = 0; i < _vectorizerAttribute.Dim; i++) + { + reader.Read(); // each item + } + + reader.Read(); // end array + return res; + } + + /// + public override void Write(Utf8JsonWriter writer, object value, JsonSerializerOptions options) + { + writer.WriteStartObject(); + writer.WritePropertyName("Value"); + writer.WriteStringValue(JsonSerializer.Serialize(value)); + var bytes = _vectorizerAttribute.Vectorize(value); + var jagged = SplitIntoJaggedArray(bytes, _vectorizerAttribute.VectorType == VectorType.FLOAT32 ? 4 : 8); + writer.WritePropertyName("Vector"); + if (_vectorizerAttribute.VectorType == VectorType.FLOAT32) + { + var floats = jagged.Select(a => BitConverter.ToSingle(a, 0)).ToArray(); + writer.WriteStartArray(); + foreach (var f in floats) + { + writer.WriteNumberValue(f); + } + + writer.WriteEndArray(); + } + else + { + var doubles = jagged.Select(BitConverter.ToDouble).ToArray(); + writer.WriteStartArray(); + foreach (var d in doubles) + { + writer.WriteNumberValue(d); + } + + writer.WriteEndArray(); + } + + writer.WriteEndObject(); + } + + /// + public override bool CanConvert(Type typeToConvert) => true; + + /// + /// Converts input bytes to Jagged array. + /// + /// the bytes to parse. + /// Size of the jagged arrays. + /// A jagged array of bytes. + /// thrown if the vector is not correctly balanced. + internal static byte[][] SplitIntoJaggedArray(byte[] bytes, int numBytesPerArray) + { + if (bytes.Length % numBytesPerArray != 0) + { + throw new ArgumentException("Unbalanced vector."); + } + + var result = new byte[bytes.Length / numBytesPerArray][]; + for (var i = 0; i < bytes.Length; i += numBytesPerArray) + { + result[i / numBytesPerArray] = bytes.Skip(i).Take(numBytesPerArray).ToArray(); + } + + return result; + } + } +} \ No newline at end of file diff --git a/src/Redis.OM/Modeling/Vectors/VectorType.cs b/src/Redis.OM/Modeling/Vectors/VectorType.cs new file mode 100644 index 00000000..dedf8e71 --- /dev/null +++ b/src/Redis.OM/Modeling/Vectors/VectorType.cs @@ -0,0 +1,42 @@ +using System; + +namespace Redis.OM.Modeling +{ + /// + /// Type of Vector. + /// + public enum VectorType + { + /// + /// Float 32s. + /// + FLOAT32, + + /// + /// Float 64s. + /// + FLOAT64, + } + + /// + /// Extensions for VectorType. + /// + internal static class VectorTypeExtensions + { + /// + /// Gets the Vector type as a Redis usable string. + /// + /// The Vector type. + /// A Redis Usable string. + /// Thrown if illegal value for Vector type is encountered. + internal static string AsRedisString(this VectorType vectorType) + { + return vectorType switch + { + VectorType.FLOAT32 => "FLOAT32", + VectorType.FLOAT64 => "FLOAT64", + _ => throw new ArgumentOutOfRangeException(nameof(vectorType)) + }; + } + } +} \ No newline at end of file diff --git a/src/Redis.OM/Modeling/Vectors/VectorizerAttribute.cs b/src/Redis.OM/Modeling/Vectors/VectorizerAttribute.cs new file mode 100644 index 00000000..3317dc41 --- /dev/null +++ b/src/Redis.OM/Modeling/Vectors/VectorizerAttribute.cs @@ -0,0 +1,38 @@ +using System; +using System.Text.Json.Serialization; + +namespace Redis.OM.Modeling +{ + /// + /// Method for converting a field into a vector. + /// + public abstract class VectorizerAttribute : JsonConverterAttribute + { + /// + /// Gets the vector Type generated by the vectorizer. + /// + public abstract VectorType VectorType { get; } + + /// + /// Gets the vector dimension of the vectors generated by the vectorizer. + /// + public abstract int Dim { get; } + + /// + /// Converts the provided object to a vector. + /// + /// the object to convert. + /// A byte array containing the vectorized data. + public abstract byte[] Vectorize(object obj); + + /// + /// Creates the json converter fulfilled by this attribute. + /// + /// The type to convert. + /// The Json Converter. + public override JsonConverter? CreateConverter(Type typeToConvert) + { + return new VectorJsonConverter(this); + } + } +} \ No newline at end of file diff --git a/src/Redis.OM/RedisObjectHandler.cs b/src/Redis.OM/RedisObjectHandler.cs index 55270a63..687ff77f 100644 --- a/src/Redis.OM/RedisObjectHandler.cs +++ b/src/Redis.OM/RedisObjectHandler.cs @@ -6,6 +6,7 @@ using System.Runtime.CompilerServices; using System.Text; using System.Text.Json; +using System.Web; using Redis.OM.Contracts; using Redis.OM.Modeling; @@ -332,6 +333,17 @@ internal static IDictionary BuildHashSet(this object obj) var propertyName = property.Name; ExtractPropertyName(property, ref propertyName); + if (property.GetCustomAttributes().Any()) + { + var val = property.GetValue(obj); + var vectorizer = property.GetCustomAttributes().First(); + var vector = vectorizer.Vectorize(val); + var vectorStr = "\\x" + string.Join("\\x", vector.Select(x => Convert.ToString(x, 16).PadLeft(2, '0'))); + hash.Add($"{propertyName}.Vector", vectorStr); + hash.Add($"{propertyName}.Value", JsonSerializer.Serialize(val)); + continue; + } + if (type.IsPrimitive || type == typeof(decimal) || type == typeof(string) || type == typeof(GeoLoc) || type == typeof(Ulid) || type == typeof(Guid)) { var val = property.GetValue(obj); @@ -361,6 +373,11 @@ internal static IDictionary BuildHashSet(this object obj) hash.Add(propertyName, new DateTimeOffset(val).ToUnixTimeMilliseconds().ToString()); } } + else if (type.GetInterfaces().Any(x => x.IsGenericType && x.GetGenericTypeDefinition() == typeof(IEnumerable<>)) && property.GetCustomAttributes().Any()) + { + var innerType = GetEnumerableType(property); + hash.Add(propertyName, PrimitiveCollectionToVectorBytes(property, obj, innerType)); + } else if (type.GetInterfaces().Any(x => x.IsGenericType && x.GetGenericTypeDefinition() == typeof(IEnumerable<>))) { IEnumerable e; @@ -424,37 +441,51 @@ private static string SendToJson(IDictionary hash, Type t) var type = Nullable.GetUnderlyingType(property.PropertyType) ?? property.PropertyType; var propertyName = property.Name; ExtractPropertyName(property, ref propertyName); - if (!hash.Any(x => x.Key.StartsWith(propertyName))) + var isVectorized = property.GetCustomAttributes().Any(); + var lookupPropertyName = propertyName + (isVectorized ? ".Value" : string.Empty); + var vectorPropertyName = $"{propertyName}.Vector"; + if (isVectorized && !hash.ContainsKey($"{propertyName}.Value") && !hash.ContainsKey($"{propertyName}.Vector")) + { + continue; + } + + if (isVectorized) + { + ret += $"\"{propertyName}\":{{"; + propertyName = "Value"; + } + + if (!hash.Any(x => x.Key.StartsWith(lookupPropertyName))) { continue; } if (type == typeof(bool) || type == typeof(bool?)) { - if (!hash.ContainsKey(propertyName)) + if (!hash.ContainsKey(lookupPropertyName)) { continue; } - ret += $"\"{propertyName}\":{hash[propertyName].ToLower()},"; + ret += $"\"{propertyName}\":{hash[lookupPropertyName].ToLower()},"; } else if (type.IsPrimitive || type == typeof(decimal) || type.IsEnum) { - if (!hash.ContainsKey(propertyName)) + if (!hash.ContainsKey(lookupPropertyName)) { continue; } - ret += $"\"{propertyName}\":{hash[propertyName]},"; + ret += $"\"{propertyName}\":{hash[lookupPropertyName]},"; } else if (type == typeof(string) || type == typeof(GeoLoc) || type == typeof(DateTime) || type == typeof(DateTime?) || type == typeof(DateTimeOffset) || type == typeof(Guid) || type == typeof(Guid?) || type == typeof(Ulid) || type == typeof(Ulid?)) { - if (!hash.ContainsKey(propertyName)) + if (!hash.ContainsKey(lookupPropertyName)) { continue; } - ret += $"\"{propertyName}\":\"{hash[propertyName]}\","; + ret += $"\"{propertyName}\":\"{HttpUtility.JavaScriptStringEncode(hash[lookupPropertyName])}\","; } else if (type.GetInterfaces().Any(x => x.IsGenericType && x.GetGenericTypeDefinition() == typeof(IEnumerable<>))) { @@ -525,10 +556,30 @@ private static string SendToJson(IDictionary hash, Type t) else if (hash.ContainsKey(propertyName)) { ret += $"\"{propertyName}\":"; - ret += hash[propertyName]; + ret += hash[lookupPropertyName]; ret += ","; } } + + if (isVectorized) + { + var vectorizer = property.GetCustomAttributes().First(); + var encoded = VectorJsonConverter.SplitIntoJaggedArray(Encoding.UTF8.GetBytes(hash[vectorPropertyName]), vectorizer.VectorType == VectorType.FLOAT32 ? 4 : 8); + string arrString; + if (vectorizer.VectorType == VectorType.FLOAT32) + { + var floats = encoded.Select(a => BitConverter.ToSingle(a, 0).ToString(CultureInfo.InvariantCulture)).ToArray(); + arrString = string.Join(",", floats); + } + else + { + var doubles = encoded.Select(a => BitConverter.ToDouble(a, 0).ToString(CultureInfo.InvariantCulture)).ToArray(); + arrString = string.Join(",", doubles); + } + + var valueStr = $"[{arrString}]"; + ret += $"\"Vector\":{valueStr}}}"; + } } ret = ret.TrimEnd(','); @@ -553,6 +604,21 @@ private static Type GetEnumerableType(PropertyInfo pi) return type; } + private static string PrimitiveCollectionToVectorBytes(PropertyInfo pi, object obj, Type type) + { + if (type == typeof(double)) + { + return Encoding.UTF8.GetString(((IEnumerable)pi.GetValue(obj)).SelectMany(BitConverter.GetBytes).ToArray()); + } + + if (type == typeof(float)) + { + return Encoding.UTF8.GetString(((IEnumerable)pi.GetValue(obj)).SelectMany(BitConverter.GetBytes).ToArray()); + } + + throw new ArgumentException("Could not pull a usable type out from property info"); + } + private static IEnumerable PrimitiveCollectionToStrings(PropertyInfo pi, object obj, Type type) { if (type == typeof(bool)) diff --git a/src/Redis.OM/stylecop.ruleset b/src/Redis.OM/stylecop.ruleset index 59ef8844..90642d0c 100644 --- a/src/Redis.OM/stylecop.ruleset +++ b/src/Redis.OM/stylecop.ruleset @@ -18,6 +18,7 @@ + \ No newline at end of file diff --git a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/ObjectWithVector.cs b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/ObjectWithVector.cs new file mode 100644 index 00000000..25ff3fea --- /dev/null +++ b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/ObjectWithVector.cs @@ -0,0 +1,31 @@ +using Redis.OM.Modeling; + +namespace Redis.OM.Unit.Tests; + +[Document(StorageType = StorageType.Json)] +public class ObjectWithVector +{ + [RedisIdField] + public string Id { get; set; } + + [Vector(Algorithm = VectorAlgorithm.HNSW, Dim = 10)] + public double[] SimpleHnswVector { get; set; } + + [Vector(Algorithm = VectorAlgorithm.FLAT)] + [SimpleVectorizer] + public string SimpleVectorizedVector { get; set; } +} + +[Document(StorageType = StorageType.Hash)] +public class ObjectWithVectorHash +{ + [RedisIdField] + public string Id { get; set; } + + [Vector(Algorithm = VectorAlgorithm.HNSW, Dim = 10)] + public double[] SimpleHnswVector { get; set; } + + [Vector(Algorithm = VectorAlgorithm.FLAT)] + [SimpleVectorizer] + public string SimpleVectorizedVector { get; set; } +} \ No newline at end of file diff --git a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/SimpleVectorizer.cs b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/SimpleVectorizer.cs new file mode 100644 index 00000000..e773b7c9 --- /dev/null +++ b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/SimpleVectorizer.cs @@ -0,0 +1,22 @@ +using System; +using System.Linq; +using Redis.OM.Modeling; + +namespace Redis.OM.Unit.Tests; + +public class SimpleVectorizer : VectorizerAttribute +{ + public override VectorType VectorType => VectorType.FLOAT32; + public override int Dim => 30; + + public override byte[] Vectorize(object obj) + { + var floats = new float[30]; + for (var i = 0; i < 30; i++) + { + floats[i] = i; + } + + return floats.SelectMany(BitConverter.GetBytes).ToArray(); + } +} \ No newline at end of file diff --git a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorFunctionalTests.cs b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorFunctionalTests.cs new file mode 100644 index 00000000..96901f2a --- /dev/null +++ b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorFunctionalTests.cs @@ -0,0 +1,53 @@ +using System.Text; +using Redis.OM.Contracts; +using Xunit; + +namespace Redis.OM.Unit.Tests; + +[Collection("Redis")] +public class VectorFunctionalTests +{ + private IRedisConnection _connection = null; + + public VectorFunctionalTests(RedisSetup setup) + { + _connection = setup.Connection; + } + + [Fact] + public void Insert() + { + var simpleHnswJsonStr = new StringBuilder(); + var vectorizedFlatVectorJsonStr = new StringBuilder(); + simpleHnswJsonStr.Append('['); + vectorizedFlatVectorJsonStr.Append('['); + var simpleHnswHash = new double[10]; + var vectorizedFlatHashVector = new float[30]; + for (var i = 0; i < 10; i++) + { + simpleHnswHash[i] = i; + } + for (var i = 0; i < 30; i++) + { + vectorizedFlatHashVector[i] = i; + } + + simpleHnswJsonStr.Append(string.Join(',', simpleHnswHash)); + vectorizedFlatVectorJsonStr.Append(string.Join(',', vectorizedFlatHashVector)); + simpleHnswJsonStr.Append(']'); + vectorizedFlatVectorJsonStr.Append(']'); + + var hashObj = new ObjectWithVectorHash() + { + Id = "foo", + SimpleHnswVector = simpleHnswHash, + SimpleVectorizedVector = "foobar" + }; + + var key = _connection.Set(hashObj); + var res = _connection.Get(key); + Assert.Equal("foobar", res.SimpleVectorizedVector); + } + + +} \ No newline at end of file diff --git a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorTests.cs b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorTests.cs new file mode 100644 index 00000000..d87f114a --- /dev/null +++ b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorTests.cs @@ -0,0 +1,104 @@ +using System; +using System.Linq; +using System.Text; +using System.Text.Json; +using NSubstitute; +using NSubstitute.ClearExtensions; +using Redis.OM.Contracts; +using Xunit; + +namespace Redis.OM.Unit.Tests; + +public class VectorIndexCreationTests +{ + private readonly IRedisConnection _substitute = Substitute.For(); + + [Fact] + public void CreateIndexWithVector() + { + _substitute.ClearSubstitute(); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(new RedisReply("OK")); + + _substitute.CreateIndex(typeof(ObjectWithVector)); + _substitute.CreateIndex(typeof(ObjectWithVectorHash)); + _substitute.Received().Execute( + "FT.CREATE", + $"{nameof(ObjectWithVector).ToLower()}-idx", + "ON", + "Json", + "PREFIX", + "1", + $"Redis.OM.Unit.Tests.{nameof(ObjectWithVector)}:", + "SCHEMA", + "$.SimpleHnswVector", "AS", "SimpleHnswVector", "VECTOR", "HNSW", "6", "TYPE", "FLOAT64", "DIM", "10", "DISTANCE_METRIC", "L2", + "$.SimpleVectorizedVector.Vector", "AS","SimpleVectorizedVector", "VECTOR", "FLAT", "6", "TYPE", "FLOAT32", "DIM", "30", "DISTANCE_METRIC", "L2" + ); + + _substitute.Received().Execute( + "FT.CREATE", + $"{nameof(ObjectWithVectorHash).ToLower()}-idx", + "ON", + "Hash", + "PREFIX", + "1", + $"Redis.OM.Unit.Tests.{nameof(ObjectWithVectorHash)}:", + "SCHEMA", + "SimpleHnswVector", "VECTOR", "HNSW", "6", "TYPE", "FLOAT64", "DIM", "10", "DISTANCE_METRIC", "L2", + "SimpleVectorizedVector.Vector", "VECTOR", "FLAT", "6", "TYPE", "FLOAT32", "DIM", "30", "DISTANCE_METRIC", "L2" + ); + } + + [Fact] + public void InsertVectors() + { + var simpleHnswJsonStr = new StringBuilder(); + var vectorizedFlatVectorJsonStr = new StringBuilder(); + simpleHnswJsonStr.Append('['); + vectorizedFlatVectorJsonStr.Append('['); + var simpleHnswHash = new double[10]; + var vectorizedFlatHashVector = new float[30]; + for (var i = 0; i < 10; i++) + { + simpleHnswHash[i] = i; + } + for (var i = 0; i < 30; i++) + { + vectorizedFlatHashVector[i] = i; + } + + simpleHnswJsonStr.Append(string.Join(',', simpleHnswHash)); + vectorizedFlatVectorJsonStr.Append(string.Join(',', vectorizedFlatHashVector)); + simpleHnswJsonStr.Append(']'); + vectorizedFlatVectorJsonStr.Append(']'); + + var byteStringSimpleHnsw = Encoding.UTF8.GetString(simpleHnswHash.SelectMany(BitConverter.GetBytes).ToArray()); + var byteStringVectorizedFlashHash = Encoding.UTF8.GetString(vectorizedFlatHashVector.SelectMany(BitConverter.GetBytes).ToArray()); + + var hashObj = new ObjectWithVectorHash() + { + Id = "foo", + SimpleHnswVector = simpleHnswHash, + SimpleVectorizedVector = "foobar" + }; + + var jsonObj = new ObjectWithVector() + { + Id = "foo", + SimpleHnswVector = simpleHnswHash, + SimpleVectorizedVector = "foobar" + }; + + var json = + $"{{\"Id\":\"foo\",\"SimpleHnswVector\":{simpleHnswJsonStr},\"SimpleVectorizedVector\":{{\"Value\":\"\\u0022foobar\\u0022\",\"Vector\":{vectorizedFlatVectorJsonStr}}}}}"; + + _substitute.Execute("HSET", Arg.Any()).Returns(new RedisReply("3")); + _substitute.Execute("JSON.SET", Arg.Any()).Returns(new RedisReply("OK")); + _substitute.Set(hashObj); + _substitute.Set(jsonObj); + _substitute.Received().Execute("HSET", "Redis.OM.Unit.Tests.ObjectWithVectorHash:foo", "Id", "foo", "SimpleHnswVector", + byteStringSimpleHnsw, "SimpleVectorizedVector.Vector", byteStringVectorizedFlashHash, "SimpleVectorizedVector.Value", "foobar"); + _substitute.Received().Execute("JSON.SET", "Redis.OM.Unit.Tests.ObjectWithVector:foo", ".", json); + var deseralized = JsonSerializer.Deserialize(json); + Assert.Equal("foobar", deseralized.SimpleVectorizedVector); + } +} \ No newline at end of file From 241645caa5a7fccec84ba71f1aede1615167575f Mon Sep 17 00:00:00 2001 From: slorello89 Date: Fri, 6 Oct 2023 16:09:16 -0400 Subject: [PATCH 02/36] working out vector strings --- src/Redis.OM/Modeling/Vectors/VectorUtils.cs | 162 ++++++++++++++++++ src/Redis.OM/RedisObjectHandler.cs | 12 +- .../VectorTests/VectorFunctionalTests.cs | 3 +- .../VectorTests/VectorTests.cs | 9 + 4 files changed, 177 insertions(+), 9 deletions(-) create mode 100644 src/Redis.OM/Modeling/Vectors/VectorUtils.cs diff --git a/src/Redis.OM/Modeling/Vectors/VectorUtils.cs b/src/Redis.OM/Modeling/Vectors/VectorUtils.cs new file mode 100644 index 00000000..b9165fad --- /dev/null +++ b/src/Redis.OM/Modeling/Vectors/VectorUtils.cs @@ -0,0 +1,162 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Text; + +namespace Redis.OM.Modeling +{ + /// + /// Helper utilities for handling vectors in Redis. + /// + public static class VectorUtils + { + /// + /// Converts array of doubles to a vector string. + /// + /// the doubles. + /// the vector string. + public static string ToVecString(IEnumerable doubles) + { + var bytes = doubles.SelectMany(BitConverter.GetBytes).ToArray(); + return BytesToVecStr(bytes); + } + + /// + /// Converts array of floats to a vector string. + /// + /// the floats. + /// the vector string. + public static string ToVecString(IEnumerable floats) + { + var bytes = floats.SelectMany(BitConverter.GetBytes).ToArray(); + return BytesToVecStr(bytes); + } + + /// + /// Converts the double to a binary safe redis vector string. + /// + /// the double. + /// The binary safe redis vector string. + public static string DoubleToVecStr(double d) + { + return BytesToVecStr(BitConverter.GetBytes(d)); + } + + /// + /// Converts the bytes to a binary safe redis string. + /// + /// the bytes to convert. + /// the binary safe redis String. + public static string BytesToVecStr(byte[] bytes) + { + var sb = new StringBuilder(); + foreach (var b in bytes) + { + switch (b) + { + case 0x5c: + sb.Append("\\\\"); + break; + case > 0x20 and <= 0x7f: + sb.Append((char)b); + break; + default: + sb.Append($"\\x{Convert.ToString(b, 16).PadLeft(2, '0')}"); + break; + } + } + + return sb.ToString(); + } + + /// + /// Converts Vector String to array of doubles. + /// + /// the vector string. + /// the doubles. + /// Thrown if unbalanced. + public static double[] VecStrToDoubles(string str) + { + var bytes = VecStrToBytes(str); + if (bytes.Length % 8 != 0) + { + throw new ArgumentException("Unbalanced Vector String"); + } + + var doubles = new double[bytes.Length / 8]; + for (var i = 0; i < bytes.Length; i += 8) + { + doubles[i / 8] = BitConverter.ToDouble(bytes, i); + } + + return doubles; + } + + /// + /// Parses a vector string to an array of floats. + /// + /// the string. + /// The floats. + /// thrown if unbalanced. + public static float[] VectorStrToFloats(string str) + { + var bytes = VecStrToBytes(str); + if (bytes.Length % 4 != 0) + { + throw new ArgumentException("Unbalanced Vector String"); + } + + var floats = new float[bytes.Length / 4]; + for (var i = 0; i < bytes.Length; i += 4) + { + floats[i / 4] = BitConverter.ToSingle(bytes, i); + } + + return floats; + } + + /// + /// Converts binary safe Redis blob to double. + /// + /// the string. + /// the double the string represents. + public static double DoubleFromVecStr(string str) + { + var bytes = VecStrToBytes(str); + return BitConverter.ToDouble(bytes, 0); + } + + /// + /// Converts the binary safe vector string from Redis to an array of bytes. + /// + /// the string to convert back to bytes. + /// the bytes from the string. + public static byte[] VecStrToBytes(string str) + { + var bytes = new List(); + var i = 0; + while (i < str.Length) + { + if (str[i] == '\\' && i + 1 < str.Length && str[i + 1] == '\\') + { + bytes.Add((byte)'\\'); + i += 2; + } + else if (str[i] == '\\' && i + 3 < str.Length && str[i + 1] == 'x') + { + // byte literal, interpret from hex. + bytes.Add(byte.Parse(str.Substring(i + 2, 2), NumberStyles.HexNumber)); + i += 4; + } + else + { + bytes.Add((byte)str[i]); + i++; + } + } + + return bytes.ToArray(); + } + } +} \ No newline at end of file diff --git a/src/Redis.OM/RedisObjectHandler.cs b/src/Redis.OM/RedisObjectHandler.cs index 687ff77f..82f521c7 100644 --- a/src/Redis.OM/RedisObjectHandler.cs +++ b/src/Redis.OM/RedisObjectHandler.cs @@ -338,8 +338,7 @@ internal static IDictionary BuildHashSet(this object obj) var val = property.GetValue(obj); var vectorizer = property.GetCustomAttributes().First(); var vector = vectorizer.Vectorize(val); - var vectorStr = "\\x" + string.Join("\\x", vector.Select(x => Convert.ToString(x, 16).PadLeft(2, '0'))); - hash.Add($"{propertyName}.Vector", vectorStr); + hash.Add($"{propertyName}.Vector", VectorUtils.BytesToVecStr(vector)); hash.Add($"{propertyName}.Value", JsonSerializer.Serialize(val)); continue; } @@ -564,16 +563,15 @@ private static string SendToJson(IDictionary hash, Type t) if (isVectorized) { var vectorizer = property.GetCustomAttributes().First(); - var encoded = VectorJsonConverter.SplitIntoJaggedArray(Encoding.UTF8.GetBytes(hash[vectorPropertyName]), vectorizer.VectorType == VectorType.FLOAT32 ? 4 : 8); string arrString; if (vectorizer.VectorType == VectorType.FLOAT32) { - var floats = encoded.Select(a => BitConverter.ToSingle(a, 0).ToString(CultureInfo.InvariantCulture)).ToArray(); + var floats = VectorUtils.VectorStrToFloats(hash[vectorPropertyName]); arrString = string.Join(",", floats); } else { - var doubles = encoded.Select(a => BitConverter.ToDouble(a, 0).ToString(CultureInfo.InvariantCulture)).ToArray(); + var doubles = VectorUtils.VecStrToDoubles(hash[vectorPropertyName]); arrString = string.Join(",", doubles); } @@ -608,12 +606,12 @@ private static string PrimitiveCollectionToVectorBytes(PropertyInfo pi, object o { if (type == typeof(double)) { - return Encoding.UTF8.GetString(((IEnumerable)pi.GetValue(obj)).SelectMany(BitConverter.GetBytes).ToArray()); + return VectorUtils.ToVecString((IEnumerable)pi.GetValue(obj)); } if (type == typeof(float)) { - return Encoding.UTF8.GetString(((IEnumerable)pi.GetValue(obj)).SelectMany(BitConverter.GetBytes).ToArray()); + return VectorUtils.ToVecString((IEnumerable)pi.GetValue(obj)); } throw new ArgumentException("Could not pull a usable type out from property info"); diff --git a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorFunctionalTests.cs b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorFunctionalTests.cs index 96901f2a..9f21d8ff 100644 --- a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorFunctionalTests.cs +++ b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorFunctionalTests.cs @@ -27,6 +27,7 @@ public void Insert() { simpleHnswHash[i] = i; } + for (var i = 0; i < 30; i++) { vectorizedFlatHashVector[i] = i; @@ -48,6 +49,4 @@ public void Insert() var res = _connection.Get(key); Assert.Equal("foobar", res.SimpleVectorizedVector); } - - } \ No newline at end of file diff --git a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorTests.cs b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorTests.cs index d87f114a..c02ef6ef 100644 --- a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorTests.cs +++ b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorTests.cs @@ -5,6 +5,7 @@ using NSubstitute; using NSubstitute.ClearExtensions; using Redis.OM.Contracts; +using Redis.OM.Modeling; using Xunit; namespace Redis.OM.Unit.Tests; @@ -48,6 +49,14 @@ public void CreateIndexWithVector() ); } + [Fact] + public void TestBinConversions() + { + var piStr = VectorUtils.DoubleToVecStr(Math.PI); + var pi = VectorUtils.DoubleFromVecStr(piStr); + Assert.Equal(Math.PI, pi); + } + [Fact] public void InsertVectors() { From 9150f41e3ed6ca9d472840caf0a5f8184078fe1d Mon Sep 17 00:00:00 2001 From: slorello89 Date: Tue, 10 Oct 2023 10:01:40 -0400 Subject: [PATCH 03/36] string arrays to object arrays --- src/Redis.OM.POC/RedisCommands.cs | 4 +- src/Redis.OM.POC/RedisConnection.cs | 12 +- src/Redis.OM.POC/RedisHash.cs | 2 +- src/Redis.OM/Contracts/IRedisConnection.cs | 8 +- src/Redis.OM/Contracts/IRedisHydrateable.cs | 2 +- .../Modeling/RedisCollectionStateManager.cs | 10 +- src/Redis.OM/Modeling/Vectors/VectorUtils.cs | 9 +- src/Redis.OM/RedisCommands.cs | 40 +-- src/Redis.OM/RedisConnection.cs | 18 +- src/Redis.OM/RedisObjectHandler.cs | 14 +- .../RediSearchTests/AggregationSetTests.cs | 76 ++--- .../RediSearchTests/SearchTests.cs | 316 +++++++++--------- .../VectorTests/ObjectWithVector.cs | 7 + .../VectorTests/VectorFunctionalTests.cs | 31 ++ .../VectorTests/VectorTests.cs | 14 +- 15 files changed, 304 insertions(+), 259 deletions(-) diff --git a/src/Redis.OM.POC/RedisCommands.cs b/src/Redis.OM.POC/RedisCommands.cs index 10d6cc33..19a608e0 100644 --- a/src/Redis.OM.POC/RedisCommands.cs +++ b/src/Redis.OM.POC/RedisCommands.cs @@ -565,7 +565,7 @@ public static long XAck(this IRedisConnection connection, string streamId, strin public static async Task XAddAsync(this IRedisConnection connection, string streamId, object message, string messageId = "*", int maxLen = -1, string minId = "", bool trimApprox = true, bool makeStream = true) { var kvps = message.BuildHashSet(); - var args = new List { streamId }; + var args = new List { streamId }; if (!makeStream) { args.Add("NOMKSTREAM"); @@ -611,7 +611,7 @@ public static long XAck(this IRedisConnection connection, string streamId, strin public static string? XAdd(this IRedisConnection connection, string streamId, object message, string messageId = "*", int maxLen = -1, string minId = "", bool trimApprox = true, bool makeStream = true) { var kvps = message.BuildHashSet(); - var args = new List { streamId }; + var args = new List { streamId }; if (!makeStream) { args.Add("NOMKSTREAM"); diff --git a/src/Redis.OM.POC/RedisConnection.cs b/src/Redis.OM.POC/RedisConnection.cs index 0a1859bb..6ce3df34 100644 --- a/src/Redis.OM.POC/RedisConnection.cs +++ b/src/Redis.OM.POC/RedisConnection.cs @@ -42,19 +42,19 @@ public RedisList GetList(string listName, uint chunkSize = 100) return new RedisList(this, listName, chunkSize); } - public RedisReply Execute(string command, params string[] args) + public RedisReply Execute(string command, params object[] args) { - var commandBytes = RespHelper.BuildCommand(command, args); + var commandBytes = RespHelper.BuildCommand(command, args.Select(x=>x.ToString()).ToArray()); _socket.Send(commandBytes); return RespHelper.GetNextReplyFromSocket(_socket); } - public async Task ExecuteAsync(string command, params string[] args) + public async Task ExecuteAsync(string command, params object[] args) { await _semaphoreSlim.WaitAsync(); try { - var commandBytes = new ArraySegment(RespHelper.BuildCommand(command, args)); + var commandBytes = new ArraySegment(RespHelper.BuildCommand(command, args.Select(x=>x.ToString()).ToArray())); await _socket.SendAsync(commandBytes, SocketFlags.None); return await RespHelper.GetNextReplyFromSocketAsync(_socket); } @@ -66,7 +66,7 @@ public async Task ExecuteAsync(string command, params string[] args) } /// - public RedisReply[] ExecuteInTransaction(Tuple[] commandArgsTuples) + public RedisReply[] ExecuteInTransaction(Tuple[] commandArgsTuples) { var transaction = _db.CreateTransaction(); var tasks = new List>(); @@ -81,7 +81,7 @@ public RedisReply[] ExecuteInTransaction(Tuple[] commandArgsTu } /// - public async Task ExecuteInTransactionAsync(Tuple[] commandArgsTuples) + public async Task ExecuteInTransactionAsync(Tuple[] commandArgsTuples) { var transaction = _db.CreateTransaction(); var tasks = new List>(); diff --git a/src/Redis.OM.POC/RedisHash.cs b/src/Redis.OM.POC/RedisHash.cs index fed046e4..c4919a6d 100644 --- a/src/Redis.OM.POC/RedisHash.cs +++ b/src/Redis.OM.POC/RedisHash.cs @@ -23,7 +23,7 @@ public RedisHash(IRedisConnection connection, string keyName) public string this[string key] { get => _connection.HMGet(_keyName, key).FirstOrDefault() ?? ""; - set => _connection.HSet(_keyName, new KeyValuePair(key,value)); + set => _connection.HSet(_keyName, new KeyValuePair(key,value)); } public ICollection Keys => new RedisHashScanner(_keyName, this, _connection, false); diff --git a/src/Redis.OM/Contracts/IRedisConnection.cs b/src/Redis.OM/Contracts/IRedisConnection.cs index fa275546..d21b8e39 100644 --- a/src/Redis.OM/Contracts/IRedisConnection.cs +++ b/src/Redis.OM/Contracts/IRedisConnection.cs @@ -14,7 +14,7 @@ public interface IRedisConnection : IDisposable /// The command name. /// The arguments. /// A redis Reply. - RedisReply Execute(string command, params string[] args); + RedisReply Execute(string command, params object[] args); /// /// Executes a command. @@ -22,7 +22,7 @@ public interface IRedisConnection : IDisposable /// The command name. /// The arguments. /// A redis Reply. - Task ExecuteAsync(string command, params string[] args); + Task ExecuteAsync(string command, params object[] args); /// /// Executes the contained commands within the context of a transaction. @@ -30,7 +30,7 @@ public interface IRedisConnection : IDisposable /// each tuple represents a command and /// it's arguments to execute inside a transaction. /// A redis Reply. - Task ExecuteInTransactionAsync(Tuple[] commandArgsTuples); + Task ExecuteInTransactionAsync(Tuple[] commandArgsTuples); /// /// Executes the contained commands within the context of a transaction. @@ -38,6 +38,6 @@ public interface IRedisConnection : IDisposable /// each tuple represents a command and /// it's arguments to execute inside a transaction. /// A redis Reply. - RedisReply[] ExecuteInTransaction(Tuple[] commandArgsTuples); + RedisReply[] ExecuteInTransaction(Tuple[] commandArgsTuples); } } diff --git a/src/Redis.OM/Contracts/IRedisHydrateable.cs b/src/Redis.OM/Contracts/IRedisHydrateable.cs index 7f1e6791..27e6499c 100644 --- a/src/Redis.OM/Contracts/IRedisHydrateable.cs +++ b/src/Redis.OM/Contracts/IRedisHydrateable.cs @@ -17,6 +17,6 @@ public interface IRedisHydrateable /// Converts object to dictionary for Redis. /// /// A dictionary for Redis. - IDictionary BuildHashSet(); + IDictionary BuildHashSet(); } } diff --git a/src/Redis.OM/Modeling/RedisCollectionStateManager.cs b/src/Redis.OM/Modeling/RedisCollectionStateManager.cs index f4ec792e..0a39a135 100644 --- a/src/Redis.OM/Modeling/RedisCollectionStateManager.cs +++ b/src/Redis.OM/Modeling/RedisCollectionStateManager.cs @@ -112,10 +112,11 @@ internal bool TryDetectDifferencesSingle(string key, object value, out IList)Snapshot[key]; + var snapshotHash = (IDictionary)Snapshot[key]; var deletedKeys = snapshotHash.Keys.Except(dataHash.Keys).Select(x => new KeyValuePair(x, string.Empty)); var modifiedKeys = dataHash.Where(x => - !snapshotHash.Keys.Contains(x.Key) || snapshotHash[x.Key] != x.Value); + !snapshotHash.Keys.Contains(x.Key) || snapshotHash[x.Key] != x.Value).Select(x => + new KeyValuePair(x.Key, x.Value.ToString())); differences = new List { new HashDiff(modifiedKeys, deletedKeys.Select(x => x.Key)), @@ -158,10 +159,11 @@ internal IDictionary> DetectDifferences() if (Data.ContainsKey(key)) { var dataHash = Data[key] !.BuildHashSet(); - var snapshotHash = (IDictionary)Snapshot[key]; + var snapshotHash = (IDictionary)Snapshot[key]; var deletedKeys = snapshotHash.Keys.Except(dataHash.Keys).Select(x => new KeyValuePair(x, string.Empty)); var modifiedKeys = dataHash.Where(x => - !snapshotHash.Keys.Contains(x.Key) || snapshotHash[x.Key] != x.Value); + !snapshotHash.Keys.Contains(x.Key) || snapshotHash[x.Key] != x.Value).Select(x => + new KeyValuePair(x.Key, x.Value.ToString())); var diff = new List { new HashDiff(modifiedKeys, deletedKeys.Select(x => x.Key)), diff --git a/src/Redis.OM/Modeling/Vectors/VectorUtils.cs b/src/Redis.OM/Modeling/Vectors/VectorUtils.cs index b9165fad..d60771d5 100644 --- a/src/Redis.OM/Modeling/Vectors/VectorUtils.cs +++ b/src/Redis.OM/Modeling/Vectors/VectorUtils.cs @@ -55,10 +55,13 @@ public static string BytesToVecStr(byte[] bytes) { switch (b) { - case 0x5c: - sb.Append("\\\\"); + case 0x08: + sb.Append("\\b"); break; - case > 0x20 and <= 0x7f: + case 0x22: + sb.Append("\""); + break; + case >= 0x20 and <= 0x7f: sb.Append((char)b); break; default: diff --git a/src/Redis.OM/RedisCommands.cs b/src/Redis.OM/RedisCommands.cs index 3d1700ee..b96cb803 100644 --- a/src/Redis.OM/RedisCommands.cs +++ b/src/Redis.OM/RedisCommands.cs @@ -82,9 +82,9 @@ public static async Task SetAsync(this IRedisConnection connection, obje /// the key. /// the field value pairs to set. /// How many new fields were created. - public static async Task HSetAsync(this IRedisConnection connection, string key, params KeyValuePair[] fieldValues) + public static async Task HSetAsync(this IRedisConnection connection, string key, params KeyValuePair[] fieldValues) { - var args = new List { key }; + var args = new List { key }; foreach (var kvp in fieldValues) { args.Add(kvp.Key); @@ -102,9 +102,9 @@ public static async Task HSetAsync(this IRedisConnection connection, string /// the the timespan to set for your (TTL). /// the field value pairs to set. /// How many new fields were created. - public static async Task HSetAsync(this IRedisConnection connection, string key, TimeSpan timeSpan, params KeyValuePair[] fieldValues) + public static async Task HSetAsync(this IRedisConnection connection, string key, TimeSpan timeSpan, params KeyValuePair[] fieldValues) { - var args = new List { key }; + var args = new List { key }; foreach (var kvp in fieldValues) { args.Add(kvp.Key); @@ -224,9 +224,9 @@ public static async Task JsonSetAsync(this IRedisConnection connection, st /// the the timespan to set for your (TTL). /// the field value pairs to set. /// How many new fields were created. - public static int HSet(this IRedisConnection connection, string key, TimeSpan timeSpan, params KeyValuePair[] fieldValues) + public static int HSet(this IRedisConnection connection, string key, TimeSpan timeSpan, params KeyValuePair[] fieldValues) { - var args = new List(); + var args = new List(); args.Add(key); foreach (var kvp in fieldValues) { @@ -244,9 +244,9 @@ public static int HSet(this IRedisConnection connection, string key, TimeSpan ti /// the key. /// the field value pairs to set. /// How many new fields were created. - public static int HSet(this IRedisConnection connection, string key, params KeyValuePair[] fieldValues) + public static int HSet(this IRedisConnection connection, string key, params KeyValuePair[] fieldValues) { - var args = new List { key }; + var args = new List { key }; foreach (var kvp in fieldValues) { args.Add(kvp.Key); @@ -413,7 +413,7 @@ public static string Set(this IRedisConnection connection, object obj) } var kvps = obj.BuildHashSet(); - var argsList = new List(); + var argsList = new List(); int? res = null; argsList.Add(timespan != null ? ((long)timespan.Value.TotalMilliseconds).ToString() : "-1"); foreach (var kvp in kvps) @@ -464,7 +464,7 @@ public static string Set(this IRedisConnection connection, object obj) } var kvps = obj.BuildHashSet(); - var argsList = new List(); + var argsList = new List(); int? res = null; argsList.Add(timespan != null ? ((long)timespan.Value.TotalMilliseconds).ToString() : "-1"); foreach (var kvp in kvps) @@ -638,7 +638,7 @@ public static async Task> HGetAllAsync(this IRedisCo /// the full script. /// the result. /// Thrown if the script cannot be resolved either the script is empty or the script name has not been encountered. - public static async Task CreateAndEvalAsync(this IRedisConnection connection, string scriptName, string[] keys, string[] argv, string fullScript = "") + public static async Task CreateAndEvalAsync(this IRedisConnection connection, string scriptName, string[] keys, object[] argv, string fullScript = "") { string sha; if (!Scripts.ShaCollection.TryGetValue(scriptName, out sha)) @@ -659,7 +659,7 @@ public static async Task> HGetAllAsync(this IRedisCo Scripts.ShaCollection[scriptName] = sha; } - var args = new List + var args = new List { sha, keys.Count().ToString(), @@ -691,7 +691,7 @@ public static async Task> HGetAllAsync(this IRedisCo /// the full script. /// the result. /// Thrown if the script cannot be resolved either the script is empty or the script name has not been encountered. - public static int? CreateAndEval(this IRedisConnection connection, string scriptName, string[] keys, string[] argv, string fullScript = "") + public static int? CreateAndEval(this IRedisConnection connection, string scriptName, string[] keys, object[] argv, string fullScript = "") { if (!Scripts.ShaCollection.ContainsKey(scriptName)) { @@ -712,7 +712,7 @@ public static async Task> HGetAllAsync(this IRedisCo Scripts.ShaCollection[scriptName] = sha; } - var args = new List + var args = new List { Scripts.ShaCollection[scriptName], keys.Count().ToString(), @@ -784,7 +784,7 @@ internal static void UnlinkAndSet(this IRedisConnection connection, string ke else { var hash = value.BuildHashSet(); - var args = new List((hash.Keys.Count * 2) + 1); + var args = new List((hash.Keys.Count * 2) + 1); args.Add(hash.Keys.Count.ToString()); foreach (var pair in hash) { @@ -815,7 +815,7 @@ internal static async Task UnlinkAndSetAsync(this IRedisConnection connection else { var hash = value.BuildHashSet(); - var args = new List((hash.Keys.Count * 2) + 1); + var args = new List((hash.Keys.Count * 2) + 1); args.Add(hash.Keys.Count.ToString()); foreach (var pair in hash) { @@ -830,24 +830,24 @@ internal static async Task UnlinkAndSetAsync(this IRedisConnection connection private static RedisReply[] SendCommandWithExpiry( this IRedisConnection connection, string command, - string[] args, + object[] args, string keyToExpire, TimeSpan ts) { var commandTuple = Tuple.Create(command, args); - var expireTuple = Tuple.Create("PEXPIRE", new[] { keyToExpire, ((long)ts.TotalMilliseconds).ToString(CultureInfo.InvariantCulture) }); + var expireTuple = Tuple.Create("PEXPIRE", new object[] { keyToExpire, ((long)ts.TotalMilliseconds).ToString(CultureInfo.InvariantCulture) }); return connection.ExecuteInTransaction(new[] { commandTuple, expireTuple }); } private static Task SendCommandWithExpiryAsync( this IRedisConnection connection, string command, - string[] args, + object[] args, string keyToExpire, TimeSpan ts) { var commandTuple = Tuple.Create(command, args); - var expireTuple = Tuple.Create("PEXPIRE", new[] { keyToExpire, ((long)ts.TotalMilliseconds).ToString(CultureInfo.InvariantCulture) }); + var expireTuple = Tuple.Create("PEXPIRE", new object[] { keyToExpire, ((long)ts.TotalMilliseconds).ToString(CultureInfo.InvariantCulture) }); return connection.ExecuteInTransactionAsync(new[] { commandTuple, expireTuple }); } } diff --git a/src/Redis.OM/RedisConnection.cs b/src/Redis.OM/RedisConnection.cs index 997f4ea9..5a36b7c3 100644 --- a/src/Redis.OM/RedisConnection.cs +++ b/src/Redis.OM/RedisConnection.cs @@ -24,7 +24,7 @@ internal RedisConnection(IDatabase db) } /// - public RedisReply Execute(string command, params string[] args) + public RedisReply Execute(string command, params object[] args) { try { @@ -38,11 +38,11 @@ public RedisReply Execute(string command, params string[] args) } /// - public async Task ExecuteAsync(string command, params string[] args) + public async Task ExecuteAsync(string command, params object[] args) { try { - var result = await _db.ExecuteAsync(command, args); + var result = await _db.ExecuteAsync(command, args).ConfigureAwait(false); return new RedisReply(result); } catch (Exception ex) @@ -52,7 +52,7 @@ public async Task ExecuteAsync(string command, params string[] args) } /// - public RedisReply[] ExecuteInTransaction(Tuple[] commandArgsTuples) + public async Task ExecuteInTransactionAsync(Tuple[] commandArgsTuples) { var transaction = _db.CreateTransaction(); var tasks = new List>(); @@ -61,13 +61,13 @@ public RedisReply[] ExecuteInTransaction(Tuple[] commandArgsTu tasks.Add(transaction.ExecuteAsync(tuple.Item1, tuple.Item2)); } - transaction.Execute(); - Task.WhenAll(tasks).Wait(); + await transaction.ExecuteAsync().ConfigureAwait(false); + await Task.WhenAll(tasks).ConfigureAwait(false); return tasks.Select(x => new RedisReply(x.Result)).ToArray(); } /// - public async Task ExecuteInTransactionAsync(Tuple[] commandArgsTuples) + public RedisReply[] ExecuteInTransaction(Tuple[] commandArgsTuples) { var transaction = _db.CreateTransaction(); var tasks = new List>(); @@ -76,8 +76,8 @@ public async Task ExecuteInTransactionAsync(Tuple new RedisReply(x.Result)).ToArray(); } diff --git a/src/Redis.OM/RedisObjectHandler.cs b/src/Redis.OM/RedisObjectHandler.cs index 82f521c7..3628a2f9 100644 --- a/src/Redis.OM/RedisObjectHandler.cs +++ b/src/Redis.OM/RedisObjectHandler.cs @@ -315,18 +315,18 @@ internal static T ToObject(this RedisReply val) /// /// object to be turned into a hash set. /// A hash set generated from the object. - internal static IDictionary BuildHashSet(this object obj) + internal static IDictionary BuildHashSet(this object obj) { if (obj is IRedisHydrateable hydrateable) { - return hydrateable.BuildHashSet(); + return hydrateable.BuildHashSet().ToDictionary(x => x.Key, x => (object)x.Value); } var properties = obj .GetType() .GetProperties() .Where(x => x.GetValue(obj) != null); - var hash = new Dictionary(); + var hash = new Dictionary(); foreach (var property in properties) { var type = Nullable.GetUnderlyingType(property.PropertyType) ?? property.PropertyType; @@ -338,7 +338,7 @@ internal static IDictionary BuildHashSet(this object obj) var val = property.GetValue(obj); var vectorizer = property.GetCustomAttributes().First(); var vector = vectorizer.Vectorize(val); - hash.Add($"{propertyName}.Vector", VectorUtils.BytesToVecStr(vector)); + hash.Add($"{propertyName}.Vector", vector); hash.Add($"{propertyName}.Value", JsonSerializer.Serialize(val)); continue; } @@ -602,16 +602,16 @@ private static Type GetEnumerableType(PropertyInfo pi) return type; } - private static string PrimitiveCollectionToVectorBytes(PropertyInfo pi, object obj, Type type) + private static byte[] PrimitiveCollectionToVectorBytes(PropertyInfo pi, object obj, Type type) { if (type == typeof(double)) { - return VectorUtils.ToVecString((IEnumerable)pi.GetValue(obj)); + return ((IEnumerable)pi.GetValue(obj)).SelectMany(BitConverter.GetBytes).ToArray(); } if (type == typeof(float)) { - return VectorUtils.ToVecString((IEnumerable)pi.GetValue(obj)); + return ((IEnumerable)pi.GetValue(obj)).SelectMany(BitConverter.GetBytes).ToArray(); } throw new ArgumentException("Could not pull a usable type out from property info"); diff --git a/test/Redis.OM.Unit.Tests/RediSearchTests/AggregationSetTests.cs b/test/Redis.OM.Unit.Tests/RediSearchTests/AggregationSetTests.cs index 55e25b25..26fe2379 100644 --- a/test/Redis.OM.Unit.Tests/RediSearchTests/AggregationSetTests.cs +++ b/test/Redis.OM.Unit.Tests/RediSearchTests/AggregationSetTests.cs @@ -349,8 +349,8 @@ public void TestConfigurableChunkSize() public void TestLoad() { var collection = new RedisAggregationSet(_substitute, true, chunkSize: 10000); - _substitute.Execute("FT.AGGREGATE", Arg.Any()).Returns(MockedResult); - _substitute.Execute("FT.CURSOR", Arg.Any()).Returns(MockedResultCursorEnd); + _substitute.Execute("FT.AGGREGATE", Arg.Any()).Returns(MockedResult); + _substitute.Execute("FT.CURSOR", Arg.Any()).Returns(MockedResultCursorEnd); _ = collection.Load(x => x.RecordShell.Name).ToList(); _substitute.Received().Execute("FT.AGGREGATE","person-idx","*","LOAD","1","Name","WITHCURSOR", "COUNT","10000"); } @@ -359,8 +359,8 @@ public void TestLoad() public void TestMultiVariant() { var collection = new RedisAggregationSet(_substitute, true, chunkSize: 10000); - _substitute.Execute("FT.AGGREGATE", Arg.Any()).Returns(MockedResult); - _substitute.Execute("FT.CURSOR", Arg.Any()).Returns(MockedResultCursorEnd); + _substitute.Execute("FT.AGGREGATE", Arg.Any()).Returns(MockedResult); + _substitute.Execute("FT.CURSOR", Arg.Any()).Returns(MockedResultCursorEnd); _ = collection.Load(x => new {x.RecordShell.Name, x.RecordShell.Age}).ToList(); _substitute.Received().Execute("FT.AGGREGATE","person-idx","*","LOAD","2","Name", "Age","WITHCURSOR", "COUNT","10000"); } @@ -369,8 +369,8 @@ public void TestMultiVariant() public void TestLoadAll() { var collection = new RedisAggregationSet(_substitute, true, chunkSize: 10000); - _substitute.Execute("FT.AGGREGATE", Arg.Any()).Returns(MockedResult); - _substitute.Execute("FT.CURSOR", Arg.Any()).Returns(MockedResultCursorEnd); + _substitute.Execute("FT.AGGREGATE", Arg.Any()).Returns(MockedResult); + _substitute.Execute("FT.CURSOR", Arg.Any()).Returns(MockedResultCursorEnd); _ = collection.LoadAll().ToList(); _substitute.Received().Execute("FT.AGGREGATE", "person-idx", "*", "LOAD", "*", "WITHCURSOR", "COUNT", "10000"); } @@ -379,8 +379,8 @@ public void TestLoadAll() public void TestMultipleOrderBys() { var collection = new RedisAggregationSet(_substitute, true, chunkSize: 10000); - _substitute.Execute("FT.AGGREGATE", Arg.Any()).Returns(MockedResult); - _substitute.Execute("FT.CURSOR", Arg.Any()).Returns(MockedResultCursorEnd); + _substitute.Execute("FT.AGGREGATE", Arg.Any()).Returns(MockedResult); + _substitute.Execute("FT.CURSOR", Arg.Any()).Returns(MockedResultCursorEnd); _ = collection.OrderBy(x => x.RecordShell.Name).OrderByDescending(x => x.RecordShell.Age).ToList(); _substitute.Received().Execute("FT.AGGREGATE","person-idx", "*", "SORTBY", "4", "@Name", "ASC", "@Age", "DESC", "WITHCURSOR", "COUNT", "10000"); } @@ -389,8 +389,8 @@ public void TestMultipleOrderBys() public void TestRightSideStringTypeFilter() { var collection = new RedisAggregationSet(_substitute, true, chunkSize: 10000); - _substitute.Execute("FT.AGGREGATE", Arg.Any()).Returns(MockedResult); - _substitute.Execute("FT.CURSOR", Arg.Any()).Returns(MockedResultCursorEnd); + _substitute.Execute("FT.AGGREGATE", Arg.Any()).Returns(MockedResult); + _substitute.Execute("FT.CURSOR", Arg.Any()).Returns(MockedResultCursorEnd); _ = collection.Apply(x => string.Format("{0} {1}", x.RecordShell.FirstName, x.RecordShell.LastName), "FullName").Filter(p => p.Aggregations["FullName"] == "Bruce Wayne").ToList(); _substitute.Received().Execute("FT.AGGREGATE", "person-idx", "*", "APPLY", "format(\"%s %s\",@FirstName,@LastName)", "AS", "FullName", "FILTER", "@FullName == 'Bruce Wayne'", "WITHCURSOR", "COUNT", "10000"); @@ -400,8 +400,8 @@ public void TestRightSideStringTypeFilter() public void TestNestedOrderBy() { var collection = new RedisAggregationSet(_substitute, true, chunkSize: 10000); - _substitute.Execute("FT.AGGREGATE", Arg.Any()).Returns(MockedResult); - _substitute.Execute("FT.CURSOR", Arg.Any()).Returns(MockedResultCursorEnd); + _substitute.Execute("FT.AGGREGATE", Arg.Any()).Returns(MockedResult); + _substitute.Execute("FT.CURSOR", Arg.Any()).Returns(MockedResultCursorEnd); _ = collection.OrderBy(x => x.RecordShell.Address.State).ToList(); _substitute.Received().Execute("FT.AGGREGATE","person-idx", "*", "SORTBY", "2", "@Address_State", "ASC", "WITHCURSOR", "COUNT", "10000"); } @@ -410,8 +410,8 @@ public void TestNestedOrderBy() public void TestNestedGroup() { var collection = new RedisAggregationSet(_substitute, true, chunkSize: 10000); - _substitute.Execute("FT.AGGREGATE", Arg.Any()).Returns(MockedResult); - _substitute.Execute("FT.CURSOR", Arg.Any()).Returns(MockedResultCursorEnd); + _substitute.Execute("FT.AGGREGATE", Arg.Any()).Returns(MockedResult); + _substitute.Execute("FT.CURSOR", Arg.Any()).Returns(MockedResultCursorEnd); _ = collection.GroupBy(x => x.RecordShell.Address.State).ToList(); _substitute.Received().Execute("FT.AGGREGATE","person-idx", "*", "GROUPBY", "1", "@Address_State", "WITHCURSOR", "COUNT", "10000"); } @@ -420,8 +420,8 @@ public void TestNestedGroup() public void TestNestedGroupMulti() { var collection = new RedisAggregationSet(_substitute, true, chunkSize: 10000); - _substitute.Execute("FT.AGGREGATE", Arg.Any()).Returns(MockedResult); - _substitute.Execute("FT.CURSOR", Arg.Any()).Returns(MockedResultCursorEnd); + _substitute.Execute("FT.AGGREGATE", Arg.Any()).Returns(MockedResult); + _substitute.Execute("FT.CURSOR", Arg.Any()).Returns(MockedResultCursorEnd); _ = collection.GroupBy(x => new {x.RecordShell.Address.State, x.RecordShell.Address.ForwardingAddress.City}).ToList(); _substitute.Received().Execute("FT.AGGREGATE","person-idx", "*", "GROUPBY", "2", "@Address_State", "@Address_ForwardingAddress_City", "WITHCURSOR", "COUNT", "10000"); } @@ -430,8 +430,8 @@ public void TestNestedGroupMulti() public void TestNestedApply() { var collection = new RedisAggregationSet(_substitute, true, chunkSize: 10000); - _substitute.Execute("FT.AGGREGATE", Arg.Any()).Returns(MockedResult); - _substitute.Execute("FT.CURSOR", Arg.Any()).Returns(MockedResultCursorEnd); + _substitute.Execute("FT.AGGREGATE", Arg.Any()).Returns(MockedResult); + _substitute.Execute("FT.CURSOR", Arg.Any()).Returns(MockedResultCursorEnd); _ = collection.Apply(x => x.RecordShell.Address.HouseNumber + 4, "house_num_modified").ToList(); _substitute.Received().Execute("FT.AGGREGATE","person-idx", "*", "APPLY", "@Address_HouseNumber + 4", "AS", "house_num_modified", "WITHCURSOR", "COUNT", "10000"); } @@ -440,8 +440,8 @@ public void TestNestedApply() public void TestMissedBinExpression() { var collection = new RedisAggregationSet(_substitute, true, chunkSize: 10000); - _substitute.Execute("FT.AGGREGATE", Arg.Any()).Returns(MockedResult); - _substitute.Execute("FT.CURSOR", Arg.Any()).Returns(MockedResultCursorEnd); + _substitute.Execute("FT.AGGREGATE", Arg.Any()).Returns(MockedResult); + _substitute.Execute("FT.CURSOR", Arg.Any()).Returns(MockedResultCursorEnd); _ = collection.Apply(x => x.RecordShell.Address.HouseNumber + 4, "house_num_modified") .Apply(x=>x.RecordShell.Age + x["house_num_modified"] * 4 + x.RecordShell.Sales, "arbitrary_calculation").ToList(); _substitute.Received().Execute("FT.AGGREGATE","person-idx", "*", "APPLY", "@Address_HouseNumber + 4", "AS", "house_num_modified", "APPLY", "@Age + @house_num_modified * 4 + @Sales", "AS", "arbitrary_calculation", "WITHCURSOR", "COUNT", "10000"); @@ -456,8 +456,8 @@ public void TestWhereByComplexObjectOnTheRightSide() LastName = "Bond" }; var collection = new RedisAggregationSet(_substitute, true, chunkSize: 10000); - _substitute.Execute("FT.AGGREGATE", Arg.Any()).Returns(MockedResult); - _substitute.Execute("FT.CURSOR", Arg.Any()).Returns(MockedResultCursorEnd); + _substitute.Execute("FT.AGGREGATE", Arg.Any()).Returns(MockedResult); + _substitute.Execute("FT.CURSOR", Arg.Any()).Returns(MockedResultCursorEnd); _ = collection.Where(x =>x.RecordShell.FirstName==customerFilter.FirstName) .ToList(); _substitute.Received().Execute("FT.AGGREGATE", "person-idx", "@FirstName:{James}", "WITHCURSOR", "COUNT", "10000"); } @@ -471,8 +471,8 @@ public void TestSequentialWhereClauseTranslation() LastName = "Bond" }; var collection = new RedisAggregationSet(_substitute, true, chunkSize: 10000); - _substitute.Execute("FT.AGGREGATE", Arg.Any()).Returns(MockedResult); - _substitute.Execute("FT.CURSOR", Arg.Any()).Returns(MockedResultCursorEnd); + _substitute.Execute("FT.AGGREGATE", Arg.Any()).Returns(MockedResult); + _substitute.Execute("FT.CURSOR", Arg.Any()).Returns(MockedResultCursorEnd); _ = collection.Where(x => x.RecordShell.FirstName == customerFilter.FirstName).Where(p=>p.RecordShell.LastName==customerFilter.LastName).ToList(); _substitute.Received().Execute("FT.AGGREGATE", "person-idx", "@LastName:{Bond} @FirstName:{James}", "WITHCURSOR", "COUNT", "10000"); } @@ -480,8 +480,8 @@ public void TestSequentialWhereClauseTranslation() public void TestSkipTakeTranslatedLimit() { var collection = new RedisAggregationSet(_substitute, true, chunkSize: 10000); - _substitute.Execute("FT.AGGREGATE", Arg.Any()).Returns(MockedResult); - _substitute.Execute("FT.CURSOR", Arg.Any()).Returns(MockedResultCursorEnd); + _substitute.Execute("FT.AGGREGATE", Arg.Any()).Returns(MockedResult); + _substitute.Execute("FT.CURSOR", Arg.Any()).Returns(MockedResultCursorEnd); _ = collection.OrderByDescending(p=>p.RecordShell.Age).Skip(0).Take(10).ToList(); _substitute.Received().Execute("FT.AGGREGATE", "person-idx","*","SORTBY","2","@Age","DESC","LIMIT","0","10", "WITHCURSOR", "COUNT", "10000"); } @@ -490,8 +490,8 @@ public void TestSkipTakeTranslatedLimit() public void RightBinExpressionOperator() { var collection = new RedisAggregationSet(_substitute, true, chunkSize: 10000); - _substitute.Execute("FT.AGGREGATE", Arg.Any()).Returns(MockedResult); - _substitute.Execute("FT.CURSOR", Arg.Any()).Returns(MockedResultCursorEnd); + _substitute.Execute("FT.AGGREGATE", Arg.Any()).Returns(MockedResult); + _substitute.Execute("FT.CURSOR", Arg.Any()).Returns(MockedResultCursorEnd); Expression, bool>> query = a => a.RecordShell!.Age == 0 && (a.RecordShell!.Age == 2 || a.RecordShell!.Age == 50); _ = collection.Where(query).ToList(); @@ -502,8 +502,8 @@ public void RightBinExpressionOperator() public void RightBinExpressionWithUniaryOperator() { var collection = new RedisAggregationSet(_substitute, true, chunkSize: 10000); - _substitute.Execute("FT.AGGREGATE", Arg.Any()).Returns(MockedResult); - _substitute.Execute("FT.CURSOR", Arg.Any()).Returns(MockedResultCursorEnd); + _substitute.Execute("FT.AGGREGATE", Arg.Any()).Returns(MockedResult); + _substitute.Execute("FT.CURSOR", Arg.Any()).Returns(MockedResultCursorEnd); Expression, bool>> query = a => a.RecordShell!.Name.Contains("Steve") && (a.RecordShell!.Age == 2 || a.RecordShell!.Age == 50); _ = collection.Where(query).ToList(); @@ -514,8 +514,8 @@ public void RightBinExpressionWithUniaryOperator() public void LeftBinExpressionWithUniaryOperator() { var collection = new RedisAggregationSet(_substitute, true, chunkSize: 10000); - _substitute.Execute("FT.AGGREGATE", Arg.Any()).Returns(MockedResult); - _substitute.Execute("FT.CURSOR", Arg.Any()).Returns(MockedResultCursorEnd); + _substitute.Execute("FT.AGGREGATE", Arg.Any()).Returns(MockedResult); + _substitute.Execute("FT.CURSOR", Arg.Any()).Returns(MockedResultCursorEnd); Expression, bool>> query = a => (a.RecordShell!.Age == 2 || a.RecordShell!.Age == 50) && a.RecordShell!.Name.Contains("Steve"); _ = collection.Where(query).ToList(); @@ -531,8 +531,8 @@ public void PunctuationMarkInTagQuery() LastName = "White" }; var collection = new RedisAggregationSet(_substitute, true, chunkSize: 10000); - _substitute.Execute("FT.AGGREGATE", Arg.Any()).Returns(MockedResult); - _substitute.Execute("FT.CURSOR", Arg.Any()).Returns(MockedResultCursorEnd); + _substitute.Execute("FT.AGGREGATE", Arg.Any()).Returns(MockedResult); + _substitute.Execute("FT.CURSOR", Arg.Any()).Returns(MockedResultCursorEnd); Expression, bool>> query = a => a.RecordShell.FirstName == customerFilter.FirstName; _ = collection.Where(query).ToList(); _substitute.Received().Execute("FT.AGGREGATE", "person-idx", "@FirstName:{Walter\\-Junior}", "WITHCURSOR", "COUNT", "10000"); @@ -542,8 +542,8 @@ public void PunctuationMarkInTagQuery() public void CustomPropertyNamesInQuery() { //Arrange - _substitute.Execute("FT.AGGREGATE", Arg.Any()).Returns(MockedResult); - _substitute.Execute("FT.CURSOR", Arg.Any()).Returns(MockedResultCursorEnd); + _substitute.Execute("FT.AGGREGATE", Arg.Any()).Returns(MockedResult); + _substitute.Execute("FT.CURSOR", Arg.Any()).Returns(MockedResultCursorEnd); var collection = new RedisAggregationSet(_substitute, true, 10); @@ -568,8 +568,8 @@ public void DateTimeQuery() var dto = DateTimeOffset.Now.Subtract(TimeSpan.FromHours(3)); var dtoMs = dto.ToUnixTimeMilliseconds(); var collection = new RedisAggregationSet(_substitute, true, chunkSize: 10000); - _substitute.Execute("FT.AGGREGATE", Arg.Any()).Returns(MockedResult); - _substitute.Execute("FT.CURSOR", Arg.Any()).Returns(MockedResultCursorEnd); + _substitute.Execute("FT.AGGREGATE", Arg.Any()).Returns(MockedResult); + _substitute.Execute("FT.CURSOR", Arg.Any()).Returns(MockedResultCursorEnd); Expression, bool>> query = a => a.RecordShell.Timestamp > dt; _ = collection.Where(query).ToList(); diff --git a/test/Redis.OM.Unit.Tests/RediSearchTests/SearchTests.cs b/test/Redis.OM.Unit.Tests/RediSearchTests/SearchTests.cs index 66522715..a8ae6df7 100644 --- a/test/Redis.OM.Unit.Tests/RediSearchTests/SearchTests.cs +++ b/test/Redis.OM.Unit.Tests/RediSearchTests/SearchTests.cs @@ -50,7 +50,7 @@ public class SearchTests public void TestBasicQuery() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); _ = collection.Where(x => x.Age < 33).ToList(); @@ -66,7 +66,7 @@ public void TestBasicQuery() public void TestBasicNegationQuery() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); _ = collection.Where(x => !(x.Age < 33)).ToList(); _substitute.Received().Execute( @@ -82,7 +82,7 @@ public void TestBasicNegationQuery() public void TestBasicQueryWithVariable() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var y = 33; var collection = new RedisCollection(_substitute); _ = collection.Where(x => x.Age < y).ToList(); @@ -99,7 +99,7 @@ public void TestBasicQueryWithVariable() public void TestFirstOrDefaultWithMixedLocals() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var heightList = new List { 70.0, 68.0 }; var y = 33; foreach (var height in heightList) @@ -120,7 +120,7 @@ public void TestFirstOrDefaultWithMixedLocals() public void TestBasicQueryWithExactNumericMatch() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var y = 33; var collection = new RedisCollection(_substitute); _ = collection.Where(x => x.Age == y).ToList(); @@ -137,7 +137,7 @@ public void TestBasicQueryWithExactNumericMatch() public void TestBasicFirstOrDefaultQuery() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var y = 33; var collection = new RedisCollection(_substitute); _ = collection.FirstOrDefault(x => x.Age == y); @@ -154,7 +154,7 @@ public void TestBasicFirstOrDefaultQuery() public void TestBasicQueryNoNameIndex() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var y = 33; var collection = new RedisCollection(_substitute); _ = collection.FirstOrDefault(x => x.Age == y); @@ -171,7 +171,7 @@ public void TestBasicQueryNoNameIndex() public void TestBasicOrQuery() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); _ = collection.Where(x => x.Age < 33 || x.TagField == "Steve").ToList(); @@ -188,7 +188,7 @@ public void TestBasicOrQuery() public void TestBasicOrQueryTwoTags() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); _ = collection.Where(x => x.TagField == "Bob" || x.TagField == "Steve").ToList(); @@ -205,7 +205,7 @@ public void TestBasicOrQueryTwoTags() public void TestBasicOrQueryWithNegation() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); _ = collection.Where(x => x.Age < 33 || x.TagField != "Steve" || x.Name != "Steve").ToList(); @@ -222,7 +222,7 @@ public void TestBasicOrQueryWithNegation() public void TestBasicAndQuery() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); _ = collection.Where(x => x.Age < 33 && x.TagField == "Steve").ToList(); @@ -239,7 +239,7 @@ public void TestBasicAndQuery() public void TestBasicTagQuery() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); _ = collection.Where(x => x.Age < 33 && x.TagField == "Steve").ToList(); @@ -256,7 +256,7 @@ public void TestBasicTagQuery() public void TestBasicThreeClauseQuery() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); _ = collection.Where(x => x.Age < 33 && x.TagField == "Steve" && x.Height >= 70).ToList(); @@ -273,7 +273,7 @@ public void TestBasicThreeClauseQuery() public void TestGroupedThreeClauseQuery() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); _ = collection.Where(x => x.Age < 33 && (x.TagField == "Steve" || x.Height >= 70)).ToList(); @@ -290,7 +290,7 @@ public void TestGroupedThreeClauseQuery() public void TestBasicQueryWithContains() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); _ = collection.Where(x => x.Name.Contains("Ste")).ToList(); @@ -307,7 +307,7 @@ public void TestBasicQueryWithContains() public void TestBasicQueryWithStartsWith() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); _ = collection.Where(x => x.Name.StartsWith("Ste")).ToList(); @@ -324,7 +324,7 @@ public void TestBasicQueryWithStartsWith() public void TestFuzzy() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); _ = collection.Where(x => x.Name.FuzzyMatch("Ste", 2)).ToList(); @@ -341,7 +341,7 @@ public void TestFuzzy() public void TestMatchStartsWith() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); _ = collection.Where(x => x.Name.MatchStartsWith("Ste")).ToList(); @@ -358,7 +358,7 @@ public void TestMatchStartsWith() public void TestMatchEndsWith() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); _ = collection.Where(x => x.Name.MatchEndsWith("Ste")).ToList(); @@ -375,7 +375,7 @@ public void TestMatchEndsWith() public void TestMatchContains() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); _ = collection.Where(x => x.Name.MatchContains("Ste")).ToList(); @@ -392,7 +392,7 @@ public void TestMatchContains() public void TestTagStartsWith() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); _ = collection.Where(x => x.TagField.StartsWith("Ste")).ToList(); @@ -409,7 +409,7 @@ public void TestTagStartsWith() public void TestTagEndsWith() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); _ = collection.Where(x => x.TagField.EndsWith("Ste")).ToList(); @@ -426,7 +426,7 @@ public void TestTagEndsWith() public void TestTextEndsWith() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); _ = collection.Where(x => x.Name.EndsWith("Ste")).ToList(); @@ -443,7 +443,7 @@ public void TestTextEndsWith() public void TestTextStartsWith() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); _ = collection.Where(x => x.Name.StartsWith("Ste")).ToList(); @@ -460,7 +460,7 @@ public void TestTextStartsWith() public void TestBasicQueryWithEndsWith() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); _ = collection.Where(x => x.Name.EndsWith("Ste")).ToList(); @@ -477,7 +477,7 @@ public void TestBasicQueryWithEndsWith() public void TestBasicQueryFromPropertyOfModel() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); var modelObject = new Person() { Name = "Steve" }; @@ -495,7 +495,7 @@ public void TestBasicQueryFromPropertyOfModel() public void TestBasicQueryFromPropertyOfModelWithStringInterpolation() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); var modelObject = new Person() { Name = "Steve" }; @@ -513,7 +513,7 @@ public void TestBasicQueryFromPropertyOfModelWithStringInterpolation() public void TestBasicQueryFromPropertyOfModelWithStringFormatFourArgs() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); var modelObject = new Person() { Name = "Steve" }; @@ -533,7 +533,7 @@ public void TestBasicQueryFromPropertyOfModelWithStringFormatFourArgs() public void TestBasicQueryWithContainsWithNegation() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); _ = collection.Where(x => !x.Name.Contains("Ste")).ToList(); @@ -550,7 +550,7 @@ public void TestBasicQueryWithContainsWithNegation() public void TestTwoPredicateQueryWithContains() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); _ = collection.Where(x => x.Name.Contains("Ste") || x.TagField == "John").ToList(); @@ -567,7 +567,7 @@ public void TestTwoPredicateQueryWithContains() public void TestTwoPredicateQueryWithPrefixMatching() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); _ = collection.Where(x => x.Name.Contains("Ste*") || x.TagField == "John").ToList(); @@ -584,7 +584,7 @@ public void TestTwoPredicateQueryWithPrefixMatching() public void TestGeoFilter() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); _ = collection.GeoFilter(x => x.Home, 5, 6.7, 50, GeoLocDistanceUnit.Kilometers).ToList(); @@ -609,7 +609,7 @@ public void TestGeoFilter() public void TestGeoFilterWithWhereClause() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); var res = collection.Where(x => x.TagField == "Steve").GeoFilter(x => x.Home, 5, 6.7, 50, GeoLocDistanceUnit.Kilometers).ToList(); @@ -634,7 +634,7 @@ public void TestGeoFilterWithWhereClause() public void TestSelect() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); _ = collection.Select(x => x.Name).ToList(); @@ -655,7 +655,7 @@ public void TestSelect() public void TestSelectComplexAnonType() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); _ = collection.Select(x => new { x.Name }).ToList(); @@ -677,7 +677,7 @@ public void TestSelectComplexAnonType() public void TextEqualityExpression() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); _ = collection.Where(x => x.Name == "Steve").ToList(); @@ -694,7 +694,7 @@ public void TextEqualityExpression() public void TestPaginationChunkSizesSinglePredicate() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute, 1000); _ = collection.Where(x => x.Name == "Steve").ToList(); @@ -711,7 +711,7 @@ public void TestPaginationChunkSizesSinglePredicate() public void TestPaginationChunkSizesMultiplePredicates() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute, 1000); _ = collection.Where(x => x.TagField == "Steve").GeoFilter(x => x.Home, 5, 6.7, 50, GeoLocDistanceUnit.Kilometers).ToList(); @@ -735,7 +735,7 @@ public void TestPaginationChunkSizesMultiplePredicates() public void TestNestedObjectStringSearch() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute, 1000); _ = collection.Where(x => x.Address.City == "Newark").ToList(); @@ -753,7 +753,7 @@ public void TestNestedObjectStringSearch() public void TestNestedObjectStringSearchNested2Levels() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute, 1000); _ = collection.Where(x => x.Address.ForwardingAddress.City == "Newark").ToList(); @@ -771,7 +771,7 @@ public void TestNestedObjectStringSearchNested2Levels() public void TestNestedObjectNumericSearch() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute, 1000); _ = collection.Where(x => x.Address.HouseNumber == 4).ToList(); @@ -789,7 +789,7 @@ public void TestNestedObjectNumericSearch() public void TestNestedObjectNumericSearch2Levels() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute, 1000); _ = collection.Where(x => x.Address.ForwardingAddress.HouseNumber == 4).ToList(); @@ -807,7 +807,7 @@ public void TestNestedObjectNumericSearch2Levels() public void TestNestedQueryOfGeo() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute, 1000); _ = collection.GeoFilter(x => x.Address.Location, 5, 6.7, 50, GeoLocDistanceUnit.Kilometers).ToList(); @@ -831,7 +831,7 @@ public void TestNestedQueryOfGeo() public void TestNestedQueryOfGeo2Levels() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute, 1000); _ = collection.GeoFilter(x => x.Address.ForwardingAddress.Location, 5, 6.7, 50, GeoLocDistanceUnit.Kilometers).ToList(); @@ -855,7 +855,7 @@ public void TestNestedQueryOfGeo2Levels() public void TestArrayContains() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute, 1000); _ = collection.Where(x => x.NickNames.Contains("Steve")).ToList(); @@ -873,7 +873,7 @@ public void TestArrayContains() public void TestArrayContainsSpecialChar() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute, 1000); _ = collection.Where(x => x.NickNames.Contains("Steve@redis.com")).ToList(); @@ -891,7 +891,7 @@ public void TestArrayContainsSpecialChar() public void TestArrayContainsVar() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute, 1000); var steve = "Steve"; @@ -910,7 +910,7 @@ public void TestArrayContainsVar() public void TestArrayContainsNested() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute, 1000); _ = collection.Where(x => x.Mother.NickNames.Contains("Di")).ToList(); @@ -927,10 +927,10 @@ public void TestArrayContainsNested() [Fact] public async Task TestUpdateJson() { - _substitute.ExecuteAsync("FT.SEARCH", Arg.Any()).Returns(_mockReply); + _substitute.ExecuteAsync("FT.SEARCH", Arg.Any()).Returns(_mockReply); - _substitute.ExecuteAsync("EVALSHA", Arg.Any()).Returns(Task.FromResult(new RedisReply("42"))); - _substitute.ExecuteAsync("SCRIPT", Arg.Any()) + _substitute.ExecuteAsync("EVALSHA", Arg.Any()).Returns(Task.FromResult(new RedisReply("42"))); + _substitute.ExecuteAsync("SCRIPT", Arg.Any()) .Returns(Task.FromResult(new RedisReply("cbbf1c4fab5064f419e469cc51c563f8bf51e6fb"))); var collection = new RedisCollection(_substitute); var steve = await collection.FirstAsync(x => x.Name == "Steve"); @@ -944,12 +944,12 @@ public async Task TestUpdateJson() public async Task TestUpdateJsonUnloadedScriptAsync() { - _substitute.ExecuteAsync("FT.SEARCH", Arg.Any()).Returns(_mockReply); + _substitute.ExecuteAsync("FT.SEARCH", Arg.Any()).Returns(_mockReply); - _substitute.ExecuteAsync("EVALSHA", Arg.Any()) + _substitute.ExecuteAsync("EVALSHA", Arg.Any()) .Throws(new RedisServerException("Failed on EVALSHA")); - _substitute.ExecuteAsync("EVAL", Arg.Any()).Returns(Task.FromResult(new RedisReply("42"))); - _substitute.ExecuteAsync("SCRIPT", Arg.Any()).Returns(Task.FromResult(new RedisReply("cbbf1c4fab5064f419e469cc51c563f8bf51e6fb"))); + _substitute.ExecuteAsync("EVAL", Arg.Any()).Returns(Task.FromResult(new RedisReply("42"))); + _substitute.ExecuteAsync("SCRIPT", Arg.Any()).Returns(Task.FromResult(new RedisReply("cbbf1c4fab5064f419e469cc51c563f8bf51e6fb"))); var collection = new RedisCollection(_substitute); var steve = await collection.FirstAsync(x => x.Name == "Steve"); steve.Age = 33; @@ -962,12 +962,12 @@ public async Task TestUpdateJsonUnloadedScriptAsync() [Fact] public void TestUpdateJsonUnloadedScript() { - _substitute.Execute("FT.SEARCH", Arg.Any()).Returns(_mockReply); + _substitute.Execute("FT.SEARCH", Arg.Any()).Returns(_mockReply); - _substitute.Execute("EVALSHA", Arg.Any()) + _substitute.Execute("EVALSHA", Arg.Any()) .Throws(new RedisServerException("Failed on EVALSHA")); - _substitute.Execute("EVAL", Arg.Any()).Returns(new RedisReply("42")); - _substitute.Execute("SCRIPT", Arg.Any()).Returns(new RedisReply("cbbf1c4fab5064f419e469cc51c563f8bf51e6fb")); + _substitute.Execute("EVAL", Arg.Any()).Returns(new RedisReply("42")); + _substitute.Execute("SCRIPT", Arg.Any()).Returns(new RedisReply("cbbf1c4fab5064f419e469cc51c563f8bf51e6fb")); var collection = new RedisCollection(_substitute); var steve = collection.First(x => x.Name == "Steve"); steve.Age = 33; @@ -980,9 +980,9 @@ public void TestUpdateJsonUnloadedScript() [Fact] public async Task TestUpdateJsonName() { - _substitute.ExecuteAsync("FT.SEARCH", Arg.Any()).Returns(_mockReply); - _substitute.ExecuteAsync("EVALSHA", Arg.Any()).Returns(new RedisReply("42")); - _substitute.ExecuteAsync("SCRIPT", Arg.Any()).Returns(new RedisReply("42")); + _substitute.ExecuteAsync("FT.SEARCH", Arg.Any()).Returns(_mockReply); + _substitute.ExecuteAsync("EVALSHA", Arg.Any()).Returns(new RedisReply("42")); + _substitute.ExecuteAsync("SCRIPT", Arg.Any()).Returns(new RedisReply("42")); var collection = new RedisCollection(_substitute); var steve = await collection.FirstAsync(x => x.Name == "Steve"); steve.Name = "Bob"; @@ -994,9 +994,9 @@ public async Task TestUpdateJsonName() [Fact] public async Task TestUpdateJsonNestedObject() { - _substitute.ExecuteAsync("FT.SEARCH", Arg.Any()).Returns(_mockReply); - _substitute.ExecuteAsync("EVALSHA", Arg.Any()).Returns(new RedisReply("42")); - _substitute.ExecuteAsync("SCRIPT", Arg.Any()).Returns(new RedisReply("cbbf1c4fab5064f419e469cc51c563f8bf51e6fb")); + _substitute.ExecuteAsync("FT.SEARCH", Arg.Any()).Returns(_mockReply); + _substitute.ExecuteAsync("EVALSHA", Arg.Any()).Returns(new RedisReply("42")); + _substitute.ExecuteAsync("SCRIPT", Arg.Any()).Returns(new RedisReply("cbbf1c4fab5064f419e469cc51c563f8bf51e6fb")); var collection = new RedisCollection(_substitute); var steve = await collection.FirstAsync(x => x.Name == "Steve"); steve.Address = new Address { State = "Florida" }; @@ -1015,9 +1015,9 @@ public async Task TestUpdateJsonNestedObject() [Fact] public async Task TestUpdateJsonWithDouble() { - _substitute.ExecuteAsync("FT.SEARCH", Arg.Any()).Returns(_mockReply); - _substitute.ExecuteAsync("EVALSHA", Arg.Any()).Returns(new RedisReply("42")); - _substitute.ExecuteAsync("SCRIPT", Arg.Any()).Returns(new RedisReply("42")); + _substitute.ExecuteAsync("FT.SEARCH", Arg.Any()).Returns(_mockReply); + _substitute.ExecuteAsync("EVALSHA", Arg.Any()).Returns(new RedisReply("42")); + _substitute.ExecuteAsync("SCRIPT", Arg.Any()).Returns(new RedisReply("42")); var collection = new RedisCollection(_substitute); var steve = await collection.FirstAsync(x => x.Name == "Steve"); steve.Age = 33; @@ -1031,8 +1031,8 @@ public async Task TestUpdateJsonWithDouble() public async Task TestDeleteAsync() { const string key = "Redis.OM.Unit.Tests.RediSearchTests.Person:01FVN836BNQGYMT80V7RCVY73N"; - _substitute.ExecuteAsync("FT.SEARCH", Arg.Any()).Returns(_mockReply); - _substitute.ExecuteAsync("UNLINK", Arg.Any()).Returns("1"); + _substitute.ExecuteAsync("FT.SEARCH", Arg.Any()).Returns(_mockReply); + _substitute.ExecuteAsync("UNLINK", Arg.Any()).Returns("1"); var collection = new RedisCollection(_substitute); var steve = await collection.FirstAsync(x => x.Name == "Steve"); Assert.True(collection.StateManager.Data.ContainsKey(key)); @@ -1047,8 +1047,8 @@ public async Task TestDeleteAsync() public void TestDelete() { const string key = "Redis.OM.Unit.Tests.RediSearchTests.Person:01FVN836BNQGYMT80V7RCVY73N"; - _substitute.Execute("FT.SEARCH", Arg.Any()).Returns(_mockReply); - _substitute.Execute("UNLINK", Arg.Any()).Returns(new RedisReply("1")); + _substitute.Execute("FT.SEARCH", Arg.Any()).Returns(_mockReply); + _substitute.Execute("UNLINK", Arg.Any()).Returns(new RedisReply("1")); var collection = new RedisCollection(_substitute); var steve = collection.First(x => x.Name == "Steve"); Assert.True(collection.StateManager.Data.ContainsKey(key)); @@ -1064,7 +1064,7 @@ public void TestDelete() [InlineData(false)] public async Task TestFirstAsync(bool useExpression) { - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); var expectedPredicate = useExpression ? "(@TagField:{bob})" : "*"; @@ -1093,7 +1093,7 @@ await _substitute.Received().ExecuteAsync( public async Task TestFirstAsyncNone(bool useExpression) { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReplyNone); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReplyNone); var expectedPredicate = useExpression ? "(@TagField:{bob})" : "*"; @@ -1121,7 +1121,7 @@ await _substitute.Received().ExecuteAsync( public async Task TestFirstOrDefaultAsync(bool useExpression) { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); var expectedPredicate = useExpression ? "(@TagField:{bob})" : "*"; @@ -1151,7 +1151,7 @@ await _substitute.Received().ExecuteAsync( [InlineData(false)] public async Task TestFirstOrDefaultAsyncNone(bool useExpression) { - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReplyNone); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReplyNone); var collection = new RedisCollection(_substitute); @@ -1183,7 +1183,7 @@ await _substitute.Received().ExecuteAsync( public async Task TestSingleAsync(bool useExpression) { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); var expectedPredicate = useExpression ? "(@TagField:{bob})" : "*"; var collection = new RedisCollection(_substitute); @@ -1212,7 +1212,7 @@ await _substitute.Received().ExecuteAsync( public async Task TestSingleAsyncNone(bool useExpression) { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReplyNone); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReplyNone); var collection = new RedisCollection(_substitute); var expectedPredicate = useExpression ? "(@TagField:{bob})" : "*"; @@ -1240,7 +1240,7 @@ await _substitute.Received().ExecuteAsync( public async Task TestSingleAsyncTwo(bool useExpression) { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply2Count); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply2Count); var collection = new RedisCollection(_substitute); var expectedPredicate = useExpression ? "(@TagField:{bob})" : "*"; @@ -1268,7 +1268,7 @@ await _substitute.Received().ExecuteAsync( public async Task TestSingleOrDefaultAsync(bool useExpression) { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); @@ -1300,7 +1300,7 @@ await _substitute.Received().ExecuteAsync( public async Task TestSingleOrDefaultAsyncNone(bool useExpression) { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReplyNone); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReplyNone); var collection = new RedisCollection(_substitute); @@ -1332,7 +1332,7 @@ await _substitute.Received().ExecuteAsync( public async Task TestSingleOrDefaultAsyncTwo(bool useExpression) { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply2Count); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply2Count); var collection = new RedisCollection(_substitute); @@ -1364,7 +1364,7 @@ await _substitute.Received().ExecuteAsync( public async Task TestAnyAsync(bool useExpression) { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); var expectedPredicate = useExpression ? "(@TagField:{bob})" : "*"; @@ -1388,7 +1388,7 @@ await _substitute.Received().ExecuteAsync( public async Task TestAnyAsyncNone(bool useExpression) { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReplyNone); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReplyNone); var collection = new RedisCollection(_substitute); @@ -1411,7 +1411,7 @@ await _substitute.Received().ExecuteAsync( public async Task TestCountAsync(bool useExpression) { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); @@ -1433,7 +1433,7 @@ await _substitute.Received().ExecuteAsync( public async Task TestCount2Async(bool useExpression) { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply2Count); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply2Count); var collection = new RedisCollection(_substitute); var expectedPredicate = useExpression ? "(@TagField:{bob})" : "*"; @@ -1452,7 +1452,7 @@ await _substitute.Received().ExecuteAsync( public async Task TestOrderByWithAsync() { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply2Count); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply2Count); var collection = new RedisCollection(_substitute); var expectedPredicate = "*"; _ = await collection.OrderBy(x => x.Age).ToListAsync(); @@ -1472,7 +1472,7 @@ await _substitute.Received().ExecuteAsync( public async Task TestOrderByDescendingWithAsync() { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply2Count); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply2Count); var collection = new RedisCollection(_substitute); var expectedPredicate = "*"; _ = await collection.OrderByDescending(x => x.Age).ToListAsync(); @@ -1492,7 +1492,7 @@ await _substitute.Received().ExecuteAsync( public async Task CombinedExpressionsWithFirstOrDefaultAsync() { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply2Count); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply2Count); var collection = new RedisCollection(_substitute); @@ -1512,7 +1512,7 @@ await _substitute.Received().ExecuteAsync( public async Task CombinedExpressionsWithFirstAsync() { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply2Count); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply2Count); var collection = new RedisCollection(_substitute); @@ -1532,7 +1532,7 @@ await _substitute.Received().ExecuteAsync( public async Task CombinedExpressionsWithAnyAsync() { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply2Count); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply2Count); var collection = new RedisCollection(_substitute); @@ -1552,7 +1552,7 @@ await _substitute.Received().ExecuteAsync( public async Task CombinedExpressionsSingleOrDefaultAsync() { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); @@ -1572,7 +1572,7 @@ await _substitute.Received().ExecuteAsync( public async Task CombinedExpressionsSingleAsync() { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); @@ -1592,7 +1592,7 @@ await _substitute.Received().ExecuteAsync( public async Task CombinedExpressionsCountAsync() { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); @@ -1612,7 +1612,7 @@ await _substitute.Received().ExecuteAsync( public async Task TestCombinedExpressionWithExpressionFirstOrDefaultAsync() { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply2Count); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply2Count); var collection = new RedisCollection(_substitute); @@ -1638,7 +1638,7 @@ await _substitute.Received().ExecuteAsync( public async Task TestCombinedExpressionWithExpressionFirstAsync() { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); @@ -1664,7 +1664,7 @@ await _substitute.Received().ExecuteAsync( public async Task TestCombinedExpressionWithExpressionAnyAsync() { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); @@ -1690,7 +1690,7 @@ await _substitute.Received().ExecuteAsync( public async Task TestCombinedExpressionWithExpressionSingleAsync() { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); @@ -1716,7 +1716,7 @@ await _substitute.Received().ExecuteAsync( public async Task TestCombinedExpressionWithExpressionSingleOrDefaultAsync() { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); @@ -1742,7 +1742,7 @@ await _substitute.Received().ExecuteAsync( public async Task TestCombinedExpressionWithExpressionCountAsync() { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); @@ -1768,7 +1768,7 @@ await _substitute.Received().ExecuteAsync( public async Task TestCreateIndexWithNoStopwords() { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns("OK"); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns("OK"); await _substitute.CreateIndexAsync(typeof(ObjectWithZeroStopwords)); @@ -1789,7 +1789,7 @@ await _substitute.Received().ExecuteAsync( public async Task TestCreateIndexWithTwoStopwords() { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns("OK"); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns("OK"); await _substitute.CreateIndexAsync(typeof(ObjectWithTwoStopwords)); @@ -1809,7 +1809,7 @@ await _substitute.Received().ExecuteAsync( public async Task TestCreateIndexWithStringLikeValueTypes() { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns("OK"); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns("OK"); await _substitute.CreateIndexAsync(typeof(ObjectWithStringLikeValueTypes)); @@ -1834,7 +1834,7 @@ await _substitute.Received().ExecuteAsync("FT.CREATE", public async Task TestCreateIndexWithStringLikeValueTypesHash() { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns("OK"); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns("OK"); await _substitute.CreateIndexAsync(typeof(ObjectWithStringLikeValueTypesHash)); @@ -1857,7 +1857,7 @@ await _substitute.Received().ExecuteAsync("FT.CREATE", public async Task TestCreateIndexWithDatetimeValue() { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns("OK"); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns("OK"); await _substitute.CreateIndexAsync(typeof(ObjectWithDateTime)); await _substitute.CreateIndexAsync(typeof(ObjectWithDateTimeHash)); @@ -1891,7 +1891,7 @@ await _substitute.Received().ExecuteAsync("FT.CREATE", public async Task TestQueryOfUlid() { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); @@ -1907,7 +1907,7 @@ public async Task TestQueryOfUlid() public async Task TestQueryOfGuid() { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); @@ -1924,7 +1924,7 @@ public async Task TestQueryOfGuid() public async Task TestQueryOfBoolean() { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); @@ -1941,7 +1941,7 @@ public async Task TestQueryOfBoolean() public async Task TestQueryOfEnum() { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); @@ -1958,7 +1958,7 @@ public async Task TestQueryOfEnum() public async Task TestQueryOfEnumHash() { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); @@ -1973,7 +1973,7 @@ public async Task TestQueryOfEnumHash() public async Task TestGreaterThanEnumQuery() { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); @@ -1987,7 +1987,7 @@ public async Task TestGreaterThanEnumQuery() [Fact] public async Task TestIndexCreationWithEmbeddedListOfDocuments() { - _substitute.ExecuteAsync("FT.CREATE", Arg.Any()).Returns("OK"); + _substitute.ExecuteAsync("FT.CREATE", Arg.Any()).Returns("OK"); await _substitute.CreateIndexAsync(typeof(ObjectWithEmbeddedArrayOfObjects)); await _substitute.Received().ExecuteAsync("FT.CREATE", "objectwithembeddedarrayofobjects-idx", @@ -2012,7 +2012,7 @@ await _substitute.Received().ExecuteAsync("FT.CREATE", public async Task TestAnyQueryForArrayOfEmbeddedObjects() { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); @@ -2032,7 +2032,7 @@ await _substitute.Received().ExecuteAsync( public async Task TestAnyQueryForArrayOfEmbeddedObjectsEnum() { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); @@ -2052,7 +2052,7 @@ await _substitute.Received().ExecuteAsync( public async Task TestAnyQueryForArrayOfEmbeddedObjectsExtraPredicate() { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); @@ -2072,7 +2072,7 @@ await _substitute.Received().ExecuteAsync( public async Task TestAnyQueryForArrayOfEmbeddedObjectsMultipleAnyCalls() { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); @@ -2092,7 +2092,7 @@ await _substitute.Received().ExecuteAsync( public async Task TestAnyQueryForArrayOfEmbeddedObjectsMultiplePredicatesInsideAny() { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); @@ -2112,7 +2112,7 @@ await _substitute.Received().ExecuteAsync( public async Task TestAnyQueryForArrayOfEmbeddedObjectsOtherTypes() { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); var boolean = true; var ulid = Ulid.NewUlid(); @@ -2136,7 +2136,7 @@ await _substitute.Received().ExecuteAsync( public async Task TestAnyQueryForListOfEmbeddedObjects() { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); @@ -2156,7 +2156,7 @@ await _substitute.Received().ExecuteAsync( public async Task TestAnyQueryForArrayOfEmbeddedObjectsMultiVariant() { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); @@ -2176,7 +2176,7 @@ await _substitute.Received().ExecuteAsync( public async Task SearchWithMultipleWhereClauses() { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); @@ -2198,7 +2198,7 @@ await _substitute.Received().ExecuteAsync( public async Task TestAsyncMaterializationMethodsWithCombinedQueries() { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); _ = await collection.Where(x => x.TagField == "CountAsync") .CountAsync(x => x.Age == 32); @@ -2262,7 +2262,7 @@ await _substitute.Received().ExecuteAsync( public void TestMaterializationMethodsWithCombinedQueries() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute).Where(x => x.Age == 32); _ = collection.Count(x => x.TagField == "Count"); _ = collection.Any(x => x.TagField == "Any"); @@ -2321,7 +2321,7 @@ public void SearchTagFieldContains() { var potentialTagFieldValues = new [] { "Steve", "Alice", "Bob" }; _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute).Where(x => potentialTagFieldValues.Contains(x.TagField)); _ = collection.ToList(); _substitute.Received().Execute( @@ -2338,7 +2338,7 @@ public void SearchTextFieldContains() { var potentialTextFieldValues = new [] { "Steve", "Alice", "Bob" }; _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute).Where(x => potentialTextFieldValues.Contains(x.Name)); _ = collection.ToList(); _substitute.Received().Execute( @@ -2355,7 +2355,7 @@ public void SearchNumericFieldContains() { var potentialTagFieldValues = new int?[] { 35, 50, 60 }; _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute).Where(x => potentialTagFieldValues.Contains(x.Age)); _ = collection.ToList(); _substitute.Received().Execute( @@ -2371,7 +2371,7 @@ public void SearchNumericFieldContains() public void Issue201() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var p1 = new Person() { Name = "Steve" }; var collection = new RedisCollection(_substitute, 1000); @@ -2391,7 +2391,7 @@ public void Issue201() public void RangeOnDateTimeWithMultiplePredicates() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var fromDto = DateTimeOffset.UtcNow.Subtract(TimeSpan.FromHours(4)); var toDto = DateTimeOffset.UtcNow; @@ -2469,7 +2469,7 @@ public void RangeOnDateTimeWithMultiplePredicates() public void RangeOnDatetime() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var timestamp = DateTime.Now; var timeAnHourAgo = timestamp.Subtract(TimeSpan.FromHours(1)); @@ -2559,7 +2559,7 @@ public void RangeOnDatetime() public async Task RangeOnDatetimeAsync() { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); var timestamp = DateTime.Now; var timeAnHourAgo = timestamp.Subtract(TimeSpan.FromHours(1)); @@ -2648,7 +2648,7 @@ await _substitute.Received().ExecuteAsync( public async Task RangeOnDatetimeAsyncHash() { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); var timestamp = DateTime.Now; var timeAnHourAgo = timestamp.Subtract(TimeSpan.FromHours(1)); @@ -2751,7 +2751,7 @@ public void SearchNumericFieldListContains() { var potentialTagFieldValues = new List { 35, 50, 60 }; _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute).Where(x => potentialTagFieldValues.Contains(x.Age)); _ = collection.ToList(); _substitute.Received().Execute( @@ -2768,7 +2768,7 @@ public void SearchTagFieldAndTextListContains() { var potentialTagFieldValues = new List { "Steve", "Alice", "Bob" }; _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute).Where(x => potentialTagFieldValues.Contains(x.TagField) || potentialTagFieldValues.Contains(x.Name)); _ = collection.ToList(); _substitute.Received().Execute( @@ -2789,7 +2789,7 @@ public void TestNullResponseDoc() var query = new RedisQuery("fake-idx"); _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()) + _substitute.Execute(Arg.Any(), Arg.Any()) .Returns((RedisReply)res); _ = _substitute.Search(query); @@ -2800,7 +2800,7 @@ public void SearchTagFieldAndTextListContainsWithEscapes() { var potentialTagFieldValues = new List { "steve@example.com", "alice@example.com", "bob@example.com" }; _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute).Where(x => potentialTagFieldValues.Contains(x.TagField) || potentialTagFieldValues.Contains(x.Name)); _ = collection.ToList(); _substitute.Received().Execute( @@ -2816,7 +2816,7 @@ public void SearchTagFieldAndTextListContainsWithEscapes() public void SearchWithEmptyAny() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); var any = collection.Any(); _substitute.Received().Execute( @@ -2844,7 +2844,7 @@ public void SearchWithEmptyAny() public void TestContainsFromLocal() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); const string steve = "steve"; _ = collection.Where(x => x.NickNamesList.Contains(steve)).ToList(); @@ -2868,7 +2868,7 @@ public void SearchGuidFieldContains() var guid3Str = ExpressionParserUtilities.EscapeTagField(guid3.ToString()); var potentialFieldValues = new [] { guid1, guid2, guid3 }; _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute).Where(x => potentialFieldValues.Contains(x.Guid)); _ = collection.ToList(); _substitute.Received().Execute( @@ -2884,7 +2884,7 @@ public void SearchGuidFieldContains() public void TestContainsFromProperty() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); var steve = new Person { @@ -2909,7 +2909,7 @@ public void SearchUlidFieldContains() var potentialFieldValues = new [] { ulid1, ulid2, ulid3 }; _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute).Where(x => potentialFieldValues.Contains(x.Ulid)); _ = collection.ToList(); _substitute.Received().Execute( @@ -2930,7 +2930,7 @@ public void SearchEnumFieldContains() var potentialFieldValues = new [] { enum1, enum2, enum3 }; _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute).Where(x => potentialFieldValues.Contains(x.AnEnum)); _ = collection.ToList(); _substitute.Received().Execute( @@ -2951,7 +2951,7 @@ public void SearchNumericEnumFieldContains() var potentialFieldValues = new [] { enum1, enum2, enum3 }; _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute).Where(x => potentialFieldValues.Contains(x.AnEnumAsInt)); _ = collection.ToList(); _substitute.Received().Execute( @@ -2972,7 +2972,7 @@ public void SearchEnumFieldContainsList() var potentialFieldValues = new List { enum1, enum2, enum3 }; _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute).Where(x => potentialFieldValues.Contains(x.AnEnum)); _ = collection.ToList(); _substitute.Received().Execute( @@ -2993,7 +2993,7 @@ public void SearchNumericEnumFieldContainsList() var potentialFieldValues = new List { enum1, enum2, enum3 }; _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute).Where(x => potentialFieldValues.Contains(x.AnEnumAsInt)); _ = collection.ToList(); _substitute.Received().Execute( @@ -3014,7 +3014,7 @@ public void SearchEnumFieldContainsListAsProperty() var potentialFieldValues = new { list = new List { enum1, enum2, enum3 } }; _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute).Where(x => potentialFieldValues.list.Contains(x.AnEnum)); _ = collection.ToList(); _substitute.Received().Execute( @@ -3035,7 +3035,7 @@ public void SearchNumericEnumFieldContainsListAsProperty() var potentialFieldValues = new { list = new List { enum1, enum2, enum3 } }; _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute).Where(x => potentialFieldValues.list.Contains(x.AnEnumAsInt)); _ = collection.ToList(); _substitute.Received().Execute( @@ -3051,7 +3051,7 @@ public void SearchNumericEnumFieldContainsListAsProperty() public void TestNestedOrderBy() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); _ = new RedisCollection(_substitute).OrderBy(x => x.Address.State).ToList(); _substitute.Received().Execute("FT.SEARCH", "person-idx", "*", "LIMIT", "0", "100", "SORTBY", "Address_State", "ASC"); } @@ -3060,7 +3060,7 @@ public void TestNestedOrderBy() public void TestGeoFilterNested() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); _ = collection.GeoFilter(x => x.Address.Location, 5, 6.7, 50, GeoLocDistanceUnit.Kilometers).ToList(); @@ -3084,7 +3084,7 @@ public void TestGeoFilterNested() public void TestSelectWithWhere() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); _ = collection.Where(x => x.Age == 33).Select(x => x.Name).ToList(); _substitute.Received().Execute( @@ -3104,7 +3104,7 @@ public void TestNullableEnumQueries() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); _ = collection.Where(x => x.AnEnum == AnEnum.one && x.NullableStringEnum == AnEnum.two).ToList(); @@ -3122,7 +3122,7 @@ public void TestNullableEnumQueries() public void TestEscapeForwardSlash() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); _ = collection.Where(x => x.TagField == "a/test/string").ToList(); @@ -3140,7 +3140,7 @@ public void TestEscapeForwardSlash() public void TestMixedNestingIndexCreation() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(new RedisReply("OK")); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(new RedisReply("OK")); _substitute.CreateIndex(typeof(ComplexObjectWithCascadeAndJsonPath)); @@ -3166,7 +3166,7 @@ public void TestMixedNestingIndexCreation() public void TestMixedNestingQuerying() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); @@ -3204,7 +3204,7 @@ public void TestMixedNestingQuerying() [Fact] public async Task TestCreateIndexWithJsonPropertyName() { - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns("OK"); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns("OK"); await _substitute.CreateIndexAsync(typeof(ObjectWithPropertyNamesDefined)); @@ -3223,7 +3223,7 @@ await _substitute.Received().ExecuteAsync( public void QueryNamedPropertiesJson() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); _ = collection.FirstOrDefault(x => x.Key == "hello"); @@ -3242,7 +3242,7 @@ public void QueryNamedPropertiesJson() public void TestMultipleContains() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); Expression> whereExpressionFail = a => !a.FirstName.Contains("Andrey") && !a.LastName.Contains("Bred"); @@ -3272,7 +3272,7 @@ public void TestMultipleContains() public void TestSelectNestedObject() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); _ = collection.Select(x => x.Address).ToList(); @@ -3317,7 +3317,7 @@ public void NonNullableNumericFieldContains() var floats = new [] { 25.5F, 26, 27 }; var ushorts = new ushort[] { 28, 29, 30 }; _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute).Where(x => ints.Contains(x.Integer)); _ = collection.ToList(); var expected = $"@{nameof(ObjectWithNumerics.Integer)}:[1 1]|@{nameof(ObjectWithNumerics.Integer)}:[2 2]|@{nameof(ObjectWithNumerics.Integer)}:[3 3]"; diff --git a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/ObjectWithVector.cs b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/ObjectWithVector.cs index 25ff3fea..d4025443 100644 --- a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/ObjectWithVector.cs +++ b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/ObjectWithVector.cs @@ -28,4 +28,11 @@ public class ObjectWithVectorHash [Vector(Algorithm = VectorAlgorithm.FLAT)] [SimpleVectorizer] public string SimpleVectorizedVector { get; set; } +} + +[Document] +public class ToyVector +{ + [RedisIdField] public string Id { get; set; } + [Vector(Dim=6)]public double[] SimpleVector { get; set; } } \ No newline at end of file diff --git a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorFunctionalTests.cs b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorFunctionalTests.cs index 9f21d8ff..ae011415 100644 --- a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorFunctionalTests.cs +++ b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorFunctionalTests.cs @@ -1,5 +1,6 @@ using System.Text; using Redis.OM.Contracts; +using Redis.OM.Modeling; using Xunit; namespace Redis.OM.Unit.Tests; @@ -14,6 +15,36 @@ public VectorFunctionalTests(RedisSetup setup) _connection = setup.Connection; } + [Fact] + public void Dave() + { + _connection.CreateIndex(typeof(ToyVector)); + + var doubles = VectorUtils.VecStrToDoubles("I'm_sorry_Dave,_I'm_afraid_I_can't_do_that......"); + var obj = new ToyVector() + { + Id = "foo", + SimpleVector = doubles + }; + _connection.Set(obj); + } + + [Fact] + public void TestIndex() + { + _connection.CreateIndex(typeof(ObjectWithVectorHash)); + + var doubles = new double[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; + var obj = new ObjectWithVectorHash + { + Id = "foo", + SimpleHnswVector = doubles, + SimpleVectorizedVector = "foo", + }; + + _connection.Set(obj); + } + [Fact] public void Insert() { diff --git a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorTests.cs b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorTests.cs index c02ef6ef..915de57a 100644 --- a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorTests.cs +++ b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorTests.cs @@ -74,14 +74,16 @@ public void InsertVectors() { vectorizedFlatHashVector[i] = i; } + + simpleHnswJsonStr.Append(string.Join(',', simpleHnswHash)); vectorizedFlatVectorJsonStr.Append(string.Join(',', vectorizedFlatHashVector)); simpleHnswJsonStr.Append(']'); vectorizedFlatVectorJsonStr.Append(']'); - - var byteStringSimpleHnsw = Encoding.UTF8.GetString(simpleHnswHash.SelectMany(BitConverter.GetBytes).ToArray()); - var byteStringVectorizedFlashHash = Encoding.UTF8.GetString(vectorizedFlatHashVector.SelectMany(BitConverter.GetBytes).ToArray()); + + var simpleHnswBytes = simpleHnswHash.SelectMany(BitConverter.GetBytes).ToArray(); + var flatVectorizedBytes = vectorizedFlatHashVector.SelectMany(BitConverter.GetBytes).ToArray(); var hashObj = new ObjectWithVectorHash() { @@ -100,12 +102,12 @@ public void InsertVectors() var json = $"{{\"Id\":\"foo\",\"SimpleHnswVector\":{simpleHnswJsonStr},\"SimpleVectorizedVector\":{{\"Value\":\"\\u0022foobar\\u0022\",\"Vector\":{vectorizedFlatVectorJsonStr}}}}}"; - _substitute.Execute("HSET", Arg.Any()).Returns(new RedisReply("3")); - _substitute.Execute("JSON.SET", Arg.Any()).Returns(new RedisReply("OK")); + _substitute.Execute("HSET", Arg.Any()).Returns(new RedisReply("3")); + _substitute.Execute("JSON.SET", Arg.Any()).Returns(new RedisReply("OK")); _substitute.Set(hashObj); _substitute.Set(jsonObj); _substitute.Received().Execute("HSET", "Redis.OM.Unit.Tests.ObjectWithVectorHash:foo", "Id", "foo", "SimpleHnswVector", - byteStringSimpleHnsw, "SimpleVectorizedVector.Vector", byteStringVectorizedFlashHash, "SimpleVectorizedVector.Value", "foobar"); + Arg.Is(x=>x.SequenceEqual(simpleHnswBytes)), "SimpleVectorizedVector.Vector", Arg.Is(x=>x.SequenceEqual(flatVectorizedBytes)), "SimpleVectorizedVector.Value", "\"foobar\""); _substitute.Received().Execute("JSON.SET", "Redis.OM.Unit.Tests.ObjectWithVector:foo", ".", json); var deseralized = JsonSerializer.Deserialize(json); Assert.Equal("foobar", deseralized.SimpleVectorizedVector); From a377faf793b34a830620388a85ecc2ef6607add0 Mon Sep 17 00:00:00 2001 From: slorello89 Date: Tue, 10 Oct 2023 15:16:27 -0400 Subject: [PATCH 04/36] seralization/deserialization of vectors --- src/Redis.OM/Modeling/Vectors/VectorUtils.cs | 12 +++--- src/Redis.OM/RedisCommands.cs | 4 +- src/Redis.OM/RedisObjectHandler.cs | 37 +++++++++++++++---- src/Redis.OM/RedisReply.cs | 30 +++++++++------ .../VectorTests/VectorFunctionalTests.cs | 16 +++++++- 5 files changed, 72 insertions(+), 27 deletions(-) diff --git a/src/Redis.OM/Modeling/Vectors/VectorUtils.cs b/src/Redis.OM/Modeling/Vectors/VectorUtils.cs index d60771d5..40057b9f 100644 --- a/src/Redis.OM/Modeling/Vectors/VectorUtils.cs +++ b/src/Redis.OM/Modeling/Vectors/VectorUtils.cs @@ -76,12 +76,12 @@ public static string BytesToVecStr(byte[] bytes) /// /// Converts Vector String to array of doubles. /// - /// the vector string. + /// the reply. /// the doubles. /// Thrown if unbalanced. - public static double[] VecStrToDoubles(string str) + public static double[] VecStrToDoubles(RedisReply reply) { - var bytes = VecStrToBytes(str); + var bytes = (byte[]?)reply ?? throw new InvalidCastException("Could not convert result to raw result."); if (bytes.Length % 8 != 0) { throw new ArgumentException("Unbalanced Vector String"); @@ -99,12 +99,12 @@ public static double[] VecStrToDoubles(string str) /// /// Parses a vector string to an array of floats. /// - /// the string. + /// the reply. /// The floats. /// thrown if unbalanced. - public static float[] VectorStrToFloats(string str) + public static float[] VectorStrToFloats(RedisReply reply) { - var bytes = VecStrToBytes(str); + var bytes = (byte[]?)reply ?? throw new InvalidCastException("Could not convert result to raw result."); if (bytes.Length % 4 != 0) { throw new ArgumentException("Unbalanced Vector String"); diff --git a/src/Redis.OM/RedisCommands.cs b/src/Redis.OM/RedisCommands.cs index b96cb803..a54b8c50 100644 --- a/src/Redis.OM/RedisCommands.cs +++ b/src/Redis.OM/RedisCommands.cs @@ -598,9 +598,9 @@ public static string Set(this IRedisConnection connection, object obj, TimeSpan /// the connection. /// the key name. /// the object serialized into a dictionary. - public static IDictionary HGetAll(this IRedisConnection connection, string keyName) + public static IDictionary HGetAll(this IRedisConnection connection, string keyName) { - var ret = new Dictionary(); + var ret = new Dictionary(); var res = connection.Execute("HGETALL", keyName).ToArray(); for (var i = 0; i < res.Length; i += 2) { diff --git a/src/Redis.OM/RedisObjectHandler.cs b/src/Redis.OM/RedisObjectHandler.cs index 3628a2f9..d4b95e86 100644 --- a/src/Redis.OM/RedisObjectHandler.cs +++ b/src/Redis.OM/RedisObjectHandler.cs @@ -40,6 +40,7 @@ internal static class RedisObjectHandler internal static T FromHashSet(IDictionary hash) where T : notnull { + var dict = hash.ToDictionary(x => x.Key, x => new RedisReply(x.Value.ToString())); if (typeof(IRedisHydrateable).IsAssignableFrom(typeof(T))) { var obj = Activator.CreateInstance(); @@ -59,7 +60,7 @@ internal static T FromHashSet(IDictionary hash) } else { - asJson = SendToJson(hash, typeof(T)); + asJson = SendToJson(dict, typeof(T)); } return JsonSerializer.Deserialize(asJson, RedisSerializationSettings.JsonSerializerOptions) ?? throw new Exception("Deserialization fail"); @@ -90,7 +91,7 @@ internal static T FromHashSet(IDictionary hash) } else if (attr != null) { - asJson = SendToJson(stringDictionary, typeof(T)); + asJson = SendToJson(hash, typeof(T)); } else { @@ -106,7 +107,7 @@ internal static T FromHashSet(IDictionary hash) /// The hash. /// The type to deserialize to. /// the deserialized object. - internal static object? FromHashSet(IDictionary hash, Type type) + internal static object? FromHashSet(IDictionary hash, Type type) { var asJson = SendToJson(hash, type); return JsonSerializer.Deserialize(asJson, type); @@ -426,7 +427,7 @@ internal static IDictionary BuildHashSet(this object obj) return hash; } - private static string SendToJson(IDictionary hash, Type t) + private static string SendToJson(IDictionary hash, Type t) { var properties = t.GetProperties(); if ((!properties.Any() || t == typeof(Ulid) || t == typeof(Ulid?)) && hash.Count == 1) @@ -466,7 +467,7 @@ private static string SendToJson(IDictionary hash, Type t) continue; } - ret += $"\"{propertyName}\":{hash[lookupPropertyName].ToLower()},"; + ret += $"\"{propertyName}\":{((string)hash[lookupPropertyName]).ToLower()},"; } else if (type.IsPrimitive || type == typeof(decimal) || type.IsEnum) { @@ -486,6 +487,28 @@ private static string SendToJson(IDictionary hash, Type t) ret += $"\"{propertyName}\":\"{HttpUtility.JavaScriptStringEncode(hash[lookupPropertyName])}\","; } + else if ((type == typeof(double[]) || type == typeof(float[])) && property.GetCustomAttributes().Any()) + { + if (!hash.ContainsKey(lookupPropertyName)) + { + continue; + } + + string arrString; + if (type == typeof(float[])) + { + var floats = VectorUtils.VectorStrToFloats(hash[lookupPropertyName]); + arrString = string.Join(",", floats); + } + else + { + var doubles = VectorUtils.VecStrToDoubles(hash[lookupPropertyName]); + arrString = string.Join(",", doubles); + } + + var valueStr = $"[{arrString}]"; + ret += $"\"{lookupPropertyName}\":{valueStr},"; + } else if (type.GetInterfaces().Any(x => x.IsGenericType && x.GetGenericTypeDefinition() == typeof(IEnumerable<>))) { var entries = hash.Where(x => x.Key.StartsWith($"{propertyName}[")) @@ -511,7 +534,7 @@ private static string SendToJson(IDictionary hash, Type t) if (innerType == typeof(bool) || innerType == typeof(bool?)) { var val = entries[$"{propertyName}[{i}]"]; - ret += $"{val.ToLower()},"; + ret += $"{((string)val).ToLower()},"; } else if (innerType.IsPrimitive || innerType == typeof(decimal)) { @@ -544,7 +567,7 @@ private static string SendToJson(IDictionary hash, Type t) else { var entries = hash.Where(x => x.Key.StartsWith($"{propertyName}.")) - .Select(x => new KeyValuePair(x.Key.Substring($"{propertyName}.".Length), x.Value)) + .Select(x => new KeyValuePair(x.Key.Substring($"{propertyName}.".Length), x.Value)) .ToDictionary(x => x.Key, x => x.Value); if (entries.Any()) { diff --git a/src/Redis.OM/RedisReply.cs b/src/Redis.OM/RedisReply.cs index f29bd38a..de2db574 100644 --- a/src/Redis.OM/RedisReply.cs +++ b/src/Redis.OM/RedisReply.cs @@ -1,6 +1,7 @@ using System; using System.Globalization; using System.Linq; +using System.Text; using StackExchange.Redis; namespace Redis.OM @@ -15,7 +16,7 @@ public class RedisReply : IConvertible #pragma warning restore SA1018 private readonly double? _internalDouble; private readonly int? _internalInt; - private readonly string? _internalString; + private readonly byte[]? _raw; private readonly long? _internalLong; /// @@ -30,11 +31,11 @@ public RedisReply(RedisResult result) break; case ResultType.SimpleString: case ResultType.BulkString: - _internalString = (string)result; + _raw = (byte[])result; break; case ResultType.Error: Error = true; - _internalString = result.ToString(); + _raw = (byte[])result; break; case ResultType.Integer: _internalLong = (long)result; @@ -60,7 +61,7 @@ internal RedisReply(double val) /// the value. internal RedisReply(string val) { - _internalString = val; + _raw = Encoding.UTF8.GetBytes(val); } /// @@ -108,7 +109,7 @@ public static implicit operator double(RedisReply v) return (double)v._internalDouble; } - if (v._internalString != null && double.TryParse(v._internalString, NumberStyles.Number, CultureInfo.InvariantCulture, out var ret)) + if (v._raw != null && double.TryParse(Encoding.UTF8.GetString(v._raw), NumberStyles.Number, CultureInfo.InvariantCulture, out var ret)) { return ret; } @@ -126,6 +127,13 @@ public static implicit operator double(RedisReply v) throw new InvalidCastException("Could not cast to double"); } + /// + /// implicitly converts the reply to a byte array. + /// + /// the . + /// the byte array. + public static implicit operator byte[]?(RedisReply v) => v._raw; + /// /// implicitly converts the reply to a double. /// @@ -160,7 +168,7 @@ public static implicit operator double(RedisReply v) /// /// the reply. /// the string. - public static implicit operator string(RedisReply v) => v._internalString ?? string.Empty; + public static implicit operator string(RedisReply v) => v._raw is not null ? Encoding.UTF8.GetString(v._raw) : string.Empty; /// /// implicitly converts a string into a redis reply. @@ -182,7 +190,7 @@ public static implicit operator int(RedisReply v) return (int)v._internalInt; } - if (v._internalString != null && int.TryParse(v._internalString, out var ret)) + if (v._raw != null && int.TryParse(Encoding.UTF8.GetString(v._raw), out var ret)) { return ret; } @@ -227,7 +235,7 @@ public static implicit operator long(RedisReply v) return (long)v._internalLong; } - if (v._internalString != null && long.TryParse(v._internalString, out var ret)) + if (v._raw != null && long.TryParse(Encoding.UTF8.GetString(v._raw), out var ret)) { return ret; } @@ -293,9 +301,9 @@ public override string ToString() return _internalInt.ToString(); } - if (_internalString != null) + if (_raw != null) { - return _internalString; + return Encoding.UTF8.GetString(_raw); } return base.ToString(); @@ -325,7 +333,7 @@ public TypeCode GetTypeCode() return TypeCode.Int64; } - if (_internalString != null) + if (_raw != null) { return TypeCode.String; } diff --git a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorFunctionalTests.cs b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorFunctionalTests.cs index ae011415..5c0f8dcb 100644 --- a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorFunctionalTests.cs +++ b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorFunctionalTests.cs @@ -42,7 +42,21 @@ public void TestIndex() SimpleVectorizedVector = "foo", }; - _connection.Set(obj); + var key = _connection.Set(obj); + var res = _connection.Get(key); + Assert.Equal(doubles, res.SimpleHnswVector); + + key = _connection.Set(new ObjectWithVector() + { + Id = "foo", + SimpleHnswVector = doubles, + SimpleVectorizedVector = "foobarbaz" + }); + + var jsonRes = _connection.Get(key); + + Assert.Equal(doubles, jsonRes.SimpleHnswVector); + Assert.Equal("foobarbaz", jsonRes.SimpleVectorizedVector); } [Fact] From bc6ce0591450c36b26700e0bd84b0cda868bbdb9 Mon Sep 17 00:00:00 2001 From: slorello89 Date: Thu, 12 Oct 2023 12:51:03 -0400 Subject: [PATCH 05/36] initial queries working --- src/Redis.OM/Common/ExpressionTranslator.cs | 40 ++++++++++++++- src/Redis.OM/Modeling/Vectors/VectorResult.cs | 30 +++++++++++ src/Redis.OM/Modeling/Vectors/VectorUtils.cs | 23 +++++++++ .../Modeling/Vectors/VectorizerAttribute.cs | 5 ++ src/Redis.OM/SearchExtensions.cs | 23 +++++++++ .../Searching/Query/NearestNeighbors.cs | 36 +++++++++++++ src/Redis.OM/Searching/Query/RedisQuery.cs | 26 ++++++++-- .../VectorTests/ObjectWithVector.cs | 2 +- .../VectorTests/VectorFunctionalTests.cs | 51 ++++++++++++++++++- .../VectorTests/VectorTests.cs | 33 ++++++++++-- 10 files changed, 259 insertions(+), 10 deletions(-) create mode 100644 src/Redis.OM/Modeling/Vectors/VectorResult.cs create mode 100644 src/Redis.OM/Searching/Query/NearestNeighbors.cs diff --git a/src/Redis.OM/Common/ExpressionTranslator.cs b/src/Redis.OM/Common/ExpressionTranslator.cs index 13215c5a..f4428cd7 100644 --- a/src/Redis.OM/Common/ExpressionTranslator.cs +++ b/src/Redis.OM/Common/ExpressionTranslator.cs @@ -30,7 +30,7 @@ public static RedisAggregation BuildAggregationFromExpression(Expression express var attr = type.GetCustomAttribute(); if (attr == null) { - throw new InvalidOperationException("Aggregations can only be perfomred on objects decorated with a RedisObjectDefinitionAttribute that specifies a particular index"); + throw new InvalidOperationException("Aggregations can only be performed on objects decorated with a RedisObjectDefinitionAttribute that specifies a particular index"); } var indexName = string.IsNullOrEmpty(attr.IndexName) ? $"{type.Name.ToLower()}-idx" : attr.IndexName; @@ -229,6 +229,9 @@ internal static RedisQuery BuildQueryFromExpression(Expression expression, Type case "Where": query.QueryText = TranslateWhereMethod(exp); break; + case "NearestNeighbors": + query.NearestNeighbors = ParseNearestNeighborsFromExpression(exp); + break; } } @@ -248,6 +251,41 @@ internal static RedisQuery BuildQueryFromExpression(Expression expression, Type return query; } + /// + /// Builds a Nearest Neighbor query from provided expression. + /// + /// The expression. + /// The nearest neighbor query. + internal static NearestNeighbors ParseNearestNeighborsFromExpression(MethodCallExpression expression) + { + var memberExpression = (MemberExpression)((LambdaExpression)((UnaryExpression)expression.Arguments[1]).Operand).Body; + var attr = memberExpression.Member.GetCustomAttributes().FirstOrDefault() ?? throw new ArgumentException($"Could not find Vector attribute on {memberExpression.Member.Name}."); + var vectorizer = memberExpression.Member.GetCustomAttributes().FirstOrDefault(); + var propertyName = !string.IsNullOrEmpty(attr.PropertyName) ? attr.PropertyName : memberExpression.Member.Name; + var numNeighbors = (int)((ConstantExpression)expression.Arguments[2]).Value; + var value = ((ConstantExpression)expression.Arguments[3]).Value ?? throw new InvalidOperationException("Provided vector property was null"); + byte[] bytes; + + if (vectorizer is not null) + { + bytes = vectorizer.Vectorize(value); + } + else if (memberExpression.Type == typeof(float[])) + { + bytes = ((float[])value).SelectMany(BitConverter.GetBytes).ToArray(); + } + else if (memberExpression.Type == typeof(double[])) + { + bytes = ((double[])value).SelectMany(BitConverter.GetBytes).ToArray(); + } + else + { + throw new ArgumentException($"{memberExpression.Type} was not valid without a Vectorizer"); + } + + return new NearestNeighbors(propertyName, numNeighbors, bytes); + } + /// /// Get's the index field type for the given member info. /// diff --git a/src/Redis.OM/Modeling/Vectors/VectorResult.cs b/src/Redis.OM/Modeling/Vectors/VectorResult.cs new file mode 100644 index 00000000..01ec75d3 --- /dev/null +++ b/src/Redis.OM/Modeling/Vectors/VectorResult.cs @@ -0,0 +1,30 @@ +namespace Redis.OM.Modeling +{ + /// + /// A result from a vector search. + /// + /// The Document type. + public class VectorResult + { + /// + /// Initializes a new instance of the class. + /// + /// the score. + /// the document. + internal VectorResult(double score, T document) + { + Score = score; + Document = document; + } + + /// + /// Gets the distance score between this document and the queried vector. + /// + public double Score { get; } + + /// + /// Gets the document part of the result. + /// + public T Document { get; } + } +} \ No newline at end of file diff --git a/src/Redis.OM/Modeling/Vectors/VectorUtils.cs b/src/Redis.OM/Modeling/Vectors/VectorUtils.cs index 40057b9f..c0ec6c54 100644 --- a/src/Redis.OM/Modeling/Vectors/VectorUtils.cs +++ b/src/Redis.OM/Modeling/Vectors/VectorUtils.cs @@ -96,6 +96,29 @@ public static double[] VecStrToDoubles(RedisReply reply) return doubles; } + /// + /// Converts Vector String to array of doubles. + /// + /// the reply. + /// the doubles. + /// Thrown if unbalanced. + public static double[] VecStrToDoubles(string reply) + { + var bytes = Encoding.ASCII.GetBytes(reply); + if (bytes.Length % 8 != 0) + { + throw new ArgumentException("Unbalanced Vector String"); + } + + var doubles = new double[bytes.Length / 8]; + for (var i = 0; i < bytes.Length; i += 8) + { + doubles[i / 8] = BitConverter.ToDouble(bytes, i); + } + + return doubles; + } + /// /// Parses a vector string to an array of floats. /// diff --git a/src/Redis.OM/Modeling/Vectors/VectorizerAttribute.cs b/src/Redis.OM/Modeling/Vectors/VectorizerAttribute.cs index 3317dc41..815b0c98 100644 --- a/src/Redis.OM/Modeling/Vectors/VectorizerAttribute.cs +++ b/src/Redis.OM/Modeling/Vectors/VectorizerAttribute.cs @@ -8,6 +8,11 @@ namespace Redis.OM.Modeling /// public abstract class VectorizerAttribute : JsonConverterAttribute { + /// + /// Gets the number of bytes per vector. + /// + public int BytesPerVector => VectorType == VectorType.FLOAT32 ? 4 * Dim : 8 * Dim; + /// /// Gets the vector Type generated by the vectorizer. /// diff --git a/src/Redis.OM/SearchExtensions.cs b/src/Redis.OM/SearchExtensions.cs index 8d08a37d..af993cd4 100644 --- a/src/Redis.OM/SearchExtensions.cs +++ b/src/Redis.OM/SearchExtensions.cs @@ -100,6 +100,29 @@ public static IRedisCollection Where(this IRedisCollection source, Expr return new RedisCollection((RedisQueryProvider)source.Provider, exp, source.StateManager, combined, source.SaveState, source.ChunkSize); } + /// + /// Finds nearest neighbors to provided vector. + /// + /// The source. + /// The expression yielding the field to search on. + /// Number of neighbors to search for. + /// The vector or item to search on. + /// The indexed type. + /// The type of the vector. + /// A Redis Collection with a nearest neighbors expression attached to it. + public static IRedisCollection NearestNeighbors(this IRedisCollection source, Expression> expression, int numNeighbors, TKnnType item) + where T : notnull + { + var collection = (RedisCollection)source; + var booleanExpression = collection.BooleanExpression; + + var exp = Expression.Call( + null, + GetMethodInfo(NearestNeighbors, source, expression, numNeighbors, item), + new[] { source.Expression, Expression.Quote(expression), Expression.Constant(numNeighbors), Expression.Constant(item) }); + return new RedisCollection((RedisQueryProvider)source.Provider, exp, source.StateManager, booleanExpression, source.SaveState, source.ChunkSize); + } + /// /// Specifies which items to pull out of Redis. /// diff --git a/src/Redis.OM/Searching/Query/NearestNeighbors.cs b/src/Redis.OM/Searching/Query/NearestNeighbors.cs new file mode 100644 index 00000000..f2791e96 --- /dev/null +++ b/src/Redis.OM/Searching/Query/NearestNeighbors.cs @@ -0,0 +1,36 @@ +namespace Redis.OM.Searching.Query +{ + /// + /// Components of a KNN search. + /// + public class NearestNeighbors + { + /// + /// Initializes a new instance of the class. + /// + /// The property name to search on. + /// The number of nearest neighbors. + /// The vector blob. + public NearestNeighbors(string propertyName, int numNeighbors, byte[] vectorBlob) + { + PropertyName = propertyName; + NumNeighbors = numNeighbors; + VectorBlob = vectorBlob; + } + + /// + /// Gets the name of the property to perform the vector search on. + /// + public string PropertyName { get; } + + /// + /// Gets the number of neighbors to find. + /// + public int NumNeighbors { get; } + + /// + /// Gets the Vector blob to search on. + /// + public byte[] VectorBlob { get; } + } +} \ No newline at end of file diff --git a/src/Redis.OM/Searching/Query/RedisQuery.cs b/src/Redis.OM/Searching/Query/RedisQuery.cs index 2ee18626..6f549fa0 100644 --- a/src/Redis.OM/Searching/Query/RedisQuery.cs +++ b/src/Redis.OM/Searching/Query/RedisQuery.cs @@ -18,6 +18,11 @@ public RedisQuery(string index) this.Index = index; } + /// + /// Gets or sets the nearest neighbors query. + /// + public NearestNeighbors? NearestNeighbors { get; set; } + /// /// Gets or sets the flags for the query options. /// @@ -63,16 +68,31 @@ public RedisQuery(string index) /// /// the serialized arguments. /// thrown if the index is null. - internal string[] SerializeQuery() + internal object[] SerializeQuery() { - var ret = new List(); + var ret = new List(); if (string.IsNullOrEmpty(Index)) { throw new ArgumentException("Index cannot be null"); } ret.Add(Index); - ret.Add(QueryText); + if (NearestNeighbors is null) + { + ret.Add(QueryText); + } + else + { + var queryText = $"({QueryText})=>[KNN {NearestNeighbors.NumNeighbors} @{NearestNeighbors.PropertyName} $V]"; + ret.Add(queryText); + ret.Add("PARAMS"); + ret.Add(2); + ret.Add("V"); + ret.Add(NearestNeighbors.VectorBlob); + ret.Add("DIALECT"); + ret.Add(2); + } + foreach (var flag in (QueryFlags[])Enum.GetValues(typeof(QueryFlags))) { if ((Flags & (long)flag) == (long)flag) diff --git a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/ObjectWithVector.cs b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/ObjectWithVector.cs index d4025443..457eed78 100644 --- a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/ObjectWithVector.cs +++ b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/ObjectWithVector.cs @@ -30,7 +30,7 @@ public class ObjectWithVectorHash public string SimpleVectorizedVector { get; set; } } -[Document] +[Document(StorageType = StorageType.Json, Prefixes = new []{"Simple"})] public class ToyVector { [RedisIdField] public string Id { get; set; } diff --git a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorFunctionalTests.cs b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorFunctionalTests.cs index 5c0f8dcb..cb813ed5 100644 --- a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorFunctionalTests.cs +++ b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorFunctionalTests.cs @@ -1,6 +1,8 @@ +using System.Linq; using System.Text; using Redis.OM.Contracts; using Redis.OM.Modeling; +using Redis.OM.Searching; using Xunit; namespace Redis.OM.Unit.Tests; @@ -15,20 +17,65 @@ public VectorFunctionalTests(RedisSetup setup) _connection = setup.Connection; } + [Fact] + public void BasicQuery() + { + _connection.CreateIndex(typeof(ObjectWithVector)); + var collection = new RedisCollection(_connection); + collection.Insert(new ObjectWithVector + { + Id = "helloWorld", + SimpleHnswVector = Enumerable.Range(0, 10).Select(x => (double)x).ToArray(), + SimpleVectorizedVector = "FooBarBaz" + }); + + var res = collection.NearestNeighbors(x => x.SimpleVectorizedVector, 1, "FooBarBaz").First(); + Assert.Equal("helloWorld", res.Id); + } + + [Fact] + public void Overflow() + { + var doubles = new double[] + { + 1.79769313486231570e+308, 1.79769313486231570e+308, 1.79769313486231570e+308, 1.79769313486231570e+308, + 1.79769313486231570e+308, 1.79769313486231570e+308 + }; + + var lowerDoubles = new double[] + { -1.79769E+308, -1.79769E+308, -1.79769E+308, -1.79769E+308, -1.79769E+308, -1.79769E+308 }; + _connection.CreateIndex(typeof(ToyVector)); + var obj = new ToyVector() + { + Id = "1", + SimpleVector = doubles + }; + + var collection = new RedisCollection(_connection); + collection.NearestNeighbors(x => x.SimpleVector, 1, lowerDoubles).First(); + } + [Fact] public void Dave() { _connection.CreateIndex(typeof(ToyVector)); - var doubles = VectorUtils.VecStrToDoubles("I'm_sorry_Dave,_I'm_afraid_I_can't_do_that......"); + // var doubles = VectorUtils.VecStrToDoubles("This vector's json result gets blown out oddly.."); + var doubles = VectorUtils.VecStrToDoubles("I'm sorry Dave, I'm afraid I can't do that......"); + // var doubles = new double[] { 0, 1, 2, 3, 4, 5 }; var obj = new ToyVector() { - Id = "foo", + Id = "1", SimpleVector = doubles }; _connection.Set(obj); + + var collection = new RedisCollection(_connection); + collection.NearestNeighbors(x => x.SimpleVector, 1, doubles).First(); } + + [Fact] public void TestIndex() { diff --git a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorTests.cs b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorTests.cs index 915de57a..d44d56f5 100644 --- a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorTests.cs +++ b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorTests.cs @@ -1,11 +1,13 @@ using System; using System.Linq; +using System.Linq.Expressions; using System.Text; using System.Text.Json; using NSubstitute; using NSubstitute.ClearExtensions; using Redis.OM.Contracts; using Redis.OM.Modeling; +using Redis.OM.Searching; using Xunit; namespace Redis.OM.Unit.Tests; @@ -18,7 +20,7 @@ public class VectorIndexCreationTests public void CreateIndexWithVector() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(new RedisReply("OK")); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(new RedisReply("OK")); _substitute.CreateIndex(typeof(ObjectWithVector)); _substitute.CreateIndex(typeof(ObjectWithVectorHash)); @@ -49,6 +51,33 @@ public void CreateIndexWithVector() ); } + [Fact] + public void SimpleKnnQuery() + { + _substitute.ClearSubstitute(); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(new RedisReply(0)); + var collection = new RedisCollection(_substitute); + var compVector = new double[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; + float[] floats = Enumerable.Range(0, 30).Select(x => (float)x).ToArray(); + var blob = compVector.SelectMany(BitConverter.GetBytes).ToArray(); + var floatBlob = floats.SelectMany(BitConverter.GetBytes).ToArray(); + _ = collection.NearestNeighbors(x=>x.SimpleHnswVector, 5, compVector).ToList(); + + _substitute.Received().Execute("FT.SEARCH", + $"{nameof(ObjectWithVector).ToLower()}-idx", + "(*)=>[KNN 5 @SimpleHnswVector $V]", + "PARAMS", 2, "V", Arg.Is(b=>b.SequenceEqual(blob)), "DIALECT", 2, "LIMIT", "0", "100"); + + _substitute.ClearSubstitute(); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(new RedisReply(0)); + _ = collection.NearestNeighbors(x => x.SimpleVectorizedVector, 8, "hello world").ToArray(); + + _substitute.Received().Execute("FT.SEARCH", + $"{nameof(ObjectWithVector).ToLower()}-idx", + "(*)=>[KNN 8 @SimpleVectorizedVector $V]", + "PARAMS", 2, "V", Arg.Is(b=>b.SequenceEqual(floatBlob)), "DIALECT", 2, "LIMIT", "0", "100"); + } + [Fact] public void TestBinConversions() { @@ -74,8 +103,6 @@ public void InsertVectors() { vectorizedFlatHashVector[i] = i; } - - simpleHnswJsonStr.Append(string.Join(',', simpleHnswHash)); vectorizedFlatVectorJsonStr.Append(string.Join(',', vectorizedFlatHashVector)); From c257207392d5286eefdd0a140d7a95e56f93200c Mon Sep 17 00:00:00 2001 From: slorello89 Date: Thu, 19 Oct 2023 09:08:37 -0400 Subject: [PATCH 06/36] Vector Range, score binding --- .../AggregationPredicates/QueryPredicate.cs | 4 +- .../Common/ExpressionParserUtilities.cs | 90 +++++++++++++++--- src/Redis.OM/Common/ExpressionTranslator.cs | 50 +++++----- .../Modeling/Vectors/JsonScoreConverter.cs | 27 ++++++ src/Redis.OM/Modeling/Vectors/VectorResult.cs | 18 ++-- .../Modeling/Vectors/VectorScoreField.cs | 18 ++++ src/Redis.OM/Modeling/Vectors/VectorScores.cs | 47 ++++++++++ src/Redis.OM/RedisObjectHandler.cs | 42 ++++++++- src/Redis.OM/Searching/Query/RedisQuery.cs | 32 +++++-- src/Redis.OM/Searching/RedisCollection.cs | 2 +- .../Searching/RedisCollectionEnumerator.cs | 2 +- src/Redis.OM/Vectors.cs | 33 +++++++ .../VectorTests/ObjectWithVector.cs | 5 + .../VectorTests/VectorFunctionalTests.cs | 91 +++++++++++-------- .../VectorTests/VectorTests.cs | 54 ++++++++++- 15 files changed, 409 insertions(+), 106 deletions(-) create mode 100644 src/Redis.OM/Modeling/Vectors/JsonScoreConverter.cs create mode 100644 src/Redis.OM/Modeling/Vectors/VectorScoreField.cs create mode 100644 src/Redis.OM/Modeling/Vectors/VectorScores.cs create mode 100644 src/Redis.OM/Vectors.cs diff --git a/src/Redis.OM/Aggregation/AggregationPredicates/QueryPredicate.cs b/src/Redis.OM/Aggregation/AggregationPredicates/QueryPredicate.cs index 5a6a27ef..87263bf8 100644 --- a/src/Redis.OM/Aggregation/AggregationPredicates/QueryPredicate.cs +++ b/src/Redis.OM/Aggregation/AggregationPredicates/QueryPredicate.cs @@ -81,7 +81,7 @@ protected override void ValidateAndPushOperand(Expression expression, Stack()); // hack - will need to revisit when integrating vectors into aggregations. stack.Push(BuildQueryPredicate(binaryExpression.NodeType, memberExpression, System.Linq.Expressions.Expression.Constant(val))); } } @@ -92,7 +92,7 @@ protected override void ValidateAndPushOperand(Expression expression, Stack())); } else if (expression is UnaryExpression uni) { diff --git a/src/Redis.OM/Common/ExpressionParserUtilities.cs b/src/Redis.OM/Common/ExpressionParserUtilities.cs index aec69fbb..2b02c2ec 100644 --- a/src/Redis.OM/Common/ExpressionParserUtilities.cs +++ b/src/Redis.OM/Common/ExpressionParserUtilities.cs @@ -1,6 +1,7 @@ using System; using System.Collections; using System.Collections.Generic; +using System.Collections.ObjectModel; using System.Globalization; using System.Linq; using System.Linq.Expressions; @@ -11,6 +12,7 @@ using Redis.OM.Aggregation; using Redis.OM.Aggregation.AggregationPredicates; using Redis.OM.Modeling; +using Redis.OM.Modeling.Vectors; using Redis.OM.Searching.Query; namespace Redis.OM.Common @@ -72,18 +74,19 @@ internal static string GetOperandString(MethodCallExpression exp) /// Gets the operand string from a search. /// /// expression. + /// The parameters. /// Treat enum as an integer. /// Whether or not to negate the result. /// the operand string. /// thrown if expression is un-parseable. - internal static string GetOperandStringForQueryArgs(Expression exp, bool treatEnumsAsInt = false, bool negate = false) + internal static string GetOperandStringForQueryArgs(Expression exp, List parameters, bool treatEnumsAsInt = false, bool negate = false) { var res = exp switch { ConstantExpression constExp => $"{constExp.Value}", MemberExpression member => GetOperandStringForMember(member, treatEnumsAsInt), - MethodCallExpression method => TranslateMethodStandardQuerySyntax(method), - UnaryExpression unary => GetOperandStringForQueryArgs(unary.Operand, treatEnumsAsInt, unary.NodeType == ExpressionType.Not), + MethodCallExpression method => TranslateMethodStandardQuerySyntax(method, parameters), + UnaryExpression unary => GetOperandStringForQueryArgs(unary.Operand, parameters, treatEnumsAsInt, unary.NodeType == ExpressionType.Not), _ => throw new ArgumentException("Unrecognized Expression type") }; @@ -171,20 +174,22 @@ internal static string ParseBinaryExpression(BinaryExpression rootBinaryExpressi /// Translates the method expression. /// /// the expression. + /// The parameters to be passed into the query. /// The expression translated. /// thrown if the method isn't recognized. - internal static string TranslateMethodExpressions(MethodCallExpression exp) + internal static string TranslateMethodExpressions(MethodCallExpression exp, List parameters) { return exp.Method.Name switch { - "Contains" => TranslateContainsStandardQuerySyntax(exp), + "Contains" => TranslateContainsStandardQuerySyntax(exp, parameters), nameof(StringExtension.FuzzyMatch) => TranslateFuzzyMatch(exp), nameof(StringExtension.MatchContains) => TranslateMatchContains(exp), nameof(StringExtension.MatchStartsWith) => TranslateMatchStartsWith(exp), nameof(StringExtension.MatchEndsWith) => TranslateMatchEndsWith(exp), + nameof(VectorExtensions.VectorRange) => TranslateVectorRange(exp, parameters), nameof(string.StartsWith) => TranslateStartsWith(exp), nameof(string.EndsWith) => TranslateEndsWith(exp), - "Any" => TranslateAnyForEmbeddedObjects(exp), + "Any" => TranslateAnyForEmbeddedObjects(exp, parameters), _ => throw new ArgumentException($"Unrecognized method for query translation:{exp.Method.Name}") }; } @@ -640,16 +645,17 @@ private static IEnumerable SplitBinaryExpression(BinaryExpress while (true); } - private static string TranslateMethodStandardQuerySyntax(MethodCallExpression exp) + private static string TranslateMethodStandardQuerySyntax(MethodCallExpression exp, List parameters) { return exp.Method.Name switch { nameof(StringExtension.FuzzyMatch) => TranslateFuzzyMatch(exp), nameof(string.Format) => TranslateFormatMethodStandardQuerySyntax(exp), - nameof(string.Contains) => TranslateContainsStandardQuerySyntax(exp), + nameof(string.Contains) => TranslateContainsStandardQuerySyntax(exp, parameters), nameof(string.StartsWith) => TranslateStartsWith(exp), nameof(string.EndsWith) => TranslateEndsWith(exp), - "Any" => TranslateAnyForEmbeddedObjects(exp), + nameof(VectorExtensions.VectorRange) => TranslateVectorRange(exp, parameters), + "Any" => TranslateAnyForEmbeddedObjects(exp, parameters), _ => throw new InvalidOperationException($"Unable to parse method {exp.Method.Name}") }; } @@ -786,7 +792,7 @@ private static string TranslateFuzzyMatch(MethodCallExpression exp) }; } - private static string TranslateContainsStandardQuerySyntax(MethodCallExpression exp) + private static string TranslateContainsStandardQuerySyntax(MethodCallExpression exp, List parameters) { MemberExpression? expression = null; Type type; @@ -797,7 +803,7 @@ private static string TranslateContainsStandardQuerySyntax(MethodCallExpression { var propertyExpression = (MemberExpression)exp.Arguments.Last(); var valuesExpression = (MemberExpression)exp.Arguments.First(); - literal = GetOperandStringForQueryArgs(propertyExpression); + literal = GetOperandStringForQueryArgs(propertyExpression, parameters); if (!literal.StartsWith("@")) { if (exp.Arguments.Count == 1 && exp.Object != null) @@ -835,7 +841,7 @@ private static string TranslateContainsStandardQuerySyntax(MethodCallExpression type = Nullable.GetUnderlyingType(propertyExpression.Type) ?? propertyExpression.Type; memberName = GetOperandStringForMember(propertyExpression); var treatEnumsAsInts = type.IsEnum && !(propertyExpression.Member.GetCustomAttributes(typeof(JsonConverterAttribute)).FirstOrDefault() is JsonConverterAttribute converter && converter.ConverterType == typeof(JsonStringEnumConverter)); - literal = GetOperandStringForQueryArgs(valuesExpression, treatEnumsAsInts); + literal = GetOperandStringForQueryArgs(valuesExpression, parameters, treatEnumsAsInts); if ((type == typeof(string) || type == typeof(string[]) || type == typeof(List) || type == typeof(Guid) || type == typeof(Ulid) || (type.IsEnum && !treatEnumsAsInts)) && attribute is IndexedAttribute) { @@ -875,7 +881,7 @@ private static string TranslateContainsStandardQuerySyntax(MethodCallExpression type = Nullable.GetUnderlyingType(expression.Type) ?? expression.Type; memberName = GetOperandStringForMember(expression); - literal = GetOperandStringForQueryArgs(exp.Arguments.Last()); + literal = GetOperandStringForQueryArgs(exp.Arguments.Last(), parameters); if (searchFieldAttribute is not null && searchFieldAttribute is SearchableAttribute) { @@ -885,12 +891,12 @@ private static string TranslateContainsStandardQuerySyntax(MethodCallExpression return (type == typeof(string)) ? $"({memberName}:{{*{EscapeTagField(literal)}*}})" : $"({memberName}:{{{EscapeTagField(literal)}}})"; } - private static string TranslateAnyForEmbeddedObjects(MethodCallExpression exp) + private static string TranslateAnyForEmbeddedObjects(MethodCallExpression exp, List parameters) { var type = exp.Arguments.Last().Type; var prefix = GetOperandString(exp.Arguments[0]); var lambda = (LambdaExpression)exp.Arguments.Last(); - var tempQuery = ExpressionTranslator.TranslateBinaryExpression((BinaryExpression)lambda.Body); + var tempQuery = ExpressionTranslator.TranslateBinaryExpression((BinaryExpression)lambda.Body, parameters); return tempQuery.Replace("@", $"{prefix}_"); } @@ -927,5 +933,59 @@ private static string GetConstantStringForArgs(ConstantExpression constExp) return $"{valueAsString}"; } + + private static object GetOperand(Expression expression) + { + return expression switch + { + MemberExpression me => GetValue(me.Member, ((ConstantExpression)me.Expression).Value), + ConstantExpression ce => ce.Value, + _ => throw new InvalidOperationException("Could not determine value.") + }; + } + + private static string TranslateVectorRange(MethodCallExpression exp, List parameters) + { + if (exp.Arguments[0] is not MemberExpression member) + { + throw new InvalidOperationException("Vector Range must be called on a member"); + } + + var field = GetOperandStringForMember(member); + var vectorizer = member.Member.GetCustomAttributes().FirstOrDefault(); + byte[] bytes; + var operand = GetOperand(exp.Arguments[1]); + + if (vectorizer is not null) + { + bytes = vectorizer.Vectorize(operand); + } + else if (member.Type == typeof(double[])) + { + bytes = ((double[])operand).SelectMany(BitConverter.GetBytes).ToArray(); + } + else if (member.Type == typeof(float[])) + { + bytes = ((float[])operand).SelectMany(BitConverter.GetBytes).ToArray(); + } + else + { + throw new InvalidOperationException( + $"Attempting to run a vector range on a {member.Type} with no provided vectorizer"); + } + + var distance = (double)((ConstantExpression)exp.Arguments[2]).Value; + var distanceArgName = parameters.Count.ToString(); + parameters.Add(distance); + var vectorArgName = parameters.Count.ToString(); + parameters.Add(bytes); + if (exp.Arguments.Count > 3) + { + var scoreName = $"{GetOperand(exp.Arguments[3])}{VectorScores.RangeScoreSuffix}"; + return $"{field}:[VECTOR_RANGE ${distanceArgName} ${vectorArgName}]=>{{$YIELD_DISTANCE_AS: {scoreName}}}"; + } + + return $"{field}:[VECTOR_RANGE ${distanceArgName} ${vectorArgName}]"; + } } } \ No newline at end of file diff --git a/src/Redis.OM/Common/ExpressionTranslator.cs b/src/Redis.OM/Common/ExpressionTranslator.cs index f4428cd7..1984a480 100644 --- a/src/Redis.OM/Common/ExpressionTranslator.cs +++ b/src/Redis.OM/Common/ExpressionTranslator.cs @@ -8,6 +8,7 @@ using Redis.OM.Aggregation; using Redis.OM.Aggregation.AggregationPredicates; using Redis.OM.Modeling; +using Redis.OM.Modeling.Vectors; using Redis.OM.Searching; using Redis.OM.Searching.Query; @@ -183,6 +184,7 @@ internal static RedisQuery BuildQueryFromExpression(Expression expression, Type throw new InvalidOperationException("Searches can only be performed on objects decorated with a RedisObjectDefinitionAttribute that specifies a particular index"); } + var parameters = new List(); var indexName = string.IsNullOrEmpty(attr.IndexName) ? $"{type.Name.ToLower()}-idx" : attr.IndexName; var query = new RedisQuery(indexName!) { QueryText = "*" }; switch (expression) @@ -227,7 +229,7 @@ internal static RedisQuery BuildQueryFromExpression(Expression expression, Type query.GeoFilter = ExpressionParserUtilities.TranslateGeoFilter(exp); break; case "Where": - query.QueryText = TranslateWhereMethod(exp); + query.QueryText = TranslateWhereMethod(exp, parameters); break; case "NearestNeighbors": query.NearestNeighbors = ParseNearestNeighborsFromExpression(exp); @@ -239,15 +241,17 @@ internal static RedisQuery BuildQueryFromExpression(Expression expression, Type } case LambdaExpression lambda: - query.QueryText = BuildQueryFromExpression(lambda.Body); + query.QueryText = BuildQueryFromExpression(lambda.Body, parameters); break; } if (mainBooleanExpression != null) { - query.QueryText = BuildQueryFromExpression(((LambdaExpression)mainBooleanExpression).Body); + parameters = new List(); + query.QueryText = BuildQueryFromExpression(((LambdaExpression)mainBooleanExpression).Body, parameters); } + query.Parameters = parameters; return query; } @@ -314,40 +318,41 @@ internal static SearchFieldType DetermineIndexFieldsType(MemberInfo member) /// Translates a binary expression. /// /// The Binary Expression. + /// The parameters of the query. /// The query string formatted from the binary expression. /// Thrown if expression is not parsable because of the arguments passed into it. - internal static string TranslateBinaryExpression(BinaryExpression binExpression) + internal static string TranslateBinaryExpression(BinaryExpression binExpression, List parameters) { var sb = new StringBuilder(); if (binExpression.Left is BinaryExpression leftBin && binExpression.Right is BinaryExpression rightBin) { sb.Append("("); - sb.Append(TranslateBinaryExpression(leftBin)); + sb.Append(TranslateBinaryExpression(leftBin, parameters)); sb.Append(SplitPredicateSeporators(binExpression.NodeType)); - sb.Append(TranslateBinaryExpression(rightBin)); + sb.Append(TranslateBinaryExpression(rightBin, parameters)); sb.Append(")"); } else if (binExpression.Left is BinaryExpression left) { sb.Append("("); - sb.Append(TranslateBinaryExpression(left)); + sb.Append(TranslateBinaryExpression(left, parameters)); sb.Append(SplitPredicateSeporators(binExpression.NodeType)); - sb.Append(ExpressionParserUtilities.GetOperandStringForQueryArgs(binExpression.Right)); + sb.Append(ExpressionParserUtilities.GetOperandStringForQueryArgs(binExpression.Right, parameters)); sb.Append(")"); } else if (binExpression.Right is BinaryExpression right) { sb.Append("("); - sb.Append(ExpressionParserUtilities.GetOperandStringForQueryArgs(binExpression.Left)); + sb.Append(ExpressionParserUtilities.GetOperandStringForQueryArgs(binExpression.Left, parameters)); sb.Append(SplitPredicateSeporators(binExpression.NodeType)); - sb.Append(TranslateBinaryExpression(right)); + sb.Append(TranslateBinaryExpression(right, parameters)); sb.Append(")"); } else { - var leftContent = ExpressionParserUtilities.GetOperandStringForQueryArgs(binExpression.Left); + var leftContent = ExpressionParserUtilities.GetOperandStringForQueryArgs(binExpression.Left, parameters); - var rightContent = ExpressionParserUtilities.GetOperandStringForQueryArgs(binExpression.Right); + var rightContent = ExpressionParserUtilities.GetOperandStringForQueryArgs(binExpression.Right, parameters); if (binExpression.Left is MemberExpression member) { @@ -715,26 +720,26 @@ private static RedisSortBy TranslateOrderByMethod(MethodCallExpression expressio var predicate = (UnaryExpression)expression.Arguments[1]; var lambda = (LambdaExpression)predicate.Operand; var memberExpression = (MemberExpression)lambda.Body; - sb.Field = ExpressionParserUtilities.GetSearchFieldNameFromMember(memberExpression); + sb.Field = memberExpression.Member.Name == nameof(VectorScores.NearestNeighborsScore) ? VectorScores.NearestNeighborScoreName : ExpressionParserUtilities.GetSearchFieldNameFromMember(memberExpression); sb.Direction = ascending ? SortDirection.Ascending : SortDirection.Descending; return sb; } - private static string BuildQueryFromExpression(Expression exp) + private static string BuildQueryFromExpression(Expression exp, List parameters) { if (exp is BinaryExpression binExp) { - return TranslateBinaryExpression(binExp); + return TranslateBinaryExpression(binExp, parameters); } if (exp is MethodCallExpression method) { - return ExpressionParserUtilities.TranslateMethodExpressions(method); + return ExpressionParserUtilities.TranslateMethodExpressions(method, parameters); } if (exp is UnaryExpression uni) { - var operandString = BuildQueryFromExpression(uni.Operand); + var operandString = BuildQueryFromExpression(uni.Operand, parameters); if (uni.NodeType == ExpressionType.Not) { operandString = $"-{operandString}"; @@ -752,18 +757,11 @@ private static string BuildQueryFromExpression(Expression exp) throw new ArgumentException("Unparseable Lambda Body detected"); } - private static string TranslateFirstMethod(MethodCallExpression expression) + private static string TranslateWhereMethod(MethodCallExpression expression, List parameters) { var predicate = (UnaryExpression)expression.Arguments[1]; var lambda = (LambdaExpression)predicate.Operand; - return BuildQueryFromExpression(lambda.Body); - } - - private static string TranslateWhereMethod(MethodCallExpression expression) - { - var predicate = (UnaryExpression)expression.Arguments[1]; - var lambda = (LambdaExpression)predicate.Operand; - return BuildQueryFromExpression(lambda.Body); + return BuildQueryFromExpression(lambda.Body, parameters); } private static string BuildQueryPredicate(ExpressionType expType, string left, string right, MemberExpression memberExpression) diff --git a/src/Redis.OM/Modeling/Vectors/JsonScoreConverter.cs b/src/Redis.OM/Modeling/Vectors/JsonScoreConverter.cs new file mode 100644 index 00000000..e4c70284 --- /dev/null +++ b/src/Redis.OM/Modeling/Vectors/JsonScoreConverter.cs @@ -0,0 +1,27 @@ +using System; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Redis.OM.Modeling +{ + /// + /// ignores the Json Score field. + /// + internal class JsonScoreConverter : JsonConverter + { + /// + public override bool CanConvert(Type typeToConvert) => typeToConvert == typeof(double) || typeToConvert == typeof(double?); + + /// + public override object? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + return -1.0; + } + + /// + public override void Write(Utf8JsonWriter writer, object value, JsonSerializerOptions options) + { + writer.WriteNumberValue(-1.0); + } + } +} \ No newline at end of file diff --git a/src/Redis.OM/Modeling/Vectors/VectorResult.cs b/src/Redis.OM/Modeling/Vectors/VectorResult.cs index 01ec75d3..29ea8c0d 100644 --- a/src/Redis.OM/Modeling/Vectors/VectorResult.cs +++ b/src/Redis.OM/Modeling/Vectors/VectorResult.cs @@ -1,30 +1,30 @@ -namespace Redis.OM.Modeling +namespace Redis.OM { /// - /// A result from a vector search. + /// Represents a vector result with its score and the document associated with it. /// - /// The Document type. + /// the document type. public class VectorResult { /// /// Initializes a new instance of the class. /// - /// the score. /// the document. - internal VectorResult(double score, T document) + /// the score. + internal VectorResult(T document, double score) { Score = score; Document = document; } /// - /// Gets the distance score between this document and the queried vector. + /// Gets the document. /// - public double Score { get; } + public T Document { get; } /// - /// Gets the document part of the result. + /// Gets the score. /// - public T Document { get; } + public double Score { get; } } } \ No newline at end of file diff --git a/src/Redis.OM/Modeling/Vectors/VectorScoreField.cs b/src/Redis.OM/Modeling/Vectors/VectorScoreField.cs new file mode 100644 index 00000000..9584e7f0 --- /dev/null +++ b/src/Redis.OM/Modeling/Vectors/VectorScoreField.cs @@ -0,0 +1,18 @@ +using System; +using System.Text.Json.Serialization; + +namespace Redis.OM.Modeling +{ + /// + /// Attribute to decorate vector score field. A field decorated with this will have the sentinel value -1 when + /// the score is not present in the result. + /// + public class KnnVectorScore : JsonConverterAttribute + { + /// + public override JsonConverter? CreateConverter(Type typeToConvert) + { + return new JsonScoreConverter(); + } + } +} \ No newline at end of file diff --git a/src/Redis.OM/Modeling/Vectors/VectorScores.cs b/src/Redis.OM/Modeling/Vectors/VectorScores.cs new file mode 100644 index 00000000..d7aaefdd --- /dev/null +++ b/src/Redis.OM/Modeling/Vectors/VectorScores.cs @@ -0,0 +1,47 @@ +using System.Collections.Generic; +using System.Linq; +using System.Text.Json.Serialization; + +namespace Redis.OM.Modeling.Vectors +{ + /// + /// A collector for vector scores. + /// + public class VectorScores + { + /// + /// The Range score suffix. + /// + internal const string RangeScoreSuffix = "_RangeScore"; + + /// + /// The Nearest neighbor score name. + /// + internal const string NearestNeighborScoreName = "KnnNeighborScore"; + + /// + /// Initializes a new instance of the class. + /// + internal VectorScores() + { + } + + /// + /// Gets the nearest neighbor score. + /// + [JsonIgnore] + public double? NearestNeighborsScore { get; internal set; } + + /// + /// Gets the first score from the vector ranges. + /// + [JsonIgnore] + public double? RangeScore => RangeScores.FirstOrDefault().Value; + + /// + /// Gets or sets the range score dictionary. + /// + [JsonIgnore] + internal Dictionary RangeScores { get; set; } = new (); + } +} \ No newline at end of file diff --git a/src/Redis.OM/RedisObjectHandler.cs b/src/Redis.OM/RedisObjectHandler.cs index d4b95e86..7aa07510 100644 --- a/src/Redis.OM/RedisObjectHandler.cs +++ b/src/Redis.OM/RedisObjectHandler.cs @@ -9,6 +9,7 @@ using System.Web; using Redis.OM.Contracts; using Redis.OM.Modeling; +using Redis.OM.Modeling.Vectors; [assembly: InternalsVisibleTo("Redis.OM.POC")] @@ -63,7 +64,30 @@ internal static T FromHashSet(IDictionary hash) asJson = SendToJson(dict, typeof(T)); } - return JsonSerializer.Deserialize(asJson, RedisSerializationSettings.JsonSerializerOptions) ?? throw new Exception("Deserialization fail"); + var res = JsonSerializer.Deserialize(asJson, RedisSerializationSettings.JsonSerializerOptions) ?? throw new Exception("Deserialization fail"); + if (hash.ContainsKey(VectorScores.NearestNeighborScoreName) || hash.Keys.Any(x => x.EndsWith(VectorScores.RangeScoreSuffix))) + { + var vectorScores = new VectorScores(); + if (hash.ContainsKey(VectorScores.NearestNeighborScoreName)) + { + vectorScores.NearestNeighborsScore = ParseScoreFromString(hash[VectorScores.NearestNeighborScoreName]); + } + + foreach (var key in hash.Keys.Where(x => x.EndsWith(VectorScores.RangeScoreSuffix))) + { + var strippedKey = key.Substring(0, key.Length - VectorScores.RangeScoreSuffix.Length); + var score = ParseScoreFromString(hash[key]); + vectorScores.RangeScores.Add(strippedKey, score); + } + + var scoreProperties = typeof(T).GetProperties().Where(x => x.PropertyType == typeof(VectorScores)); + foreach (var p in scoreProperties) + { + p.SetValue(res, vectorScores); + } + } + + return res; } /// @@ -608,6 +632,22 @@ private static string SendToJson(IDictionary hash, Type t) return ret; } + private static double ParseScoreFromString(string scoreStr) + { + if (double.TryParse(scoreStr, NumberStyles.Any, CultureInfo.InvariantCulture, out var score)) + { + return score; + } + + return scoreStr switch + { + "inf" => double.PositiveInfinity, + "-inf" => double.NegativeInfinity, + "nan" => double.NaN, + _ => throw new ArgumentException($"Could not parse score from {scoreStr}", nameof(scoreStr)) + }; + } + private static Type GetEnumerableType(PropertyInfo pi) { var type = pi.PropertyType.GetElementType(); diff --git a/src/Redis.OM/Searching/Query/RedisQuery.cs b/src/Redis.OM/Searching/Query/RedisQuery.cs index 6f549fa0..f0e345bc 100644 --- a/src/Redis.OM/Searching/Query/RedisQuery.cs +++ b/src/Redis.OM/Searching/Query/RedisQuery.cs @@ -1,5 +1,7 @@ using System; using System.Collections.Generic; +using System.Linq; +using Redis.OM.Modeling.Vectors; namespace Redis.OM.Searching.Query { @@ -23,6 +25,11 @@ public RedisQuery(string index) /// public NearestNeighbors? NearestNeighbors { get; set; } + /// + /// Gets or sets the parameters for the query. + /// + public List Parameters { get; set; } = new (); + /// /// Gets or sets the flags for the query options. /// @@ -70,6 +77,7 @@ public RedisQuery(string index) /// thrown if the index is null. internal object[] SerializeQuery() { + var parameters = new List(Parameters); var ret = new List(); if (string.IsNullOrEmpty(Index)) { @@ -77,18 +85,28 @@ internal object[] SerializeQuery() } ret.Add(Index); - if (NearestNeighbors is null) + + if (NearestNeighbors is not null) { - ret.Add(QueryText); + var queryText = $"({QueryText})=>[KNN {NearestNeighbors.NumNeighbors} @{NearestNeighbors.PropertyName} ${parameters.Count} AS {VectorScores.NearestNeighborScoreName}]"; + ret.Add(queryText); + parameters.Add(NearestNeighbors.VectorBlob); } else { - var queryText = $"({QueryText})=>[KNN {NearestNeighbors.NumNeighbors} @{NearestNeighbors.PropertyName} $V]"; - ret.Add(queryText); + ret.Add(QueryText); + } + + if (parameters.Any()) + { ret.Add("PARAMS"); - ret.Add(2); - ret.Add("V"); - ret.Add(NearestNeighbors.VectorBlob); + ret.Add(parameters.Count * 2); + for (var i = 0; i < parameters.Count; i++) + { + ret.Add(i.ToString()); + ret.Add(parameters[i]); + } + ret.Add("DIALECT"); ret.Add(2); } diff --git a/src/Redis.OM/Searching/RedisCollection.cs b/src/Redis.OM/Searching/RedisCollection.cs index 57b0d851..b73b9f6f 100644 --- a/src/Redis.OM/Searching/RedisCollection.cs +++ b/src/Redis.OM/Searching/RedisCollection.cs @@ -549,7 +549,7 @@ public T Single(Expression> expression) } /// - public IEnumerator GetEnumerator() + public virtual IEnumerator GetEnumerator() { StateManager.Clear(); return new RedisCollectionEnumerator(Expression, _connection, ChunkSize, StateManager, BooleanExpression, SaveState, RootType, typeof(T)); diff --git a/src/Redis.OM/Searching/RedisCollectionEnumerator.cs b/src/Redis.OM/Searching/RedisCollectionEnumerator.cs index dc8a29e2..19cd349d 100644 --- a/src/Redis.OM/Searching/RedisCollectionEnumerator.cs +++ b/src/Redis.OM/Searching/RedisCollectionEnumerator.cs @@ -164,7 +164,7 @@ private async ValueTask GetNextChunkAsync() _query!.Limit!.Offset = _query.Limit.Offset + _query.Limit.Number; } - _records = await _connection.SearchAsync(_query); + _records = await _connection.SearchAsync(_query).ConfigureAwait(false); _index = 0; _started = true; ConcatenateRecords(); diff --git a/src/Redis.OM/Vectors.cs b/src/Redis.OM/Vectors.cs new file mode 100644 index 00000000..df6868e7 --- /dev/null +++ b/src/Redis.OM/Vectors.cs @@ -0,0 +1,33 @@ +using System; + +namespace Redis.OM +{ + /// + /// Container class for Vector extensions. + /// + public static class VectorExtensions + { + /// + /// Placeholder method to allow you to perform vector range operations. Only meant to be run + /// in context of a query. + /// + /// The vector field. + /// The comparison object. + /// The allowable distance from the provided object. + /// The type to compare. + /// Whether the queried vector is within the allowable distance. + public static bool VectorRange(this T obj, object comparisonObject, double range) => throw new NotImplementedException("This method is only meant to be run within a query of Redis."); + + /// + /// Placeholder method to allow you to perform vector range operations. Only meant to be run + /// in context of a query. + /// + /// The vector field. + /// The comparison object. + /// The allowable distance from the provided object. + /// The name of the score in the output. + /// The type to compare. + /// Whether the queried vector is within the allowable distance. + public static bool VectorRange(this T obj, object comparisonObject, double range, string scoreName) => throw new NotImplementedException("This method is only meant to be run within a query of Redis."); + } +} \ No newline at end of file diff --git a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/ObjectWithVector.cs b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/ObjectWithVector.cs index 457eed78..e94c8301 100644 --- a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/ObjectWithVector.cs +++ b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/ObjectWithVector.cs @@ -1,4 +1,7 @@ +using System.Text.Json.Serialization; using Redis.OM.Modeling; +using Redis.OM.Modeling.Vectors; +using StackExchange.Redis; namespace Redis.OM.Unit.Tests; @@ -14,6 +17,8 @@ public class ObjectWithVector [Vector(Algorithm = VectorAlgorithm.FLAT)] [SimpleVectorizer] public string SimpleVectorizedVector { get; set; } + + public VectorScores VectorScoreField { get; set; } } [Document(StorageType = StorageType.Hash)] diff --git a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorFunctionalTests.cs b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorFunctionalTests.cs index cb813ed5..ba1586a3 100644 --- a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorFunctionalTests.cs +++ b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorFunctionalTests.cs @@ -1,7 +1,6 @@ using System.Linq; using System.Text; using Redis.OM.Contracts; -using Redis.OM.Modeling; using Redis.OM.Searching; using Xunit; @@ -10,7 +9,7 @@ namespace Redis.OM.Unit.Tests; [Collection("Redis")] public class VectorFunctionalTests { - private IRedisConnection _connection = null; + private readonly IRedisConnection _connection; public VectorFunctionalTests(RedisSetup setup) { @@ -18,7 +17,7 @@ public VectorFunctionalTests(RedisSetup setup) } [Fact] - public void BasicQuery() + public void BasicRangeQuery() { _connection.CreateIndex(typeof(ObjectWithVector)); var collection = new RedisCollection(_connection); @@ -28,54 +27,68 @@ public void BasicQuery() SimpleHnswVector = Enumerable.Range(0, 10).Select(x => (double)x).ToArray(), SimpleVectorizedVector = "FooBarBaz" }); - - var res = collection.NearestNeighbors(x => x.SimpleVectorizedVector, 1, "FooBarBaz").First(); + var queryVector = Enumerable.Range(0, 10).Select(x => (double)x).ToArray(); + var res = collection.First(x => x.SimpleHnswVector.VectorRange(queryVector, 5)); Assert.Equal("helloWorld", res.Id); } [Fact] - public void Overflow() + public void MultiRangeOnSameProperty() { - var doubles = new double[] + _connection.CreateIndex(typeof(ObjectWithVector)); + var collection = new RedisCollection(_connection); + collection.Insert(new ObjectWithVector { - 1.79769313486231570e+308, 1.79769313486231570e+308, 1.79769313486231570e+308, 1.79769313486231570e+308, - 1.79769313486231570e+308, 1.79769313486231570e+308 - }; + Id = "helloWorld", + SimpleHnswVector = Enumerable.Range(0, 10).Select(x => (double)x).ToArray(), + SimpleVectorizedVector = "FooBarBaz" + }); + var queryVector = Enumerable.Range(0, 10).Select(x => (double)x).ToArray(); + var res = collection.First(x => x.SimpleHnswVector.VectorRange(queryVector, 5) && x.SimpleHnswVector.VectorRange(queryVector, 6)); + Assert.Equal("helloWorld", res.Id); + } - var lowerDoubles = new double[] - { -1.79769E+308, -1.79769E+308, -1.79769E+308, -1.79769E+308, -1.79769E+308, -1.79769E+308 }; - _connection.CreateIndex(typeof(ToyVector)); - var obj = new ToyVector() + [Fact] + public void RangeAndKnn() + { + _connection.CreateIndex(typeof(ObjectWithVector)); + var collection = new RedisCollection(_connection); + collection.Insert(new ObjectWithVector { - Id = "1", - SimpleVector = doubles - }; - - var collection = new RedisCollection(_connection); - collection.NearestNeighbors(x => x.SimpleVector, 1, lowerDoubles).First(); + Id = "helloWorld", + SimpleHnswVector = Enumerable.Range(0, 10).Select(x => (double)x).ToArray(), + SimpleVectorizedVector = "FooBarBaz" + }); + var queryVector = Enumerable.Range(0, 10).Select(x => (double)x).ToArray(); + queryVector[0] += 2; + var res = collection.NearestNeighbors(x=>x.SimpleVectorizedVector, 1, "FooBarBaz") + .First(x => x.SimpleHnswVector.VectorRange(queryVector, 5, "range")); + Assert.Equal("helloWorld", res.Id); + Assert.Equal(4, res.VectorScoreField.RangeScore); + Assert.Equal(0, res.VectorScoreField.NearestNeighborsScore); } - + [Fact] - public void Dave() + public void BasicQuery() { - _connection.CreateIndex(typeof(ToyVector)); - - // var doubles = VectorUtils.VecStrToDoubles("This vector's json result gets blown out oddly.."); - var doubles = VectorUtils.VecStrToDoubles("I'm sorry Dave, I'm afraid I can't do that......"); - // var doubles = new double[] { 0, 1, 2, 3, 4, 5 }; - var obj = new ToyVector() + _connection.CreateIndex(typeof(ObjectWithVector)); + var collection = new RedisCollection(_connection); + collection.Insert(new ObjectWithVector { - Id = "1", - SimpleVector = doubles - }; - _connection.Set(obj); + Id = "helloWorld", + SimpleHnswVector = Enumerable.Range(0, 10).Select(x => (double)x).ToArray(), + SimpleVectorizedVector = "FooBarBaz" + }); + var queryVector = Enumerable.Range(0, 10).Select(x => (double)x).ToArray(); + queryVector[0] += 2; - var collection = new RedisCollection(_connection); - collection.NearestNeighbors(x => x.SimpleVector, 1, doubles).First(); + var res = collection.NearestNeighbors(x => x.SimpleVectorizedVector, 1, "FooBarBaz").First(); + Assert.Equal("helloWorld", res.Id); + Assert.Equal(0, res.VectorScoreField.NearestNeighborsScore); + res = collection.NearestNeighbors(x => x.SimpleHnswVector, 1, queryVector).First(); + Assert.Equal(4, res.VectorScoreField.NearestNeighborsScore); } - - [Fact] public void TestIndex() { @@ -84,7 +97,7 @@ public void TestIndex() var doubles = new double[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; var obj = new ObjectWithVectorHash { - Id = "foo", + Id = "helloWorld", SimpleHnswVector = doubles, SimpleVectorizedVector = "foo", }; @@ -95,7 +108,7 @@ public void TestIndex() key = _connection.Set(new ObjectWithVector() { - Id = "foo", + Id = "helloWorld", SimpleHnswVector = doubles, SimpleVectorizedVector = "foobarbaz" }); @@ -132,7 +145,7 @@ public void Insert() var hashObj = new ObjectWithVectorHash() { - Id = "foo", + Id = "helloWorld", SimpleHnswVector = simpleHnswHash, SimpleVectorizedVector = "foobar" }; diff --git a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorTests.cs b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorTests.cs index d44d56f5..cf107938 100644 --- a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorTests.cs +++ b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorTests.cs @@ -1,12 +1,12 @@ using System; using System.Linq; -using System.Linq.Expressions; using System.Text; using System.Text.Json; using NSubstitute; using NSubstitute.ClearExtensions; using Redis.OM.Contracts; using Redis.OM.Modeling; +using Redis.OM.Modeling.Vectors; using Redis.OM.Searching; using Xunit; @@ -65,8 +65,8 @@ public void SimpleKnnQuery() _substitute.Received().Execute("FT.SEARCH", $"{nameof(ObjectWithVector).ToLower()}-idx", - "(*)=>[KNN 5 @SimpleHnswVector $V]", - "PARAMS", 2, "V", Arg.Is(b=>b.SequenceEqual(blob)), "DIALECT", 2, "LIMIT", "0", "100"); + $"(*)=>[KNN 5 @SimpleHnswVector $0 AS {VectorScores.NearestNeighborScoreName}]", + "PARAMS", 2, "0", Arg.Is(b=>b.SequenceEqual(blob)), "DIALECT", 2, "LIMIT", "0", "100"); _substitute.ClearSubstitute(); _substitute.Execute(Arg.Any(), Arg.Any()).Returns(new RedisReply(0)); @@ -74,10 +74,54 @@ public void SimpleKnnQuery() _substitute.Received().Execute("FT.SEARCH", $"{nameof(ObjectWithVector).ToLower()}-idx", - "(*)=>[KNN 8 @SimpleVectorizedVector $V]", - "PARAMS", 2, "V", Arg.Is(b=>b.SequenceEqual(floatBlob)), "DIALECT", 2, "LIMIT", "0", "100"); + $"(*)=>[KNN 8 @SimpleVectorizedVector $0 AS {VectorScores.NearestNeighborScoreName}]", + "PARAMS", 2, "0", Arg.Is(b=>b.SequenceEqual(floatBlob)), "DIALECT", 2, "LIMIT", "0", "100"); } + [Fact] + public void SimpleRangeQuery() + { + _substitute.ClearSubstitute(); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(new RedisReply(0)); + var collection = new RedisCollection(_substitute); + var compVector = new double[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; + float[] floats = Enumerable.Range(0, 30).Select(x => (float)x).ToArray(); + var floatBytes = floats.SelectMany(BitConverter.GetBytes).ToArray(); + _ = collection.Where(x => x.SimpleVectorizedVector.VectorRange("foobar", .3)).ToList(); + _substitute.Received().Execute("FT.SEARCH", $"{nameof(ObjectWithVector).ToLower()}-idx", + "@SimpleVectorizedVector:[VECTOR_RANGE $0 $1]", "PARAMS", 4, "0", .3, "1", Arg.Is(b => b.SequenceEqual(floatBytes)), + "DIALECT", 2, "LIMIT", "0", "100"); + + } + + [Fact] + public void SimpleKnnQueryWithSortBy() + { + _substitute.ClearSubstitute(); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(new RedisReply(0)); + var collection = new RedisCollection(_substitute); + var compVector = new double[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; + float[] floats = Enumerable.Range(0, 30).Select(x => (float)x).ToArray(); + var blob = compVector.SelectMany(BitConverter.GetBytes).ToArray(); + var floatBlob = floats.SelectMany(BitConverter.GetBytes).ToArray(); + _ = collection.NearestNeighbors(x=>x.SimpleHnswVector, 5, compVector).OrderBy(x=>x.VectorScoreField.NearestNeighborsScore).ToList(); + + _substitute.Received().Execute("FT.SEARCH", + $"{nameof(ObjectWithVector).ToLower()}-idx", + $"(*)=>[KNN 5 @SimpleHnswVector $0 AS {VectorScores.NearestNeighborScoreName}]", + "PARAMS", 2, "0", Arg.Is(b=>b.SequenceEqual(blob)), "DIALECT", 2, "LIMIT", "0", "100", "SORTBY", VectorScores.NearestNeighborScoreName, "ASC"); + + _substitute.ClearSubstitute(); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(new RedisReply(0)); + _ = collection.NearestNeighbors(x => x.SimpleVectorizedVector, 8, "hello world").OrderByDescending(x=>x.VectorScoreField.NearestNeighborsScore).ToArray(); + + _substitute.Received().Execute("FT.SEARCH", + $"{nameof(ObjectWithVector).ToLower()}-idx", + $"(*)=>[KNN 8 @SimpleVectorizedVector $0 AS {VectorScores.NearestNeighborScoreName}]", + "PARAMS", 2, "0", Arg.Is(b=>b.SequenceEqual(floatBlob)), "DIALECT", 2, "LIMIT", "0", "100", "SORTBY", VectorScores.NearestNeighborScoreName, "DESC"); + } + + [Fact] public void TestBinConversions() { From feb09464b2332df590906287dedeaaf7a8ff2dee Mon Sep 17 00:00:00 2001 From: slorello89 Date: Thu, 19 Oct 2023 14:50:10 -0400 Subject: [PATCH 07/36] removing extra FromHashSet method --- src/Redis.OM.POC/XRangeResponse.cs | 15 +---- src/Redis.OM/Modeling/Vectors/VectorScores.cs | 3 +- src/Redis.OM/RedisCommands.cs | 4 +- src/Redis.OM/RedisObjectHandler.cs | 58 ++++--------------- src/Redis.OM/RedisReply.cs | 9 +++ src/Redis.OM/Searching/SearchResponse.cs | 6 +- .../BasicTypeWithGeoLoc.cs | 1 + test/Redis.OM.Unit.Tests/GeoLocTests.cs | 2 +- .../VectorTests/ObjectWithVector.cs | 4 +- .../VectorTests/VectorFunctionalTests.cs | 27 +++++++-- .../VectorTests/VectorTests.cs | 4 +- 11 files changed, 60 insertions(+), 73 deletions(-) diff --git a/src/Redis.OM.POC/XRangeResponse.cs b/src/Redis.OM.POC/XRangeResponse.cs index 09b6ed36..b651529b 100644 --- a/src/Redis.OM.POC/XRangeResponse.cs +++ b/src/Redis.OM.POC/XRangeResponse.cs @@ -13,15 +13,6 @@ public class XRangeResponse { public IDictionary Messages { get; set; } - public XRangeResponse(StreamEntry[] entries) - { - Messages = new Dictionary(); - foreach(var entry in entries) - { - var innerDict = entry.Values.ToDictionary(x => x.Name.ToString(), x => x.Value.ToString()); - Messages.Add(entry.Id, (T)RedisObjectHandler.FromHashSet(innerDict)); - } - } public XRangeResponse(RedisResult[] vals, string streamName) { Messages = new Dictionary(); @@ -43,10 +34,10 @@ public XRangeResponse(RedisResult[] vals, string streamName) { var id = (string)((RedisResult[])obj.ToArray()[0])[i]; var pairs = ((RedisResult[])((RedisResult[])obj.ToArray()[0])[i + 1]); - var messageDict = new Dictionary(); + var messageDict = new Dictionary(); for (var j = 0; j < pairs.Length; j += 2) { - messageDict.Add(((string)pairs[j]), ((string)pairs[j + 1])); + messageDict.Add(((string)pairs[j]), new RedisReply(pairs[j + 1])); } Messages.Add(id, (T)RedisObjectHandler.FromHashSet(messageDict)); } @@ -70,7 +61,7 @@ public XRangeResponse(RedisReply[] vals, string streamName) { var id = (string)obj.ToArray()[0].ToArray()[i]; var pairs = obj.ToArray()[0].ToArray()[i + 1].ToArray(); - var messageDict = new Dictionary(); + var messageDict = new Dictionary(); for (var j = 0; j < pairs.Length; j+=2) { messageDict.Add(pairs[j], pairs[j + 1]); diff --git a/src/Redis.OM/Modeling/Vectors/VectorScores.cs b/src/Redis.OM/Modeling/Vectors/VectorScores.cs index d7aaefdd..b6faefbb 100644 --- a/src/Redis.OM/Modeling/Vectors/VectorScores.cs +++ b/src/Redis.OM/Modeling/Vectors/VectorScores.cs @@ -5,7 +5,8 @@ namespace Redis.OM.Modeling.Vectors { /// - /// A collector for vector scores. + /// A collector for vector scores, binding this to your model causes Redis OM to bind all scores resulting from + /// a vector query to it. Otherwise it will be ignored when it is added to Redis. /// public class VectorScores { diff --git a/src/Redis.OM/RedisCommands.cs b/src/Redis.OM/RedisCommands.cs index a54b8c50..2db3bce3 100644 --- a/src/Redis.OM/RedisCommands.cs +++ b/src/Redis.OM/RedisCommands.cs @@ -616,9 +616,9 @@ public static IDictionary HGetAll(this IRedisConnection conn /// the connection. /// the key name. /// the object serialized into a dictionary. - public static async Task> HGetAllAsync(this IRedisConnection connection, string keyName) + public static async Task> HGetAllAsync(this IRedisConnection connection, string keyName) { - var ret = new Dictionary(); + var ret = new Dictionary(); var res = (await connection.ExecuteAsync("HGETALL", keyName)).ToArray(); for (var i = 0; i < res.Length; i += 2) { diff --git a/src/Redis.OM/RedisObjectHandler.cs b/src/Redis.OM/RedisObjectHandler.cs index 7aa07510..37ad0c76 100644 --- a/src/Redis.OM/RedisObjectHandler.cs +++ b/src/Redis.OM/RedisObjectHandler.cs @@ -32,21 +32,20 @@ internal static class RedisObjectHandler }; /// - /// Builds object from provided hash set. + /// Tries to parse the hash set into a fully or partially hydrated object. /// - /// Hash set to build item from. - /// The type to construct. - /// An instance of the requested object. - /// Throws an exception if Deserialization fails. - internal static T FromHashSet(IDictionary hash) - where T : notnull + /// The hash to generate the object from. + /// The type to convert to. + /// A fully or partially hydrated object. + /// Thrown if deserialization fails. + /// Thrown if documentAttribute not decorating type. + internal static T FromHashSet(IDictionary hash) { - var dict = hash.ToDictionary(x => x.Key, x => new RedisReply(x.Value.ToString())); + var stringDictionary = hash.ToDictionary(x => x.Key, x => x.Value.ToString()); if (typeof(IRedisHydrateable).IsAssignableFrom(typeof(T))) { var obj = Activator.CreateInstance(); - ((IRedisHydrateable)obj).Hydrate(hash); - return obj; + ((IRedisHydrateable)obj!).Hydrate(stringDictionary); } var attr = Attribute.GetCustomAttribute(typeof(T), typeof(DocumentAttribute)) as DocumentAttribute; @@ -61,7 +60,7 @@ internal static T FromHashSet(IDictionary hash) } else { - asJson = SendToJson(dict, typeof(T)); + asJson = SendToJson(hash, typeof(T)); } var res = JsonSerializer.Deserialize(asJson, RedisSerializationSettings.JsonSerializerOptions) ?? throw new Exception("Deserialization fail"); @@ -90,41 +89,6 @@ internal static T FromHashSet(IDictionary hash) return res; } - /// - /// Tries to parse the hash set into a fully or partially hydrated object. - /// - /// The hash to generate the object from. - /// The type to convert to. - /// A fully or partially hydrated object. - /// Thrown if deserialization fails. - /// Thrown if documentAttribute not decorating type. - internal static T FromHashSet(IDictionary hash) - { - var stringDictionary = hash.ToDictionary(x => x.Key, x => x.Value.ToString()); - if (typeof(IRedisHydrateable).IsAssignableFrom(typeof(T))) - { - var obj = Activator.CreateInstance(); - ((IRedisHydrateable)obj!).Hydrate(stringDictionary); - } - - var attr = Attribute.GetCustomAttribute(typeof(T), typeof(DocumentAttribute)) as DocumentAttribute; - string asJson; - if (attr != null && attr.StorageType == StorageType.Json && hash.ContainsKey("$")) - { - asJson = hash["$"]; - } - else if (attr != null) - { - asJson = SendToJson(hash, typeof(T)); - } - else - { - throw new ArgumentException("Type must be decorated with a DocumentAttribute"); - } - - return JsonSerializer.Deserialize(asJson, RedisSerializationSettings.JsonSerializerOptions) ?? throw new Exception("Deserialization fail"); - } - /// /// Turns hash set into a basic object. To be used when you won't know the type at compile time. /// @@ -325,7 +289,7 @@ internal static void ExtractPropertyName(PropertyInfo property, ref string prope internal static T ToObject(this RedisReply val) where T : notnull { - var hash = new Dictionary(); + var hash = new Dictionary(); var vals = val.ToArray(); for (var i = 0; i < vals.Length; i += 2) { diff --git a/src/Redis.OM/RedisReply.cs b/src/Redis.OM/RedisReply.cs index de2db574..7bb5b458 100644 --- a/src/Redis.OM/RedisReply.cs +++ b/src/Redis.OM/RedisReply.cs @@ -46,6 +46,15 @@ public RedisReply(RedisResult result) } } + /// + /// Initializes a new instance of the class. + /// + /// the raw bytes. + internal RedisReply(byte[] raw) + { + _raw = raw; + } + /// /// Initializes a new instance of the class. /// diff --git a/src/Redis.OM/Searching/SearchResponse.cs b/src/Redis.OM/Searching/SearchResponse.cs index b60125c3..57e340da 100644 --- a/src/Redis.OM/Searching/SearchResponse.cs +++ b/src/Redis.OM/Searching/SearchResponse.cs @@ -1,7 +1,6 @@ using System; using System.Collections.Generic; using System.Linq; -using Redis.OM; namespace Redis.OM.Searching { @@ -54,7 +53,8 @@ public IDictionary DocumentsAs() var dict = new Dictionary(); foreach (var kvp in Documents) { - var obj = RedisObjectHandler.FromHashSet(kvp.Value); + var rrDict = kvp.Value.ToDictionary(x => x.Key, x => (RedisReply)x.Value); + var obj = RedisObjectHandler.FromHashSet(rrDict); dict.Add(kvp.Key, obj); } @@ -112,7 +112,7 @@ public SearchResponse(RedisReply val) for (var i = 1; i < vals.Count(); i += 2) { var docId = (string)vals[i]; - var documentHash = new Dictionary(); + var documentHash = new Dictionary(); var docArray = vals[i + 1].ToArray(); if (docArray.Length > 1) { diff --git a/test/Redis.OM.Unit.Tests/BasicTypeWithGeoLoc.cs b/test/Redis.OM.Unit.Tests/BasicTypeWithGeoLoc.cs index 07649716..b518f8a9 100644 --- a/test/Redis.OM.Unit.Tests/BasicTypeWithGeoLoc.cs +++ b/test/Redis.OM.Unit.Tests/BasicTypeWithGeoLoc.cs @@ -2,6 +2,7 @@ namespace Redis.OM.Unit.Tests { + [Document] public class BasicTypeWithGeoLoc { public string Name { get; set; } diff --git a/test/Redis.OM.Unit.Tests/GeoLocTests.cs b/test/Redis.OM.Unit.Tests/GeoLocTests.cs index d6b74d57..9a950a9f 100644 --- a/test/Redis.OM.Unit.Tests/GeoLocTests.cs +++ b/test/Redis.OM.Unit.Tests/GeoLocTests.cs @@ -20,7 +20,7 @@ public void TestParsingFromJson() [Fact] public void TestParsingFromFormattedHash() { - var hash = new Dictionary + var hash = new Dictionary { {"Name", "Foo"}, {"Location", "32.5,22.4"} diff --git a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/ObjectWithVector.cs b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/ObjectWithVector.cs index e94c8301..c68466ba 100644 --- a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/ObjectWithVector.cs +++ b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/ObjectWithVector.cs @@ -18,7 +18,7 @@ public class ObjectWithVector [SimpleVectorizer] public string SimpleVectorizedVector { get; set; } - public VectorScores VectorScoreField { get; set; } + public VectorScores VectorScores { get; set; } } [Document(StorageType = StorageType.Hash)] @@ -33,6 +33,8 @@ public class ObjectWithVectorHash [Vector(Algorithm = VectorAlgorithm.FLAT)] [SimpleVectorizer] public string SimpleVectorizedVector { get; set; } + + public VectorScores VectorScores { get; set; } } [Document(StorageType = StorageType.Json, Prefixes = new []{"Simple"})] diff --git a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorFunctionalTests.cs b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorFunctionalTests.cs index ba1586a3..5fc7b186 100644 --- a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorFunctionalTests.cs +++ b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorFunctionalTests.cs @@ -64,8 +64,8 @@ public void RangeAndKnn() var res = collection.NearestNeighbors(x=>x.SimpleVectorizedVector, 1, "FooBarBaz") .First(x => x.SimpleHnswVector.VectorRange(queryVector, 5, "range")); Assert.Equal("helloWorld", res.Id); - Assert.Equal(4, res.VectorScoreField.RangeScore); - Assert.Equal(0, res.VectorScoreField.NearestNeighborsScore); + Assert.Equal(4, res.VectorScores.RangeScore); + Assert.Equal(0, res.VectorScores.NearestNeighborsScore); } [Fact] @@ -84,11 +84,30 @@ public void BasicQuery() var res = collection.NearestNeighbors(x => x.SimpleVectorizedVector, 1, "FooBarBaz").First(); Assert.Equal("helloWorld", res.Id); - Assert.Equal(0, res.VectorScoreField.NearestNeighborsScore); + Assert.Equal(0, res.VectorScores.NearestNeighborsScore); res = collection.NearestNeighbors(x => x.SimpleHnswVector, 1, queryVector).First(); - Assert.Equal(4, res.VectorScoreField.NearestNeighborsScore); + Assert.Equal(4, res.VectorScores.NearestNeighborsScore); } + [Fact] + public void ScoresOnHash() + { + _connection.DropIndexAndAssociatedRecords(typeof(ObjectWithVectorHash)); + _connection.CreateIndex(typeof(ObjectWithVectorHash)); + var doubles = new double[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; + var obj = new ObjectWithVectorHash + { + Id = "helloWorld", + SimpleHnswVector = doubles, + SimpleVectorizedVector = "foo", + }; + var collection = new RedisCollection(_connection); + collection.Insert(obj); + var res = collection.NearestNeighbors(x => x.SimpleHnswVector, 5, doubles).First(); + + Assert.Equal(0, res.VectorScores.NearestNeighborsScore); + } + [Fact] public void TestIndex() { diff --git a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorTests.cs b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorTests.cs index cf107938..c8ea6c0c 100644 --- a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorTests.cs +++ b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorTests.cs @@ -104,7 +104,7 @@ public void SimpleKnnQueryWithSortBy() float[] floats = Enumerable.Range(0, 30).Select(x => (float)x).ToArray(); var blob = compVector.SelectMany(BitConverter.GetBytes).ToArray(); var floatBlob = floats.SelectMany(BitConverter.GetBytes).ToArray(); - _ = collection.NearestNeighbors(x=>x.SimpleHnswVector, 5, compVector).OrderBy(x=>x.VectorScoreField.NearestNeighborsScore).ToList(); + _ = collection.NearestNeighbors(x=>x.SimpleHnswVector, 5, compVector).OrderBy(x=>x.VectorScores.NearestNeighborsScore).ToList(); _substitute.Received().Execute("FT.SEARCH", $"{nameof(ObjectWithVector).ToLower()}-idx", @@ -113,7 +113,7 @@ public void SimpleKnnQueryWithSortBy() _substitute.ClearSubstitute(); _substitute.Execute(Arg.Any(), Arg.Any()).Returns(new RedisReply(0)); - _ = collection.NearestNeighbors(x => x.SimpleVectorizedVector, 8, "hello world").OrderByDescending(x=>x.VectorScoreField.NearestNeighborsScore).ToArray(); + _ = collection.NearestNeighbors(x => x.SimpleVectorizedVector, 8, "hello world").OrderByDescending(x=>x.VectorScores.NearestNeighborsScore).ToArray(); _substitute.Received().Execute("FT.SEARCH", $"{nameof(ObjectWithVector).ToLower()}-idx", From 04fbb90e7dbf5390dbb6c2f7840c057add2ee36f Mon Sep 17 00:00:00 2001 From: slorello89 Date: Thu, 19 Oct 2023 16:24:39 -0400 Subject: [PATCH 08/36] hybrid queries --- .../VectorTests/ObjectWithVector.cs | 8 +++++++ .../VectorTests/VectorFunctionalTests.cs | 21 +++++++++++++++++++ .../VectorTests/VectorTests.cs | 17 +++++++++++++++ 3 files changed, 46 insertions(+) diff --git a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/ObjectWithVector.cs b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/ObjectWithVector.cs index c68466ba..d17c3c80 100644 --- a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/ObjectWithVector.cs +++ b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/ObjectWithVector.cs @@ -11,6 +11,10 @@ public class ObjectWithVector [RedisIdField] public string Id { get; set; } + [Indexed] public string Name { get; set; } + + [Indexed] public int Num { get; set; } + [Vector(Algorithm = VectorAlgorithm.HNSW, Dim = 10)] public double[] SimpleHnswVector { get; set; } @@ -27,6 +31,10 @@ public class ObjectWithVectorHash [RedisIdField] public string Id { get; set; } + [Indexed] public string Name { get; set; } + + [Indexed] public int Num { get; set; } + [Vector(Algorithm = VectorAlgorithm.HNSW, Dim = 10)] public double[] SimpleHnswVector { get; set; } diff --git a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorFunctionalTests.cs b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorFunctionalTests.cs index 5fc7b186..5ac9eece 100644 --- a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorFunctionalTests.cs +++ b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorFunctionalTests.cs @@ -108,6 +108,27 @@ public void ScoresOnHash() Assert.Equal(0, res.VectorScores.NearestNeighborsScore); } + [Fact] + public void HybridQueryTest() + { + _connection.DropIndexAndAssociatedRecords(typeof(ObjectWithVectorHash)); + _connection.CreateIndex(typeof(ObjectWithVectorHash)); + var doubles = new double[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; + var obj = new ObjectWithVectorHash + { + Id = "theOneWithStuff", + SimpleHnswVector = doubles, + Name = "Steve", + Num = 6, + SimpleVectorizedVector = "foo", + }; + var collection = new RedisCollection(_connection); + collection.Insert(obj); + var res = collection.Where(x=>x.Name == "Steve" && x.Num == 6).NearestNeighbors(x => x.SimpleHnswVector, 5, doubles).First(); + + Assert.Equal(0, res.VectorScores.NearestNeighborsScore); + } + [Fact] public void TestIndex() { diff --git a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorTests.cs b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorTests.cs index c8ea6c0c..0ca117b5 100644 --- a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorTests.cs +++ b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorTests.cs @@ -183,4 +183,21 @@ public void InsertVectors() var deseralized = JsonSerializer.Deserialize(json); Assert.Equal("foobar", deseralized.SimpleVectorizedVector); } + + [Fact] + public void HybridQuery() + { + _substitute.ClearSubstitute(); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(new RedisReply(0)); + var collection = new RedisCollection(_substitute); + var compVector = new double[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; + var blob = compVector.SelectMany(BitConverter.GetBytes); + _ = collection.Where(x => x.Name == "Steve" && x.Num < 5) + .NearestNeighbors(x => x.SimpleHnswVector, 2, compVector).ToList(); + _substitute.Received().Execute("FT.SEARCH", + $"{nameof(ObjectWithVector).ToLower()}-idx", + $"(((@Name:{{Steve}}) (@Num:[-inf (5])))=>[KNN 2 @SimpleHnswVector $0 AS {VectorScores.NearestNeighborScoreName}]", + "PARAMS", 2, "0", Arg.Is(b=>b.SequenceEqual(blob)), "DIALECT", 2, "LIMIT", "0", "100"); + + } } \ No newline at end of file From 2a9ad737e389db8cdf3f1a8d83c3c6f528877f89 Mon Sep 17 00:00:00 2001 From: slorello89 Date: Wed, 25 Oct 2023 15:24:00 -0400 Subject: [PATCH 09/36] external-vectorizers --- .gitignore | 2 + .../OpenAISentenceVectorizer.cs | 73 ++++++++++++++++ .../Redis.OM.Vectorizers.OpenAI.csproj | 15 ++++ Redis.OM.sln | 45 ++++++++++ .../Configuration.cs | 55 ++++++++++++ .../Redis.OM.Vectorizers.Core.csproj | 15 ++++ .../HuggingFaceApiSentenceVectorizer.cs | 65 +++++++++++++++ .../Redis.OM.Vectorizers.HuggingFace.csproj | 22 +++++ .../VectorTests/HuggingFaceVectors.cs | 24 ++++++ .../VectorTests/OpenAIVectors.cs | 24 ++++++ .../VectorTests/VectorFunctionalTests.cs | 83 +++++++++++++++++++ .../Redis.OM.Unit.Tests.csproj | 8 ++ test/Redis.OM.Unit.Tests/appsettings.json | 4 + 13 files changed, 435 insertions(+) create mode 100644 Redis.OM.Vectorizers.OpenAI/OpenAISentenceVectorizer.cs create mode 100644 Redis.OM.Vectorizers.OpenAI/Redis.OM.Vectorizers.OpenAI.csproj create mode 100644 src/Redis.OM.Vectorizers.Core/Configuration.cs create mode 100644 src/Redis.OM.Vectorizers.Core/Redis.OM.Vectorizers.Core.csproj create mode 100644 src/Redis.OM.Vectorizers.HuggingFace/HuggingFaceApiSentenceVectorizer.cs create mode 100644 src/Redis.OM.Vectorizers.HuggingFace/Redis.OM.Vectorizers.HuggingFace.csproj create mode 100644 test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/HuggingFaceVectors.cs create mode 100644 test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/OpenAIVectors.cs create mode 100644 test/Redis.OM.Unit.Tests/appsettings.json diff --git a/.gitignore b/.gitignore index 8279cee0..c0196859 100644 --- a/.gitignore +++ b/.gitignore @@ -388,3 +388,5 @@ FodyWeavers.xsd # JetBrains Rider .idea/ *.sln.iml + +test/Redis.OM.Unit.Tests/appsettings.json.local \ No newline at end of file diff --git a/Redis.OM.Vectorizers.OpenAI/OpenAISentenceVectorizer.cs b/Redis.OM.Vectorizers.OpenAI/OpenAISentenceVectorizer.cs new file mode 100644 index 00000000..d5c6fdad --- /dev/null +++ b/Redis.OM.Vectorizers.OpenAI/OpenAISentenceVectorizer.cs @@ -0,0 +1,73 @@ +using System.Net.Http.Json; +using System.Text.Json; +using Redis.OM.Modeling; +using Redis.OM.Vectorizers.HuggingFace; + +namespace Redis.OM.OpenAI; + +public class OpenAISentenceVectorizer : VectorizerAttribute +{ + private const string DefaultModel = "text-embedding-ada-002"; + public string ModelId { get; set; } = DefaultModel; + public override VectorType VectorType { get; } + + public override int Dim + { + get + { + if (ModelId == DefaultModel) + { + return 1536; + } + + return GetFloats("this is a test string").Length; + } + } + + public override byte[] Vectorize(object obj) + { + var s = (string)obj; + var floats = GetFloats(s); + return floats.SelectMany(BitConverter.GetBytes).ToArray(); + } + + internal float[] GetFloats(string s) + { + var client = Configuration.Instance.Client; + var requestContent = JsonContent.Create( + new + { + input = s, + model = ModelId + }); + + var request = new HttpRequestMessage + { + Method = HttpMethod.Post, + RequestUri = new Uri($"{Configuration.Instance.OpenAiApiUrl}/v1/embeddings"), + Content = requestContent, + Headers = { { "Authorization", $"Bearer {Configuration.Instance.OpenAiAuthorizationToken}" } } + }; + + var res = client.SendAsync(request).Result; + if (!res.IsSuccessStatusCode) + { + throw new HttpRequestException( + $"Open AI did not respond with a positive error code: {res.StatusCode}, {res.ReasonPhrase}"); + } + var jsonObj = res.Content.ReadFromJsonAsync().Result; + + + if (!jsonObj.TryGetProperty("data", out var data)) + { + throw new Exception("Malformed Response"); + } + + if (data.GetArrayLength() < 1 || !data[0].TryGetProperty("embedding", out var embedding)) + { + throw new Exception("Malformed Response"); + } + + return embedding.Deserialize()!; + } +} \ No newline at end of file diff --git a/Redis.OM.Vectorizers.OpenAI/Redis.OM.Vectorizers.OpenAI.csproj b/Redis.OM.Vectorizers.OpenAI/Redis.OM.Vectorizers.OpenAI.csproj new file mode 100644 index 00000000..9b77ddc7 --- /dev/null +++ b/Redis.OM.Vectorizers.OpenAI/Redis.OM.Vectorizers.OpenAI.csproj @@ -0,0 +1,15 @@ + + + + net6.0 + enable + enable + Redis.OM.OpenAI + + + + + + + + diff --git a/Redis.OM.sln b/Redis.OM.sln index 9924b2ca..ac06d3cb 100644 --- a/Redis.OM.sln +++ b/Redis.OM.sln @@ -10,6 +10,12 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Redis.OM.POC", "src\Redis.O EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Redis.OM.Unit.Tests", "test\Redis.OM.Unit.Tests\Redis.OM.Unit.Tests.csproj", "{570BF479-BCF4-4D1B-A702-2234CA0A3E7D}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Redis.OM.Vectorizers.HuggingFace", "src\Redis.OM.Vectorizers.HuggingFace\Redis.OM.Vectorizers.HuggingFace.csproj", "{329F600A-3AF7-456A-8652-A79A39804EE5}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Redis.OM.Vectorizers.OpenAI", "Redis.OM.Vectorizers.OpenAI\Redis.OM.Vectorizers.OpenAI.csproj", "{D9797D53-E8E2-4D60-9B07-2CA087E6CD19}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Redis.OM.Vectorizers.Core", "src\Redis.OM.Vectorizers.Core\Redis.OM.Vectorizers.Core.csproj", "{4B9F4623-3126-48B7-B690-F28F702A4717}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -56,6 +62,42 @@ Global {570BF479-BCF4-4D1B-A702-2234CA0A3E7D}.Release|x64.Build.0 = Release|Any CPU {570BF479-BCF4-4D1B-A702-2234CA0A3E7D}.Release|x86.ActiveCfg = Release|Any CPU {570BF479-BCF4-4D1B-A702-2234CA0A3E7D}.Release|x86.Build.0 = Release|Any CPU + {329F600A-3AF7-456A-8652-A79A39804EE5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {329F600A-3AF7-456A-8652-A79A39804EE5}.Debug|Any CPU.Build.0 = Debug|Any CPU + {329F600A-3AF7-456A-8652-A79A39804EE5}.Debug|x64.ActiveCfg = Debug|Any CPU + {329F600A-3AF7-456A-8652-A79A39804EE5}.Debug|x64.Build.0 = Debug|Any CPU + {329F600A-3AF7-456A-8652-A79A39804EE5}.Debug|x86.ActiveCfg = Debug|Any CPU + {329F600A-3AF7-456A-8652-A79A39804EE5}.Debug|x86.Build.0 = Debug|Any CPU + {329F600A-3AF7-456A-8652-A79A39804EE5}.Release|Any CPU.ActiveCfg = Release|Any CPU + {329F600A-3AF7-456A-8652-A79A39804EE5}.Release|Any CPU.Build.0 = Release|Any CPU + {329F600A-3AF7-456A-8652-A79A39804EE5}.Release|x64.ActiveCfg = Release|Any CPU + {329F600A-3AF7-456A-8652-A79A39804EE5}.Release|x64.Build.0 = Release|Any CPU + {329F600A-3AF7-456A-8652-A79A39804EE5}.Release|x86.ActiveCfg = Release|Any CPU + {329F600A-3AF7-456A-8652-A79A39804EE5}.Release|x86.Build.0 = Release|Any CPU + {D9797D53-E8E2-4D60-9B07-2CA087E6CD19}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {D9797D53-E8E2-4D60-9B07-2CA087E6CD19}.Debug|Any CPU.Build.0 = Debug|Any CPU + {D9797D53-E8E2-4D60-9B07-2CA087E6CD19}.Debug|x64.ActiveCfg = Debug|Any CPU + {D9797D53-E8E2-4D60-9B07-2CA087E6CD19}.Debug|x64.Build.0 = Debug|Any CPU + {D9797D53-E8E2-4D60-9B07-2CA087E6CD19}.Debug|x86.ActiveCfg = Debug|Any CPU + {D9797D53-E8E2-4D60-9B07-2CA087E6CD19}.Debug|x86.Build.0 = Debug|Any CPU + {D9797D53-E8E2-4D60-9B07-2CA087E6CD19}.Release|Any CPU.ActiveCfg = Release|Any CPU + {D9797D53-E8E2-4D60-9B07-2CA087E6CD19}.Release|Any CPU.Build.0 = Release|Any CPU + {D9797D53-E8E2-4D60-9B07-2CA087E6CD19}.Release|x64.ActiveCfg = Release|Any CPU + {D9797D53-E8E2-4D60-9B07-2CA087E6CD19}.Release|x64.Build.0 = Release|Any CPU + {D9797D53-E8E2-4D60-9B07-2CA087E6CD19}.Release|x86.ActiveCfg = Release|Any CPU + {D9797D53-E8E2-4D60-9B07-2CA087E6CD19}.Release|x86.Build.0 = Release|Any CPU + {4B9F4623-3126-48B7-B690-F28F702A4717}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {4B9F4623-3126-48B7-B690-F28F702A4717}.Debug|Any CPU.Build.0 = Debug|Any CPU + {4B9F4623-3126-48B7-B690-F28F702A4717}.Debug|x64.ActiveCfg = Debug|Any CPU + {4B9F4623-3126-48B7-B690-F28F702A4717}.Debug|x64.Build.0 = Debug|Any CPU + {4B9F4623-3126-48B7-B690-F28F702A4717}.Debug|x86.ActiveCfg = Debug|Any CPU + {4B9F4623-3126-48B7-B690-F28F702A4717}.Debug|x86.Build.0 = Debug|Any CPU + {4B9F4623-3126-48B7-B690-F28F702A4717}.Release|Any CPU.ActiveCfg = Release|Any CPU + {4B9F4623-3126-48B7-B690-F28F702A4717}.Release|Any CPU.Build.0 = Release|Any CPU + {4B9F4623-3126-48B7-B690-F28F702A4717}.Release|x64.ActiveCfg = Release|Any CPU + {4B9F4623-3126-48B7-B690-F28F702A4717}.Release|x64.Build.0 = Release|Any CPU + {4B9F4623-3126-48B7-B690-F28F702A4717}.Release|x86.ActiveCfg = Release|Any CPU + {4B9F4623-3126-48B7-B690-F28F702A4717}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -63,6 +105,9 @@ Global GlobalSection(NestedProjects) = preSolution {7994382C-28EF-4F55-9B6D-810D35247816} = {8D9ECCFF-E022-4B68-BB43-228CEA248DEC} {E3A31119-E4F1-4793-B5C2-ED2D51502B01} = {8D9ECCFF-E022-4B68-BB43-228CEA248DEC} + {329F600A-3AF7-456A-8652-A79A39804EE5} = {8D9ECCFF-E022-4B68-BB43-228CEA248DEC} + {D9797D53-E8E2-4D60-9B07-2CA087E6CD19} = {8D9ECCFF-E022-4B68-BB43-228CEA248DEC} + {4B9F4623-3126-48B7-B690-F28F702A4717} = {8D9ECCFF-E022-4B68-BB43-228CEA248DEC} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {E5752441-184B-4F17-BAD0-93823AC68607} diff --git a/src/Redis.OM.Vectorizers.Core/Configuration.cs b/src/Redis.OM.Vectorizers.Core/Configuration.cs new file mode 100644 index 00000000..705fe2a9 --- /dev/null +++ b/src/Redis.OM.Vectorizers.Core/Configuration.cs @@ -0,0 +1,55 @@ +using System.Net.Http.Headers; +using Microsoft.Extensions.Configuration; + +namespace Redis.OM.Vectorizers.HuggingFace; + +public class Configuration +{ + public string? this[string str] => _settings[str]; + public string HuggingFaceAuthorizationToken => _settings["REDIS_OM_HF_TOKEN"] ?? string.Empty; + public string OpenAiAuthorizationToken => _settings["REDIS_OM_OAI_TOKEN"] ?? string.Empty; + public string ModelId => _settings["REDIS_OM_HF_MODEL_ID"] ?? string.Empty; + public string HuggingFaceBaseAddress => _settings["REDIS_OM_HF_FEATURE_EXTRACTION_URL"] ?? string.Empty; + + private const string DefaultHuggingFaceApiUrl = "https://api-inference.huggingface.co"; + + private const string DefaultOpenAiApiUrl = "https://api.openai.com"; + public string OpenAiApiUrl => _settings["REDIS_OM_OAI_API_URL"] ?? String.Empty; + + private readonly IConfiguration _settings; + + private static readonly object LockObject = new (); + private static Configuration? _instance; + + public readonly HttpClient Client; + + public static Configuration Instance + { + get + { + if (_instance is not null) return _instance; + lock (LockObject) + { + _instance ??= new Configuration(); + } + + return _instance; + } + } + + internal Configuration() + { + var builder = new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary + { + {"REDIS_OM_HF_FEATURE_EXTRACTION_URL", DefaultHuggingFaceApiUrl}, + {"REDIS_OM_OAI_API_URL", DefaultOpenAiApiUrl}, + {"REDIS_OM_HF_TOKEN", Environment.GetEnvironmentVariable("REDIS_OM_HF_TOKEN")}, + {"REDIS_OM_OAI_TOKEN", Environment.GetEnvironmentVariable("REDIS_OM_OAI_TOKEN")} + }) + .AddJsonFile("settings.json", true, true) + .AddJsonFile("appsettings.json", true, true); + _settings = builder.Build(); + Client = new HttpClient(); + } +} \ No newline at end of file diff --git a/src/Redis.OM.Vectorizers.Core/Redis.OM.Vectorizers.Core.csproj b/src/Redis.OM.Vectorizers.Core/Redis.OM.Vectorizers.Core.csproj new file mode 100644 index 00000000..b7cacc4f --- /dev/null +++ b/src/Redis.OM.Vectorizers.Core/Redis.OM.Vectorizers.Core.csproj @@ -0,0 +1,15 @@ + + + + net6.0 + enable + enable + + + + + + + + + diff --git a/src/Redis.OM.Vectorizers.HuggingFace/HuggingFaceApiSentenceVectorizer.cs b/src/Redis.OM.Vectorizers.HuggingFace/HuggingFaceApiSentenceVectorizer.cs new file mode 100644 index 00000000..b88b34e6 --- /dev/null +++ b/src/Redis.OM.Vectorizers.HuggingFace/HuggingFaceApiSentenceVectorizer.cs @@ -0,0 +1,65 @@ +using System.Net.Http.Json; +using Redis.OM.Modeling; + +namespace Redis.OM.Vectorizers.HuggingFace; + +public class HuggingFaceApiSentenceVectorizer : VectorizerAttribute +{ + public string? ModelId { get; set; } + public override VectorType VectorType => VectorType.FLOAT32; + private int? _dim; + + public override int Dim + { + get + { + if (_dim is not null) return _dim.Value; + const string testString = "This is a vector dimensionality probing query"; + var floats = GetFloats(testString); + _dim = floats.Length; + + return _dim.Value; + } + } + + public override byte[] Vectorize(object obj) + { + var s = (string)obj; + var floats = GetFloats(s); + return floats.SelectMany(BitConverter.GetBytes).ToArray(); + } + + public float[] GetFloats(string s) + { + var client = Configuration.Instance.Client; + var modelId = ModelId ?? Configuration.Instance["REDIS_OM_HF_MODEL_ID"]; + if (modelId is null) throw new InvalidOperationException("Model Id Required to use Hugging Face API."); + + var requestContent = JsonContent.Create(new + { + inputs = new string[] { s }, + options = new { wait_for_model = true } + }); + + var request = new HttpRequestMessage() + { + Method = HttpMethod.Post, + Content = requestContent, + RequestUri = + new Uri($"{Configuration.Instance.HuggingFaceBaseAddress}/pipeline/feature-extraction/{modelId}"), + Headers = + { + { "Authorization", $"Bearer {Configuration.Instance.HuggingFaceAuthorizationToken}" } + } + }; + + var res = client.SendAsync(request).Result; + var floats = res.Content.ReadFromJsonAsync().Result; + if (floats is null) + { + throw new Exception("Did not receive a response back from HuggingFace"); + } + + return floats.First(); + } +} \ No newline at end of file diff --git a/src/Redis.OM.Vectorizers.HuggingFace/Redis.OM.Vectorizers.HuggingFace.csproj b/src/Redis.OM.Vectorizers.HuggingFace/Redis.OM.Vectorizers.HuggingFace.csproj new file mode 100644 index 00000000..f1aea53d --- /dev/null +++ b/src/Redis.OM.Vectorizers.HuggingFace/Redis.OM.Vectorizers.HuggingFace.csproj @@ -0,0 +1,22 @@ + + + + net6.0 + enable + enable + Redis.OM.HuggingFace + + + + + + + + + + + + + + + diff --git a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/HuggingFaceVectors.cs b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/HuggingFaceVectors.cs new file mode 100644 index 00000000..9b27e3f3 --- /dev/null +++ b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/HuggingFaceVectors.cs @@ -0,0 +1,24 @@ +using Redis.OM.Vectorizers.HuggingFace; +using Redis.OM.Modeling; +using Redis.OM.Modeling.Vectors; + +namespace Redis.OM.Unit.Tests; + +[Document(StorageType = StorageType.Json)] +public class HuggingFaceVectors +{ + [RedisIdField] + public string Id { get; set; } + + [Vector] + [HuggingFaceApiSentenceVectorizer(ModelId = "sentence-transformers/all-MiniLM-L6-v2")] + public string Sentence { get; set; } + + [Indexed] + public string Name { get; set; } + + [Indexed] + public int Age { get; set; } + + public VectorScores VectorScore { get; set; } +} \ No newline at end of file diff --git a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/OpenAIVectors.cs b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/OpenAIVectors.cs new file mode 100644 index 00000000..2a15594c --- /dev/null +++ b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/OpenAIVectors.cs @@ -0,0 +1,24 @@ +using Redis.OM.Modeling; +using Redis.OM.Modeling.Vectors; +using Redis.OM.OpenAI; + +namespace Redis.OM.Unit.Tests; + +[Document(StorageType = StorageType.Json)] +public class OpenAIVectors +{ + [RedisIdField] + public string Id { get; set; } + + [Vector] + [OpenAISentenceVectorizer] + public string Sentence { get; set; } + + [Indexed] + public string Name { get; set; } + + [Indexed] + public int Age { get; set; } + + public VectorScores VectorScore { get; set; } +} \ No newline at end of file diff --git a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorFunctionalTests.cs b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorFunctionalTests.cs index 5ac9eece..5bf86b37 100644 --- a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorFunctionalTests.cs +++ b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorFunctionalTests.cs @@ -1,3 +1,4 @@ +using System; using System.Linq; using System.Text; using Redis.OM.Contracts; @@ -16,6 +17,88 @@ public VectorFunctionalTests(RedisSetup setup) _connection = setup.Connection; } + [Fact] + public void TestHuggingFaceVectorizer() + { + _connection.DropIndexAndAssociatedRecords(typeof(HuggingFaceVectors)); + _connection.CreateIndex(typeof(HuggingFaceVectors)); + var collection = new RedisCollection(_connection); + var obj = new HuggingFaceVectors + { + Age = 45, + Sentence = "Hello World this is Hal.", + Name = "Hal" + }; + + collection.Insert(obj); + var res = collection.NearestNeighbors(x => x.Sentence, 2, "Hello World this is Hal.").First(); + Assert.Equal(obj.Id, res.Id); + Assert.Equal(0, res.VectorScore.NearestNeighborsScore); + Assert.Equal(obj.Sentence, res.Sentence); + } + + [Fact] + public void TestParis() + { + _connection.DropIndexAndAssociatedRecords(typeof(HuggingFaceVectors)); + _connection.CreateIndex(typeof(HuggingFaceVectors)); + var collection = new RedisCollection(_connection); + var obj = new HuggingFaceVectors + { + Age = 2259, + Sentence = "What is the capital of France?", + Name = "Paris" + }; + + collection.Insert(obj); + var res = collection + .First(x => x.Sentence.VectorRange("What really is the capital of France?", .1, "range") && x.Age > 1000); + res = collection.NearestNeighbors(x => x.Sentence, 2, "What really is the capital of France?").First(x => x.Age > 1000); + Assert.Equal(obj.Id, res.Id); + Assert.True(res.VectorScore.RangeScore < .1); + Assert.Equal(obj.Sentence, res.Sentence); + } + + [Fact] + public void TestOpenAIVectorizer() + { + _connection.DropIndexAndAssociatedRecords(typeof(OpenAIVectors)); + _connection.CreateIndex(typeof(OpenAIVectors)); + var collection = new RedisCollection(_connection); + var obj = new OpenAIVectors + { + Age = 45, + Sentence = "Hello World this is Hal.", + Name = "Hal" + }; + + collection.Insert(obj); + var res = collection.NearestNeighbors(x => x.Sentence, 2, "Hello World this is Hal.").First(); + Assert.Equal(obj.Id, res.Id); + Assert.True(res.VectorScore.NearestNeighborsScore < .01); + Assert.Equal(obj.Sentence, res.Sentence); + } + + [Fact] + public void TestOpenAIVectorRange() + { + _connection.DropIndexAndAssociatedRecords(typeof(OpenAIVectors)); + _connection.CreateIndex(typeof(OpenAIVectors)); + var collection = new RedisCollection(_connection); + var obj = new OpenAIVectors + { + Age = 2259, + Sentence = "What is the capital of France?", + Name = "Paris" + }; + + collection.Insert(obj); + var res = collection.First(x => x.Sentence.VectorRange("What really is the capital of France?", 1, "range")); + Assert.Equal(obj.Id, res.Id); + Assert.True(res.VectorScore.RangeScore < .1); + Assert.Equal(obj.Sentence, res.Sentence); + } + [Fact] public void BasicRangeQuery() { diff --git a/test/Redis.OM.Unit.Tests/Redis.OM.Unit.Tests.csproj b/test/Redis.OM.Unit.Tests/Redis.OM.Unit.Tests.csproj index 72fa0e34..18ad7c13 100644 --- a/test/Redis.OM.Unit.Tests/Redis.OM.Unit.Tests.csproj +++ b/test/Redis.OM.Unit.Tests/Redis.OM.Unit.Tests.csproj @@ -14,7 +14,15 @@ + + + + + Always + + + diff --git a/test/Redis.OM.Unit.Tests/appsettings.json b/test/Redis.OM.Unit.Tests/appsettings.json new file mode 100644 index 00000000..98c74579 --- /dev/null +++ b/test/Redis.OM.Unit.Tests/appsettings.json @@ -0,0 +1,4 @@ +{ + "REDIS_OM_HF_TOKEN":"REDIS_OM_HF_TOKEN", + "REDIS_OM_OAI_TOKEN": "REDIS_OM_OAI_TOKEN" +} \ No newline at end of file From f2866f0b90ca01f1ce4845b72192ac7141f23814 Mon Sep 17 00:00:00 2001 From: slorello89 Date: Thu, 26 Oct 2023 10:23:50 -0400 Subject: [PATCH 10/36] semantic cache start --- src/Redis.OM/Contracts/ISemanticCache.cs | 82 +++++++++ src/Redis.OM/Contracts/IVectorizer.cs | 27 +++ src/Redis.OM/SemanticCache.cs | 203 +++++++++++++++++++++++ src/Redis.OM/SemanticCacheResponse.cs | 50 ++++++ 4 files changed, 362 insertions(+) create mode 100644 src/Redis.OM/Contracts/ISemanticCache.cs create mode 100644 src/Redis.OM/Contracts/IVectorizer.cs create mode 100644 src/Redis.OM/SemanticCache.cs create mode 100644 src/Redis.OM/SemanticCacheResponse.cs diff --git a/src/Redis.OM/Contracts/ISemanticCache.cs b/src/Redis.OM/Contracts/ISemanticCache.cs new file mode 100644 index 00000000..05849ca3 --- /dev/null +++ b/src/Redis.OM/Contracts/ISemanticCache.cs @@ -0,0 +1,82 @@ +using System.Collections.Generic; +using System.Threading.Tasks; + +namespace Redis.OM.Contracts +{ + /// + /// An interface for interacting with a Semantic Cache. + /// + public interface ISemanticCache + { + /// + /// Gets the index name of the cache. + /// + string IndexName { get; } + + /// + /// Gets the prefix to be used for the keys. + /// + string Prefix { get; } + + /// + /// Gets the threshold to be used for the keys. + /// + double Threshold { get; } + + /// + /// Gets the Time to live for the keys added to the cache. + /// + long? Ttl { get; } + + /// + /// Gets the vectorizer to use for the Semantic Cache. + /// + IVectorizer Vectorizer { get; } + + /// + /// Checks the cache to see if any close prompts have been added. + /// + /// The prompt. + /// How many results to pull back at most (defaults to 10). + /// The responses. + SemanticCacheResponse[] Check(string prompt, int maxNumResults = 10); + + /// + /// Checks the cache to see if any close prompts have been added. + /// + /// The prompt. + /// How many results to pull back at most (defaults to 10). + /// The responses. + Task CheckAsync(string prompt, int maxNumResults = 10); + + /// + /// Stores the Prompt/response/metadata in Redis. + /// + /// The prompt. + /// The response. + /// The metadata. + void Store(string prompt, string response, object? metadata); + + /// + /// Stores the Prompt/response/metadata in Redis. + /// + /// The prompt. + /// The response. + /// The metadata. + /// A representing the asynchronous operation. + Task StoreAsync(string prompt, string response, object? metadata); + + /// + /// Deletes the cache from Redis. + /// + /// Whether or not to drop the records associated with the cache. Defaults to true. + void DeleteCache(bool dropRecords = true); + + /// + /// Deletes the cache from Redis. + /// + /// Whether or not to drop the records associated with the cache. Defaults to true. + /// A representing the asynchronous operation. + Task DeleteCacheAsync(bool dropRecords = true); + } +} \ No newline at end of file diff --git a/src/Redis.OM/Contracts/IVectorizer.cs b/src/Redis.OM/Contracts/IVectorizer.cs new file mode 100644 index 00000000..495f1f3b --- /dev/null +++ b/src/Redis.OM/Contracts/IVectorizer.cs @@ -0,0 +1,27 @@ +using Redis.OM.Modeling; + +namespace Redis.OM.Contracts +{ + /// + /// Converter of objects into vectors. + /// + public interface IVectorizer + { + /// + /// Gets the vector Type generated by the vectorizer. + /// + VectorType VectorType { get; } + + /// + /// Gets the vector dimension of the vectors generated by the vectorizer. + /// + int Dim { get; } + + /// + /// Converts the provided object to a vector. + /// + /// the object to convert. + /// A byte array containing the vectorized data. + byte[] Vectorize(object obj); + } +} \ No newline at end of file diff --git a/src/Redis.OM/SemanticCache.cs b/src/Redis.OM/SemanticCache.cs new file mode 100644 index 00000000..7f50e1e0 --- /dev/null +++ b/src/Redis.OM/SemanticCache.cs @@ -0,0 +1,203 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Security.Cryptography; +using System.Text; +using System.Threading.Tasks; +using Redis.OM.Contracts; +using Redis.OM.Searching.Query; + +namespace Redis.OM +{ + /// + /// A semantic cache for Large Language Models. + /// + public class SemanticCache : ISemanticCache + { + private readonly IRedisConnection _connection; + + /// + /// Initializes a new instance of the class. + /// + /// The index name. + /// The prefix for indexed items. + /// The threshold to check against.. + /// The Time To Live for a record inserted. + /// The vectorizer to use. + /// The connection to redis. + internal SemanticCache(string indexName, string prefix, double threshold, long? ttl, IVectorizer vectorizer, IRedisConnection connection) + { + IndexName = indexName; + Prefix = prefix; + Threshold = threshold; + Ttl = ttl; + Vectorizer = vectorizer; + _connection = connection; + } + + /// + public string IndexName { get; } + + /// + public string Prefix { get; } + + /// + public double Threshold { get; } + + /// + public long? Ttl { get; } + + /// + public IVectorizer Vectorizer { get; } + + /// + public SemanticCacheResponse[] Check(string prompt, int maxNumResults = 10) + { + var query = BuildCheckQuery(prompt, maxNumResults); + var res = (RedisReply[])_connection.Execute("FT.SEARCH", query.SerializeQuery()); + return BuildResponse(res); + } + + /// + public async Task CheckAsync(string prompt, int maxNumResults = 10) + { + var query = BuildCheckQuery(prompt, maxNumResults); + var res = (RedisReply[])await _connection.ExecuteAsync("FT.SEARCH", query.SerializeQuery()).ConfigureAwait(false); + return BuildResponse(res); + } + + /// + public void Store(string prompt, string response, object? metadata) + { + var key = $"{Prefix}:{Sha256Hash(prompt)}"; + var hash = BuildDocumentHash(prompt, response, metadata); + if (Ttl is not null) + { + _connection.HSet(key, TimeSpan.FromMilliseconds((double)Ttl), hash.ToArray()); + } + else + { + _connection.HSet(key, hash.ToArray()); + } + } + + /// + public Task StoreAsync(string prompt, string response, object? metadata) + { + var key = $"{Prefix}:{Sha256Hash(prompt)}"; + var hash = BuildDocumentHash(prompt, response, metadata); + return Ttl is not null ? _connection.HSetAsync(key, TimeSpan.FromMilliseconds((double)Ttl), hash.ToArray()) : _connection.HSetAsync(key, hash.ToArray()); + } + + /// + public void DeleteCache(bool dropRecords = true) + { + try + { + if (dropRecords) + { + _connection.Execute("FT.DROPINDEX", IndexName, "DD"); + } + else + { + _connection.Execute("FT.DROPINDEX", IndexName); + } + } + catch (Exception ex) + { + if (!ex.Message.Contains("Unknown Index name")) + { + throw; + } + } + } + + /// + public Task DeleteCacheAsync(bool dropRecords = true) + { + try + { + return dropRecords ? _connection.ExecuteAsync("FT.DROPINDEX", IndexName, "DD") : _connection.ExecuteAsync("FT.DROPINDEX", IndexName); + } + catch (Exception ex) + { + if (!ex.Message.Contains("Unknown Index name")) + { + throw; + } + } + + return Task.CompletedTask; + } + + private static string Sha256Hash(string value) + { + StringBuilder sb = new StringBuilder(); + + using (var hash = SHA256.Create()) + { + Encoding enc = Encoding.UTF8; + byte[] result = hash.ComputeHash(enc.GetBytes(value)); + + foreach (byte b in result) + { + sb.Append(b.ToString("x2")); + } + } + + return sb.ToString(); + } + + private RedisQuery BuildCheckQuery(string prompt, int maxNumResults) + { + var query = new RedisQuery(IndexName); + query.QueryText = "@embedding:[VECTOR_RANGE $0 $1]=>[$YIELD_DISTANCE_AS: semantic_score]"; + query.Parameters.Add(Threshold); + query.Parameters.Add(Vectorizer.Vectorize(prompt)); + if (maxNumResults != 10) + { + query.Limit = new SearchLimit { Number = maxNumResults, Offset = 0 }; + } + + return query; + } + + private SemanticCacheResponse[] BuildResponse(RedisReply[] res) + { + List results = new List(); + for (int i = 1; i < res.Length; i += 2) + { + var key = (string)res[i]; + var hashArr = (RedisReply[])res[i + 1]; + Dictionary hash = new Dictionary(); + for (var j = 0; j < hashArr.Length; j += 2) + { + hash.Add(hashArr[j], hashArr[j + 1]); + } + + var score = double.Parse(hash["semantic_score"], CultureInfo.InvariantCulture); + var response = hash["response"]; + hash.TryGetValue("metadata", out var metadata); + results.Add(new SemanticCacheResponse(key, response, score, metadata)); + } + + return results.ToArray(); + } + + private Dictionary BuildDocumentHash(string prompt, string response, object? metadata) + { + var bytes = Vectorizer.Vectorize(prompt); + Dictionary hash = new Dictionary(); + hash.Add("embedding", bytes); + hash.Add("response", response); + hash.Add("prompt", prompt); + if (metadata is not null) + { + hash.Add("metadata", metadata); + } + + return hash; + } + } +} \ No newline at end of file diff --git a/src/Redis.OM/SemanticCacheResponse.cs b/src/Redis.OM/SemanticCacheResponse.cs new file mode 100644 index 00000000..38e5cd37 --- /dev/null +++ b/src/Redis.OM/SemanticCacheResponse.cs @@ -0,0 +1,50 @@ +namespace Redis.OM +{ + /// + /// A response to a Semantic Cache Query. + /// + public class SemanticCacheResponse + { + /// + /// Initializes a new instance of the class. + /// + /// The key. + /// The string response. + /// The Score. + /// The metadata. + internal SemanticCacheResponse(string key, string response, double score, object? metaData) + { + Key = key; + Response = response; + Score = score; + MetaData = metaData; + } + + /// + /// Gets the key. + /// + public string Key { get; } + + /// + /// Gets the response. + /// + public string Response { get; } + + /// + /// Gets the score. + /// + public double Score { get; } + + /// + /// Gets the metadata. + /// + public object? MetaData { get; } + + /// + /// Converts response to string implicitly. + /// + /// The response. + /// The response string. + public static implicit operator string(SemanticCacheResponse response) => response.Response; + } +} \ No newline at end of file From 21957f5d98f65044b03daf3df37d86111c143823 Mon Sep 17 00:00:00 2001 From: slorello89 Date: Thu, 26 Oct 2023 12:54:23 -0400 Subject: [PATCH 11/36] more vectorizers --- .../OpenAISentenceVectorizer.cs | 73 +++---------------- .../OpenAISentenceVectorizerAttribute.cs | 73 +++++++++++++++++++ .../HuggingFaceApiSentenceVectorizer.cs | 47 +++++------- ...ggingFaceApiSentenceVectorizerAttribute.cs | 65 +++++++++++++++++ src/Redis.OM/Contracts/ISemanticCache.cs | 2 +- src/Redis.OM/Contracts/IVectorizer.cs | 5 +- src/Redis.OM/SemanticCache.cs | 4 +- 7 files changed, 175 insertions(+), 94 deletions(-) create mode 100644 Redis.OM.Vectorizers.OpenAI/OpenAISentenceVectorizerAttribute.cs create mode 100644 src/Redis.OM.Vectorizers.HuggingFace/HuggingFaceApiSentenceVectorizerAttribute.cs diff --git a/Redis.OM.Vectorizers.OpenAI/OpenAISentenceVectorizer.cs b/Redis.OM.Vectorizers.OpenAI/OpenAISentenceVectorizer.cs index d5c6fdad..7dce1d93 100644 --- a/Redis.OM.Vectorizers.OpenAI/OpenAISentenceVectorizer.cs +++ b/Redis.OM.Vectorizers.OpenAI/OpenAISentenceVectorizer.cs @@ -1,73 +1,24 @@ -using System.Net.Http.Json; -using System.Text.Json; +using Redis.OM.Contracts; using Redis.OM.Modeling; -using Redis.OM.Vectorizers.HuggingFace; namespace Redis.OM.OpenAI; -public class OpenAISentenceVectorizer : VectorizerAttribute +public class OpenAISentenceVectorizer : IVectorizer { - private const string DefaultModel = "text-embedding-ada-002"; - public string ModelId { get; set; } = DefaultModel; - public override VectorType VectorType { get; } + private readonly string _openAIAuthToken; + private readonly string _model; - public override int Dim + public OpenAISentenceVectorizer(string openAIAuthToken, string model = "text-embedding-ada-002", int dim = 1536) { - get - { - if (ModelId == DefaultModel) - { - return 1536; - } - - return GetFloats("this is a test string").Length; - } + _openAIAuthToken = openAIAuthToken; + _model = model; + Dim = dim; } - public override byte[] Vectorize(object obj) + public VectorType VectorType => VectorType.FLOAT32; + public int Dim { get; } + public byte[] Vectorize(string obj) { - var s = (string)obj; - var floats = GetFloats(s); - return floats.SelectMany(BitConverter.GetBytes).ToArray(); - } - - internal float[] GetFloats(string s) - { - var client = Configuration.Instance.Client; - var requestContent = JsonContent.Create( - new - { - input = s, - model = ModelId - }); - - var request = new HttpRequestMessage - { - Method = HttpMethod.Post, - RequestUri = new Uri($"{Configuration.Instance.OpenAiApiUrl}/v1/embeddings"), - Content = requestContent, - Headers = { { "Authorization", $"Bearer {Configuration.Instance.OpenAiAuthorizationToken}" } } - }; - - var res = client.SendAsync(request).Result; - if (!res.IsSuccessStatusCode) - { - throw new HttpRequestException( - $"Open AI did not respond with a positive error code: {res.StatusCode}, {res.ReasonPhrase}"); - } - var jsonObj = res.Content.ReadFromJsonAsync().Result; - - - if (!jsonObj.TryGetProperty("data", out var data)) - { - throw new Exception("Malformed Response"); - } - - if (data.GetArrayLength() < 1 || !data[0].TryGetProperty("embedding", out var embedding)) - { - throw new Exception("Malformed Response"); - } - - return embedding.Deserialize()!; + throw new NotImplementedException(); } } \ No newline at end of file diff --git a/Redis.OM.Vectorizers.OpenAI/OpenAISentenceVectorizerAttribute.cs b/Redis.OM.Vectorizers.OpenAI/OpenAISentenceVectorizerAttribute.cs new file mode 100644 index 00000000..b8c24f2c --- /dev/null +++ b/Redis.OM.Vectorizers.OpenAI/OpenAISentenceVectorizerAttribute.cs @@ -0,0 +1,73 @@ +using System.Net.Http.Json; +using System.Text.Json; +using Redis.OM.Modeling; +using Redis.OM.Vectorizers.HuggingFace; + +namespace Redis.OM.OpenAI; + +public class OpenAISentenceVectorizerAttribute : VectorizerAttribute +{ + private const string DefaultModel = "text-embedding-ada-002"; + public string ModelId { get; set; } = DefaultModel; + public override VectorType VectorType { get; } + + public override int Dim + { + get + { + if (ModelId == DefaultModel) + { + return 1536; + } + + return GetFloats("this is a test string").Length; + } + } + + public override byte[] Vectorize(object obj) + { + var s = (string)obj; + var floats = GetFloats(s); + return floats.SelectMany(BitConverter.GetBytes).ToArray(); + } + + internal float[] GetFloats(string s) + { + var client = Configuration.Instance.Client; + var requestContent = JsonContent.Create( + new + { + input = s, + model = ModelId + }); + + var request = new HttpRequestMessage + { + Method = HttpMethod.Post, + RequestUri = new Uri($"{Configuration.Instance.OpenAiApiUrl}/v1/embeddings"), + Content = requestContent, + Headers = { { "Authorization", $"Bearer {Configuration.Instance.OpenAiAuthorizationToken}" } } + }; + + var res = client.SendAsync(request).Result; + if (!res.IsSuccessStatusCode) + { + throw new HttpRequestException( + $"Open AI did not respond with a positive error code: {res.StatusCode}, {res.ReasonPhrase}"); + } + var jsonObj = res.Content.ReadFromJsonAsync().Result; + + + if (!jsonObj.TryGetProperty("data", out var data)) + { + throw new Exception("Malformed Response"); + } + + if (data.GetArrayLength() < 1 || !data[0].TryGetProperty("embedding", out var embedding)) + { + throw new Exception("Malformed Response"); + } + + return embedding.Deserialize()!; + } +} \ No newline at end of file diff --git a/src/Redis.OM.Vectorizers.HuggingFace/HuggingFaceApiSentenceVectorizer.cs b/src/Redis.OM.Vectorizers.HuggingFace/HuggingFaceApiSentenceVectorizer.cs index b88b34e6..92343c3d 100644 --- a/src/Redis.OM.Vectorizers.HuggingFace/HuggingFaceApiSentenceVectorizer.cs +++ b/src/Redis.OM.Vectorizers.HuggingFace/HuggingFaceApiSentenceVectorizer.cs @@ -1,43 +1,34 @@ -using System.Net.Http.Json; +using System.Net.Http.Json; +using Redis.OM.Contracts; using Redis.OM.Modeling; namespace Redis.OM.Vectorizers.HuggingFace; -public class HuggingFaceApiSentenceVectorizer : VectorizerAttribute +public class HuggingFaceApiSentenceVectorizer : IVectorizer { - public string? ModelId { get; set; } - public override VectorType VectorType => VectorType.FLOAT32; - private int? _dim; - - public override int Dim + public HuggingFaceApiSentenceVectorizer(string authToken, string modelId, int dim) { - get - { - if (_dim is not null) return _dim.Value; - const string testString = "This is a vector dimensionality probing query"; - var floats = GetFloats(testString); - _dim = floats.Length; - - return _dim.Value; - } + _huggingFaceAuthToken = authToken; + ModelId = modelId; + Dim = dim; } - - public override byte[] Vectorize(object obj) + + private readonly string _huggingFaceAuthToken; + public string ModelId { get; } + public VectorType VectorType => VectorType.FLOAT32; + + public int Dim { get; } + public byte[] Vectorize(string str) { - var s = (string)obj; - var floats = GetFloats(s); - return floats.SelectMany(BitConverter.GetBytes).ToArray(); + return GetFloats(str).SelectMany(BitConverter.GetBytes).ToArray(); } - + public float[] GetFloats(string s) { var client = Configuration.Instance.Client; - var modelId = ModelId ?? Configuration.Instance["REDIS_OM_HF_MODEL_ID"]; - if (modelId is null) throw new InvalidOperationException("Model Id Required to use Hugging Face API."); - var requestContent = JsonContent.Create(new { - inputs = new string[] { s }, + inputs = new [] { s }, options = new { wait_for_model = true } }); @@ -46,10 +37,10 @@ public float[] GetFloats(string s) Method = HttpMethod.Post, Content = requestContent, RequestUri = - new Uri($"{Configuration.Instance.HuggingFaceBaseAddress}/pipeline/feature-extraction/{modelId}"), + new Uri($"{Configuration.Instance.HuggingFaceBaseAddress}/pipeline/feature-extraction/{ModelId}"), Headers = { - { "Authorization", $"Bearer {Configuration.Instance.HuggingFaceAuthorizationToken}" } + { "Authorization", $"Bearer {_huggingFaceAuthToken}" } } }; diff --git a/src/Redis.OM.Vectorizers.HuggingFace/HuggingFaceApiSentenceVectorizerAttribute.cs b/src/Redis.OM.Vectorizers.HuggingFace/HuggingFaceApiSentenceVectorizerAttribute.cs new file mode 100644 index 00000000..0fc589ab --- /dev/null +++ b/src/Redis.OM.Vectorizers.HuggingFace/HuggingFaceApiSentenceVectorizerAttribute.cs @@ -0,0 +1,65 @@ +using System.Net.Http.Json; +using Redis.OM.Modeling; + +namespace Redis.OM.Vectorizers.HuggingFace; + +public class HuggingFaceApiSentenceVectorizerAttribute : VectorizerAttribute +{ + public string? ModelId { get; set; } + public override VectorType VectorType => VectorType.FLOAT32; + private int? _dim; + + public override int Dim + { + get + { + if (_dim is not null) return _dim.Value; + const string testString = "This is a vector dimensionality probing query"; + var floats = GetFloats(testString); + _dim = floats.Length; + + return _dim.Value; + } + } + + public override byte[] Vectorize(object obj) + { + var s = (string)obj; + var floats = GetFloats(s); + return floats.SelectMany(BitConverter.GetBytes).ToArray(); + } + + public float[] GetFloats(string s) + { + var client = Configuration.Instance.Client; + var modelId = ModelId ?? Configuration.Instance["REDIS_OM_HF_MODEL_ID"]; + if (modelId is null) throw new InvalidOperationException("Model Id Required to use Hugging Face API."); + + var requestContent = JsonContent.Create(new + { + inputs = new string[] { s }, + options = new { wait_for_model = true } + }); + + var request = new HttpRequestMessage() + { + Method = HttpMethod.Post, + Content = requestContent, + RequestUri = + new Uri($"{Configuration.Instance.HuggingFaceBaseAddress}/pipeline/feature-extraction/{modelId}"), + Headers = + { + { "Authorization", $"Bearer {Configuration.Instance.HuggingFaceAuthorizationToken}" } + } + }; + + var res = client.SendAsync(request).Result; + var floats = res.Content.ReadFromJsonAsync().Result; + if (floats is null) + { + throw new Exception("Did not receive a response back from HuggingFace"); + } + + return floats.First(); + } +} \ No newline at end of file diff --git a/src/Redis.OM/Contracts/ISemanticCache.cs b/src/Redis.OM/Contracts/ISemanticCache.cs index 05849ca3..93842e16 100644 --- a/src/Redis.OM/Contracts/ISemanticCache.cs +++ b/src/Redis.OM/Contracts/ISemanticCache.cs @@ -31,7 +31,7 @@ public interface ISemanticCache /// /// Gets the vectorizer to use for the Semantic Cache. /// - IVectorizer Vectorizer { get; } + IVectorizer Vectorizer { get; } /// /// Checks the cache to see if any close prompts have been added. diff --git a/src/Redis.OM/Contracts/IVectorizer.cs b/src/Redis.OM/Contracts/IVectorizer.cs index 495f1f3b..41b0c685 100644 --- a/src/Redis.OM/Contracts/IVectorizer.cs +++ b/src/Redis.OM/Contracts/IVectorizer.cs @@ -5,7 +5,8 @@ namespace Redis.OM.Contracts /// /// Converter of objects into vectors. /// - public interface IVectorizer + /// The type to be vectorized. + public interface IVectorizer { /// /// Gets the vector Type generated by the vectorizer. @@ -22,6 +23,6 @@ public interface IVectorizer /// /// the object to convert. /// A byte array containing the vectorized data. - byte[] Vectorize(object obj); + byte[] Vectorize(T obj); } } \ No newline at end of file diff --git a/src/Redis.OM/SemanticCache.cs b/src/Redis.OM/SemanticCache.cs index 7f50e1e0..3dc58b76 100644 --- a/src/Redis.OM/SemanticCache.cs +++ b/src/Redis.OM/SemanticCache.cs @@ -26,7 +26,7 @@ public class SemanticCache : ISemanticCache /// The Time To Live for a record inserted. /// The vectorizer to use. /// The connection to redis. - internal SemanticCache(string indexName, string prefix, double threshold, long? ttl, IVectorizer vectorizer, IRedisConnection connection) + internal SemanticCache(string indexName, string prefix, double threshold, long? ttl, IVectorizer vectorizer, IRedisConnection connection) { IndexName = indexName; Prefix = prefix; @@ -49,7 +49,7 @@ internal SemanticCache(string indexName, string prefix, double threshold, long? public long? Ttl { get; } /// - public IVectorizer Vectorizer { get; } + public IVectorizer Vectorizer { get; } /// public SemanticCacheResponse[] Check(string prompt, int maxNumResults = 10) From 4fbd0bf5cf7c5e32f970775d4c1acca828cda386 Mon Sep 17 00:00:00 2001 From: slorello89 Date: Thu, 26 Oct 2023 16:53:00 -0400 Subject: [PATCH 12/36] some semantic caching --- .../OpenAISentenceVectorizer.cs | 49 +++++++++++++++- .../OpenAISentenceVectorizerAttribute.cs | 3 +- .../RedisConnectionProviderExtensions.cs | 20 +++++++ .../Configuration.cs | 2 +- .../RedisConnectionProviderExtensions.cs | 21 +++++++ src/Redis.OM/Contracts/ISemanticCache.cs | 19 +++++-- src/Redis.OM/RediSearchCommands.cs | 50 ++++++++++++++++ src/Redis.OM/SemanticCache.cs | 57 +++++++++++++++++-- .../VectorTests/OpenAIVectors.cs | 1 - .../VectorTests/SemanticCachingTests.cs | 40 +++++++++++++ .../RedisSetupCollection.cs | 22 +++---- 11 files changed, 253 insertions(+), 31 deletions(-) create mode 100644 Redis.OM.Vectorizers.OpenAI/RedisConnectionProviderExtensions.cs create mode 100644 src/Redis.OM.Vectorizers.HuggingFace/RedisConnectionProviderExtensions.cs create mode 100644 test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/SemanticCachingTests.cs diff --git a/Redis.OM.Vectorizers.OpenAI/OpenAISentenceVectorizer.cs b/Redis.OM.Vectorizers.OpenAI/OpenAISentenceVectorizer.cs index 7dce1d93..f8cf2eb7 100644 --- a/Redis.OM.Vectorizers.OpenAI/OpenAISentenceVectorizer.cs +++ b/Redis.OM.Vectorizers.OpenAI/OpenAISentenceVectorizer.cs @@ -1,7 +1,9 @@ +using System.Net.Http.Json; +using System.Text.Json; using Redis.OM.Contracts; using Redis.OM.Modeling; -namespace Redis.OM.OpenAI; +namespace Redis.OM; public class OpenAISentenceVectorizer : IVectorizer { @@ -17,8 +19,49 @@ public OpenAISentenceVectorizer(string openAIAuthToken, string model = "text-emb public VectorType VectorType => VectorType.FLOAT32; public int Dim { get; } - public byte[] Vectorize(string obj) + public byte[] Vectorize(string str) { - throw new NotImplementedException(); + var floats = GetFloats(str); + return floats.SelectMany(BitConverter.GetBytes).ToArray(); + } + + internal float[] GetFloats(string s) + { + var client = Configuration.Instance.Client; + var requestContent = JsonContent.Create( + new + { + input = s, + model = _model + }); + + var request = new HttpRequestMessage + { + Method = HttpMethod.Post, + RequestUri = new Uri($"{Configuration.Instance.OpenAiApiUrl}/v1/embeddings"), + Content = requestContent, + Headers = { { "Authorization", $"Bearer {_openAIAuthToken}" } } + }; + + var res = client.SendAsync(request).Result; + if (!res.IsSuccessStatusCode) + { + throw new HttpRequestException( + $"Open AI did not respond with a positive error code: {res.StatusCode}, {res.ReasonPhrase}"); + } + var jsonObj = res.Content.ReadFromJsonAsync().Result; + + + if (!jsonObj.TryGetProperty("data", out var data)) + { + throw new Exception("Malformed Response"); + } + + if (data.GetArrayLength() < 1 || !data[0].TryGetProperty("embedding", out var embedding)) + { + throw new Exception("Malformed Response"); + } + + return embedding.Deserialize()!; } } \ No newline at end of file diff --git a/Redis.OM.Vectorizers.OpenAI/OpenAISentenceVectorizerAttribute.cs b/Redis.OM.Vectorizers.OpenAI/OpenAISentenceVectorizerAttribute.cs index b8c24f2c..1806e2e3 100644 --- a/Redis.OM.Vectorizers.OpenAI/OpenAISentenceVectorizerAttribute.cs +++ b/Redis.OM.Vectorizers.OpenAI/OpenAISentenceVectorizerAttribute.cs @@ -1,9 +1,8 @@ using System.Net.Http.Json; using System.Text.Json; using Redis.OM.Modeling; -using Redis.OM.Vectorizers.HuggingFace; -namespace Redis.OM.OpenAI; +namespace Redis.OM; public class OpenAISentenceVectorizerAttribute : VectorizerAttribute { diff --git a/Redis.OM.Vectorizers.OpenAI/RedisConnectionProviderExtensions.cs b/Redis.OM.Vectorizers.OpenAI/RedisConnectionProviderExtensions.cs new file mode 100644 index 00000000..0789642e --- /dev/null +++ b/Redis.OM.Vectorizers.OpenAI/RedisConnectionProviderExtensions.cs @@ -0,0 +1,20 @@ +using Redis.OM.Contracts; + +namespace Redis.OM; + +public static class RedisConnectionProviderExtensions +{ + public static ISemanticCache OpenAISemanticCache(this IRedisConnectionProvider provider, string openAIAuthToken, double threshold = .15, string indexName = "OpenAISemanticCache", string? prefix = null, long? ttl = null) + { + var vectorizer = new OpenAISentenceVectorizer(openAIAuthToken); + var connection = provider.Connection; + var info = connection.GetIndexInfo(indexName); + var cache = new SemanticCache(indexName, prefix ?? indexName, threshold, ttl, vectorizer, connection); + if (info is null) + { + cache.CreateIndex(); + } + + return cache; + } +} \ No newline at end of file diff --git a/src/Redis.OM.Vectorizers.Core/Configuration.cs b/src/Redis.OM.Vectorizers.Core/Configuration.cs index 705fe2a9..e29a7e8d 100644 --- a/src/Redis.OM.Vectorizers.Core/Configuration.cs +++ b/src/Redis.OM.Vectorizers.Core/Configuration.cs @@ -1,7 +1,7 @@ using System.Net.Http.Headers; using Microsoft.Extensions.Configuration; -namespace Redis.OM.Vectorizers.HuggingFace; +namespace Redis.OM; public class Configuration { diff --git a/src/Redis.OM.Vectorizers.HuggingFace/RedisConnectionProviderExtensions.cs b/src/Redis.OM.Vectorizers.HuggingFace/RedisConnectionProviderExtensions.cs new file mode 100644 index 00000000..39e9d624 --- /dev/null +++ b/src/Redis.OM.Vectorizers.HuggingFace/RedisConnectionProviderExtensions.cs @@ -0,0 +1,21 @@ +using Redis.OM.Contracts; +using Redis.OM.Vectorizers.HuggingFace; + +namespace Redis.OM; + +public static class RedisConnectionProviderExtensions +{ + public static ISemanticCache HuggingFaceSemanticCache(this IRedisConnectionProvider provider, string huggingFaceAuthToken, double threshold = .15, string modelId = "sentence-transformers/all-mpnet-base-v2", int dim = 768, string indexName = "HuggingFaceSemanticCache", string? prefix = null, long? ttl = null) + { + var vectorizer = new HuggingFaceApiSentenceVectorizer(huggingFaceAuthToken, modelId, dim); + var connection = provider.Connection; + var info = connection.GetIndexInfo(indexName); + var cache = new SemanticCache(indexName, prefix ?? indexName, threshold, ttl, vectorizer, connection); + if (info is null) + { + cache.CreateIndex(); + } + + return cache; + } +} \ No newline at end of file diff --git a/src/Redis.OM/Contracts/ISemanticCache.cs b/src/Redis.OM/Contracts/ISemanticCache.cs index 93842e16..3dad9f69 100644 --- a/src/Redis.OM/Contracts/ISemanticCache.cs +++ b/src/Redis.OM/Contracts/ISemanticCache.cs @@ -39,7 +39,7 @@ public interface ISemanticCache /// The prompt. /// How many results to pull back at most (defaults to 10). /// The responses. - SemanticCacheResponse[] Check(string prompt, int maxNumResults = 10); + SemanticCacheResponse[] GetSimilar(string prompt, int maxNumResults = 10); /// /// Checks the cache to see if any close prompts have been added. @@ -47,7 +47,7 @@ public interface ISemanticCache /// The prompt. /// How many results to pull back at most (defaults to 10). /// The responses. - Task CheckAsync(string prompt, int maxNumResults = 10); + Task GetSimilarAsync(string prompt, int maxNumResults = 10); /// /// Stores the Prompt/response/metadata in Redis. @@ -55,7 +55,7 @@ public interface ISemanticCache /// The prompt. /// The response. /// The metadata. - void Store(string prompt, string response, object? metadata); + void Store(string prompt, string response, object? metadata = null); /// /// Stores the Prompt/response/metadata in Redis. @@ -64,7 +64,7 @@ public interface ISemanticCache /// The response. /// The metadata. /// A representing the asynchronous operation. - Task StoreAsync(string prompt, string response, object? metadata); + Task StoreAsync(string prompt, string response, object? metadata = null); /// /// Deletes the cache from Redis. @@ -78,5 +78,16 @@ public interface ISemanticCache /// Whether or not to drop the records associated with the cache. Defaults to true. /// A representing the asynchronous operation. Task DeleteCacheAsync(bool dropRecords = true); + + /// + /// Creates the index for Semantic Cache. + /// + void CreateIndex(); + + /// + /// Creates the index for the Semantic Cache. + /// + /// A representing the asynchronous operation. + Task CreateIndexAsync(); } } \ No newline at end of file diff --git a/src/Redis.OM/RediSearchCommands.cs b/src/Redis.OM/RediSearchCommands.cs index 1b75b776..fad794ff 100644 --- a/src/Redis.OM/RediSearchCommands.cs +++ b/src/Redis.OM/RediSearchCommands.cs @@ -91,6 +91,56 @@ public static async Task CreateIndexAsync(this IRedisConnection connection } } + /// + /// Get index information. + /// + /// the connection. + /// The index name. + /// Strong-typed result of FT.INFO idx. + public static RedisIndexInfo? GetIndexInfo(this IRedisConnection connection, string indexName) + { + try + { + var redisReply = connection.Execute("FT.INFO", indexName); + var redisIndexInfo = new RedisIndexInfo(redisReply); + return redisIndexInfo; + } + catch (Exception ex) + { + if (ex.Message.ToLower().Contains("unknown index name")) + { + return null; + } + + throw; + } + } + + /// + /// Get index information. + /// + /// the connection. + /// the index name. + /// Strong-typed result of FT.INFO idx. + public static async Task GetIndexInfoAsync(this IRedisConnection connection, string indexName) + { + try + { + var redisReply = await connection.ExecuteAsync("FT.INFO", indexName).ConfigureAwait(false); + var redisIndexInfo = new RedisIndexInfo(redisReply); + return redisIndexInfo; + } + catch (Exception ex) + { + if (ex.Message.ToLower().Contains("unknown index name")) + { + return null; + } + + throw; + } + } + /// /// Get index information. /// diff --git a/src/Redis.OM/SemanticCache.cs b/src/Redis.OM/SemanticCache.cs index 3dc58b76..4aa3e6c6 100644 --- a/src/Redis.OM/SemanticCache.cs +++ b/src/Redis.OM/SemanticCache.cs @@ -2,10 +2,12 @@ using System.Collections.Generic; using System.Globalization; using System.Linq; +using System.Runtime.CompilerServices; using System.Security.Cryptography; using System.Text; using System.Threading.Tasks; using Redis.OM.Contracts; +using Redis.OM.Modeling; using Redis.OM.Searching.Query; namespace Redis.OM @@ -26,7 +28,7 @@ public class SemanticCache : ISemanticCache /// The Time To Live for a record inserted. /// The vectorizer to use. /// The connection to redis. - internal SemanticCache(string indexName, string prefix, double threshold, long? ttl, IVectorizer vectorizer, IRedisConnection connection) + public SemanticCache(string indexName, string prefix, double threshold, long? ttl, IVectorizer vectorizer, IRedisConnection connection) { IndexName = indexName; Prefix = prefix; @@ -52,7 +54,7 @@ internal SemanticCache(string indexName, string prefix, double threshold, long? public IVectorizer Vectorizer { get; } /// - public SemanticCacheResponse[] Check(string prompt, int maxNumResults = 10) + public SemanticCacheResponse[] GetSimilar(string prompt, int maxNumResults = 10) { var query = BuildCheckQuery(prompt, maxNumResults); var res = (RedisReply[])_connection.Execute("FT.SEARCH", query.SerializeQuery()); @@ -60,7 +62,7 @@ public SemanticCacheResponse[] Check(string prompt, int maxNumResults = 10) } /// - public async Task CheckAsync(string prompt, int maxNumResults = 10) + public async Task GetSimilarAsync(string prompt, int maxNumResults = 10) { var query = BuildCheckQuery(prompt, maxNumResults); var res = (RedisReply[])await _connection.ExecuteAsync("FT.SEARCH", query.SerializeQuery()).ConfigureAwait(false); @@ -68,7 +70,7 @@ public async Task CheckAsync(string prompt, int maxNumR } /// - public void Store(string prompt, string response, object? metadata) + public void Store(string prompt, string response, object? metadata = null) { var key = $"{Prefix}:{Sha256Hash(prompt)}"; var hash = BuildDocumentHash(prompt, response, metadata); @@ -83,7 +85,7 @@ public void Store(string prompt, string response, object? metadata) } /// - public Task StoreAsync(string prompt, string response, object? metadata) + public Task StoreAsync(string prompt, string response, object? metadata = null) { var key = $"{Prefix}:{Sha256Hash(prompt)}"; var hash = BuildDocumentHash(prompt, response, metadata); @@ -131,6 +133,44 @@ public Task DeleteCacheAsync(bool dropRecords = true) return Task.CompletedTask; } + /// + public void CreateIndex() + { + try + { + var serializedParams = SerializedIndexArgs(); + _connection.Execute("FT.CREATE", serializedParams); + } + catch (Exception ex) + { + if (ex.Message.Contains("Index already exists")) + { + return; + } + + throw; + } + } + + /// + public Task CreateIndexAsync() + { + try + { + var serializedParams = SerializedIndexArgs(); + return _connection.ExecuteAsync("FT.CREATE", serializedParams); + } + catch (Exception ex) + { + if (ex.Message.Contains("Index already exists")) + { + return Task.CompletedTask; + } + + throw; + } + } + private static string Sha256Hash(string value) { StringBuilder sb = new StringBuilder(); @@ -152,7 +192,7 @@ private static string Sha256Hash(string value) private RedisQuery BuildCheckQuery(string prompt, int maxNumResults) { var query = new RedisQuery(IndexName); - query.QueryText = "@embedding:[VECTOR_RANGE $0 $1]=>[$YIELD_DISTANCE_AS: semantic_score]"; + query.QueryText = "@embedding:[VECTOR_RANGE $0 $1]=>{$YIELD_DISTANCE_AS: semantic_score}"; query.Parameters.Add(Threshold); query.Parameters.Add(Vectorizer.Vectorize(prompt)); if (maxNumResults != 10) @@ -199,5 +239,10 @@ private Dictionary BuildDocumentHash(string prompt, string respo return hash; } + + private object[] SerializedIndexArgs() + { + return new object[] { IndexName, nameof(Prefix), 1, Prefix, "SCHEMA", "embedding", "VECTOR", "FLAT", 6, "DIM", Vectorizer.Dim, "TYPE", Vectorizer.VectorType.AsRedisString(), "DISTANCE_METRIC", "COSINE", }; + } } } \ No newline at end of file diff --git a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/OpenAIVectors.cs b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/OpenAIVectors.cs index 2a15594c..5b4c3998 100644 --- a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/OpenAIVectors.cs +++ b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/OpenAIVectors.cs @@ -1,6 +1,5 @@ using Redis.OM.Modeling; using Redis.OM.Modeling.Vectors; -using Redis.OM.OpenAI; namespace Redis.OM.Unit.Tests; diff --git a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/SemanticCachingTests.cs b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/SemanticCachingTests.cs new file mode 100644 index 00000000..f9dc8fa6 --- /dev/null +++ b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/SemanticCachingTests.cs @@ -0,0 +1,40 @@ +using System; +using System.Linq; +using Redis.OM.Contracts; +using Xunit; + +namespace Redis.OM.Unit.Tests; + +[Collection("Redis")] +public class SemanticCachingTests +{ + private readonly IRedisConnectionProvider _provider; + public SemanticCachingTests(RedisSetup setup) + { + _provider = setup.Provider; + } + + [Fact] + public void OpenAISemanticCache() + { + var token = Environment.GetEnvironmentVariable("REDIS_OM_OAI_TOKEN"); + Assert.NotNull(token); + var cache = _provider.OpenAISemanticCache(token); + cache.Store("What is the capital of France?", "Paris"); + var res = cache.GetSimilar("What really is the capital of France?").First(); + Assert.Equal("Paris",res.Response); + Assert.True(res.Score < .15); + } + + [Fact] + public void HuggingFaceSemanticCache() + { + var token = Environment.GetEnvironmentVariable("REDIS_OM_HF_TOKEN"); + Assert.NotNull(token); + var cache = _provider.HuggingFaceSemanticCache(token); + cache.Store("What is the capital of France?", "Paris"); + var res = cache.GetSimilar("What really is the capital of France?").First(); + Assert.Equal("Paris",res.Response); + Assert.True(res.Score < .15); + } +} \ No newline at end of file diff --git a/test/Redis.OM.Unit.Tests/RedisSetupCollection.cs b/test/Redis.OM.Unit.Tests/RedisSetupCollection.cs index 2e533f2a..374b1292 100644 --- a/test/Redis.OM.Unit.Tests/RedisSetupCollection.cs +++ b/test/Redis.OM.Unit.Tests/RedisSetupCollection.cs @@ -32,24 +32,18 @@ public RedisSetup() Connection.CreateIndex(typeof(ObjectWithDateTimeOffsetJson)); } - private IRedisConnection _connection = null; - public IRedisConnection Connection - { - get - { - if (_connection == null) - _connection = GetConnection(); - return _connection; - } - } + private IRedisConnectionProvider _provider; - private IRedisConnection GetConnection() + public IRedisConnectionProvider Provider => _provider ??= GetProvider(); + + public IRedisConnection Connection => Provider.Connection; + + private IRedisConnectionProvider GetProvider() { var host = Environment.GetEnvironmentVariable("STANDALONE_HOST_PORT") ?? "localhost:6379"; var connectionString = $"redis://{host}"; - var provider = new RedisConnectionProvider(connectionString); - return provider.Connection; - } + return new RedisConnectionProvider(connectionString); + } public void Dispose() { From 30aca011a565090e809a1010deaf9889194927aa Mon Sep 17 00:00:00 2001 From: slorello89 Date: Mon, 30 Oct 2023 13:35:31 -0400 Subject: [PATCH 13/36] azure openai vectorizer. --- .../OpenAISentenceVectorizer.cs | 20 +++--- .../OpenAISentenceVectorizerAttribute.cs | 54 ++-------------- .../Redis.OM.Vectorizers.OpenAI.csproj | 2 +- .../RedisConnectionProviderExtensions.cs | 1 + Redis.OM.sln | 24 ++++++- .../AzureOpenAISentenceVectorizer.cs | 62 +++++++++++++++++++ .../AzureOpenAISentenceVectorizerAttribute.cs | 30 +++++++++ .../Redis.OM.Vectorizers.AzureOpenAI.csproj | 14 +++++ .../RedisConnectionProviderExtensions.cs | 20 ++++++ .../Configuration.cs | 4 +- .../Redis.OM.Vectorizers.Core.csproj | 1 + .../RedisOMHttpUtil.cs | 9 +++ .../HuggingFaceApiSentenceVectorizer.cs | 13 ++-- ...ggingFaceApiSentenceVectorizerAttribute.cs | 35 ++--------- .../Redis.OM.Vectorizers.HuggingFace.csproj | 2 +- .../VectorTests/OpenAIVectors.cs | 1 + .../VectorTests/SemanticCachingTests.cs | 25 ++++++++ .../Redis.OM.Unit.Tests.csproj | 1 + 18 files changed, 215 insertions(+), 103 deletions(-) create mode 100644 src/Redis.OM.Vectorizers.AzureOpenAI/AzureOpenAISentenceVectorizer.cs create mode 100644 src/Redis.OM.Vectorizers.AzureOpenAI/AzureOpenAISentenceVectorizerAttribute.cs create mode 100644 src/Redis.OM.Vectorizers.AzureOpenAI/Redis.OM.Vectorizers.AzureOpenAI.csproj create mode 100644 src/Redis.OM.Vectorizers.AzureOpenAI/RedisConnectionProviderExtensions.cs create mode 100644 src/Redis.OM.Vectorizers.Core/RedisOMHttpUtil.cs diff --git a/Redis.OM.Vectorizers.OpenAI/OpenAISentenceVectorizer.cs b/Redis.OM.Vectorizers.OpenAI/OpenAISentenceVectorizer.cs index f8cf2eb7..c5c0d233 100644 --- a/Redis.OM.Vectorizers.OpenAI/OpenAISentenceVectorizer.cs +++ b/Redis.OM.Vectorizers.OpenAI/OpenAISentenceVectorizer.cs @@ -3,7 +3,7 @@ using Redis.OM.Contracts; using Redis.OM.Modeling; -namespace Redis.OM; +namespace Redis.OM.Vectorizers.OpenAI; public class OpenAISentenceVectorizer : IVectorizer { @@ -21,35 +21,31 @@ public OpenAISentenceVectorizer(string openAIAuthToken, string model = "text-emb public int Dim { get; } public byte[] Vectorize(string str) { - var floats = GetFloats(str); + var floats = GetFloats(str, _model, _openAIAuthToken); return floats.SelectMany(BitConverter.GetBytes).ToArray(); } - internal float[] GetFloats(string s) + internal static float[] GetFloats(string s, string model, string openAIAuthToken) { var client = Configuration.Instance.Client; - var requestContent = JsonContent.Create( - new - { - input = s, - model = _model - }); + var requestContent = JsonContent.Create(new { input = s, model = model }); var request = new HttpRequestMessage { Method = HttpMethod.Post, RequestUri = new Uri($"{Configuration.Instance.OpenAiApiUrl}/v1/embeddings"), Content = requestContent, - Headers = { { "Authorization", $"Bearer {_openAIAuthToken}" } } + Headers = { { "Authorization", $"Bearer {openAIAuthToken}" } } }; - var res = client.SendAsync(request).Result; + var res = client.Send(request); if (!res.IsSuccessStatusCode) { throw new HttpRequestException( $"Open AI did not respond with a positive error code: {res.StatusCode}, {res.ReasonPhrase}"); } - var jsonObj = res.Content.ReadFromJsonAsync().Result; + + var jsonObj = JsonSerializer.Deserialize(RedisOMHttpUtil.ReadJsonSync(res)); if (!jsonObj.TryGetProperty("data", out var data)) diff --git a/Redis.OM.Vectorizers.OpenAI/OpenAISentenceVectorizerAttribute.cs b/Redis.OM.Vectorizers.OpenAI/OpenAISentenceVectorizerAttribute.cs index 1806e2e3..5d36d3f5 100644 --- a/Redis.OM.Vectorizers.OpenAI/OpenAISentenceVectorizerAttribute.cs +++ b/Redis.OM.Vectorizers.OpenAI/OpenAISentenceVectorizerAttribute.cs @@ -2,26 +2,15 @@ using System.Text.Json; using Redis.OM.Modeling; -namespace Redis.OM; +namespace Redis.OM.Vectorizers.OpenAI; public class OpenAISentenceVectorizerAttribute : VectorizerAttribute { private const string DefaultModel = "text-embedding-ada-002"; public string ModelId { get; set; } = DefaultModel; - public override VectorType VectorType { get; } + public override VectorType VectorType => VectorType.FLOAT32; - public override int Dim - { - get - { - if (ModelId == DefaultModel) - { - return 1536; - } - - return GetFloats("this is a test string").Length; - } - } + public override int Dim => ModelId == DefaultModel ? 1536 : GetFloats("Probing model dimensions").Length; public override byte[] Vectorize(object obj) { @@ -32,41 +21,6 @@ public override byte[] Vectorize(object obj) internal float[] GetFloats(string s) { - var client = Configuration.Instance.Client; - var requestContent = JsonContent.Create( - new - { - input = s, - model = ModelId - }); - - var request = new HttpRequestMessage - { - Method = HttpMethod.Post, - RequestUri = new Uri($"{Configuration.Instance.OpenAiApiUrl}/v1/embeddings"), - Content = requestContent, - Headers = { { "Authorization", $"Bearer {Configuration.Instance.OpenAiAuthorizationToken}" } } - }; - - var res = client.SendAsync(request).Result; - if (!res.IsSuccessStatusCode) - { - throw new HttpRequestException( - $"Open AI did not respond with a positive error code: {res.StatusCode}, {res.ReasonPhrase}"); - } - var jsonObj = res.Content.ReadFromJsonAsync().Result; - - - if (!jsonObj.TryGetProperty("data", out var data)) - { - throw new Exception("Malformed Response"); - } - - if (data.GetArrayLength() < 1 || !data[0].TryGetProperty("embedding", out var embedding)) - { - throw new Exception("Malformed Response"); - } - - return embedding.Deserialize()!; + return OpenAISentenceVectorizer.GetFloats(s, ModelId, Configuration.Instance.OpenAiAuthorizationToken); } } \ No newline at end of file diff --git a/Redis.OM.Vectorizers.OpenAI/Redis.OM.Vectorizers.OpenAI.csproj b/Redis.OM.Vectorizers.OpenAI/Redis.OM.Vectorizers.OpenAI.csproj index 9b77ddc7..f25305d1 100644 --- a/Redis.OM.Vectorizers.OpenAI/Redis.OM.Vectorizers.OpenAI.csproj +++ b/Redis.OM.Vectorizers.OpenAI/Redis.OM.Vectorizers.OpenAI.csproj @@ -4,7 +4,7 @@ net6.0 enable enable - Redis.OM.OpenAI + Redis.OM.Vectorizers.OpenAI diff --git a/Redis.OM.Vectorizers.OpenAI/RedisConnectionProviderExtensions.cs b/Redis.OM.Vectorizers.OpenAI/RedisConnectionProviderExtensions.cs index 0789642e..71e81531 100644 --- a/Redis.OM.Vectorizers.OpenAI/RedisConnectionProviderExtensions.cs +++ b/Redis.OM.Vectorizers.OpenAI/RedisConnectionProviderExtensions.cs @@ -1,4 +1,5 @@ using Redis.OM.Contracts; +using Redis.OM.Vectorizers.OpenAI; namespace Redis.OM; diff --git a/Redis.OM.sln b/Redis.OM.sln index ac06d3cb..5156672f 100644 --- a/Redis.OM.sln +++ b/Redis.OM.sln @@ -16,6 +16,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Redis.OM.Vectorizers.OpenAI EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Redis.OM.Vectorizers.Core", "src\Redis.OM.Vectorizers.Core\Redis.OM.Vectorizers.Core.csproj", "{4B9F4623-3126-48B7-B690-F28F702A4717}" EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Vectorizers", "Vectorizers", "{452DC80B-8195-44E8-A376-C246619492A8}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Redis.OM.Vectorizers.AzureOpenAI", "src\Redis.OM.Vectorizers.AzureOpenAI\Redis.OM.Vectorizers.AzureOpenAI.csproj", "{E64C4D96-9AA1-4C4A-895E-E7A232F29D0D}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -98,6 +102,18 @@ Global {4B9F4623-3126-48B7-B690-F28F702A4717}.Release|x64.Build.0 = Release|Any CPU {4B9F4623-3126-48B7-B690-F28F702A4717}.Release|x86.ActiveCfg = Release|Any CPU {4B9F4623-3126-48B7-B690-F28F702A4717}.Release|x86.Build.0 = Release|Any CPU + {E64C4D96-9AA1-4C4A-895E-E7A232F29D0D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {E64C4D96-9AA1-4C4A-895E-E7A232F29D0D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E64C4D96-9AA1-4C4A-895E-E7A232F29D0D}.Debug|x64.ActiveCfg = Debug|Any CPU + {E64C4D96-9AA1-4C4A-895E-E7A232F29D0D}.Debug|x64.Build.0 = Debug|Any CPU + {E64C4D96-9AA1-4C4A-895E-E7A232F29D0D}.Debug|x86.ActiveCfg = Debug|Any CPU + {E64C4D96-9AA1-4C4A-895E-E7A232F29D0D}.Debug|x86.Build.0 = Debug|Any CPU + {E64C4D96-9AA1-4C4A-895E-E7A232F29D0D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E64C4D96-9AA1-4C4A-895E-E7A232F29D0D}.Release|Any CPU.Build.0 = Release|Any CPU + {E64C4D96-9AA1-4C4A-895E-E7A232F29D0D}.Release|x64.ActiveCfg = Release|Any CPU + {E64C4D96-9AA1-4C4A-895E-E7A232F29D0D}.Release|x64.Build.0 = Release|Any CPU + {E64C4D96-9AA1-4C4A-895E-E7A232F29D0D}.Release|x86.ActiveCfg = Release|Any CPU + {E64C4D96-9AA1-4C4A-895E-E7A232F29D0D}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -105,9 +121,11 @@ Global GlobalSection(NestedProjects) = preSolution {7994382C-28EF-4F55-9B6D-810D35247816} = {8D9ECCFF-E022-4B68-BB43-228CEA248DEC} {E3A31119-E4F1-4793-B5C2-ED2D51502B01} = {8D9ECCFF-E022-4B68-BB43-228CEA248DEC} - {329F600A-3AF7-456A-8652-A79A39804EE5} = {8D9ECCFF-E022-4B68-BB43-228CEA248DEC} - {D9797D53-E8E2-4D60-9B07-2CA087E6CD19} = {8D9ECCFF-E022-4B68-BB43-228CEA248DEC} - {4B9F4623-3126-48B7-B690-F28F702A4717} = {8D9ECCFF-E022-4B68-BB43-228CEA248DEC} + {452DC80B-8195-44E8-A376-C246619492A8} = {8D9ECCFF-E022-4B68-BB43-228CEA248DEC} + {4B9F4623-3126-48B7-B690-F28F702A4717} = {452DC80B-8195-44E8-A376-C246619492A8} + {329F600A-3AF7-456A-8652-A79A39804EE5} = {452DC80B-8195-44E8-A376-C246619492A8} + {D9797D53-E8E2-4D60-9B07-2CA087E6CD19} = {452DC80B-8195-44E8-A376-C246619492A8} + {E64C4D96-9AA1-4C4A-895E-E7A232F29D0D} = {452DC80B-8195-44E8-A376-C246619492A8} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {E5752441-184B-4F17-BAD0-93823AC68607} diff --git a/src/Redis.OM.Vectorizers.AzureOpenAI/AzureOpenAISentenceVectorizer.cs b/src/Redis.OM.Vectorizers.AzureOpenAI/AzureOpenAISentenceVectorizer.cs new file mode 100644 index 00000000..22d10a00 --- /dev/null +++ b/src/Redis.OM.Vectorizers.AzureOpenAI/AzureOpenAISentenceVectorizer.cs @@ -0,0 +1,62 @@ + +using System.Net.Http.Json; +using System.Text.Json; +using Redis.OM.Contracts; +using Redis.OM.Modeling; + +namespace Redis.OM.Vectorizers.AzureOpenAI; + +public class AzureOpenAISentenceVectorizer : IVectorizer +{ + private readonly string _apiKey; + private readonly string _resourceName; + private readonly string _deploymentName; + + public AzureOpenAISentenceVectorizer(string apiKey, string resourceName, string deploymentName, int dim) + { + _apiKey = apiKey; + _resourceName = resourceName; + _deploymentName = deploymentName; + Dim = dim; + } + + public VectorType VectorType => VectorType.FLOAT32; + public int Dim { get; } + public byte[] Vectorize(string str) => GetFloats(str, _resourceName, _deploymentName, _apiKey).SelectMany(BitConverter.GetBytes).ToArray(); + + internal static float[] GetFloats(string s, string resourceName, string deploymentName, string apiKey) + { + var client = Configuration.Instance.Client; + var requestContent = JsonContent.Create(new { input = s }); + + var request = new HttpRequestMessage + { + Method = HttpMethod.Post, + RequestUri = new Uri( + $"https://{resourceName}.openai.azure.com/openai/deployments/{deploymentName}/embeddings?api-version=2023-05-15"), + Content = requestContent, + Headers = { { "api-key", apiKey } } + }; + + var res = client.SendAsync(request).Result; + if (!res.IsSuccessStatusCode) + { + throw new HttpRequestException( + $"Open AI did not respond with a positive error code: {res.StatusCode}, {res.ReasonPhrase}"); + } + var jsonObj = res.Content.ReadFromJsonAsync().Result; + + + if (!jsonObj.TryGetProperty("data", out var data)) + { + throw new Exception("Malformed Response"); + } + + if (data.GetArrayLength() < 1 || !data[0].TryGetProperty("embedding", out var embedding)) + { + throw new Exception("Malformed Response"); + } + + return embedding.Deserialize()!; + } +} \ No newline at end of file diff --git a/src/Redis.OM.Vectorizers.AzureOpenAI/AzureOpenAISentenceVectorizerAttribute.cs b/src/Redis.OM.Vectorizers.AzureOpenAI/AzureOpenAISentenceVectorizerAttribute.cs new file mode 100644 index 00000000..9b1e06a7 --- /dev/null +++ b/src/Redis.OM.Vectorizers.AzureOpenAI/AzureOpenAISentenceVectorizerAttribute.cs @@ -0,0 +1,30 @@ +using System.Net.Http.Json; +using System.Text.Json; +using Redis.OM.Modeling; + +namespace Redis.OM.Vectorizers.AzureOpenAI; + +public class AzureOpenAISentenceVectorizerAttribute : VectorizerAttribute +{ + public AzureOpenAISentenceVectorizerAttribute(string deploymentName, string resourceName, int dim) + { + DeploymentName = deploymentName; + ResourceName = resourceName; + Dim = dim; + } + + public string DeploymentName { get; } + public string ResourceName { get; } + public override VectorType VectorType => VectorType.FLOAT32; + public override int Dim { get; } + public override byte[] Vectorize(object obj) + { + if (obj is not string s) + { + throw new ArgumentException("Object must be a string to be embedded", nameof(obj)); + } + + var floats = AzureOpenAISentenceVectorizer.GetFloats(s, ResourceName, DeploymentName, Configuration.Instance.AzureOpenAIApiKey); + return floats.SelectMany(BitConverter.GetBytes).ToArray(); + } +} \ No newline at end of file diff --git a/src/Redis.OM.Vectorizers.AzureOpenAI/Redis.OM.Vectorizers.AzureOpenAI.csproj b/src/Redis.OM.Vectorizers.AzureOpenAI/Redis.OM.Vectorizers.AzureOpenAI.csproj new file mode 100644 index 00000000..dbe9c879 --- /dev/null +++ b/src/Redis.OM.Vectorizers.AzureOpenAI/Redis.OM.Vectorizers.AzureOpenAI.csproj @@ -0,0 +1,14 @@ + + + + net6.0 + enable + enable + + + + + + + + diff --git a/src/Redis.OM.Vectorizers.AzureOpenAI/RedisConnectionProviderExtensions.cs b/src/Redis.OM.Vectorizers.AzureOpenAI/RedisConnectionProviderExtensions.cs new file mode 100644 index 00000000..7f5b7a8c --- /dev/null +++ b/src/Redis.OM.Vectorizers.AzureOpenAI/RedisConnectionProviderExtensions.cs @@ -0,0 +1,20 @@ +using Redis.OM.Contracts; + +namespace Redis.OM.Vectorizers.AzureOpenAI; + +public static class RedisConnectionProviderExtensions +{ + public static ISemanticCache AzureOpenAISemanticCache(this IRedisConnectionProvider provider, string apiKey, string resourceName, string deploymentId, int dim, double threshold = .15, string indexName = "AzureOpenAISemanticCache", string? prefix = null, long? ttl = null) + { + var vectorizer = new AzureOpenAISentenceVectorizer(apiKey, resourceName, deploymentId, dim); + var connection = provider.Connection; + var cache = new SemanticCache(indexName, prefix ?? indexName, threshold, ttl, vectorizer, connection); + var info = connection.GetIndexInfo(indexName); + if (info is null) + { + cache.CreateIndex(); + } + + return cache; + } +} \ No newline at end of file diff --git a/src/Redis.OM.Vectorizers.Core/Configuration.cs b/src/Redis.OM.Vectorizers.Core/Configuration.cs index e29a7e8d..69c574bc 100644 --- a/src/Redis.OM.Vectorizers.Core/Configuration.cs +++ b/src/Redis.OM.Vectorizers.Core/Configuration.cs @@ -8,6 +8,7 @@ public class Configuration public string? this[string str] => _settings[str]; public string HuggingFaceAuthorizationToken => _settings["REDIS_OM_HF_TOKEN"] ?? string.Empty; public string OpenAiAuthorizationToken => _settings["REDIS_OM_OAI_TOKEN"] ?? string.Empty; + public string AzureOpenAIApiKey => _settings["REDIS_OM_AZURE_OAI_TOKEN"] ?? string.Empty; public string ModelId => _settings["REDIS_OM_HF_MODEL_ID"] ?? string.Empty; public string HuggingFaceBaseAddress => _settings["REDIS_OM_HF_FEATURE_EXTRACTION_URL"] ?? string.Empty; @@ -45,7 +46,8 @@ internal Configuration() {"REDIS_OM_HF_FEATURE_EXTRACTION_URL", DefaultHuggingFaceApiUrl}, {"REDIS_OM_OAI_API_URL", DefaultOpenAiApiUrl}, {"REDIS_OM_HF_TOKEN", Environment.GetEnvironmentVariable("REDIS_OM_HF_TOKEN")}, - {"REDIS_OM_OAI_TOKEN", Environment.GetEnvironmentVariable("REDIS_OM_OAI_TOKEN")} + {"REDIS_OM_OAI_TOKEN", Environment.GetEnvironmentVariable("REDIS_OM_OAI_TOKEN")}, + {"REDIS_OM_AZURE_OAI_TOKEN", Environment.GetEnvironmentVariable("REDIS_OM_AZURE_OAI_TOKEN")} }) .AddJsonFile("settings.json", true, true) .AddJsonFile("appsettings.json", true, true); diff --git a/src/Redis.OM.Vectorizers.Core/Redis.OM.Vectorizers.Core.csproj b/src/Redis.OM.Vectorizers.Core/Redis.OM.Vectorizers.Core.csproj index b7cacc4f..326bfb03 100644 --- a/src/Redis.OM.Vectorizers.Core/Redis.OM.Vectorizers.Core.csproj +++ b/src/Redis.OM.Vectorizers.Core/Redis.OM.Vectorizers.Core.csproj @@ -4,6 +4,7 @@ net6.0 enable enable + Redis.OM diff --git a/src/Redis.OM.Vectorizers.Core/RedisOMHttpUtil.cs b/src/Redis.OM.Vectorizers.Core/RedisOMHttpUtil.cs new file mode 100644 index 00000000..f2396516 --- /dev/null +++ b/src/Redis.OM.Vectorizers.Core/RedisOMHttpUtil.cs @@ -0,0 +1,9 @@ +namespace Redis.OM; + +public class RedisOMHttpUtil +{ + public static string ReadJsonSync(HttpResponseMessage msg) + { + return new StreamReader(msg.Content.ReadAsStream()).ReadToEnd(); + } +} \ No newline at end of file diff --git a/src/Redis.OM.Vectorizers.HuggingFace/HuggingFaceApiSentenceVectorizer.cs b/src/Redis.OM.Vectorizers.HuggingFace/HuggingFaceApiSentenceVectorizer.cs index 92343c3d..982f61f3 100644 --- a/src/Redis.OM.Vectorizers.HuggingFace/HuggingFaceApiSentenceVectorizer.cs +++ b/src/Redis.OM.Vectorizers.HuggingFace/HuggingFaceApiSentenceVectorizer.cs @@ -1,4 +1,5 @@ using System.Net.Http.Json; +using System.Text.Json; using Redis.OM.Contracts; using Redis.OM.Modeling; @@ -20,10 +21,10 @@ public HuggingFaceApiSentenceVectorizer(string authToken, string modelId, int di public int Dim { get; } public byte[] Vectorize(string str) { - return GetFloats(str).SelectMany(BitConverter.GetBytes).ToArray(); + return GetFloats(str, ModelId, _huggingFaceAuthToken).SelectMany(BitConverter.GetBytes).ToArray(); } - public float[] GetFloats(string s) + public static float[] GetFloats(string s, string modelId, string huggingFaceAuthToken) { var client = Configuration.Instance.Client; var requestContent = JsonContent.Create(new @@ -37,15 +38,15 @@ public float[] GetFloats(string s) Method = HttpMethod.Post, Content = requestContent, RequestUri = - new Uri($"{Configuration.Instance.HuggingFaceBaseAddress}/pipeline/feature-extraction/{ModelId}"), + new Uri($"{Configuration.Instance.HuggingFaceBaseAddress}/pipeline/feature-extraction/{modelId}"), Headers = { - { "Authorization", $"Bearer {_huggingFaceAuthToken}" } + { "Authorization", $"Bearer {huggingFaceAuthToken}" } } }; - var res = client.SendAsync(request).Result; - var floats = res.Content.ReadFromJsonAsync().Result; + var res = client.Send(request); + var floats = JsonSerializer.Deserialize(RedisOMHttpUtil.ReadJsonSync(res)); if (floats is null) { throw new Exception("Did not receive a response back from HuggingFace"); diff --git a/src/Redis.OM.Vectorizers.HuggingFace/HuggingFaceApiSentenceVectorizerAttribute.cs b/src/Redis.OM.Vectorizers.HuggingFace/HuggingFaceApiSentenceVectorizerAttribute.cs index 0fc589ab..1ddaa3e7 100644 --- a/src/Redis.OM.Vectorizers.HuggingFace/HuggingFaceApiSentenceVectorizerAttribute.cs +++ b/src/Redis.OM.Vectorizers.HuggingFace/HuggingFaceApiSentenceVectorizerAttribute.cs @@ -24,42 +24,19 @@ public override int Dim public override byte[] Vectorize(object obj) { - var s = (string)obj; + if (obj is not string s) + { + throw new ArgumentException("Object must be a string", nameof(obj)); + } + var floats = GetFloats(s); return floats.SelectMany(BitConverter.GetBytes).ToArray(); } public float[] GetFloats(string s) { - var client = Configuration.Instance.Client; var modelId = ModelId ?? Configuration.Instance["REDIS_OM_HF_MODEL_ID"]; if (modelId is null) throw new InvalidOperationException("Model Id Required to use Hugging Face API."); - - var requestContent = JsonContent.Create(new - { - inputs = new string[] { s }, - options = new { wait_for_model = true } - }); - - var request = new HttpRequestMessage() - { - Method = HttpMethod.Post, - Content = requestContent, - RequestUri = - new Uri($"{Configuration.Instance.HuggingFaceBaseAddress}/pipeline/feature-extraction/{modelId}"), - Headers = - { - { "Authorization", $"Bearer {Configuration.Instance.HuggingFaceAuthorizationToken}" } - } - }; - - var res = client.SendAsync(request).Result; - var floats = res.Content.ReadFromJsonAsync().Result; - if (floats is null) - { - throw new Exception("Did not receive a response back from HuggingFace"); - } - - return floats.First(); + return HuggingFaceApiSentenceVectorizer.GetFloats(s, modelId, Configuration.Instance.HuggingFaceAuthorizationToken); } } \ No newline at end of file diff --git a/src/Redis.OM.Vectorizers.HuggingFace/Redis.OM.Vectorizers.HuggingFace.csproj b/src/Redis.OM.Vectorizers.HuggingFace/Redis.OM.Vectorizers.HuggingFace.csproj index f1aea53d..6be4f8ad 100644 --- a/src/Redis.OM.Vectorizers.HuggingFace/Redis.OM.Vectorizers.HuggingFace.csproj +++ b/src/Redis.OM.Vectorizers.HuggingFace/Redis.OM.Vectorizers.HuggingFace.csproj @@ -4,7 +4,7 @@ net6.0 enable enable - Redis.OM.HuggingFace + Redis.OM diff --git a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/OpenAIVectors.cs b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/OpenAIVectors.cs index 5b4c3998..b6cf79ed 100644 --- a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/OpenAIVectors.cs +++ b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/OpenAIVectors.cs @@ -1,5 +1,6 @@ using Redis.OM.Modeling; using Redis.OM.Modeling.Vectors; +using Redis.OM.Vectorizers.OpenAI; namespace Redis.OM.Unit.Tests; diff --git a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/SemanticCachingTests.cs b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/SemanticCachingTests.cs index f9dc8fa6..afcca84a 100644 --- a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/SemanticCachingTests.cs +++ b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/SemanticCachingTests.cs @@ -1,6 +1,7 @@ using System; using System.Linq; using Redis.OM.Contracts; +using Redis.OM.Vectorizers.AzureOpenAI; using Xunit; namespace Redis.OM.Unit.Tests; @@ -37,4 +38,28 @@ public void HuggingFaceSemanticCache() Assert.Equal("Paris",res.Response); Assert.True(res.Score < .15); } + + [Fact] + public void AzureOpenAISemanticCache() + { + var token = Environment.GetEnvironmentVariable("REDIS_OM_AZURE_OAI_TOKEN"); + var resource = Environment.GetEnvironmentVariable("REDIS_OM_AZURE_OAI_RESOURCE"); + var deployment = Environment.GetEnvironmentVariable("REDIS_OM_AZURE_OAI_DEPLOYMENT"); + var dimStr = Environment.GetEnvironmentVariable("REDIS_OM_AZURE_OAI_DIM"); + if (string.IsNullOrEmpty(dimStr) || !int.TryParse(dimStr, out var dim)) + { + throw new InvalidOperationException("REDIS_OM_AZURE_OAI_DIM must contain a valid integrer value."); + } + + + Assert.NotNull(token); + Assert.NotNull(resource); + Assert.NotNull(deployment); + var cache = _provider.AzureOpenAISemanticCache(token, resource, deployment, dim); + cache.Store("What is the capital of France?", "Paris"); + var res = cache.GetSimilar("What really is the capital of France?").First(); + Assert.Equal("Paris",res.Response); + Assert.True(res.Score < .15); + + } } \ No newline at end of file diff --git a/test/Redis.OM.Unit.Tests/Redis.OM.Unit.Tests.csproj b/test/Redis.OM.Unit.Tests/Redis.OM.Unit.Tests.csproj index 18ad7c13..1d9b5830 100644 --- a/test/Redis.OM.Unit.Tests/Redis.OM.Unit.Tests.csproj +++ b/test/Redis.OM.Unit.Tests/Redis.OM.Unit.Tests.csproj @@ -15,6 +15,7 @@ + From 06b888b1519670980e00683a52666650e6165c6e Mon Sep 17 00:00:00 2001 From: slorello89 Date: Mon, 30 Oct 2023 15:22:24 -0400 Subject: [PATCH 14/36] reorganizing vectorizers --- .../Redis.OM.Vectorizers.OpenAI.csproj | 15 ------ .../RedisConnectionProviderExtensions.cs | 21 -------- Redis.OM.sln | 47 +----------------- .../Redis.OM.Vectorizers.AzureOpenAI.csproj | 14 ------ .../RedisConnectionProviderExtensions.cs | 20 -------- .../Redis.OM.Vectorizers.Core.csproj | 16 ------- .../RedisConnectionProviderExtensions.cs | 21 -------- .../AzureOpenAISentenceVectorizer.cs | 2 +- .../AzureOpenAISentenceVectorizerAttribute.cs | 2 +- .../Configuration.cs | 0 .../HuggingFaceApiSentenceVectorizer.cs | 2 +- ...ggingFaceApiSentenceVectorizerAttribute.cs | 2 +- .../OpenAISentenceVectorizer.cs | 2 +- .../OpenAISentenceVectorizerAttribute.cs | 2 +- .../Redis.OM.Vectorizers.csproj} | 10 ++-- .../RedisConnectionProviderExtensions.cs | 48 +++++++++++++++++++ .../RedisOMHttpUtil.cs | 2 +- src/Redis.OM/Redis.OM.csproj | 6 +-- test/Redis.OM.Unit.Tests/Address.cs | 1 - .../Redis.OM.Unit.Tests/ConfigurationTests.cs | 1 - test/Redis.OM.Unit.Tests/CoreTests.cs | 1 - .../RediSearchTests/Person.cs | 1 - .../VectorTests/HuggingFaceVectors.cs | 2 +- .../VectorTests/OpenAIVectors.cs | 2 +- .../VectorTests/SemanticCachingTests.cs | 2 +- .../Redis.OM.Unit.Tests.csproj | 4 +- .../SearchJsonTests/RedisJsonIndexTests.cs | 1 - 27 files changed, 67 insertions(+), 180 deletions(-) delete mode 100644 Redis.OM.Vectorizers.OpenAI/Redis.OM.Vectorizers.OpenAI.csproj delete mode 100644 Redis.OM.Vectorizers.OpenAI/RedisConnectionProviderExtensions.cs delete mode 100644 src/Redis.OM.Vectorizers.AzureOpenAI/Redis.OM.Vectorizers.AzureOpenAI.csproj delete mode 100644 src/Redis.OM.Vectorizers.AzureOpenAI/RedisConnectionProviderExtensions.cs delete mode 100644 src/Redis.OM.Vectorizers.Core/Redis.OM.Vectorizers.Core.csproj delete mode 100644 src/Redis.OM.Vectorizers.HuggingFace/RedisConnectionProviderExtensions.cs rename src/{Redis.OM.Vectorizers.AzureOpenAI => Redis.OM.Vectorizers}/AzureOpenAISentenceVectorizer.cs (97%) rename src/{Redis.OM.Vectorizers.AzureOpenAI => Redis.OM.Vectorizers}/AzureOpenAISentenceVectorizerAttribute.cs (95%) rename src/{Redis.OM.Vectorizers.Core => Redis.OM.Vectorizers}/Configuration.cs (100%) rename src/{Redis.OM.Vectorizers.HuggingFace => Redis.OM.Vectorizers}/HuggingFaceApiSentenceVectorizer.cs (97%) rename src/{Redis.OM.Vectorizers.HuggingFace => Redis.OM.Vectorizers}/HuggingFaceApiSentenceVectorizerAttribute.cs (96%) rename {Redis.OM.Vectorizers.OpenAI => src/Redis.OM.Vectorizers}/OpenAISentenceVectorizer.cs (98%) rename {Redis.OM.Vectorizers.OpenAI => src/Redis.OM.Vectorizers}/OpenAISentenceVectorizerAttribute.cs (95%) rename src/{Redis.OM.Vectorizers.HuggingFace/Redis.OM.Vectorizers.HuggingFace.csproj => Redis.OM.Vectorizers/Redis.OM.Vectorizers.csproj} (83%) create mode 100644 src/Redis.OM.Vectorizers/RedisConnectionProviderExtensions.cs rename src/{Redis.OM.Vectorizers.Core => Redis.OM.Vectorizers}/RedisOMHttpUtil.cs (84%) diff --git a/Redis.OM.Vectorizers.OpenAI/Redis.OM.Vectorizers.OpenAI.csproj b/Redis.OM.Vectorizers.OpenAI/Redis.OM.Vectorizers.OpenAI.csproj deleted file mode 100644 index f25305d1..00000000 --- a/Redis.OM.Vectorizers.OpenAI/Redis.OM.Vectorizers.OpenAI.csproj +++ /dev/null @@ -1,15 +0,0 @@ - - - - net6.0 - enable - enable - Redis.OM.Vectorizers.OpenAI - - - - - - - - diff --git a/Redis.OM.Vectorizers.OpenAI/RedisConnectionProviderExtensions.cs b/Redis.OM.Vectorizers.OpenAI/RedisConnectionProviderExtensions.cs deleted file mode 100644 index 71e81531..00000000 --- a/Redis.OM.Vectorizers.OpenAI/RedisConnectionProviderExtensions.cs +++ /dev/null @@ -1,21 +0,0 @@ -using Redis.OM.Contracts; -using Redis.OM.Vectorizers.OpenAI; - -namespace Redis.OM; - -public static class RedisConnectionProviderExtensions -{ - public static ISemanticCache OpenAISemanticCache(this IRedisConnectionProvider provider, string openAIAuthToken, double threshold = .15, string indexName = "OpenAISemanticCache", string? prefix = null, long? ttl = null) - { - var vectorizer = new OpenAISentenceVectorizer(openAIAuthToken); - var connection = provider.Connection; - var info = connection.GetIndexInfo(indexName); - var cache = new SemanticCache(indexName, prefix ?? indexName, threshold, ttl, vectorizer, connection); - if (info is null) - { - cache.CreateIndex(); - } - - return cache; - } -} \ No newline at end of file diff --git a/Redis.OM.sln b/Redis.OM.sln index 5156672f..8fa8067b 100644 --- a/Redis.OM.sln +++ b/Redis.OM.sln @@ -10,16 +10,10 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Redis.OM.POC", "src\Redis.O EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Redis.OM.Unit.Tests", "test\Redis.OM.Unit.Tests\Redis.OM.Unit.Tests.csproj", "{570BF479-BCF4-4D1B-A702-2234CA0A3E7D}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Redis.OM.Vectorizers.HuggingFace", "src\Redis.OM.Vectorizers.HuggingFace\Redis.OM.Vectorizers.HuggingFace.csproj", "{329F600A-3AF7-456A-8652-A79A39804EE5}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Redis.OM.Vectorizers.OpenAI", "Redis.OM.Vectorizers.OpenAI\Redis.OM.Vectorizers.OpenAI.csproj", "{D9797D53-E8E2-4D60-9B07-2CA087E6CD19}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Redis.OM.Vectorizers.Core", "src\Redis.OM.Vectorizers.Core\Redis.OM.Vectorizers.Core.csproj", "{4B9F4623-3126-48B7-B690-F28F702A4717}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Redis.OM.Vectorizers", "src\Redis.OM.Vectorizers\Redis.OM.Vectorizers.csproj", "{4B9F4623-3126-48B7-B690-F28F702A4717}" EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Vectorizers", "Vectorizers", "{452DC80B-8195-44E8-A376-C246619492A8}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Redis.OM.Vectorizers.AzureOpenAI", "src\Redis.OM.Vectorizers.AzureOpenAI\Redis.OM.Vectorizers.AzureOpenAI.csproj", "{E64C4D96-9AA1-4C4A-895E-E7A232F29D0D}" -EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -66,30 +60,6 @@ Global {570BF479-BCF4-4D1B-A702-2234CA0A3E7D}.Release|x64.Build.0 = Release|Any CPU {570BF479-BCF4-4D1B-A702-2234CA0A3E7D}.Release|x86.ActiveCfg = Release|Any CPU {570BF479-BCF4-4D1B-A702-2234CA0A3E7D}.Release|x86.Build.0 = Release|Any CPU - {329F600A-3AF7-456A-8652-A79A39804EE5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {329F600A-3AF7-456A-8652-A79A39804EE5}.Debug|Any CPU.Build.0 = Debug|Any CPU - {329F600A-3AF7-456A-8652-A79A39804EE5}.Debug|x64.ActiveCfg = Debug|Any CPU - {329F600A-3AF7-456A-8652-A79A39804EE5}.Debug|x64.Build.0 = Debug|Any CPU - {329F600A-3AF7-456A-8652-A79A39804EE5}.Debug|x86.ActiveCfg = Debug|Any CPU - {329F600A-3AF7-456A-8652-A79A39804EE5}.Debug|x86.Build.0 = Debug|Any CPU - {329F600A-3AF7-456A-8652-A79A39804EE5}.Release|Any CPU.ActiveCfg = Release|Any CPU - {329F600A-3AF7-456A-8652-A79A39804EE5}.Release|Any CPU.Build.0 = Release|Any CPU - {329F600A-3AF7-456A-8652-A79A39804EE5}.Release|x64.ActiveCfg = Release|Any CPU - {329F600A-3AF7-456A-8652-A79A39804EE5}.Release|x64.Build.0 = Release|Any CPU - {329F600A-3AF7-456A-8652-A79A39804EE5}.Release|x86.ActiveCfg = Release|Any CPU - {329F600A-3AF7-456A-8652-A79A39804EE5}.Release|x86.Build.0 = Release|Any CPU - {D9797D53-E8E2-4D60-9B07-2CA087E6CD19}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {D9797D53-E8E2-4D60-9B07-2CA087E6CD19}.Debug|Any CPU.Build.0 = Debug|Any CPU - {D9797D53-E8E2-4D60-9B07-2CA087E6CD19}.Debug|x64.ActiveCfg = Debug|Any CPU - {D9797D53-E8E2-4D60-9B07-2CA087E6CD19}.Debug|x64.Build.0 = Debug|Any CPU - {D9797D53-E8E2-4D60-9B07-2CA087E6CD19}.Debug|x86.ActiveCfg = Debug|Any CPU - {D9797D53-E8E2-4D60-9B07-2CA087E6CD19}.Debug|x86.Build.0 = Debug|Any CPU - {D9797D53-E8E2-4D60-9B07-2CA087E6CD19}.Release|Any CPU.ActiveCfg = Release|Any CPU - {D9797D53-E8E2-4D60-9B07-2CA087E6CD19}.Release|Any CPU.Build.0 = Release|Any CPU - {D9797D53-E8E2-4D60-9B07-2CA087E6CD19}.Release|x64.ActiveCfg = Release|Any CPU - {D9797D53-E8E2-4D60-9B07-2CA087E6CD19}.Release|x64.Build.0 = Release|Any CPU - {D9797D53-E8E2-4D60-9B07-2CA087E6CD19}.Release|x86.ActiveCfg = Release|Any CPU - {D9797D53-E8E2-4D60-9B07-2CA087E6CD19}.Release|x86.Build.0 = Release|Any CPU {4B9F4623-3126-48B7-B690-F28F702A4717}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {4B9F4623-3126-48B7-B690-F28F702A4717}.Debug|Any CPU.Build.0 = Debug|Any CPU {4B9F4623-3126-48B7-B690-F28F702A4717}.Debug|x64.ActiveCfg = Debug|Any CPU @@ -102,18 +72,6 @@ Global {4B9F4623-3126-48B7-B690-F28F702A4717}.Release|x64.Build.0 = Release|Any CPU {4B9F4623-3126-48B7-B690-F28F702A4717}.Release|x86.ActiveCfg = Release|Any CPU {4B9F4623-3126-48B7-B690-F28F702A4717}.Release|x86.Build.0 = Release|Any CPU - {E64C4D96-9AA1-4C4A-895E-E7A232F29D0D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {E64C4D96-9AA1-4C4A-895E-E7A232F29D0D}.Debug|Any CPU.Build.0 = Debug|Any CPU - {E64C4D96-9AA1-4C4A-895E-E7A232F29D0D}.Debug|x64.ActiveCfg = Debug|Any CPU - {E64C4D96-9AA1-4C4A-895E-E7A232F29D0D}.Debug|x64.Build.0 = Debug|Any CPU - {E64C4D96-9AA1-4C4A-895E-E7A232F29D0D}.Debug|x86.ActiveCfg = Debug|Any CPU - {E64C4D96-9AA1-4C4A-895E-E7A232F29D0D}.Debug|x86.Build.0 = Debug|Any CPU - {E64C4D96-9AA1-4C4A-895E-E7A232F29D0D}.Release|Any CPU.ActiveCfg = Release|Any CPU - {E64C4D96-9AA1-4C4A-895E-E7A232F29D0D}.Release|Any CPU.Build.0 = Release|Any CPU - {E64C4D96-9AA1-4C4A-895E-E7A232F29D0D}.Release|x64.ActiveCfg = Release|Any CPU - {E64C4D96-9AA1-4C4A-895E-E7A232F29D0D}.Release|x64.Build.0 = Release|Any CPU - {E64C4D96-9AA1-4C4A-895E-E7A232F29D0D}.Release|x86.ActiveCfg = Release|Any CPU - {E64C4D96-9AA1-4C4A-895E-E7A232F29D0D}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -123,9 +81,6 @@ Global {E3A31119-E4F1-4793-B5C2-ED2D51502B01} = {8D9ECCFF-E022-4B68-BB43-228CEA248DEC} {452DC80B-8195-44E8-A376-C246619492A8} = {8D9ECCFF-E022-4B68-BB43-228CEA248DEC} {4B9F4623-3126-48B7-B690-F28F702A4717} = {452DC80B-8195-44E8-A376-C246619492A8} - {329F600A-3AF7-456A-8652-A79A39804EE5} = {452DC80B-8195-44E8-A376-C246619492A8} - {D9797D53-E8E2-4D60-9B07-2CA087E6CD19} = {452DC80B-8195-44E8-A376-C246619492A8} - {E64C4D96-9AA1-4C4A-895E-E7A232F29D0D} = {452DC80B-8195-44E8-A376-C246619492A8} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {E5752441-184B-4F17-BAD0-93823AC68607} diff --git a/src/Redis.OM.Vectorizers.AzureOpenAI/Redis.OM.Vectorizers.AzureOpenAI.csproj b/src/Redis.OM.Vectorizers.AzureOpenAI/Redis.OM.Vectorizers.AzureOpenAI.csproj deleted file mode 100644 index dbe9c879..00000000 --- a/src/Redis.OM.Vectorizers.AzureOpenAI/Redis.OM.Vectorizers.AzureOpenAI.csproj +++ /dev/null @@ -1,14 +0,0 @@ - - - - net6.0 - enable - enable - - - - - - - - diff --git a/src/Redis.OM.Vectorizers.AzureOpenAI/RedisConnectionProviderExtensions.cs b/src/Redis.OM.Vectorizers.AzureOpenAI/RedisConnectionProviderExtensions.cs deleted file mode 100644 index 7f5b7a8c..00000000 --- a/src/Redis.OM.Vectorizers.AzureOpenAI/RedisConnectionProviderExtensions.cs +++ /dev/null @@ -1,20 +0,0 @@ -using Redis.OM.Contracts; - -namespace Redis.OM.Vectorizers.AzureOpenAI; - -public static class RedisConnectionProviderExtensions -{ - public static ISemanticCache AzureOpenAISemanticCache(this IRedisConnectionProvider provider, string apiKey, string resourceName, string deploymentId, int dim, double threshold = .15, string indexName = "AzureOpenAISemanticCache", string? prefix = null, long? ttl = null) - { - var vectorizer = new AzureOpenAISentenceVectorizer(apiKey, resourceName, deploymentId, dim); - var connection = provider.Connection; - var cache = new SemanticCache(indexName, prefix ?? indexName, threshold, ttl, vectorizer, connection); - var info = connection.GetIndexInfo(indexName); - if (info is null) - { - cache.CreateIndex(); - } - - return cache; - } -} \ No newline at end of file diff --git a/src/Redis.OM.Vectorizers.Core/Redis.OM.Vectorizers.Core.csproj b/src/Redis.OM.Vectorizers.Core/Redis.OM.Vectorizers.Core.csproj deleted file mode 100644 index 326bfb03..00000000 --- a/src/Redis.OM.Vectorizers.Core/Redis.OM.Vectorizers.Core.csproj +++ /dev/null @@ -1,16 +0,0 @@ - - - - net6.0 - enable - enable - Redis.OM - - - - - - - - - diff --git a/src/Redis.OM.Vectorizers.HuggingFace/RedisConnectionProviderExtensions.cs b/src/Redis.OM.Vectorizers.HuggingFace/RedisConnectionProviderExtensions.cs deleted file mode 100644 index 39e9d624..00000000 --- a/src/Redis.OM.Vectorizers.HuggingFace/RedisConnectionProviderExtensions.cs +++ /dev/null @@ -1,21 +0,0 @@ -using Redis.OM.Contracts; -using Redis.OM.Vectorizers.HuggingFace; - -namespace Redis.OM; - -public static class RedisConnectionProviderExtensions -{ - public static ISemanticCache HuggingFaceSemanticCache(this IRedisConnectionProvider provider, string huggingFaceAuthToken, double threshold = .15, string modelId = "sentence-transformers/all-mpnet-base-v2", int dim = 768, string indexName = "HuggingFaceSemanticCache", string? prefix = null, long? ttl = null) - { - var vectorizer = new HuggingFaceApiSentenceVectorizer(huggingFaceAuthToken, modelId, dim); - var connection = provider.Connection; - var info = connection.GetIndexInfo(indexName); - var cache = new SemanticCache(indexName, prefix ?? indexName, threshold, ttl, vectorizer, connection); - if (info is null) - { - cache.CreateIndex(); - } - - return cache; - } -} \ No newline at end of file diff --git a/src/Redis.OM.Vectorizers.AzureOpenAI/AzureOpenAISentenceVectorizer.cs b/src/Redis.OM.Vectorizers/AzureOpenAISentenceVectorizer.cs similarity index 97% rename from src/Redis.OM.Vectorizers.AzureOpenAI/AzureOpenAISentenceVectorizer.cs rename to src/Redis.OM.Vectorizers/AzureOpenAISentenceVectorizer.cs index 22d10a00..06359d3c 100644 --- a/src/Redis.OM.Vectorizers.AzureOpenAI/AzureOpenAISentenceVectorizer.cs +++ b/src/Redis.OM.Vectorizers/AzureOpenAISentenceVectorizer.cs @@ -4,7 +4,7 @@ using Redis.OM.Contracts; using Redis.OM.Modeling; -namespace Redis.OM.Vectorizers.AzureOpenAI; +namespace Redis.OM.Vectorizers; public class AzureOpenAISentenceVectorizer : IVectorizer { diff --git a/src/Redis.OM.Vectorizers.AzureOpenAI/AzureOpenAISentenceVectorizerAttribute.cs b/src/Redis.OM.Vectorizers/AzureOpenAISentenceVectorizerAttribute.cs similarity index 95% rename from src/Redis.OM.Vectorizers.AzureOpenAI/AzureOpenAISentenceVectorizerAttribute.cs rename to src/Redis.OM.Vectorizers/AzureOpenAISentenceVectorizerAttribute.cs index 9b1e06a7..a1b0dcbe 100644 --- a/src/Redis.OM.Vectorizers.AzureOpenAI/AzureOpenAISentenceVectorizerAttribute.cs +++ b/src/Redis.OM.Vectorizers/AzureOpenAISentenceVectorizerAttribute.cs @@ -2,7 +2,7 @@ using System.Text.Json; using Redis.OM.Modeling; -namespace Redis.OM.Vectorizers.AzureOpenAI; +namespace Redis.OM.Vectorizers; public class AzureOpenAISentenceVectorizerAttribute : VectorizerAttribute { diff --git a/src/Redis.OM.Vectorizers.Core/Configuration.cs b/src/Redis.OM.Vectorizers/Configuration.cs similarity index 100% rename from src/Redis.OM.Vectorizers.Core/Configuration.cs rename to src/Redis.OM.Vectorizers/Configuration.cs diff --git a/src/Redis.OM.Vectorizers.HuggingFace/HuggingFaceApiSentenceVectorizer.cs b/src/Redis.OM.Vectorizers/HuggingFaceApiSentenceVectorizer.cs similarity index 97% rename from src/Redis.OM.Vectorizers.HuggingFace/HuggingFaceApiSentenceVectorizer.cs rename to src/Redis.OM.Vectorizers/HuggingFaceApiSentenceVectorizer.cs index 982f61f3..ada56754 100644 --- a/src/Redis.OM.Vectorizers.HuggingFace/HuggingFaceApiSentenceVectorizer.cs +++ b/src/Redis.OM.Vectorizers/HuggingFaceApiSentenceVectorizer.cs @@ -3,7 +3,7 @@ using Redis.OM.Contracts; using Redis.OM.Modeling; -namespace Redis.OM.Vectorizers.HuggingFace; +namespace Redis.OM.Vectorizers; public class HuggingFaceApiSentenceVectorizer : IVectorizer { diff --git a/src/Redis.OM.Vectorizers.HuggingFace/HuggingFaceApiSentenceVectorizerAttribute.cs b/src/Redis.OM.Vectorizers/HuggingFaceApiSentenceVectorizerAttribute.cs similarity index 96% rename from src/Redis.OM.Vectorizers.HuggingFace/HuggingFaceApiSentenceVectorizerAttribute.cs rename to src/Redis.OM.Vectorizers/HuggingFaceApiSentenceVectorizerAttribute.cs index 1ddaa3e7..3ac689b5 100644 --- a/src/Redis.OM.Vectorizers.HuggingFace/HuggingFaceApiSentenceVectorizerAttribute.cs +++ b/src/Redis.OM.Vectorizers/HuggingFaceApiSentenceVectorizerAttribute.cs @@ -1,7 +1,7 @@ using System.Net.Http.Json; using Redis.OM.Modeling; -namespace Redis.OM.Vectorizers.HuggingFace; +namespace Redis.OM.Vectorizers; public class HuggingFaceApiSentenceVectorizerAttribute : VectorizerAttribute { diff --git a/Redis.OM.Vectorizers.OpenAI/OpenAISentenceVectorizer.cs b/src/Redis.OM.Vectorizers/OpenAISentenceVectorizer.cs similarity index 98% rename from Redis.OM.Vectorizers.OpenAI/OpenAISentenceVectorizer.cs rename to src/Redis.OM.Vectorizers/OpenAISentenceVectorizer.cs index c5c0d233..93dcf51d 100644 --- a/Redis.OM.Vectorizers.OpenAI/OpenAISentenceVectorizer.cs +++ b/src/Redis.OM.Vectorizers/OpenAISentenceVectorizer.cs @@ -3,7 +3,7 @@ using Redis.OM.Contracts; using Redis.OM.Modeling; -namespace Redis.OM.Vectorizers.OpenAI; +namespace Redis.OM.Vectorizers; public class OpenAISentenceVectorizer : IVectorizer { diff --git a/Redis.OM.Vectorizers.OpenAI/OpenAISentenceVectorizerAttribute.cs b/src/Redis.OM.Vectorizers/OpenAISentenceVectorizerAttribute.cs similarity index 95% rename from Redis.OM.Vectorizers.OpenAI/OpenAISentenceVectorizerAttribute.cs rename to src/Redis.OM.Vectorizers/OpenAISentenceVectorizerAttribute.cs index 5d36d3f5..4dd5bfca 100644 --- a/Redis.OM.Vectorizers.OpenAI/OpenAISentenceVectorizerAttribute.cs +++ b/src/Redis.OM.Vectorizers/OpenAISentenceVectorizerAttribute.cs @@ -2,7 +2,7 @@ using System.Text.Json; using Redis.OM.Modeling; -namespace Redis.OM.Vectorizers.OpenAI; +namespace Redis.OM.Vectorizers; public class OpenAISentenceVectorizerAttribute : VectorizerAttribute { diff --git a/src/Redis.OM.Vectorizers.HuggingFace/Redis.OM.Vectorizers.HuggingFace.csproj b/src/Redis.OM.Vectorizers/Redis.OM.Vectorizers.csproj similarity index 83% rename from src/Redis.OM.Vectorizers.HuggingFace/Redis.OM.Vectorizers.HuggingFace.csproj rename to src/Redis.OM.Vectorizers/Redis.OM.Vectorizers.csproj index 6be4f8ad..5345d7f1 100644 --- a/src/Redis.OM.Vectorizers.HuggingFace/Redis.OM.Vectorizers.HuggingFace.csproj +++ b/src/Redis.OM.Vectorizers/Redis.OM.Vectorizers.csproj @@ -1,17 +1,12 @@ - net6.0 + net6.0;net7.0 enable enable Redis.OM - - - - - @@ -19,4 +14,7 @@ + + + diff --git a/src/Redis.OM.Vectorizers/RedisConnectionProviderExtensions.cs b/src/Redis.OM.Vectorizers/RedisConnectionProviderExtensions.cs new file mode 100644 index 00000000..7cc54069 --- /dev/null +++ b/src/Redis.OM.Vectorizers/RedisConnectionProviderExtensions.cs @@ -0,0 +1,48 @@ +using Redis.OM.Contracts; + +namespace Redis.OM.Vectorizers; + +public static class RedisConnectionProviderExtensions +{ + public static ISemanticCache HuggingFaceSemanticCache(this IRedisConnectionProvider provider, string huggingFaceAuthToken, double threshold = .15, string modelId = "sentence-transformers/all-mpnet-base-v2", int dim = 768, string indexName = "HuggingFaceSemanticCache", string? prefix = null, long? ttl = null) + { + var vectorizer = new HuggingFaceApiSentenceVectorizer(huggingFaceAuthToken, modelId, dim); + var connection = provider.Connection; + var info = connection.GetIndexInfo(indexName); + var cache = new SemanticCache(indexName, prefix ?? indexName, threshold, ttl, vectorizer, connection); + if (info is null) + { + cache.CreateIndex(); + } + + return cache; + } + + public static ISemanticCache OpenAISemanticCache(this IRedisConnectionProvider provider, string openAIAuthToken, double threshold = .15, string indexName = "OpenAISemanticCache", string? prefix = null, long? ttl = null) + { + var vectorizer = new OpenAISentenceVectorizer(openAIAuthToken); + var connection = provider.Connection; + var info = connection.GetIndexInfo(indexName); + var cache = new SemanticCache(indexName, prefix ?? indexName, threshold, ttl, vectorizer, connection); + if (info is null) + { + cache.CreateIndex(); + } + + return cache; + } + + public static ISemanticCache AzureOpenAISemanticCache(this IRedisConnectionProvider provider, string apiKey, string resourceName, string deploymentId, int dim, double threshold = .15, string indexName = "AzureOpenAISemanticCache", string? prefix = null, long? ttl = null) + { + var vectorizer = new AzureOpenAISentenceVectorizer(apiKey, resourceName, deploymentId, dim); + var connection = provider.Connection; + var cache = new SemanticCache(indexName, prefix ?? indexName, threshold, ttl, vectorizer, connection); + var info = connection.GetIndexInfo(indexName); + if (info is null) + { + cache.CreateIndex(); + } + + return cache; + } +} \ No newline at end of file diff --git a/src/Redis.OM.Vectorizers.Core/RedisOMHttpUtil.cs b/src/Redis.OM.Vectorizers/RedisOMHttpUtil.cs similarity index 84% rename from src/Redis.OM.Vectorizers.Core/RedisOMHttpUtil.cs rename to src/Redis.OM.Vectorizers/RedisOMHttpUtil.cs index f2396516..2b2eba46 100644 --- a/src/Redis.OM.Vectorizers.Core/RedisOMHttpUtil.cs +++ b/src/Redis.OM.Vectorizers/RedisOMHttpUtil.cs @@ -1,6 +1,6 @@ namespace Redis.OM; -public class RedisOMHttpUtil +internal class RedisOMHttpUtil { public static string ReadJsonSync(HttpResponseMessage msg) { diff --git a/src/Redis.OM/Redis.OM.csproj b/src/Redis.OM/Redis.OM.csproj index e087f947..1226bcb4 100644 --- a/src/Redis.OM/Redis.OM.csproj +++ b/src/Redis.OM/Redis.OM.csproj @@ -6,9 +6,9 @@ Redis.OM enable true - 0.5.4 - 0.5.4 - https://github.com/redis/redis-om-dotnet/releases/tag/v0.5.4 + 0.6.0 + 0.6.0 + https://github.com/redis/redis-om-dotnet/releases/tag/v0.6.0 Object Mapping and More for Redis Redis OM Steve Lorello diff --git a/test/Redis.OM.Unit.Tests/Address.cs b/test/Redis.OM.Unit.Tests/Address.cs index 8deb6aa8..e6b0da15 100644 --- a/test/Redis.OM.Unit.Tests/Address.cs +++ b/test/Redis.OM.Unit.Tests/Address.cs @@ -5,7 +5,6 @@ using System.Text; using System.Text.Json.Serialization; using System.Threading.Tasks; -using Redis.OM; using Redis.OM.Modeling; namespace Redis.OM.Unit.Tests diff --git a/test/Redis.OM.Unit.Tests/ConfigurationTests.cs b/test/Redis.OM.Unit.Tests/ConfigurationTests.cs index 3586362c..979fac7f 100644 --- a/test/Redis.OM.Unit.Tests/ConfigurationTests.cs +++ b/test/Redis.OM.Unit.Tests/ConfigurationTests.cs @@ -1,6 +1,5 @@ using System.Linq; using System.Net; -using Redis.OM; using Xunit; namespace Redis.OM.Unit.Tests diff --git a/test/Redis.OM.Unit.Tests/CoreTests.cs b/test/Redis.OM.Unit.Tests/CoreTests.cs index 5a59fd7e..c94aa0c7 100644 --- a/test/Redis.OM.Unit.Tests/CoreTests.cs +++ b/test/Redis.OM.Unit.Tests/CoreTests.cs @@ -4,7 +4,6 @@ using StackExchange.Redis; using System.Linq; using System.IO; -using Redis.OM; using Redis.OM.Modeling; using System.Threading; using System.Threading.Tasks; diff --git a/test/Redis.OM.Unit.Tests/RediSearchTests/Person.cs b/test/Redis.OM.Unit.Tests/RediSearchTests/Person.cs index c57ae65a..78ff0326 100644 --- a/test/Redis.OM.Unit.Tests/RediSearchTests/Person.cs +++ b/test/Redis.OM.Unit.Tests/RediSearchTests/Person.cs @@ -1,5 +1,4 @@ using System.Collections.Generic; -using Redis.OM; using Redis.OM.Modeling; namespace Redis.OM.Unit.Tests.RediSearchTests diff --git a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/HuggingFaceVectors.cs b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/HuggingFaceVectors.cs index 9b27e3f3..e37bbaf0 100644 --- a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/HuggingFaceVectors.cs +++ b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/HuggingFaceVectors.cs @@ -1,4 +1,4 @@ -using Redis.OM.Vectorizers.HuggingFace; +using Redis.OM.Vectorizers; using Redis.OM.Modeling; using Redis.OM.Modeling.Vectors; diff --git a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/OpenAIVectors.cs b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/OpenAIVectors.cs index b6cf79ed..ef3f113f 100644 --- a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/OpenAIVectors.cs +++ b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/OpenAIVectors.cs @@ -1,6 +1,6 @@ using Redis.OM.Modeling; using Redis.OM.Modeling.Vectors; -using Redis.OM.Vectorizers.OpenAI; +using Redis.OM.Vectorizers; namespace Redis.OM.Unit.Tests; diff --git a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/SemanticCachingTests.cs b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/SemanticCachingTests.cs index afcca84a..523bd684 100644 --- a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/SemanticCachingTests.cs +++ b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/SemanticCachingTests.cs @@ -1,7 +1,7 @@ using System; using System.Linq; using Redis.OM.Contracts; -using Redis.OM.Vectorizers.AzureOpenAI; +using Redis.OM.Vectorizers; using Xunit; namespace Redis.OM.Unit.Tests; diff --git a/test/Redis.OM.Unit.Tests/Redis.OM.Unit.Tests.csproj b/test/Redis.OM.Unit.Tests/Redis.OM.Unit.Tests.csproj index 9d253caa..62b888a6 100644 --- a/test/Redis.OM.Unit.Tests/Redis.OM.Unit.Tests.csproj +++ b/test/Redis.OM.Unit.Tests/Redis.OM.Unit.Tests.csproj @@ -14,10 +14,8 @@ - - - + diff --git a/test/Redis.OM.Unit.Tests/SearchJsonTests/RedisJsonIndexTests.cs b/test/Redis.OM.Unit.Tests/SearchJsonTests/RedisJsonIndexTests.cs index c4453331..5d2a7dc2 100644 --- a/test/Redis.OM.Unit.Tests/SearchJsonTests/RedisJsonIndexTests.cs +++ b/test/Redis.OM.Unit.Tests/SearchJsonTests/RedisJsonIndexTests.cs @@ -1,6 +1,5 @@ using System; using System.Linq; -using Redis.OM; using Redis.OM.Modeling; using Xunit; From 65272dc20944a4f9ea60a188b2656243e66cebd6 Mon Sep 17 00:00:00 2001 From: slorello89 Date: Mon, 30 Oct 2023 15:37:30 -0400 Subject: [PATCH 15/36] updating nuget packaging --- .../Redis.OM.Vectorizers.csproj | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/Redis.OM.Vectorizers/Redis.OM.Vectorizers.csproj b/src/Redis.OM.Vectorizers/Redis.OM.Vectorizers.csproj index 5345d7f1..830ac576 100644 --- a/src/Redis.OM.Vectorizers/Redis.OM.Vectorizers.csproj +++ b/src/Redis.OM.Vectorizers/Redis.OM.Vectorizers.csproj @@ -5,6 +5,20 @@ enable enable Redis.OM + 0.6.0 + 0.6.0 + https://github.com/redis/redis-om-dotnet/releases/tag/v0.6.0 + Core Vectorizers for Redis OM .NET. + Redis OM Vectorizers + Steve Lorello + Redis Inc + https://github.com/redis/redis-om-dotnet + icon-square.png + MIT + https://github.com/redis/redis-om-dotnet + Github + redis redisearch indexing databases + true @@ -17,4 +31,14 @@ + + + + + + + + bin\$(Configuration)\$(TargetFramework)\$(AssemblyName).xml + + From 16c694c3e24bcc5ffdde13266539dee42b62d334 Mon Sep 17 00:00:00 2001 From: slorello89 Date: Thu, 2 Nov 2023 09:42:11 -0400 Subject: [PATCH 16/36] use Vector instead of just the given type --- .../AzureOpenAISentenceVectorizerAttribute.cs | 24 +++- ...ggingFaceApiSentenceVectorizerAttribute.cs | 38 +++++- .../OpenAISentenceVectorizerAttribute.cs | 31 ++++- .../Common/ExpressionParserUtilities.cs | 20 +-- src/Redis.OM/Common/ExpressionTranslator.cs | 16 ++- src/Redis.OM/Modeling/IndexedAttribute.cs | 72 ++++++++++ src/Redis.OM/Modeling/RedisSchemaField.cs | 50 +++---- .../Modeling/Vectors/DoubleVectorizer.cs | 29 ++++ .../Vectors/DoubleVectorizerAttribute.cs | 40 ++++++ .../Modeling/Vectors/FloatVectorizer.cs | 30 +++++ .../Vectors/FloatVectorizerAttribute.cs | 39 ++++++ .../Modeling/Vectors/VectorAttribute.cs | 89 ------------ .../Modeling/Vectors/VectorJsonConverter.cs | 117 +++++++++++++--- src/Redis.OM/Modeling/Vectors/VectorUtils.cs | 56 ++++++-- .../Modeling/Vectors/VectorizerAttribute.cs | 21 ++- src/Redis.OM/Redis.OM.csproj | 2 +- src/Redis.OM/RedisObjectHandler.cs | 70 ++++++++-- src/Redis.OM/Vector.cs | 96 +++++++++++++ src/Redis.OM/Vectors.cs | 6 +- .../VectorTests/HuggingFaceVectors.cs | 4 +- .../VectorTests/ObjectWithVector.cs | 22 +-- .../VectorTests/OpenAIVectors.cs | 4 +- .../VectorTests/SimpleVectorizer.cs | 22 --- .../VectorTests/SimpleVectorizerAttribute.cs | 40 ++++++ .../VectorTests/VectorFunctionalTests.cs | 127 +++++++++++------- .../VectorTests/VectorTests.cs | 34 +++-- 26 files changed, 814 insertions(+), 285 deletions(-) create mode 100644 src/Redis.OM/Modeling/Vectors/DoubleVectorizer.cs create mode 100644 src/Redis.OM/Modeling/Vectors/DoubleVectorizerAttribute.cs create mode 100644 src/Redis.OM/Modeling/Vectors/FloatVectorizer.cs create mode 100644 src/Redis.OM/Modeling/Vectors/FloatVectorizerAttribute.cs delete mode 100644 src/Redis.OM/Modeling/Vectors/VectorAttribute.cs create mode 100644 src/Redis.OM/Vector.cs delete mode 100644 test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/SimpleVectorizer.cs create mode 100644 test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/SimpleVectorizerAttribute.cs diff --git a/src/Redis.OM.Vectorizers/AzureOpenAISentenceVectorizerAttribute.cs b/src/Redis.OM.Vectorizers/AzureOpenAISentenceVectorizerAttribute.cs index a1b0dcbe..907b3930 100644 --- a/src/Redis.OM.Vectorizers/AzureOpenAISentenceVectorizerAttribute.cs +++ b/src/Redis.OM.Vectorizers/AzureOpenAISentenceVectorizerAttribute.cs @@ -1,22 +1,40 @@ -using System.Net.Http.Json; -using System.Text.Json; +using Redis.OM.Contracts; using Redis.OM.Modeling; namespace Redis.OM.Vectorizers; -public class AzureOpenAISentenceVectorizerAttribute : VectorizerAttribute +/// +public class AzureOpenAISentenceVectorizerAttribute : VectorizerAttribute { + /// public AzureOpenAISentenceVectorizerAttribute(string deploymentName, string resourceName, int dim) { DeploymentName = deploymentName; ResourceName = resourceName; Dim = dim; + Vectorizer = new AzureOpenAISentenceVectorizer(Configuration.Instance.AzureOpenAIApiKey, ResourceName, DeploymentName, Dim); } + /// + /// Gets the DeploymentName. + /// public string DeploymentName { get; } + + /// + /// Gets the resource name. + /// public string ResourceName { get; } + + /// + public override IVectorizer Vectorizer { get; } + + /// public override VectorType VectorType => VectorType.FLOAT32; + + /// public override int Dim { get; } + + /// public override byte[] Vectorize(object obj) { if (obj is not string s) diff --git a/src/Redis.OM.Vectorizers/HuggingFaceApiSentenceVectorizerAttribute.cs b/src/Redis.OM.Vectorizers/HuggingFaceApiSentenceVectorizerAttribute.cs index 3ac689b5..e18ed559 100644 --- a/src/Redis.OM.Vectorizers/HuggingFaceApiSentenceVectorizerAttribute.cs +++ b/src/Redis.OM.Vectorizers/HuggingFaceApiSentenceVectorizerAttribute.cs @@ -1,11 +1,40 @@ using System.Net.Http.Json; +using Redis.OM.Contracts; using Redis.OM.Modeling; namespace Redis.OM.Vectorizers; -public class HuggingFaceApiSentenceVectorizerAttribute : VectorizerAttribute +/// +/// An attribute that provides a Hugging Face API Sentence Vectorizer. +/// +public class HuggingFaceApiSentenceVectorizerAttribute : VectorizerAttribute { public string? ModelId { get; set; } + + private IVectorizer? _vectorizer; + + /// + public override IVectorizer Vectorizer + { + get + { + if (_vectorizer is null) + { + var modelId = ModelId ?? Configuration.Instance["REDIS_OM_HF_MODEL_ID"]; + if (modelId is null) + { + throw new InvalidOperationException("Need a Model ID in order to process vector"); + } + + _vectorizer = new HuggingFaceApiSentenceVectorizer(Configuration.Instance.HuggingFaceAuthorizationToken, ModelId, Dim); + } + + return _vectorizer; + } + } + + + /// public override VectorType VectorType => VectorType.FLOAT32; private int? _dim; @@ -22,6 +51,7 @@ public override int Dim } } + /// public override byte[] Vectorize(object obj) { if (obj is not string s) @@ -33,6 +63,12 @@ public override byte[] Vectorize(object obj) return floats.SelectMany(BitConverter.GetBytes).ToArray(); } + /// + /// Gets the embedded floats of the string from the HuggingFace API. + /// + /// the string. + /// the embedding's floats. + /// thrown if model id is not populated. public float[] GetFloats(string s) { var modelId = ModelId ?? Configuration.Instance["REDIS_OM_HF_MODEL_ID"]; diff --git a/src/Redis.OM.Vectorizers/OpenAISentenceVectorizerAttribute.cs b/src/Redis.OM.Vectorizers/OpenAISentenceVectorizerAttribute.cs index 4dd5bfca..177dd272 100644 --- a/src/Redis.OM.Vectorizers/OpenAISentenceVectorizerAttribute.cs +++ b/src/Redis.OM.Vectorizers/OpenAISentenceVectorizerAttribute.cs @@ -1,17 +1,38 @@ -using System.Net.Http.Json; -using System.Text.Json; +using Redis.OM.Contracts; using Redis.OM.Modeling; namespace Redis.OM.Vectorizers; -public class OpenAISentenceVectorizerAttribute : VectorizerAttribute +/// +/// An OpenAI Sentence Vectorizer. +/// +public class OpenAISentenceVectorizerAttribute : VectorizerAttribute { private const string DefaultModel = "text-embedding-ada-002"; - public string ModelId { get; set; } = DefaultModel; - public override VectorType VectorType => VectorType.FLOAT32; + /// + /// The ModelId. + /// + public string ModelId { get; } = DefaultModel; + + /// + public override VectorType VectorType => VectorType.FLOAT32; + + /// public override int Dim => ModelId == DefaultModel ? 1536 : GetFloats("Probing model dimensions").Length; + private IVectorizer? _vectorizer; + + /// + public override IVectorizer Vectorizer + { + get + { + return _vectorizer ??= new OpenAISentenceVectorizer(Configuration.Instance.OpenAiAuthorizationToken, ModelId, Dim); + } + } + + /// public override byte[] Vectorize(object obj) { var s = (string)obj; diff --git a/src/Redis.OM/Common/ExpressionParserUtilities.cs b/src/Redis.OM/Common/ExpressionParserUtilities.cs index 4ea86a00..b167ec65 100644 --- a/src/Redis.OM/Common/ExpressionParserUtilities.cs +++ b/src/Redis.OM/Common/ExpressionParserUtilities.cs @@ -1005,26 +1005,16 @@ private static string TranslateVectorRange(MethodCallExpression exp, List().FirstOrDefault(); byte[] bytes; - var operand = GetOperand(exp.Arguments[1]); - - if (vectorizer is not null) - { - bytes = vectorizer.Vectorize(operand); - } - else if (member.Type == typeof(double[])) - { - bytes = ((double[])operand).SelectMany(BitConverter.GetBytes).ToArray(); - } - else if (member.Type == typeof(float[])) - { - bytes = ((float[])operand).SelectMany(BitConverter.GetBytes).ToArray(); - } - else + var operand = (Vector)GetOperand(exp.Arguments[1]); + if (vectorizer is null) { throw new InvalidOperationException( $"Attempting to run a vector range on a {member.Type} with no provided vectorizer"); } + operand.Embed(vectorizer); + + bytes = operand.Embedding ?? throw new InvalidOperationException("Embedding was null"); var distance = (double)((ConstantExpression)exp.Arguments[2]).Value; var distanceArgName = parameters.Count.ToString(); parameters.Add(distance); diff --git a/src/Redis.OM/Common/ExpressionTranslator.cs b/src/Redis.OM/Common/ExpressionTranslator.cs index 1984a480..6de9866a 100644 --- a/src/Redis.OM/Common/ExpressionTranslator.cs +++ b/src/Redis.OM/Common/ExpressionTranslator.cs @@ -263,7 +263,7 @@ internal static RedisQuery BuildQueryFromExpression(Expression expression, Type internal static NearestNeighbors ParseNearestNeighborsFromExpression(MethodCallExpression expression) { var memberExpression = (MemberExpression)((LambdaExpression)((UnaryExpression)expression.Arguments[1]).Operand).Body; - var attr = memberExpression.Member.GetCustomAttributes().FirstOrDefault() ?? throw new ArgumentException($"Could not find Vector attribute on {memberExpression.Member.Name}."); + var attr = memberExpression.Member.GetCustomAttributes().FirstOrDefault() ?? throw new ArgumentException($"Could not find Vector attribute on {memberExpression.Member.Name}."); var vectorizer = memberExpression.Member.GetCustomAttributes().FirstOrDefault(); var propertyName = !string.IsNullOrEmpty(attr.PropertyName) ? attr.PropertyName : memberExpression.Member.Name; var numNeighbors = (int)((ConstantExpression)expression.Arguments[2]).Value; @@ -272,7 +272,19 @@ internal static NearestNeighbors ParseNearestNeighborsFromExpression(MethodCallE if (vectorizer is not null) { - bytes = vectorizer.Vectorize(value); + if (value is Vector vec) + { + if (vec.Embedding is null) + { + vec.Embed(vectorizer); + } + + bytes = vec.Embedding!; + } + else + { + bytes = vectorizer.Vectorize(value); + } } else if (memberExpression.Type == typeof(float[])) { diff --git a/src/Redis.OM/Modeling/IndexedAttribute.cs b/src/Redis.OM/Modeling/IndexedAttribute.cs index cce2df7c..d844e765 100644 --- a/src/Redis.OM/Modeling/IndexedAttribute.cs +++ b/src/Redis.OM/Modeling/IndexedAttribute.cs @@ -17,7 +17,79 @@ public sealed class IndexedAttribute : SearchFieldAttribute /// public bool CaseSensitive { get; set; } = false; + /// + /// Gets or sets the vector storage algorithm to use. Defaults to Flat (which is brute force). + /// + public VectorAlgorithm Algorithm { get; set; } = VectorAlgorithm.FLAT; + + /// + /// Gets or sets the Supported distance metric. + /// + public DistanceMetric DistanceMetric { get; set; } = DistanceMetric.L2; + + /// + /// Gets or sets the Initial vector capacity in the index affecting memory allocation size of the index. + /// + public int InitialCapacity { get; set; } + + /// + /// Gets or sets Block size to hold BLOCK_SIZE amount of vectors in a contiguous array. This is useful when the + /// index is dynamic with respect to addition and deletion. Defaults to 1024. + /// + public int BlockSize { get; set; } + + /// + /// gets or sets the number of maximum allowed outgoing edges for each node in the graph in each layer. + /// On layer zero the maximal number of outgoing edges will be 2M. Default is 16. + /// + public int M { get; set; } + + /// + /// Gets or sets the number of maximum allowed potential outgoing edges candidates for each node in the graph, + /// during the graph building. Default is 200. + /// + public int EfConstructor { get; set; } + + /// + /// Gets or sets the number of maximum top candidates to hold during the KNN search. Higher values of + /// EfRuntime lead to more accurate results at the expense of a longer runtime. Default is 10. + /// + public int EfRuntime { get; set; } + + /// + /// Gets or sets Relative factor that sets the boundaries in which a range query may search for candidates. + /// That is, vector candidates whose distance from the query vector is radius*(1 + EPSILON) are potentially + /// scanned, allowing more extensive search and more accurate results (on the expense of runtime). Default is 0.01. + /// + public double Epsilon { get; set; } + /// internal override SearchFieldType SearchFieldType => SearchFieldType.INDEXED; + + /// + /// gets the number of arguments that will be produced by this attribute. + /// + internal int NumArgs + { + get + { + var numArgs = 6; + numArgs += InitialCapacity != default ? 2 : 0; + if (Algorithm == VectorAlgorithm.FLAT) + { + numArgs += BlockSize != default ? 2 : 0; + } + + if (Algorithm == VectorAlgorithm.HNSW) + { + numArgs += M != default ? 2 : 0; + numArgs += EfConstructor != default ? 2 : 0; + numArgs += EfRuntime != default ? 2 : 0; + numArgs += Epsilon != default ? 2 : 0; + } + + return numArgs; + } + } } } diff --git a/src/Redis.OM/Modeling/RedisSchemaField.cs b/src/Redis.OM/Modeling/RedisSchemaField.cs index af3edf0e..1a938eb5 100644 --- a/src/Redis.OM/Modeling/RedisSchemaField.cs +++ b/src/Redis.OM/Modeling/RedisSchemaField.cs @@ -5,6 +5,7 @@ using System.Reflection; using System.Runtime.InteropServices.ComTypes; using System.Text.Json.Serialization; +using Redis.OM.Modeling.Vectors; namespace Redis.OM.Modeling { @@ -62,9 +63,10 @@ internal static string[] SerializeArgsJson(this PropertyInfo info, int remaining { var innerType = Nullable.GetUnderlyingType(info.PropertyType) ?? info.PropertyType; - if (attr is VectorAttribute) + if (attr is IndexedAttribute indexedAttribute && (innerType == typeof(Vector) || innerType.BaseType == typeof(Vector))) { - var pathPostfix = info.GetCustomAttributes().Any() ? ".Vector" : string.Empty; + var vectorizer = info.GetCustomAttributes().FirstOrDefault(x => x.GetType() != typeof(FloatVectorizerAttribute) && x.GetType() != typeof(DoubleVectorizerAttribute)); + var pathPostfix = vectorizer != null ? ".Vector" : string.Empty; ret.Add(!string.IsNullOrEmpty(attr.PropertyName) ? $"{pathPrefix}{attr.PropertyName}{pathPostfix}" : $"{pathPrefix}{info.Name}{pathPostfix}"); ret.Add("AS"); ret.Add(!string.IsNullOrEmpty(attr.PropertyName) ? $"{aliasPrefix}{attr.PropertyName}" : $"{aliasPrefix}{info.Name}"); @@ -108,9 +110,13 @@ internal static string[] SerializeArgs(this PropertyInfo info) } var suffix = string.Empty; - if (attr.SearchFieldType == SearchFieldType.VECTOR && info.GetCustomAttributes().Any()) + if (attr.SearchFieldType == SearchFieldType.INDEXED && (info.PropertyType == typeof(Vector) || info.PropertyType.BaseType == typeof(Vector))) { - suffix = ".Vector"; + var vectorizer = info.GetCustomAttributes().FirstOrDefault(); + if (vectorizer is not null && vectorizer is not FloatVectorizerAttribute && vectorizer is not DoubleVectorizerAttribute) + { + suffix = ".Vector"; + } } var ret = new List { !string.IsNullOrEmpty(attr.PropertyName) ? $"attr.PropertyName{suffix}" : $"{info.Name}{suffix}" }; @@ -192,6 +198,11 @@ private static string GetSearchFieldType(Type declaredType, SearchFieldAttribute return "GEO"; } + if (declaredType == typeof(Vector) || declaredType.BaseType == typeof(Vector)) + { + return "VECTOR"; + } + if (declaredType.IsEnum) { return propertyInfo.GetCustomAttributes(typeof(JsonConverterAttribute)).FirstOrDefault() is JsonConverterAttribute converter && converter.ConverterType == typeof(JsonStringEnumConverter) ? "TAG" : "NUMERIC"; @@ -202,40 +213,21 @@ private static string GetSearchFieldType(Type declaredType, SearchFieldAttribute private static bool IsEnumTypeFlags(Type type) => type.GetCustomAttributes(typeof(FlagsAttribute), false).Any(); - private static IEnumerable VectorSerialization(VectorAttribute vectorAttribute, Type declaredType, PropertyInfo propertyInfo) + private static IEnumerable VectorSerialization(IndexedAttribute vectorAttribute, PropertyInfo propertyInfo) { var vectorizer = propertyInfo.GetCustomAttributes().FirstOrDefault(); if (vectorizer is null) { - if (vectorAttribute.Dim == default) - { - throw new ArgumentException("Could not determine dimension of the vector"); - } - - if (declaredType != typeof(double[]) && declaredType != typeof(float[])) - { - throw new ArgumentException("Could not determine the Vector Type"); - } + throw new InvalidOperationException("Indexed vector fields must provide a vectorizer."); } yield return vectorAttribute.Algorithm.AsRedisString(); yield return vectorAttribute.NumArgs.ToString(); yield return "TYPE"; - if (vectorizer is not null) - { - yield return vectorizer.VectorType.AsRedisString(); - } - else if (declaredType == typeof(double[])) - { - yield return "FLOAT64"; - } - else if (declaredType == typeof(float[])) - { - yield return "FLOAT32"; - } + yield return vectorizer.VectorType.AsRedisString(); yield return "DIM"; - yield return vectorizer is null ? vectorAttribute.Dim!.ToString() : vectorizer.Dim.ToString(); + yield return vectorizer.Dim.ToString(); yield return "DISTANCE_METRIC"; yield return vectorAttribute.DistanceMetric.AsRedisString(); if (vectorAttribute.InitialCapacity != default) @@ -323,9 +315,9 @@ private static string[] CommonSerialization(SearchFieldAttribute attr, Type decl } } - if (searchFieldType == "VECTOR" && attr is VectorAttribute vector) + if (searchFieldType == "VECTOR" && attr is IndexedAttribute vector) { - ret.AddRange(VectorSerialization(vector, declaredType, propertyInfo)); + ret.AddRange(VectorSerialization(vector, propertyInfo)); } if (attr.Sortable || attr.Aggregatable) diff --git a/src/Redis.OM/Modeling/Vectors/DoubleVectorizer.cs b/src/Redis.OM/Modeling/Vectors/DoubleVectorizer.cs new file mode 100644 index 00000000..1cc84080 --- /dev/null +++ b/src/Redis.OM/Modeling/Vectors/DoubleVectorizer.cs @@ -0,0 +1,29 @@ +using System; +using System.Linq; +using Redis.OM.Contracts; + +namespace Redis.OM.Modeling.Vectors; + +/// +/// A vectorizer for double arrays. +/// +public class DoubleVectorizer : IVectorizer +{ + /// + /// Initializes a new instance of the class. + /// + /// The dimensions. + public DoubleVectorizer(int dim) + { + Dim = dim; + } + + /// + public VectorType VectorType => VectorType.FLOAT64; + + /// + public int Dim { get; } + + /// + public byte[] Vectorize(double[] obj) => obj.SelectMany(BitConverter.GetBytes).ToArray(); +} \ No newline at end of file diff --git a/src/Redis.OM/Modeling/Vectors/DoubleVectorizerAttribute.cs b/src/Redis.OM/Modeling/Vectors/DoubleVectorizerAttribute.cs new file mode 100644 index 00000000..e9148b52 --- /dev/null +++ b/src/Redis.OM/Modeling/Vectors/DoubleVectorizerAttribute.cs @@ -0,0 +1,40 @@ +using System; +using Redis.OM.Contracts; + +namespace Redis.OM.Modeling.Vectors; + +/// +/// A vectorizer attribute for doubles. +/// +public class DoubleVectorizerAttribute : VectorizerAttribute +{ + /// + /// Initializes a new instance of the class. + /// + /// the dimensions. + public DoubleVectorizerAttribute(int dim) + { + Dim = dim; + Vectorizer = new DoubleVectorizer(dim); + } + + /// + public override IVectorizer Vectorizer { get; } + + /// + public override VectorType VectorType => VectorType.FLOAT64; + + /// + public override int Dim { get; } + + /// + public override byte[] Vectorize(object obj) + { + if (obj is not double[] doubles) + { + throw new InvalidOperationException("Provided object for vectorization must be a double[]"); + } + + return Vectorizer.Vectorize(doubles); + } +} \ No newline at end of file diff --git a/src/Redis.OM/Modeling/Vectors/FloatVectorizer.cs b/src/Redis.OM/Modeling/Vectors/FloatVectorizer.cs new file mode 100644 index 00000000..9e511b1a --- /dev/null +++ b/src/Redis.OM/Modeling/Vectors/FloatVectorizer.cs @@ -0,0 +1,30 @@ +using System; +using System.Linq; +using Redis.OM.Contracts; +using Redis.OM.Modeling; + +namespace Redis.OM; + +/// +/// A vectorizer for float arrays. +/// +public class FloatVectorizer : IVectorizer +{ + /// + /// Initializes a new instance of the class. + /// + /// The dimensions. + public FloatVectorizer(int dim) + { + Dim = dim; + } + + /// + public VectorType VectorType => VectorType.FLOAT32; + + /// + public int Dim { get; } + + /// + public byte[] Vectorize(float[] obj) => obj.SelectMany(BitConverter.GetBytes).ToArray(); +} \ No newline at end of file diff --git a/src/Redis.OM/Modeling/Vectors/FloatVectorizerAttribute.cs b/src/Redis.OM/Modeling/Vectors/FloatVectorizerAttribute.cs new file mode 100644 index 00000000..bc9d952a --- /dev/null +++ b/src/Redis.OM/Modeling/Vectors/FloatVectorizerAttribute.cs @@ -0,0 +1,39 @@ +using System; +using System.Linq; +using Redis.OM.Contracts; + +namespace Redis.OM.Modeling.Vectors; + +/// +public class FloatVectorizerAttribute : VectorizerAttribute +{ + /// + /// Initializes a new instance of the class. + /// + /// The dimensions of the vector. + public FloatVectorizerAttribute(int dim) + { + Dim = dim; + Vectorizer = new FloatVectorizer(dim); + } + + /// + public override VectorType VectorType => VectorType.FLOAT32; + + /// + public override int Dim { get; } + + /// + public override IVectorizer Vectorizer { get; } + + /// + public override byte[] Vectorize(object obj) + { + if (obj is not float[] floats) + { + throw new InvalidOperationException("Must pass in an array of floats"); + } + + return Vectorizer.Vectorize(floats); + } +} \ No newline at end of file diff --git a/src/Redis.OM/Modeling/Vectors/VectorAttribute.cs b/src/Redis.OM/Modeling/Vectors/VectorAttribute.cs deleted file mode 100644 index 05d607be..00000000 --- a/src/Redis.OM/Modeling/Vectors/VectorAttribute.cs +++ /dev/null @@ -1,89 +0,0 @@ -namespace Redis.OM.Modeling -{ - /// - /// An indexed vector embedding, can be whatever type you want so long as you provide a serializer. - /// - public class VectorAttribute : SearchFieldAttribute - { - /// - /// Gets or sets the vector storage algorithm to use. Defaults to Flat (which is brute force). - /// - public VectorAlgorithm Algorithm { get; set; } = VectorAlgorithm.FLAT; - - /// - /// Gets or sets the Vector dimension specified as a positive integer. Will be inferred from the vectorizer - /// if provided. - /// - public int Dim { get; set; } - - /// - /// Gets or sets the Supported distance metric. - /// - public DistanceMetric DistanceMetric { get; set; } = DistanceMetric.L2; - - /// - /// Gets or sets the Initial vector capacity in the index affecting memory allocation size of the index. - /// - public int InitialCapacity { get; set; } - - /// - /// Gets or sets Block size to hold BLOCK_SIZE amount of vectors in a contiguous array. This is useful when the - /// index is dynamic with respect to addition and deletion. Defaults to 1024. - /// - public int BlockSize { get; set; } - - /// - /// gets or sets the number of maximum allowed outgoing edges for each node in the graph in each layer. - /// On layer zero the maximal number of outgoing edges will be 2M. Default is 16. - /// - public int M { get; set; } - - /// - /// Gets or sets the number of maximum allowed potential outgoing edges candidates for each node in the graph, - /// during the graph building. Default is 200. - /// - public int EfConstructor { get; set; } - - /// - /// Gets or sets the number of maximum top candidates to hold during the KNN search. Higher values of - /// EfRuntime lead to more accurate results at the expense of a longer runtime. Default is 10. - /// - public int EfRuntime { get; set; } - - /// - /// Gets or sets Relative factor that sets the boundaries in which a range query may search for candidates. - /// That is, vector candidates whose distance from the query vector is radius*(1 + EPSILON) are potentially - /// scanned, allowing more extensive search and more accurate results (on the expense of runtime). Default is 0.01. - /// - public double Epsilon { get; set; } - - /// - internal override SearchFieldType SearchFieldType => SearchFieldType.VECTOR; - - /// - /// gets the number of arguments that will be produced by this attribute. - /// - internal int NumArgs - { - get - { - var numArgs = 6; - numArgs += InitialCapacity != default ? 2 : 0; - if (Algorithm == VectorAlgorithm.FLAT) - { - numArgs += BlockSize != default ? 2 : 0; - } - - if (Algorithm == VectorAlgorithm.HNSW) - { - numArgs += M != default ? 2 : 0; - numArgs += EfConstructor != default ? 2 : 0; - numArgs += EfRuntime != default ? 2 : 0; - numArgs += Epsilon != default ? 2 : 0; - } - - return numArgs; - } - } - } -} \ No newline at end of file diff --git a/src/Redis.OM/Modeling/Vectors/VectorJsonConverter.cs b/src/Redis.OM/Modeling/Vectors/VectorJsonConverter.cs index 4af8d4d2..be74169c 100644 --- a/src/Redis.OM/Modeling/Vectors/VectorJsonConverter.cs +++ b/src/Redis.OM/Modeling/Vectors/VectorJsonConverter.cs @@ -2,50 +2,131 @@ using System.Linq; using System.Text.Json; using System.Text.Json.Serialization; +using Redis.OM.Modeling.Vectors; namespace Redis.OM.Modeling { /// /// Converts the provided object to a json vector. /// - internal class VectorJsonConverter : JsonConverter + /// The type. + internal class VectorJsonConverter : JsonConverter> + where T : class { - private readonly VectorizerAttribute _vectorizerAttribute; + private readonly VectorizerAttribute _vectorizerAttribute; /// - /// Initializes a new instance of the class. + /// Initializes a new instance of the class. /// /// the attribute that will be used for vectorization. - internal VectorJsonConverter(VectorizerAttribute attribute) + internal VectorJsonConverter(VectorizerAttribute attribute) { _vectorizerAttribute = attribute; } /// - public override object? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + public override Vector? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { + T res; reader.Read(); - reader.Read(); - var res = JsonSerializer.Deserialize(reader.GetString() !, typeToConvert); - reader.Read(); - reader.Read(); // Vector - reader.Read(); // start array - for (var i = 0; i < _vectorizerAttribute.Dim; i++) + byte[] embedding; + if (_vectorizerAttribute is FloatVectorizerAttribute floatVectorizer) { - reader.Read(); // each item + float[] floats = new float[floatVectorizer.Dim]; + for (var i = 0; i < floatVectorizer.Dim; i++) + { + floats[i] = reader.GetSingle(); + reader.Read(); + } + + res = (floats as T) !; + embedding = floats.GetBytes(); } + else if (_vectorizerAttribute is DoubleVectorizerAttribute doubleVectorizer) + { + double[] doubles = new double[doubleVectorizer.Dim]; + for (var i = 0; i < doubleVectorizer.Dim; i++) + { + doubles[i] = reader.GetDouble(); + reader.Read(); + } - reader.Read(); // end array - return res; + res = (doubles as T) !; + embedding = doubles.GetBytes(); + } + else + { + reader.Read(); + res = JsonSerializer.Deserialize(reader.GetString() !) !; + reader.Read(); + reader.Read(); // Vector + reader.Read(); // start array + if (_vectorizerAttribute.VectorType == VectorType.FLOAT32) + { + var floats = new float[_vectorizerAttribute.Dim]; + for (var i = 0; i < _vectorizerAttribute.Dim; i++) + { + floats[i] = reader.GetSingle(); + reader.Read(); // each item + } + + embedding = floats.GetBytes(); + } + else + { + var doubles = new double[_vectorizerAttribute.Dim]; + for (var i = 0; i < _vectorizerAttribute.Dim; i++) + { + doubles[i] = reader.GetDouble(); + reader.Read(); // each item + } + + embedding = doubles.GetBytes(); + } + + reader.Read(); // end array + } + + var vector = new Vector(res!) { Embedding = embedding }; + return vector; } /// - public override void Write(Utf8JsonWriter writer, object value, JsonSerializerOptions options) + public override void Write(Utf8JsonWriter writer, Vector value, JsonSerializerOptions options) { + if (_vectorizerAttribute is DoubleVectorizerAttribute && value is Vector doubleVector) + { + writer.WriteStartArray(); + foreach (var d in doubleVector.Value) + { + writer.WriteNumberValue(d); + } + + writer.WriteEndArray(); + return; + } + + if (_vectorizerAttribute is FloatVectorizerAttribute && value is Vector floatVector) + { + writer.WriteStartArray(); + foreach (var d in floatVector.Value) + { + writer.WriteNumberValue(d); + } + + writer.WriteEndArray(); + return; + } + writer.WriteStartObject(); writer.WritePropertyName("Value"); - writer.WriteStringValue(JsonSerializer.Serialize(value)); - var bytes = _vectorizerAttribute.Vectorize(value); + writer.WriteStringValue(JsonSerializer.Serialize(value.Obj)); + if (value.Embedding is null) + { + value.Embed(_vectorizerAttribute); + } + + var bytes = value.Embedding!; var jagged = SplitIntoJaggedArray(bytes, _vectorizerAttribute.VectorType == VectorType.FLOAT32 ? 4 : 8); writer.WritePropertyName("Vector"); if (_vectorizerAttribute.VectorType == VectorType.FLOAT32) @@ -61,7 +142,7 @@ public override void Write(Utf8JsonWriter writer, object value, JsonSerializerOp } else { - var doubles = jagged.Select(BitConverter.ToDouble).ToArray(); + var doubles = jagged.Select(x => BitConverter.ToDouble(x, 0)).ToArray(); writer.WriteStartArray(); foreach (var d in doubles) { diff --git a/src/Redis.OM/Modeling/Vectors/VectorUtils.cs b/src/Redis.OM/Modeling/Vectors/VectorUtils.cs index c0ec6c54..047f1bf1 100644 --- a/src/Redis.OM/Modeling/Vectors/VectorUtils.cs +++ b/src/Redis.OM/Modeling/Vectors/VectorUtils.cs @@ -97,14 +97,13 @@ public static double[] VecStrToDoubles(RedisReply reply) } /// - /// Converts Vector String to array of doubles. + /// converts the vector bytes to an array of doubles. /// - /// the reply. - /// the doubles. + /// the bytes. + /// The doubles. /// Thrown if unbalanced. - public static double[] VecStrToDoubles(string reply) + public static double[] VectorBytesToDoubles(byte[] bytes) { - var bytes = Encoding.ASCII.GetBytes(reply); if (bytes.Length % 8 != 0) { throw new ArgumentException("Unbalanced Vector String"); @@ -120,14 +119,25 @@ public static double[] VecStrToDoubles(string reply) } /// - /// Parses a vector string to an array of floats. + /// Converts Vector String to array of doubles. /// /// the reply. - /// The floats. - /// thrown if unbalanced. - public static float[] VectorStrToFloats(RedisReply reply) + /// the doubles. + /// Thrown if unbalanced. + public static double[] VecStrToDoubles(string reply) + { + var bytes = Encoding.ASCII.GetBytes(reply); + return VectorBytesToDoubles(bytes); + } + + /// + /// Parses the bytes into an array of floats. + /// + /// the bytes. + /// the floats. + /// Thrown if bytes are unbalanced. + public static float[] VectorBytesToFloats(byte[] bytes) { - var bytes = (byte[]?)reply ?? throw new InvalidCastException("Could not convert result to raw result."); if (bytes.Length % 4 != 0) { throw new ArgumentException("Unbalanced Vector String"); @@ -142,6 +152,18 @@ public static float[] VectorStrToFloats(RedisReply reply) return floats; } + /// + /// Parses a vector string to an array of floats. + /// + /// the reply. + /// The floats. + /// thrown if unbalanced. + public static float[] VectorStrToFloats(RedisReply reply) + { + var bytes = (byte[]?)reply ?? throw new InvalidCastException("Could not convert result to raw result."); + return VectorBytesToFloats(bytes); + } + /// /// Converts binary safe Redis blob to double. /// @@ -184,5 +206,19 @@ public static byte[] VecStrToBytes(string str) return bytes.ToArray(); } + + /// + /// Converts doubles to array of bytes. + /// + /// the doubles. + /// the array of bytes. + internal static byte[] GetBytes(this double[] doubles) => doubles.SelectMany(BitConverter.GetBytes).ToArray(); + + /// + /// Converts floats to array of bytes. + /// + /// the floats. + /// the array of bytes. + internal static byte[] GetBytes(this float[] floats) => floats.SelectMany(BitConverter.GetBytes).ToArray(); } } \ No newline at end of file diff --git a/src/Redis.OM/Modeling/Vectors/VectorizerAttribute.cs b/src/Redis.OM/Modeling/Vectors/VectorizerAttribute.cs index 815b0c98..917f07ae 100644 --- a/src/Redis.OM/Modeling/Vectors/VectorizerAttribute.cs +++ b/src/Redis.OM/Modeling/Vectors/VectorizerAttribute.cs @@ -1,10 +1,11 @@ using System; using System.Text.Json.Serialization; +using Redis.OM.Contracts; namespace Redis.OM.Modeling { /// - /// Method for converting a field into a vector. + /// A vectorizer attribute. /// public abstract class VectorizerAttribute : JsonConverterAttribute { @@ -29,6 +30,22 @@ public abstract class VectorizerAttribute : JsonConverterAttribute /// the object to convert. /// A byte array containing the vectorized data. public abstract byte[] Vectorize(object obj); + } + + /// + /// Method for converting a field into a vector. + /// + /// The type. +#pragma warning disable SA1402 + public abstract class VectorizerAttribute : VectorizerAttribute + where T : class +#pragma warning restore SA1402 + { + /// + /// Gets the vectorizer for this attribute. + /// + /// The vectorizer. + public abstract IVectorizer Vectorizer { get; } /// /// Creates the json converter fulfilled by this attribute. @@ -37,7 +54,7 @@ public abstract class VectorizerAttribute : JsonConverterAttribute /// The Json Converter. public override JsonConverter? CreateConverter(Type typeToConvert) { - return new VectorJsonConverter(this); + return new VectorJsonConverter(this); } } } \ No newline at end of file diff --git a/src/Redis.OM/Redis.OM.csproj b/src/Redis.OM/Redis.OM.csproj index 1226bcb4..93e68ddc 100644 --- a/src/Redis.OM/Redis.OM.csproj +++ b/src/Redis.OM/Redis.OM.csproj @@ -2,7 +2,7 @@ netstandard2.0 - 9.0 + 11 Redis.OM enable true diff --git a/src/Redis.OM/RedisObjectHandler.cs b/src/Redis.OM/RedisObjectHandler.cs index 37ad0c76..4f765670 100644 --- a/src/Redis.OM/RedisObjectHandler.cs +++ b/src/Redis.OM/RedisObjectHandler.cs @@ -326,9 +326,22 @@ internal static IDictionary BuildHashSet(this object obj) { var val = property.GetValue(obj); var vectorizer = property.GetCustomAttributes().First(); - var vector = vectorizer.Vectorize(val); - hash.Add($"{propertyName}.Vector", vector); - hash.Add($"{propertyName}.Value", JsonSerializer.Serialize(val)); + if (val is not Vector vector) + { + throw new InvalidOperationException("VectorizerAttribute must decorate vectors"); + } + + vector.Embed(vectorizer); + if (vectorizer is FloatVectorizerAttribute or DoubleVectorizerAttribute) + { + hash.Add(propertyName, vector.Embedding!); + } + else + { + hash.Add($"{propertyName}.Vector", vector.Embedding!); + hash.Add($"{propertyName}.Value", JsonSerializer.Serialize(vector.Obj)); + } + continue; } @@ -361,10 +374,15 @@ internal static IDictionary BuildHashSet(this object obj) hash.Add(propertyName, new DateTimeOffset(val).ToUnixTimeMilliseconds().ToString()); } } - else if (type.GetInterfaces().Any(x => x.IsGenericType && x.GetGenericTypeDefinition() == typeof(IEnumerable<>)) && property.GetCustomAttributes().Any()) + else if (type == typeof(Vector)) { - var innerType = GetEnumerableType(property); - hash.Add(propertyName, PrimitiveCollectionToVectorBytes(property, obj, innerType)); + var val = (Vector)obj; + if (val.Embedding is null) + { + throw new InvalidOperationException("Could not use null embedding."); + } + + hash.Add(propertyName, val.Embedding); } else if (type.GetInterfaces().Any(x => x.IsGenericType && x.GetGenericTypeDefinition() == typeof(IEnumerable<>))) { @@ -429,7 +447,31 @@ private static string SendToJson(IDictionary hash, Type t) var type = Nullable.GetUnderlyingType(property.PropertyType) ?? property.PropertyType; var propertyName = property.Name; ExtractPropertyName(property, ref propertyName); - var isVectorized = property.GetCustomAttributes().Any(); + var vectorizer = property.GetCustomAttributes().FirstOrDefault(); + if (vectorizer is FloatVectorizerAttribute || vectorizer is DoubleVectorizerAttribute) + { + if (hash.ContainsKey(propertyName)) + { + string arrString; + if (vectorizer.VectorType == VectorType.FLOAT32) + { + var floats = VectorUtils.VectorStrToFloats(hash[propertyName]); + arrString = string.Join(",", floats); + } + else + { + var doubles = VectorUtils.VecStrToDoubles(hash[propertyName]); + arrString = string.Join(",", doubles); + } + + var valueStr = $"[{arrString}]"; + ret += $"\"{propertyName}\":{valueStr},"; + } + + continue; + } + + var isVectorized = vectorizer != null; var lookupPropertyName = propertyName + (isVectorized ? ".Value" : string.Empty); var vectorPropertyName = $"{propertyName}.Vector"; if (isVectorized && !hash.ContainsKey($"{propertyName}.Value") && !hash.ContainsKey($"{propertyName}.Vector")) @@ -475,7 +517,7 @@ private static string SendToJson(IDictionary hash, Type t) ret += $"\"{propertyName}\":\"{HttpUtility.JavaScriptStringEncode(hash[lookupPropertyName])}\","; } - else if ((type == typeof(double[]) || type == typeof(float[])) && property.GetCustomAttributes().Any()) + else if (type == typeof(Vector)) { if (!hash.ContainsKey(lookupPropertyName)) { @@ -573,7 +615,17 @@ private static string SendToJson(IDictionary hash, Type t) if (isVectorized) { - var vectorizer = property.GetCustomAttributes().First(); + if (vectorizer is null) + { + throw new InvalidOperationException( + "Vector field must be decorated with a vectorizer attribute"); + } + + if (hash.ContainsKey(lookupPropertyName)) + { + ret += $"\"Value\":\"{HttpUtility.JavaScriptStringEncode(hash[lookupPropertyName])}\","; + } + string arrString; if (vectorizer.VectorType == VectorType.FLOAT32) { diff --git a/src/Redis.OM/Vector.cs b/src/Redis.OM/Vector.cs new file mode 100644 index 00000000..ea27c4fc --- /dev/null +++ b/src/Redis.OM/Vector.cs @@ -0,0 +1,96 @@ +using System; +using Redis.OM.Modeling; + +namespace Redis.OM +{ + /// + /// Represents a vector created from an item. + /// + public abstract class Vector + { + /// + /// Gets the Embedding. + /// + public byte[]? Embedding { get; internal set; } + + /// + /// Gets the embedding represented as an array of floats. + /// + public float[]? Floats => Embedding is not null ? VectorUtils.VectorBytesToFloats(Embedding) : null; + + /// + /// Gets the embedding represented as an array of doubles. + /// + public double[]? Doubles => Embedding is not null ? VectorUtils.VectorBytesToDoubles(Embedding) : null; + + /// + /// Gets The object backed by this vector. + /// + internal abstract object? Obj { get; } + + /// + /// Gets a vector of the type. + /// + /// The value. + /// The type. + /// A vector of the given type. + public static Vector Of(T val) + where T : class + { + return new Vector(val); + } + + /// + /// Embeds the Vector using the provided vectorizer. + /// + /// The Vectorizer. + public abstract void Embed(VectorizerAttribute attr); + } + + /// + /// Represents a vector created from an item of type T. + /// + /// The type. +#pragma warning disable SA1402 + public sealed class Vector : Vector, IEquatable> + where T : class +#pragma warning restore SA1402 + { + /// + /// Initializes a new instance of the class. + /// + /// The item the vector will represent. + public Vector(T value) + { + Value = value; + } + + /// + /// Gets the item represented by the vector. + /// + public T Value { get; } + + /// + internal override object? Obj => Value; + + /// + /// Embeds the Vector using the provided vectorizer. + /// + /// The Vectorizer. + public override void Embed(VectorizerAttribute attr) + { + if (attr is not VectorizerAttribute vectorizerAttribute) + { + throw new InvalidOperationException($"VectorizerAttribute must be of the type {typeof(T).Name}"); + } + + Embedding = vectorizerAttribute.Vectorizer.Vectorize(Value); + } + + /// + public bool Equals(Vector other) + { + return Value == other.Value && Embedding == other.Embedding; + } + } +} \ No newline at end of file diff --git a/src/Redis.OM/Vectors.cs b/src/Redis.OM/Vectors.cs index df6868e7..dd62db81 100644 --- a/src/Redis.OM/Vectors.cs +++ b/src/Redis.OM/Vectors.cs @@ -16,7 +16,8 @@ public static class VectorExtensions /// The allowable distance from the provided object. /// The type to compare. /// Whether the queried vector is within the allowable distance. - public static bool VectorRange(this T obj, object comparisonObject, double range) => throw new NotImplementedException("This method is only meant to be run within a query of Redis."); + public static bool VectorRange(this Vector obj, Vector comparisonObject, double range) + where T : class => throw new NotImplementedException("This method is only meant to be run within a query of Redis."); /// /// Placeholder method to allow you to perform vector range operations. Only meant to be run @@ -28,6 +29,7 @@ public static class VectorExtensions /// The name of the score in the output. /// The type to compare. /// Whether the queried vector is within the allowable distance. - public static bool VectorRange(this T obj, object comparisonObject, double range, string scoreName) => throw new NotImplementedException("This method is only meant to be run within a query of Redis."); + public static bool VectorRange(this Vector obj, Vector comparisonObject, double range, string scoreName) + where T : class => throw new NotImplementedException("This method is only meant to be run within a query of Redis."); } } \ No newline at end of file diff --git a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/HuggingFaceVectors.cs b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/HuggingFaceVectors.cs index e37bbaf0..df69cfa4 100644 --- a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/HuggingFaceVectors.cs +++ b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/HuggingFaceVectors.cs @@ -10,9 +10,9 @@ public class HuggingFaceVectors [RedisIdField] public string Id { get; set; } - [Vector] + [Indexed] [HuggingFaceApiSentenceVectorizer(ModelId = "sentence-transformers/all-MiniLM-L6-v2")] - public string Sentence { get; set; } + public Vector Sentence { get; set; } [Indexed] public string Name { get; set; } diff --git a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/ObjectWithVector.cs b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/ObjectWithVector.cs index d17c3c80..19fd97ab 100644 --- a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/ObjectWithVector.cs +++ b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/ObjectWithVector.cs @@ -1,7 +1,5 @@ -using System.Text.Json.Serialization; using Redis.OM.Modeling; using Redis.OM.Modeling.Vectors; -using StackExchange.Redis; namespace Redis.OM.Unit.Tests; @@ -15,12 +13,13 @@ public class ObjectWithVector [Indexed] public int Num { get; set; } - [Vector(Algorithm = VectorAlgorithm.HNSW, Dim = 10)] - public double[] SimpleHnswVector { get; set; } + [Indexed(Algorithm = VectorAlgorithm.HNSW)] + [DoubleVectorizer(10)] + public Vector SimpleHnswVector { get; set; } - [Vector(Algorithm = VectorAlgorithm.FLAT)] + [Indexed(Algorithm = VectorAlgorithm.FLAT)] [SimpleVectorizer] - public string SimpleVectorizedVector { get; set; } + public Vector SimpleVectorizedVector { get; set; } public VectorScores VectorScores { get; set; } } @@ -35,12 +34,13 @@ public class ObjectWithVectorHash [Indexed] public int Num { get; set; } - [Vector(Algorithm = VectorAlgorithm.HNSW, Dim = 10)] - public double[] SimpleHnswVector { get; set; } + [Indexed(Algorithm = VectorAlgorithm.HNSW)] + [DoubleVectorizer(10)] + public Vector SimpleHnswVector { get; set; } - [Vector(Algorithm = VectorAlgorithm.FLAT)] + [Indexed(Algorithm = VectorAlgorithm.FLAT)] [SimpleVectorizer] - public string SimpleVectorizedVector { get; set; } + public Vector SimpleVectorizedVector { get; set; } public VectorScores VectorScores { get; set; } } @@ -49,5 +49,5 @@ public class ObjectWithVectorHash public class ToyVector { [RedisIdField] public string Id { get; set; } - [Vector(Dim=6)]public double[] SimpleVector { get; set; } + [Indexed][DoubleVectorizer(6)]public Vector SimpleVector { get; set; } } \ No newline at end of file diff --git a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/OpenAIVectors.cs b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/OpenAIVectors.cs index ef3f113f..6ec4d647 100644 --- a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/OpenAIVectors.cs +++ b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/OpenAIVectors.cs @@ -10,9 +10,9 @@ public class OpenAIVectors [RedisIdField] public string Id { get; set; } - [Vector] + [Indexed] [OpenAISentenceVectorizer] - public string Sentence { get; set; } + public Vector Sentence { get; set; } [Indexed] public string Name { get; set; } diff --git a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/SimpleVectorizer.cs b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/SimpleVectorizer.cs deleted file mode 100644 index e773b7c9..00000000 --- a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/SimpleVectorizer.cs +++ /dev/null @@ -1,22 +0,0 @@ -using System; -using System.Linq; -using Redis.OM.Modeling; - -namespace Redis.OM.Unit.Tests; - -public class SimpleVectorizer : VectorizerAttribute -{ - public override VectorType VectorType => VectorType.FLOAT32; - public override int Dim => 30; - - public override byte[] Vectorize(object obj) - { - var floats = new float[30]; - for (var i = 0; i < 30; i++) - { - floats[i] = i; - } - - return floats.SelectMany(BitConverter.GetBytes).ToArray(); - } -} \ No newline at end of file diff --git a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/SimpleVectorizerAttribute.cs b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/SimpleVectorizerAttribute.cs new file mode 100644 index 00000000..1514e60f --- /dev/null +++ b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/SimpleVectorizerAttribute.cs @@ -0,0 +1,40 @@ +using System; +using System.Linq; +using Redis.OM.Contracts; +using Redis.OM.Modeling; + +namespace Redis.OM.Unit.Tests; + +public class SimpleVectorizerAttribute : VectorizerAttribute +{ + public override VectorType VectorType => VectorType.FLOAT32; + public override int Dim => 30; + + public override IVectorizer Vectorizer => new SimpleVectorizer(); + + public override byte[] Vectorize(object obj) + { + if (obj is not string s) + { + throw new Exception("Could not vectorize non-string"); + } + + return Vectorizer.Vectorize(s); + } +} + +public class SimpleVectorizer : IVectorizer +{ + public VectorType VectorType => VectorType.FLOAT32; + public int Dim => 30; + public byte[] Vectorize(string obj) + { + var floats = new float[30]; + for (var i = 0; i < 30; i++) + { + floats[i] = i; + } + + return floats.SelectMany(BitConverter.GetBytes).ToArray(); + } +} \ No newline at end of file diff --git a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorFunctionalTests.cs b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorFunctionalTests.cs index 5bf86b37..3c8a9a7d 100644 --- a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorFunctionalTests.cs +++ b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorFunctionalTests.cs @@ -23,18 +23,20 @@ public void TestHuggingFaceVectorizer() _connection.DropIndexAndAssociatedRecords(typeof(HuggingFaceVectors)); _connection.CreateIndex(typeof(HuggingFaceVectors)); var collection = new RedisCollection(_connection); + var sentenceVector = Vector.Of("Hello World this is Hal."); var obj = new HuggingFaceVectors { Age = 45, - Sentence = "Hello World this is Hal.", + Sentence = sentenceVector, Name = "Hal" }; collection.Insert(obj); - var res = collection.NearestNeighbors(x => x.Sentence, 2, "Hello World this is Hal.").First(); + var queryVector = Vector.Of("Hello World this is Hal."); + var res = collection.NearestNeighbors(x => x.Sentence, 2, queryVector).First(); Assert.Equal(obj.Id, res.Id); Assert.Equal(0, res.VectorScore.NearestNeighborsScore); - Assert.Equal(obj.Sentence, res.Sentence); + Assert.Equal(obj.Sentence.Value, res.Sentence.Value); } [Fact] @@ -43,20 +45,23 @@ public void TestParis() _connection.DropIndexAndAssociatedRecords(typeof(HuggingFaceVectors)); _connection.CreateIndex(typeof(HuggingFaceVectors)); var collection = new RedisCollection(_connection); + var sentenceVector = Vector.Of("What is the capital of France?"); var obj = new HuggingFaceVectors { Age = 2259, - Sentence = "What is the capital of France?", + Sentence = sentenceVector, Name = "Paris" }; collection.Insert(obj); + var queryVector = Vector.Of("What really is the capital of France?"); var res = collection - .First(x => x.Sentence.VectorRange("What really is the capital of France?", .1, "range") && x.Age > 1000); - res = collection.NearestNeighbors(x => x.Sentence, 2, "What really is the capital of France?").First(x => x.Age > 1000); + .First(x => x.Sentence.VectorRange(queryVector, .1, "range") && x.Age > 1000); + res = collection.NearestNeighbors(x => x.Sentence, 2, queryVector).First(x => x.Age > 1000); Assert.Equal(obj.Id, res.Id); Assert.True(res.VectorScore.RangeScore < .1); - Assert.Equal(obj.Sentence, res.Sentence); + Assert.Equal(sentenceVector.Value, res.Sentence.Value); + Assert.Equal(obj.Sentence.Embedding, res.Sentence.Embedding); } [Fact] @@ -65,18 +70,21 @@ public void TestOpenAIVectorizer() _connection.DropIndexAndAssociatedRecords(typeof(OpenAIVectors)); _connection.CreateIndex(typeof(OpenAIVectors)); var collection = new RedisCollection(_connection); + var sentenceVector = Vector.Of("Hello World this is Hal."); var obj = new OpenAIVectors { Age = 45, - Sentence = "Hello World this is Hal.", + Sentence = sentenceVector, Name = "Hal" }; collection.Insert(obj); - var res = collection.NearestNeighbors(x => x.Sentence, 2, "Hello World this is Hal.").First(); + var queryVector = Vector.Of("Hello World this is Hal."); + var res = collection.NearestNeighbors(x => x.Sentence, 2, queryVector).First(); Assert.Equal(obj.Id, res.Id); Assert.True(res.VectorScore.NearestNeighborsScore < .01); - Assert.Equal(obj.Sentence, res.Sentence); + Assert.Equal(obj.Sentence.Value, res.Sentence.Value); + Assert.Equal(obj.Sentence.Embedding, res.Sentence.Embedding); } [Fact] @@ -85,18 +93,21 @@ public void TestOpenAIVectorRange() _connection.DropIndexAndAssociatedRecords(typeof(OpenAIVectors)); _connection.CreateIndex(typeof(OpenAIVectors)); var collection = new RedisCollection(_connection); + var sentenceVector = Vector.Of("What is the capital of France?"); var obj = new OpenAIVectors { Age = 2259, - Sentence = "What is the capital of France?", + Sentence = sentenceVector, Name = "Paris" }; collection.Insert(obj); - var res = collection.First(x => x.Sentence.VectorRange("What really is the capital of France?", 1, "range")); + var queryVector = Vector.Of("What really is the capital of France?"); + var res = collection.First(x => x.Sentence.VectorRange(queryVector, 1, "range")); Assert.Equal(obj.Id, res.Id); Assert.True(res.VectorScore.RangeScore < .1); - Assert.Equal(obj.Sentence, res.Sentence); + Assert.Equal(obj.Sentence.Value, res.Sentence.Value); + Assert.Equal(obj.Sentence.Embedding, res.Sentence.Embedding); } [Fact] @@ -104,13 +115,15 @@ public void BasicRangeQuery() { _connection.CreateIndex(typeof(ObjectWithVector)); var collection = new RedisCollection(_connection); + var simpleHnswVector = Vector.Of(Enumerable.Range(0, 10).Select(x => (double)x).ToArray()); + var simpleVectorizedVector = Vector.Of("FooBarBaz"); collection.Insert(new ObjectWithVector { Id = "helloWorld", - SimpleHnswVector = Enumerable.Range(0, 10).Select(x => (double)x).ToArray(), - SimpleVectorizedVector = "FooBarBaz" + SimpleHnswVector = simpleHnswVector, + SimpleVectorizedVector = simpleVectorizedVector }); - var queryVector = Enumerable.Range(0, 10).Select(x => (double)x).ToArray(); + var queryVector = Vector.Of(Enumerable.Range(0, 10).Select(x => (double)x).ToArray()); var res = collection.First(x => x.SimpleHnswVector.VectorRange(queryVector, 5)); Assert.Equal("helloWorld", res.Id); } @@ -120,13 +133,15 @@ public void MultiRangeOnSameProperty() { _connection.CreateIndex(typeof(ObjectWithVector)); var collection = new RedisCollection(_connection); + var simpleHnswVector = Vector.Of(Enumerable.Range(0, 10).Select(x => (double)x).ToArray()); + var simpleVectorizedVector = Vector.Of("FooBarBaz"); collection.Insert(new ObjectWithVector { Id = "helloWorld", - SimpleHnswVector = Enumerable.Range(0, 10).Select(x => (double)x).ToArray(), - SimpleVectorizedVector = "FooBarBaz" + SimpleHnswVector = simpleHnswVector, + SimpleVectorizedVector = simpleVectorizedVector }); - var queryVector = Enumerable.Range(0, 10).Select(x => (double)x).ToArray(); + var queryVector = Vector.Of(Enumerable.Range(0, 10).Select(x => (double)x).ToArray()); var res = collection.First(x => x.SimpleHnswVector.VectorRange(queryVector, 5) && x.SimpleHnswVector.VectorRange(queryVector, 6)); Assert.Equal("helloWorld", res.Id); } @@ -136,15 +151,18 @@ public void RangeAndKnn() { _connection.CreateIndex(typeof(ObjectWithVector)); var collection = new RedisCollection(_connection); + var simpleHnswVector = Vector.Of(Enumerable.Range(0, 10).Select(x => (double)x).ToArray()); + var simpleVectorizedVector = Vector.Of("FooBarBaz"); collection.Insert(new ObjectWithVector { Id = "helloWorld", - SimpleHnswVector = Enumerable.Range(0, 10).Select(x => (double)x).ToArray(), - SimpleVectorizedVector = "FooBarBaz" + SimpleHnswVector = simpleHnswVector, + SimpleVectorizedVector = simpleVectorizedVector }); - var queryVector = Enumerable.Range(0, 10).Select(x => (double)x).ToArray(); - queryVector[0] += 2; - var res = collection.NearestNeighbors(x=>x.SimpleVectorizedVector, 1, "FooBarBaz") + var queryVector = Vector.Of(Enumerable.Range(0, 10).Select(x => (double)x).ToArray()); + queryVector.Value[0] += 2; + var stringQueryVector = Vector.Of("FooBarBaz"); + var res = collection.NearestNeighbors(x=>x.SimpleVectorizedVector, 1, stringQueryVector) .First(x => x.SimpleHnswVector.VectorRange(queryVector, 5, "range")); Assert.Equal("helloWorld", res.Id); Assert.Equal(4, res.VectorScores.RangeScore); @@ -156,16 +174,19 @@ public void BasicQuery() { _connection.CreateIndex(typeof(ObjectWithVector)); var collection = new RedisCollection(_connection); + var simpleHnswVector = Vector.Of(Enumerable.Range(0, 10).Select(x => (double)x).ToArray()); + var simpleVectorizedVector = Vector.Of("FooBarBaz"); collection.Insert(new ObjectWithVector { Id = "helloWorld", - SimpleHnswVector = Enumerable.Range(0, 10).Select(x => (double)x).ToArray(), - SimpleVectorizedVector = "FooBarBaz" + SimpleHnswVector = simpleHnswVector, + SimpleVectorizedVector = simpleVectorizedVector }); - var queryVector = Enumerable.Range(0, 10).Select(x => (double)x).ToArray(); - queryVector[0] += 2; + var queryVector = Vector.Of(Enumerable.Range(0, 10).Select(x => (double)x).ToArray()); + queryVector.Value[0] += 2; - var res = collection.NearestNeighbors(x => x.SimpleVectorizedVector, 1, "FooBarBaz").First(); + var stringQueryVector = Vector.Of("FooBarBaz"); + var res = collection.NearestNeighbors(x => x.SimpleVectorizedVector, 1, stringQueryVector).First(); Assert.Equal("helloWorld", res.Id); Assert.Equal(0, res.VectorScores.NearestNeighborsScore); res = collection.NearestNeighbors(x => x.SimpleHnswVector, 1, queryVector).First(); @@ -177,16 +198,17 @@ public void ScoresOnHash() { _connection.DropIndexAndAssociatedRecords(typeof(ObjectWithVectorHash)); _connection.CreateIndex(typeof(ObjectWithVectorHash)); - var doubles = new double[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; + var simpleHnswVector = Vector.Of(Enumerable.Range(0, 10).Select(x => (double)x).ToArray()); + var simpleVectorizedVector = Vector.Of("foo"); var obj = new ObjectWithVectorHash { Id = "helloWorld", - SimpleHnswVector = doubles, - SimpleVectorizedVector = "foo", + SimpleHnswVector = simpleHnswVector, + SimpleVectorizedVector = simpleVectorizedVector, }; var collection = new RedisCollection(_connection); collection.Insert(obj); - var res = collection.NearestNeighbors(x => x.SimpleHnswVector, 5, doubles).First(); + var res = collection.NearestNeighbors(x => x.SimpleHnswVector, 5, simpleHnswVector).First(); Assert.Equal(0, res.VectorScores.NearestNeighborsScore); } @@ -196,18 +218,19 @@ public void HybridQueryTest() { _connection.DropIndexAndAssociatedRecords(typeof(ObjectWithVectorHash)); _connection.CreateIndex(typeof(ObjectWithVectorHash)); - var doubles = new double[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; + var simpleHnswVector = Vector.Of(Enumerable.Range(0, 10).Select(x => (double)x).ToArray()); + var simpleVectorizedVector = Vector.Of("foo"); var obj = new ObjectWithVectorHash { Id = "theOneWithStuff", - SimpleHnswVector = doubles, + SimpleHnswVector = simpleHnswVector, Name = "Steve", Num = 6, - SimpleVectorizedVector = "foo", + SimpleVectorizedVector = simpleVectorizedVector, }; var collection = new RedisCollection(_connection); collection.Insert(obj); - var res = collection.Where(x=>x.Name == "Steve" && x.Num == 6).NearestNeighbors(x => x.SimpleHnswVector, 5, doubles).First(); + var res = collection.Where(x=>x.Name == "Steve" && x.Num == 6).NearestNeighbors(x => x.SimpleHnswVector, 5, simpleHnswVector).First(); Assert.Equal(0, res.VectorScores.NearestNeighborsScore); } @@ -217,29 +240,34 @@ public void TestIndex() { _connection.CreateIndex(typeof(ObjectWithVectorHash)); - var doubles = new double[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; + var simpleHnswVector = Vector.Of(Enumerable.Range(0, 10).Select(x => (double)x).ToArray()); + var simpleVectorizedVector = Vector.Of("foo"); var obj = new ObjectWithVectorHash { Id = "helloWorld", - SimpleHnswVector = doubles, - SimpleVectorizedVector = "foo", + SimpleHnswVector = simpleHnswVector, + SimpleVectorizedVector = simpleVectorizedVector, }; var key = _connection.Set(obj); var res = _connection.Get(key); - Assert.Equal(doubles, res.SimpleHnswVector); + Assert.Equal(simpleHnswVector.Value, res.SimpleHnswVector.Value); + Assert.Equal(simpleHnswVector.Embedding, res.SimpleHnswVector.Embedding); + simpleVectorizedVector = Vector.Of("foobarbaz"); key = _connection.Set(new ObjectWithVector() { Id = "helloWorld", - SimpleHnswVector = doubles, - SimpleVectorizedVector = "foobarbaz" + SimpleHnswVector = simpleHnswVector, + SimpleVectorizedVector = simpleVectorizedVector }); var jsonRes = _connection.Get(key); - Assert.Equal(doubles, jsonRes.SimpleHnswVector); - Assert.Equal("foobarbaz", jsonRes.SimpleVectorizedVector); + Assert.Equal(simpleHnswVector.Value, jsonRes.SimpleHnswVector.Value); + Assert.Equal(simpleHnswVector.Embedding, jsonRes.SimpleHnswVector.Embedding); + Assert.Equal(simpleVectorizedVector.Value, jsonRes.SimpleVectorizedVector.Value); + Assert.Equal(simpleVectorizedVector.Embedding, jsonRes.SimpleVectorizedVector.Embedding); } [Fact] @@ -266,15 +294,18 @@ public void Insert() simpleHnswJsonStr.Append(']'); vectorizedFlatVectorJsonStr.Append(']'); + var simpleHnswVector = Vector.Of(simpleHnswHash); + var simpleVectorizedVector = Vector.Of("foobar"); var hashObj = new ObjectWithVectorHash() { Id = "helloWorld", - SimpleHnswVector = simpleHnswHash, - SimpleVectorizedVector = "foobar" + SimpleHnswVector = simpleHnswVector, + SimpleVectorizedVector = simpleVectorizedVector }; var key = _connection.Set(hashObj); var res = _connection.Get(key); - Assert.Equal("foobar", res.SimpleVectorizedVector); + Assert.Equal(simpleVectorizedVector.Value, res.SimpleVectorizedVector.Value); + Assert.Equal(simpleVectorizedVector.Embedding, res.SimpleVectorizedVector.Embedding); } } \ No newline at end of file diff --git a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorTests.cs b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorTests.cs index 50c52244..0ab57f6d 100644 --- a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorTests.cs +++ b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorTests.cs @@ -63,9 +63,9 @@ public void SimpleKnnQuery() _substitute.ClearSubstitute(); _substitute.Execute(Arg.Any(), Arg.Any()).Returns(new RedisReply(0)); var collection = new RedisCollection(_substitute); - var compVector = new double[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; + var compVector = Vector.Of(new double[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }); float[] floats = Enumerable.Range(0, 30).Select(x => (float)x).ToArray(); - var blob = compVector.SelectMany(BitConverter.GetBytes).ToArray(); + var blob = compVector.Value.SelectMany(BitConverter.GetBytes).ToArray(); var floatBlob = floats.SelectMany(BitConverter.GetBytes).ToArray(); _ = collection.NearestNeighbors(x=>x.SimpleHnswVector, 5, compVector).ToList(); @@ -76,7 +76,8 @@ public void SimpleKnnQuery() _substitute.ClearSubstitute(); _substitute.Execute(Arg.Any(), Arg.Any()).Returns(new RedisReply(0)); - _ = collection.NearestNeighbors(x => x.SimpleVectorizedVector, 8, "hello world").ToArray(); + var queryVector = Vector.Of("hello world"); + _ = collection.NearestNeighbors(x => x.SimpleVectorizedVector, 8, queryVector).ToArray(); _substitute.Received().Execute("FT.SEARCH", $"{nameof(ObjectWithVector).ToLower()}-idx", @@ -93,7 +94,8 @@ public void SimpleRangeQuery() var compVector = new double[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; float[] floats = Enumerable.Range(0, 30).Select(x => (float)x).ToArray(); var floatBytes = floats.SelectMany(BitConverter.GetBytes).ToArray(); - _ = collection.Where(x => x.SimpleVectorizedVector.VectorRange("foobar", .3)).ToList(); + var queryVector = Vector.Of("foobar"); + _ = collection.Where(x => x.SimpleVectorizedVector.VectorRange(queryVector, .3)).ToList(); _substitute.Received().Execute("FT.SEARCH", $"{nameof(ObjectWithVector).ToLower()}-idx", "@SimpleVectorizedVector:[VECTOR_RANGE $0 $1]", "PARAMS", 4, "0", .3, "1", Arg.Is(b => b.SequenceEqual(floatBytes)), "DIALECT", 2, "LIMIT", "0", "100"); @@ -106,9 +108,9 @@ public void SimpleKnnQueryWithSortBy() _substitute.ClearSubstitute(); _substitute.Execute(Arg.Any(), Arg.Any()).Returns(new RedisReply(0)); var collection = new RedisCollection(_substitute); - var compVector = new double[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; + var compVector = Vector.Of(new double[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }); float[] floats = Enumerable.Range(0, 30).Select(x => (float)x).ToArray(); - var blob = compVector.SelectMany(BitConverter.GetBytes).ToArray(); + var blob = compVector.Value.SelectMany(BitConverter.GetBytes).ToArray(); var floatBlob = floats.SelectMany(BitConverter.GetBytes).ToArray(); _ = collection.NearestNeighbors(x=>x.SimpleHnswVector, 5, compVector).OrderBy(x=>x.VectorScores.NearestNeighborsScore).ToList(); @@ -119,7 +121,8 @@ public void SimpleKnnQueryWithSortBy() _substitute.ClearSubstitute(); _substitute.Execute(Arg.Any(), Arg.Any()).Returns(new RedisReply(0)); - _ = collection.NearestNeighbors(x => x.SimpleVectorizedVector, 8, "hello world").OrderByDescending(x=>x.VectorScores.NearestNeighborsScore).ToArray(); + var queryVector = Vector.Of("hello world"); + _ = collection.NearestNeighbors(x => x.SimpleVectorizedVector, 8, queryVector).OrderByDescending(x=>x.VectorScores.NearestNeighborsScore).ToArray(); _substitute.Received().Execute("FT.SEARCH", $"{nameof(ObjectWithVector).ToLower()}-idx", @@ -162,18 +165,21 @@ public void InsertVectors() var simpleHnswBytes = simpleHnswHash.SelectMany(BitConverter.GetBytes).ToArray(); var flatVectorizedBytes = vectorizedFlatHashVector.SelectMany(BitConverter.GetBytes).ToArray(); + var simpleHnswVector = Vector.Of(simpleHnswHash); + var simpleVectorizedVector = Vector.Of("foobar"); + var hashObj = new ObjectWithVectorHash() { Id = "foo", - SimpleHnswVector = simpleHnswHash, - SimpleVectorizedVector = "foobar" + SimpleHnswVector = simpleHnswVector, + SimpleVectorizedVector = simpleVectorizedVector }; var jsonObj = new ObjectWithVector() { Id = "foo", - SimpleHnswVector = simpleHnswHash, - SimpleVectorizedVector = "foobar" + SimpleHnswVector = simpleHnswVector, + SimpleVectorizedVector = simpleVectorizedVector }; var json = @@ -187,7 +193,7 @@ public void InsertVectors() Arg.Is(x=>x.SequenceEqual(simpleHnswBytes)), "SimpleVectorizedVector.Vector", Arg.Is(x=>x.SequenceEqual(flatVectorizedBytes)), "SimpleVectorizedVector.Value", "\"foobar\""); _substitute.Received().Execute("JSON.SET", "Redis.OM.Unit.Tests.ObjectWithVector:foo", ".", json); var deseralized = JsonSerializer.Deserialize(json); - Assert.Equal("foobar", deseralized.SimpleVectorizedVector); + Assert.Equal("foobar", deseralized.SimpleVectorizedVector.Value); } [Fact] @@ -196,8 +202,8 @@ public void HybridQuery() _substitute.ClearSubstitute(); _substitute.Execute(Arg.Any(), Arg.Any()).Returns(new RedisReply(0)); var collection = new RedisCollection(_substitute); - var compVector = new double[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; - var blob = compVector.SelectMany(BitConverter.GetBytes); + var compVector = Vector.Of(new double[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }); + var blob = compVector.Value.SelectMany(BitConverter.GetBytes); _ = collection.Where(x => x.Name == "Steve" && x.Num < 5) .NearestNeighbors(x => x.SimpleHnswVector, 2, compVector).ToList(); _substitute.Received().Execute("FT.SEARCH", From 73c6c050bd1180ff97aead8f780d838a56a66b7b Mon Sep 17 00:00:00 2001 From: slorello89 Date: Thu, 2 Nov 2023 16:24:12 -0400 Subject: [PATCH 17/36] readme updates --- README.md | 98 +++++++++++++++++++ ...Vectorizer.cs => AzureOpenAIVectorizer.cs} | 4 +- ...e.cs => AzureOpenAIVectorizerAttribute.cs} | 8 +- ...Vectorizer.cs => HuggingFaceVectorizer.cs} | 4 +- ...e.cs => HuggingFaceVectorizerAttribute.cs} | 6 +- ...tenceVectorizer.cs => OpenAIVectorizer.cs} | 4 +- ...ribute.cs => OpenAIVectorizerAttribute.cs} | 6 +- .../RedisConnectionProviderExtensions.cs | 6 +- .../VectorTests/HuggingFaceVectors.cs | 2 +- .../VectorTests/OpenAIQuery.cs | 24 +++++ .../VectorTests/OpenAIVectors.cs | 2 +- .../VectorTests/SemanticCachingTests.cs | 2 +- .../VectorTests/VectorFunctionalTests.cs | 39 ++++++++ 13 files changed, 183 insertions(+), 22 deletions(-) rename src/Redis.OM.Vectorizers/{AzureOpenAISentenceVectorizer.cs => AzureOpenAIVectorizer.cs} (91%) rename src/Redis.OM.Vectorizers/{AzureOpenAISentenceVectorizerAttribute.cs => AzureOpenAIVectorizerAttribute.cs} (68%) rename src/Redis.OM.Vectorizers/{HuggingFaceApiSentenceVectorizer.cs => HuggingFaceVectorizer.cs} (91%) rename src/Redis.OM.Vectorizers/{HuggingFaceApiSentenceVectorizerAttribute.cs => HuggingFaceVectorizerAttribute.cs} (85%) rename src/Redis.OM.Vectorizers/{OpenAISentenceVectorizer.cs => OpenAIVectorizer.cs} (91%) rename src/Redis.OM.Vectorizers/{OpenAISentenceVectorizerAttribute.cs => OpenAIVectorizerAttribute.cs} (75%) create mode 100644 test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/OpenAIQuery.cs diff --git a/README.md b/README.md index ca5c7f01..514f58cf 100644 --- a/README.md +++ b/README.md @@ -288,6 +288,104 @@ customers.Where(x => x.LastName == "Bond" && x.FirstName == "James"); customers.Where(x=>x.NickNames.Contains("Jim")); ``` +### Vectors + +Redis OM .NET also supports storing and querying Vectors stored in Redis. + +A `Vector` is a representation of an object that can be transformed into a vector by a Vectorizer. + +A `VectorizerAttribute` is the abstract class you use to decorate your Vector fields, it is responsible for defining the logic to convert your Vectors into Embeddings. In the package `Redis.OM.Vectorizers` we provide vectorizers for HuggingFace, OpenAI, and AzureOpenAI to allow you to easily integrate them into your workflows. + +#### Define a Vector in your Model. + +To define a vector in your model, simply decorate a `Vector` field with and `Indexed` and a `Vectorizer` attribute (in this case we'll use OpenAI): + +```cs +[Document(StorageType = StorageType.Json)] +public class OpenAIQuery +{ + [RedisIdField] + public string Id { get; set; } + + [Indexed(DistanceMetric = DistanceMetric.COSINE)] + [OpenAIVectorizer] + public Vector Prompt { get; set; } + + public string Response { get; set; } + + [Indexed] + public string Language { get; set; } + + [Indexed] + public DateTime TimeStamp { get; set; } +} +``` + +#### Insert Vectors into Redis + +With the vector defined in our model, all we need to do is create Vectors of the generic type, and insert them with our model. Using our `RedisCollection`, you can do this by simply using `Insert`: + +```cs +var query = new OpenAIQuery +{ + Language = "en_us", + Prompt = Vector.Of("What is the Capital of France?"), + Response = "Paris", + TimeStamp = DateTime.Now - TimeSpan.FromHours(3) +}; +collection.Insert(query); +``` + +The Vectorizer will manage the embedding generation for you without you having to intervene. + +#### Query Vectors in Redis + +To query vector fields in Redis, all you need to do is use the `VectorRange` method on a vector within our normal LINQ queries, and/or use the `NearestNeighbors` with whatever other filters you want to use, here's some examples: + +```cs +var queryPrompt = Vector.Of("What really is the Capital of France?"); + +// simple vector range, find first within .15 +var result = collection.First(x => x.Prompt.VectorRange(queryPrompt, .15)); + +// simple nearest neighbors query, finds first nearest neighbor +result = collection.NearestNeighbors(x => x.Prompt, 1, queryPrompt).First(); + +// hybrid query, pre-filters result set for english responses, then runs a nearest neighbors search. +result = collection.Where(x=>x.Language == "en_us").NearestNeighbors(x => x.Prompt, 1, queryPrompt).First(); + +// hybrid query, pre-filters responses newer than 4 hours, and finds first result within .15 +var ts = DateTimeOffset.Now - TimeSpan.FromHours(4); +result = collection.First(x=>x.TimeStamp > ts && x.Prompt.VectorRange(queryPrompt, .15)); +``` + +#### What Happens to the Embeddings? + +With Redis OM, the embeddings can be completely transparent to you, they are generated and bound to the `Vector` when you query/insert your vectors. If however you needed your embedding after the insertion/Query, they are available at `Vector.Embedding`, and be queried either as the raw bytes, as an array of doubles or as an array of floats (depending on your vectorizer). + +#### Configuration + +The Vectorizers provided by the `Redis.OM.Vectorizers` package have some configuration parameters that it will pull in either from your `appsettings.json` file, or your environment variables (with your appsettings taking precedence). + +| Configuration Parameter | Description | +|-------------------------------- |-----------------------------------------------| +| REDIS_OM_HF_TOKEN | HuggingFace Authorization token. | +| REDIS_OM_OAI_TOKEN | OpenAI Authorization token | +| REDIS_OM_OAI_API_URL | OpenAI URL | +| REDIS_OM_AZURE_OAI_TOKEN | Azure OpenAI api key | +| REDIS_OM_AZURE_OAI_RESOURCE_NAME | Azure resource name | +| REDIS_OM_AZURE_OAI_DEPLOYMENT_NAME | Azure deployment | + +### Semantic Caching + +Redis OM also provides the ability to use Semantic Caching, as well as providers for OpenAI, HuggingFace, and Azure OpenAI to perform semantic caching. To use a Semantic Cache, simply pull one out of the RedisConnectionProvider and use `Store` to insert items, and `GetSimilar` to retrieve items. For example: + +```cs +var cache = _provider.OpenAISemanticCache(token); +cache.Store("What is the capital of France?", "Paris"); +var res = cache.GetSimilar("What really is the capital of France?").First(); +``` + ### 🖩 Aggregations We can also run aggregations on the customer object, again using expressions in LINQ: diff --git a/src/Redis.OM.Vectorizers/AzureOpenAISentenceVectorizer.cs b/src/Redis.OM.Vectorizers/AzureOpenAIVectorizer.cs similarity index 91% rename from src/Redis.OM.Vectorizers/AzureOpenAISentenceVectorizer.cs rename to src/Redis.OM.Vectorizers/AzureOpenAIVectorizer.cs index 06359d3c..4ec34545 100644 --- a/src/Redis.OM.Vectorizers/AzureOpenAISentenceVectorizer.cs +++ b/src/Redis.OM.Vectorizers/AzureOpenAIVectorizer.cs @@ -6,13 +6,13 @@ namespace Redis.OM.Vectorizers; -public class AzureOpenAISentenceVectorizer : IVectorizer +public class AzureOpenAIVectorizer : IVectorizer { private readonly string _apiKey; private readonly string _resourceName; private readonly string _deploymentName; - public AzureOpenAISentenceVectorizer(string apiKey, string resourceName, string deploymentName, int dim) + public AzureOpenAIVectorizer(string apiKey, string resourceName, string deploymentName, int dim) { _apiKey = apiKey; _resourceName = resourceName; diff --git a/src/Redis.OM.Vectorizers/AzureOpenAISentenceVectorizerAttribute.cs b/src/Redis.OM.Vectorizers/AzureOpenAIVectorizerAttribute.cs similarity index 68% rename from src/Redis.OM.Vectorizers/AzureOpenAISentenceVectorizerAttribute.cs rename to src/Redis.OM.Vectorizers/AzureOpenAIVectorizerAttribute.cs index 907b3930..43344ed2 100644 --- a/src/Redis.OM.Vectorizers/AzureOpenAISentenceVectorizerAttribute.cs +++ b/src/Redis.OM.Vectorizers/AzureOpenAIVectorizerAttribute.cs @@ -4,15 +4,15 @@ namespace Redis.OM.Vectorizers; /// -public class AzureOpenAISentenceVectorizerAttribute : VectorizerAttribute +public class AzureOpenAIVectorizerAttribute : VectorizerAttribute { /// - public AzureOpenAISentenceVectorizerAttribute(string deploymentName, string resourceName, int dim) + public AzureOpenAIVectorizerAttribute(string deploymentName, string resourceName, int dim) { DeploymentName = deploymentName; ResourceName = resourceName; Dim = dim; - Vectorizer = new AzureOpenAISentenceVectorizer(Configuration.Instance.AzureOpenAIApiKey, ResourceName, DeploymentName, Dim); + Vectorizer = new AzureOpenAIVectorizer(Configuration.Instance.AzureOpenAIApiKey, ResourceName, DeploymentName, Dim); } /// @@ -42,7 +42,7 @@ public override byte[] Vectorize(object obj) throw new ArgumentException("Object must be a string to be embedded", nameof(obj)); } - var floats = AzureOpenAISentenceVectorizer.GetFloats(s, ResourceName, DeploymentName, Configuration.Instance.AzureOpenAIApiKey); + var floats = AzureOpenAIVectorizer.GetFloats(s, ResourceName, DeploymentName, Configuration.Instance.AzureOpenAIApiKey); return floats.SelectMany(BitConverter.GetBytes).ToArray(); } } \ No newline at end of file diff --git a/src/Redis.OM.Vectorizers/HuggingFaceApiSentenceVectorizer.cs b/src/Redis.OM.Vectorizers/HuggingFaceVectorizer.cs similarity index 91% rename from src/Redis.OM.Vectorizers/HuggingFaceApiSentenceVectorizer.cs rename to src/Redis.OM.Vectorizers/HuggingFaceVectorizer.cs index ada56754..c07224c3 100644 --- a/src/Redis.OM.Vectorizers/HuggingFaceApiSentenceVectorizer.cs +++ b/src/Redis.OM.Vectorizers/HuggingFaceVectorizer.cs @@ -5,9 +5,9 @@ namespace Redis.OM.Vectorizers; -public class HuggingFaceApiSentenceVectorizer : IVectorizer +public class HuggingFaceVectorizer : IVectorizer { - public HuggingFaceApiSentenceVectorizer(string authToken, string modelId, int dim) + public HuggingFaceVectorizer(string authToken, string modelId, int dim) { _huggingFaceAuthToken = authToken; ModelId = modelId; diff --git a/src/Redis.OM.Vectorizers/HuggingFaceApiSentenceVectorizerAttribute.cs b/src/Redis.OM.Vectorizers/HuggingFaceVectorizerAttribute.cs similarity index 85% rename from src/Redis.OM.Vectorizers/HuggingFaceApiSentenceVectorizerAttribute.cs rename to src/Redis.OM.Vectorizers/HuggingFaceVectorizerAttribute.cs index e18ed559..85a42af7 100644 --- a/src/Redis.OM.Vectorizers/HuggingFaceApiSentenceVectorizerAttribute.cs +++ b/src/Redis.OM.Vectorizers/HuggingFaceVectorizerAttribute.cs @@ -7,7 +7,7 @@ namespace Redis.OM.Vectorizers; /// /// An attribute that provides a Hugging Face API Sentence Vectorizer. /// -public class HuggingFaceApiSentenceVectorizerAttribute : VectorizerAttribute +public class HuggingFaceVectorizerAttribute : VectorizerAttribute { public string? ModelId { get; set; } @@ -26,7 +26,7 @@ public override IVectorizer Vectorizer throw new InvalidOperationException("Need a Model ID in order to process vector"); } - _vectorizer = new HuggingFaceApiSentenceVectorizer(Configuration.Instance.HuggingFaceAuthorizationToken, ModelId, Dim); + _vectorizer = new HuggingFaceVectorizer(Configuration.Instance.HuggingFaceAuthorizationToken, ModelId, Dim); } return _vectorizer; @@ -73,6 +73,6 @@ public float[] GetFloats(string s) { var modelId = ModelId ?? Configuration.Instance["REDIS_OM_HF_MODEL_ID"]; if (modelId is null) throw new InvalidOperationException("Model Id Required to use Hugging Face API."); - return HuggingFaceApiSentenceVectorizer.GetFloats(s, modelId, Configuration.Instance.HuggingFaceAuthorizationToken); + return HuggingFaceVectorizer.GetFloats(s, modelId, Configuration.Instance.HuggingFaceAuthorizationToken); } } \ No newline at end of file diff --git a/src/Redis.OM.Vectorizers/OpenAISentenceVectorizer.cs b/src/Redis.OM.Vectorizers/OpenAIVectorizer.cs similarity index 91% rename from src/Redis.OM.Vectorizers/OpenAISentenceVectorizer.cs rename to src/Redis.OM.Vectorizers/OpenAIVectorizer.cs index 93dcf51d..5851fe82 100644 --- a/src/Redis.OM.Vectorizers/OpenAISentenceVectorizer.cs +++ b/src/Redis.OM.Vectorizers/OpenAIVectorizer.cs @@ -5,12 +5,12 @@ namespace Redis.OM.Vectorizers; -public class OpenAISentenceVectorizer : IVectorizer +public class OpenAIVectorizer : IVectorizer { private readonly string _openAIAuthToken; private readonly string _model; - public OpenAISentenceVectorizer(string openAIAuthToken, string model = "text-embedding-ada-002", int dim = 1536) + public OpenAIVectorizer(string openAIAuthToken, string model = "text-embedding-ada-002", int dim = 1536) { _openAIAuthToken = openAIAuthToken; _model = model; diff --git a/src/Redis.OM.Vectorizers/OpenAISentenceVectorizerAttribute.cs b/src/Redis.OM.Vectorizers/OpenAIVectorizerAttribute.cs similarity index 75% rename from src/Redis.OM.Vectorizers/OpenAISentenceVectorizerAttribute.cs rename to src/Redis.OM.Vectorizers/OpenAIVectorizerAttribute.cs index 177dd272..026e5718 100644 --- a/src/Redis.OM.Vectorizers/OpenAISentenceVectorizerAttribute.cs +++ b/src/Redis.OM.Vectorizers/OpenAIVectorizerAttribute.cs @@ -6,7 +6,7 @@ namespace Redis.OM.Vectorizers; /// /// An OpenAI Sentence Vectorizer. /// -public class OpenAISentenceVectorizerAttribute : VectorizerAttribute +public class OpenAIVectorizerAttribute : VectorizerAttribute { private const string DefaultModel = "text-embedding-ada-002"; @@ -28,7 +28,7 @@ public override IVectorizer Vectorizer { get { - return _vectorizer ??= new OpenAISentenceVectorizer(Configuration.Instance.OpenAiAuthorizationToken, ModelId, Dim); + return _vectorizer ??= new OpenAIVectorizer(Configuration.Instance.OpenAiAuthorizationToken, ModelId, Dim); } } @@ -42,6 +42,6 @@ public override byte[] Vectorize(object obj) internal float[] GetFloats(string s) { - return OpenAISentenceVectorizer.GetFloats(s, ModelId, Configuration.Instance.OpenAiAuthorizationToken); + return OpenAIVectorizer.GetFloats(s, ModelId, Configuration.Instance.OpenAiAuthorizationToken); } } \ No newline at end of file diff --git a/src/Redis.OM.Vectorizers/RedisConnectionProviderExtensions.cs b/src/Redis.OM.Vectorizers/RedisConnectionProviderExtensions.cs index 7cc54069..4540b42d 100644 --- a/src/Redis.OM.Vectorizers/RedisConnectionProviderExtensions.cs +++ b/src/Redis.OM.Vectorizers/RedisConnectionProviderExtensions.cs @@ -6,7 +6,7 @@ public static class RedisConnectionProviderExtensions { public static ISemanticCache HuggingFaceSemanticCache(this IRedisConnectionProvider provider, string huggingFaceAuthToken, double threshold = .15, string modelId = "sentence-transformers/all-mpnet-base-v2", int dim = 768, string indexName = "HuggingFaceSemanticCache", string? prefix = null, long? ttl = null) { - var vectorizer = new HuggingFaceApiSentenceVectorizer(huggingFaceAuthToken, modelId, dim); + var vectorizer = new HuggingFaceVectorizer(huggingFaceAuthToken, modelId, dim); var connection = provider.Connection; var info = connection.GetIndexInfo(indexName); var cache = new SemanticCache(indexName, prefix ?? indexName, threshold, ttl, vectorizer, connection); @@ -20,7 +20,7 @@ public static ISemanticCache HuggingFaceSemanticCache(this IRedisConnectionProvi public static ISemanticCache OpenAISemanticCache(this IRedisConnectionProvider provider, string openAIAuthToken, double threshold = .15, string indexName = "OpenAISemanticCache", string? prefix = null, long? ttl = null) { - var vectorizer = new OpenAISentenceVectorizer(openAIAuthToken); + var vectorizer = new OpenAIVectorizer(openAIAuthToken); var connection = provider.Connection; var info = connection.GetIndexInfo(indexName); var cache = new SemanticCache(indexName, prefix ?? indexName, threshold, ttl, vectorizer, connection); @@ -34,7 +34,7 @@ public static ISemanticCache OpenAISemanticCache(this IRedisConnectionProvider p public static ISemanticCache AzureOpenAISemanticCache(this IRedisConnectionProvider provider, string apiKey, string resourceName, string deploymentId, int dim, double threshold = .15, string indexName = "AzureOpenAISemanticCache", string? prefix = null, long? ttl = null) { - var vectorizer = new AzureOpenAISentenceVectorizer(apiKey, resourceName, deploymentId, dim); + var vectorizer = new AzureOpenAIVectorizer(apiKey, resourceName, deploymentId, dim); var connection = provider.Connection; var cache = new SemanticCache(indexName, prefix ?? indexName, threshold, ttl, vectorizer, connection); var info = connection.GetIndexInfo(indexName); diff --git a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/HuggingFaceVectors.cs b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/HuggingFaceVectors.cs index df69cfa4..585de4fc 100644 --- a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/HuggingFaceVectors.cs +++ b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/HuggingFaceVectors.cs @@ -11,7 +11,7 @@ public class HuggingFaceVectors public string Id { get; set; } [Indexed] - [HuggingFaceApiSentenceVectorizer(ModelId = "sentence-transformers/all-MiniLM-L6-v2")] + [HuggingFaceVectorizer(ModelId = "sentence-transformers/all-MiniLM-L6-v2")] public Vector Sentence { get; set; } [Indexed] diff --git a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/OpenAIQuery.cs b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/OpenAIQuery.cs new file mode 100644 index 00000000..399ec004 --- /dev/null +++ b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/OpenAIQuery.cs @@ -0,0 +1,24 @@ +using System; +using Redis.OM.Modeling; +using Redis.OM.Vectorizers; + +namespace Redis.OM.Unit.Tests; + +[Document(StorageType = StorageType.Json)] +public class OpenAIQuery +{ + [RedisIdField] + public string Id { get; set; } + + [Indexed(DistanceMetric = DistanceMetric.COSINE)] + [OpenAIVectorizer] + public Vector Prompt { get; set; } + + public string Response { get; set; } + + [Indexed] + public string Language { get; set; } + + [Indexed] + public DateTime TimeStamp { get; set; } +} \ No newline at end of file diff --git a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/OpenAIVectors.cs b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/OpenAIVectors.cs index 6ec4d647..c99df107 100644 --- a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/OpenAIVectors.cs +++ b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/OpenAIVectors.cs @@ -11,7 +11,7 @@ public class OpenAIVectors public string Id { get; set; } [Indexed] - [OpenAISentenceVectorizer] + [OpenAIVectorizer] public Vector Sentence { get; set; } [Indexed] diff --git a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/SemanticCachingTests.cs b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/SemanticCachingTests.cs index 523bd684..674bb439 100644 --- a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/SemanticCachingTests.cs +++ b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/SemanticCachingTests.cs @@ -32,7 +32,7 @@ public void HuggingFaceSemanticCache() { var token = Environment.GetEnvironmentVariable("REDIS_OM_HF_TOKEN"); Assert.NotNull(token); - var cache = _provider.HuggingFaceSemanticCache(token); + var cache = _provider.HuggingFaceSemanticCache(token, threshold: .15); cache.Store("What is the capital of France?", "Paris"); var res = cache.GetSimilar("What really is the capital of France?").First(); Assert.Equal("Paris",res.Response); diff --git a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorFunctionalTests.cs b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorFunctionalTests.cs index 3c8a9a7d..7d002b46 100644 --- a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorFunctionalTests.cs +++ b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorFunctionalTests.cs @@ -308,4 +308,43 @@ public void Insert() Assert.Equal(simpleVectorizedVector.Value, res.SimpleVectorizedVector.Value); Assert.Equal(simpleVectorizedVector.Embedding, res.SimpleVectorizedVector.Embedding); } + + [Fact] + public void OpenAIQueryTest() + { + _connection.DropIndexAndAssociatedRecords(typeof(OpenAIQuery)); + _connection.CreateIndex(typeof(OpenAIQuery)); + + var collection = new RedisCollection(_connection); + var query = new OpenAIQuery + { + Language = "en_us", + Prompt = Vector.Of("What is the Capital of France?"), + Response = "Paris", + TimeStamp = DateTime.Now - TimeSpan.FromHours(3) + }; + collection.Insert(query); + var queryPrompt = Vector.Of("What really is the Capital of France?"); + var result = collection.First(x => x.Prompt.VectorRange(queryPrompt, .15)); + + Assert.Equal("Paris", result.Response); + Assert.NotNull(queryPrompt.Embedding); + + result = collection.NearestNeighbors(x => x.Prompt, 1, queryPrompt).First(); + Assert.Equal("Paris", result.Response); + Assert.NotNull(queryPrompt.Embedding); + + result = collection.Where(x=>x.Language == "en_us").NearestNeighbors(x => x.Prompt, 1, queryPrompt).First(); + Assert.Equal("Paris", result.Response); + Assert.NotNull(queryPrompt.Embedding); + + result = collection.First(x=>x.Language == "en_us" && x.Prompt.VectorRange(queryPrompt, .15)); + Assert.Equal("Paris", result.Response); + Assert.NotNull(queryPrompt.Embedding); + + var ts = DateTimeOffset.Now - TimeSpan.FromHours(4); + result = collection.First(x=>x.TimeStamp > ts && x.Prompt.VectorRange(queryPrompt, .15)); + Assert.Equal("Paris", result.Response); + Assert.NotNull(queryPrompt.Embedding); + } } \ No newline at end of file From 9a3a25755642fb10be033b50f613beaf41501f4e Mon Sep 17 00:00:00 2001 From: slorello89 Date: Thu, 2 Nov 2023 16:38:21 -0400 Subject: [PATCH 18/36] updating docker image to .NET 7 --- dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dockerfile b/dockerfile index c2e45881..f4fe8a0a 100644 --- a/dockerfile +++ b/dockerfile @@ -1,4 +1,4 @@ -FROM mcr.microsoft.com/dotnet/sdk:6.0 +FROM mcr.microsoft.com/dotnet/sdk:7.0 WORKDIR /app From ecc0a91bcc9746ca1bd75296cb2bfd69ebb08c1b Mon Sep 17 00:00:00 2001 From: slorello89 Date: Fri, 3 Nov 2023 08:52:40 -0400 Subject: [PATCH 19/36] test cleanup --- dockerfile | 3 +-- .../VectorTests/SemanticCachingTests.cs | 6 +++--- .../VectorTests/VectorFunctionalTests.cs | 10 +++++----- test/Redis.OM.Unit.Tests/Redis.OM.Unit.Tests.csproj | 2 +- 4 files changed, 10 insertions(+), 11 deletions(-) diff --git a/dockerfile b/dockerfile index f4fe8a0a..3cc5893c 100644 --- a/dockerfile +++ b/dockerfile @@ -1,10 +1,9 @@ FROM mcr.microsoft.com/dotnet/sdk:7.0 - WORKDIR /app ADD . /app RUN ls /app RUN dotnet restore /app/Redis.OM.sln -ENTRYPOINT ["dotnet","test"] \ No newline at end of file +ENTRYPOINT ["dotnet", "test", "--framework", "net7.0" ] \ No newline at end of file diff --git a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/SemanticCachingTests.cs b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/SemanticCachingTests.cs index 674bb439..680cd368 100644 --- a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/SemanticCachingTests.cs +++ b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/SemanticCachingTests.cs @@ -15,7 +15,7 @@ public SemanticCachingTests(RedisSetup setup) _provider = setup.Provider; } - [Fact] + [SkipIfMissingEnvVar("REDIS_OM_OAI_TOKEN")] public void OpenAISemanticCache() { var token = Environment.GetEnvironmentVariable("REDIS_OM_OAI_TOKEN"); @@ -27,7 +27,7 @@ public void OpenAISemanticCache() Assert.True(res.Score < .15); } - [Fact] + [SkipIfMissingEnvVar("REDIS_OM_HF_TOKEN")] public void HuggingFaceSemanticCache() { var token = Environment.GetEnvironmentVariable("REDIS_OM_HF_TOKEN"); @@ -39,7 +39,7 @@ public void HuggingFaceSemanticCache() Assert.True(res.Score < .15); } - [Fact] + [SkipIfMissingEnvVar("REDIS_OM_AZURE_OAI_TOKEN")] public void AzureOpenAISemanticCache() { var token = Environment.GetEnvironmentVariable("REDIS_OM_AZURE_OAI_TOKEN"); diff --git a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorFunctionalTests.cs b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorFunctionalTests.cs index 7d002b46..354fb52b 100644 --- a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorFunctionalTests.cs +++ b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorFunctionalTests.cs @@ -17,7 +17,7 @@ public VectorFunctionalTests(RedisSetup setup) _connection = setup.Connection; } - [Fact] + [SkipIfMissingEnvVar("REDIS_OM_HF_TOKEN")] public void TestHuggingFaceVectorizer() { _connection.DropIndexAndAssociatedRecords(typeof(HuggingFaceVectors)); @@ -39,7 +39,7 @@ public void TestHuggingFaceVectorizer() Assert.Equal(obj.Sentence.Value, res.Sentence.Value); } - [Fact] + [SkipIfMissingEnvVar("REDIS_OM_HF_TOKEN")] public void TestParis() { _connection.DropIndexAndAssociatedRecords(typeof(HuggingFaceVectors)); @@ -64,7 +64,7 @@ public void TestParis() Assert.Equal(obj.Sentence.Embedding, res.Sentence.Embedding); } - [Fact] + [SkipIfMissingEnvVar("REDIS_OM_OAI_TOKEN")] public void TestOpenAIVectorizer() { _connection.DropIndexAndAssociatedRecords(typeof(OpenAIVectors)); @@ -87,7 +87,7 @@ public void TestOpenAIVectorizer() Assert.Equal(obj.Sentence.Embedding, res.Sentence.Embedding); } - [Fact] + [SkipIfMissingEnvVar("REDIS_OM_OAI_TOKEN")] public void TestOpenAIVectorRange() { _connection.DropIndexAndAssociatedRecords(typeof(OpenAIVectors)); @@ -309,7 +309,7 @@ public void Insert() Assert.Equal(simpleVectorizedVector.Embedding, res.SimpleVectorizedVector.Embedding); } - [Fact] + [SkipIfMissingEnvVar("REDIS_OM_OAI_TOKEN")] public void OpenAIQueryTest() { _connection.DropIndexAndAssociatedRecords(typeof(OpenAIQuery)); diff --git a/test/Redis.OM.Unit.Tests/Redis.OM.Unit.Tests.csproj b/test/Redis.OM.Unit.Tests/Redis.OM.Unit.Tests.csproj index 62b888a6..f19b411a 100644 --- a/test/Redis.OM.Unit.Tests/Redis.OM.Unit.Tests.csproj +++ b/test/Redis.OM.Unit.Tests/Redis.OM.Unit.Tests.csproj @@ -1,6 +1,6 @@  - net6.0 + net6.0;net7.0 false From 9d09c3aa15eafefc499ab66dbc2acdaa3fc4018b Mon Sep 17 00:00:00 2001 From: slorello89 Date: Fri, 3 Nov 2023 09:27:55 -0400 Subject: [PATCH 20/36] slight tweak to NearestNeighbors/VectorRange APIs --- README.md | 4 +-- .../Common/ExpressionParserUtilities.cs | 15 ++++++++-- src/Redis.OM/SearchExtensions.cs | 28 ++++++++++++++++- src/Redis.OM/Vectors.cs | 25 ++++++++++++++++ .../VectorTests/VectorFunctionalTests.cs | 30 +++++++++++++++---- 5 files changed, 90 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 514f58cf..99db3274 100644 --- a/README.md +++ b/README.md @@ -340,10 +340,10 @@ The Vectorizer will manage the embedding generation for you without you having t #### Query Vectors in Redis -To query vector fields in Redis, all you need to do is use the `VectorRange` method on a vector within our normal LINQ queries, and/or use the `NearestNeighbors` with whatever other filters you want to use, here's some examples: +To query vector fields in Redis, all you need to do is use the `VectorRange` method on a vector within a normal LINQ query, and/or use the `NearestNeighbors` with whatever other filters you want to use, here's some examples: ```cs -var queryPrompt = Vector.Of("What really is the Capital of France?"); +var queryPrompt = "What really is the Capital of France?"; // simple vector range, find first within .15 var result = collection.First(x => x.Prompt.VectorRange(queryPrompt, .15)); diff --git a/src/Redis.OM/Common/ExpressionParserUtilities.cs b/src/Redis.OM/Common/ExpressionParserUtilities.cs index b167ec65..c2515611 100644 --- a/src/Redis.OM/Common/ExpressionParserUtilities.cs +++ b/src/Redis.OM/Common/ExpressionParserUtilities.cs @@ -1005,16 +1005,25 @@ private static string TranslateVectorRange(MethodCallExpression exp, List().FirstOrDefault(); byte[] bytes; - var operand = (Vector)GetOperand(exp.Arguments[1]); + + var operand = GetOperand(exp.Arguments[1]); if (vectorizer is null) { throw new InvalidOperationException( $"Attempting to run a vector range on a {member.Type} with no provided vectorizer"); } - operand.Embed(vectorizer); + if (operand is not Vector vector) + { + bytes = vectorizer.Vectorize(operand); + } + else + { + vector.Embed(vectorizer); + + bytes = vector.Embedding ?? throw new InvalidOperationException("Embedding was null"); + } - bytes = operand.Embedding ?? throw new InvalidOperationException("Embedding was null"); var distance = (double)((ConstantExpression)exp.Arguments[2]).Value; var distanceArgName = parameters.Count.ToString(); parameters.Add(distance); diff --git a/src/Redis.OM/SearchExtensions.cs b/src/Redis.OM/SearchExtensions.cs index af993cd4..788d0f3d 100644 --- a/src/Redis.OM/SearchExtensions.cs +++ b/src/Redis.OM/SearchExtensions.cs @@ -110,8 +110,9 @@ public static IRedisCollection Where(this IRedisCollection source, Expr /// The indexed type. /// The type of the vector. /// A Redis Collection with a nearest neighbors expression attached to it. - public static IRedisCollection NearestNeighbors(this IRedisCollection source, Expression> expression, int numNeighbors, TKnnType item) + public static IRedisCollection NearestNeighbors(this IRedisCollection source, Expression>> expression, int numNeighbors, Vector item) where T : notnull + where TKnnType : class { var collection = (RedisCollection)source; var booleanExpression = collection.BooleanExpression; @@ -123,6 +124,31 @@ public static IRedisCollection NearestNeighbors(this IRedisColle return new RedisCollection((RedisQueryProvider)source.Provider, exp, source.StateManager, booleanExpression, source.SaveState, source.ChunkSize); } + /// + /// Finds nearest neighbors to provided vector. + /// + /// The source. + /// The expression yielding the field to search on. + /// Number of neighbors to search for. + /// The vector or item to search on. + /// The indexed type. + /// The type of the vector. + /// A Redis Collection with a nearest neighbors expression attached to it. + public static IRedisCollection NearestNeighbors(this IRedisCollection source, Expression>> expression, int numNeighbors, TKnnType item) + where T : notnull + where TKnnType : class + { + var collection = (RedisCollection)source; + var booleanExpression = collection.BooleanExpression; + + var vector = Vector.Of(item); + var exp = Expression.Call( + null, + GetMethodInfo(NearestNeighbors, source, expression, numNeighbors, vector), + new[] { source.Expression, Expression.Quote(expression), Expression.Constant(numNeighbors), Expression.Constant(vector) }); + return new RedisCollection((RedisQueryProvider)source.Provider, exp, source.StateManager, booleanExpression, source.SaveState, source.ChunkSize); + } + /// /// Specifies which items to pull out of Redis. /// diff --git a/src/Redis.OM/Vectors.cs b/src/Redis.OM/Vectors.cs index dd62db81..33199769 100644 --- a/src/Redis.OM/Vectors.cs +++ b/src/Redis.OM/Vectors.cs @@ -31,5 +31,30 @@ public static bool VectorRange(this Vector obj, Vector comparisonObject /// Whether the queried vector is within the allowable distance. public static bool VectorRange(this Vector obj, Vector comparisonObject, double range, string scoreName) where T : class => throw new NotImplementedException("This method is only meant to be run within a query of Redis."); + + /// + /// Placeholder method to allow you to perform vector range operations. Only meant to be run + /// in context of a query. + /// + /// The vector field. + /// The comparison object. + /// The allowable distance from the provided object. + /// The type to compare. + /// Whether the queried vector is within the allowable distance. + public static bool VectorRange(this Vector obj, T comparisonObject, double range) + where T : class => throw new NotImplementedException("This method is only meant to be run within a query of Redis."); + + /// + /// Placeholder method to allow you to perform vector range operations. Only meant to be run + /// in context of a query. + /// + /// The vector field. + /// The comparison object. + /// The allowable distance from the provided object. + /// The name of the score in the output. + /// The type to compare. + /// Whether the queried vector is within the allowable distance. + public static bool VectorRange(this Vector obj, T comparisonObject, double range, string scoreName) + where T : class => throw new NotImplementedException("This method is only meant to be run within a query of Redis."); } } \ No newline at end of file diff --git a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorFunctionalTests.cs b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorFunctionalTests.cs index 354fb52b..9661bb50 100644 --- a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorFunctionalTests.cs +++ b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorFunctionalTests.cs @@ -148,6 +148,29 @@ public void MultiRangeOnSameProperty() [Fact] public void RangeAndKnn() + { + _connection.CreateIndex(typeof(ObjectWithVector)); + var collection = new RedisCollection(_connection); + var simpleHnswVector = Vector.Of(Enumerable.Range(0, 10).Select(x => (double)x).ToArray()); + var simpleVectorizedVector = Vector.Of("FooBarBaz"); + collection.Insert(new ObjectWithVector + { + Id = "helloWorld", + SimpleHnswVector = simpleHnswVector, + SimpleVectorizedVector = simpleVectorizedVector + }); + var queryVector =Enumerable.Range(0, 10).Select(x => (double)x).ToArray(); + queryVector[0] += 2; + var stringQueryVector = Vector.Of("FooBarBaz"); + var res = collection.NearestNeighbors(x=>x.SimpleVectorizedVector, 1, stringQueryVector) + .First(x => x.SimpleHnswVector.VectorRange(queryVector, 5, "range")); + Assert.Equal("helloWorld", res.Id); + Assert.Equal(4, res.VectorScores.RangeScore); + Assert.Equal(0, res.VectorScores.NearestNeighborsScore); + } + + [Fact] + public void RangeAndKnnWithVector() { _connection.CreateIndex(typeof(ObjectWithVector)); var collection = new RedisCollection(_connection); @@ -324,27 +347,22 @@ public void OpenAIQueryTest() TimeStamp = DateTime.Now - TimeSpan.FromHours(3) }; collection.Insert(query); - var queryPrompt = Vector.Of("What really is the Capital of France?"); + var queryPrompt ="What really is the Capital of France?"; var result = collection.First(x => x.Prompt.VectorRange(queryPrompt, .15)); Assert.Equal("Paris", result.Response); - Assert.NotNull(queryPrompt.Embedding); result = collection.NearestNeighbors(x => x.Prompt, 1, queryPrompt).First(); Assert.Equal("Paris", result.Response); - Assert.NotNull(queryPrompt.Embedding); result = collection.Where(x=>x.Language == "en_us").NearestNeighbors(x => x.Prompt, 1, queryPrompt).First(); Assert.Equal("Paris", result.Response); - Assert.NotNull(queryPrompt.Embedding); result = collection.First(x=>x.Language == "en_us" && x.Prompt.VectorRange(queryPrompt, .15)); Assert.Equal("Paris", result.Response); - Assert.NotNull(queryPrompt.Embedding); var ts = DateTimeOffset.Now - TimeSpan.FromHours(4); result = collection.First(x=>x.TimeStamp > ts && x.Prompt.VectorRange(queryPrompt, .15)); Assert.Equal("Paris", result.Response); - Assert.NotNull(queryPrompt.Embedding); } } \ No newline at end of file From 980e737427d3a99a60eeb5f4d4e581b10fcac3d7 Mon Sep 17 00:00:00 2001 From: slorello89 Date: Fri, 3 Nov 2023 15:24:45 -0400 Subject: [PATCH 21/36] updates per Tylers comments --- README.md | 15 ++++++++------- ...OpenAIQuery.cs => OpenAICompletionResponse.cs} | 4 ++-- .../VectorTests/SemanticCachingTests.cs | 2 +- .../VectorTests/VectorFunctionalTests.cs | 11 +++++++---- 4 files changed, 18 insertions(+), 14 deletions(-) rename test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/{OpenAIQuery.cs => OpenAICompletionResponse.cs} (76%) diff --git a/README.md b/README.md index 99db3274..9b87fa7b 100644 --- a/README.md +++ b/README.md @@ -294,20 +294,20 @@ Redis OM .NET also supports storing and querying Vectors stored in Redis. A `Vector` is a representation of an object that can be transformed into a vector by a Vectorizer. -A `VectorizerAttribute` is the abstract class you use to decorate your Vector fields, it is responsible for defining the logic to convert your Vectors into Embeddings. In the package `Redis.OM.Vectorizers` we provide vectorizers for HuggingFace, OpenAI, and AzureOpenAI to allow you to easily integrate them into your workflows. +A `VectorizerAttribute` is the abstract class you use to decorate your Vector fields, it is responsible for defining the logic to convert the object's that `Vector` is a container for into actual vector embeddings needed. In the package `Redis.OM.Vectorizers` we provide vectorizers for HuggingFace, OpenAI, and AzureOpenAI to allow you to easily integrate them into your workflows. #### Define a Vector in your Model. -To define a vector in your model, simply decorate a `Vector` field with and `Indexed` and a `Vectorizer` attribute (in this case we'll use OpenAI): +To define a vector in your model, simply decorate a `Vector` field with an `Indexed` attribute which defines the algorithm and algorithmic parameters and a `Vectorizer` attribute which defines the shape of the vectors, (in this case we'll use OpenAI): ```cs [Document(StorageType = StorageType.Json)] -public class OpenAIQuery +public class OpenAICompletionResponse { [RedisIdField] public string Id { get; set; } - [Indexed(DistanceMetric = DistanceMetric.COSINE)] + [Indexed(DistanceMetric = DistanceMetric.COSINE, Algorithm = VectorAlgorithm.HNSW, M = 16)] [OpenAIVectorizer] public Vector Prompt { get; set; } @@ -326,7 +326,8 @@ public class OpenAIQuery With the vector defined in our model, all we need to do is create Vectors of the generic type, and insert them with our model. Using our `RedisCollection`, you can do this by simply using `Insert`: ```cs -var query = new OpenAIQuery +var collection = _provider.RedisCollection(); +var query = new OpenAICompletionResponse { Language = "en_us", Prompt = Vector.Of("What is the Capital of France?"), @@ -340,7 +341,7 @@ The Vectorizer will manage the embedding generation for you without you having t #### Query Vectors in Redis -To query vector fields in Redis, all you need to do is use the `VectorRange` method on a vector within a normal LINQ query, and/or use the `NearestNeighbors` with whatever other filters you want to use, here's some examples: +To query vector fields in Redis, all you need to do is use the `VectorRange` method on a vector within our normal LINQ queries, and/or use the `NearestNeighbors` with whatever other filters you want to use, here's some examples: ```cs var queryPrompt = "What really is the Capital of France?"; @@ -381,7 +382,7 @@ The Vectorizers provided by the `Redis.OM.Vectorizers` package have some configu Redis OM also provides the ability to use Semantic Caching, as well as providers for OpenAI, HuggingFace, and Azure OpenAI to perform semantic caching. To use a Semantic Cache, simply pull one out of the RedisConnectionProvider and use `Store` to insert items, and `GetSimilar` to retrieve items. For example: ```cs -var cache = _provider.OpenAISemanticCache(token); +var cache = _provider.OpenAISemanticCache(token, threshold: .15); cache.Store("What is the capital of France?", "Paris"); var res = cache.GetSimilar("What really is the capital of France?").First(); ``` diff --git a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/OpenAIQuery.cs b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/OpenAICompletionResponse.cs similarity index 76% rename from test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/OpenAIQuery.cs rename to test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/OpenAICompletionResponse.cs index 399ec004..a8ef2dab 100644 --- a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/OpenAIQuery.cs +++ b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/OpenAICompletionResponse.cs @@ -5,12 +5,12 @@ namespace Redis.OM.Unit.Tests; [Document(StorageType = StorageType.Json)] -public class OpenAIQuery +public class OpenAICompletionResponse { [RedisIdField] public string Id { get; set; } - [Indexed(DistanceMetric = DistanceMetric.COSINE)] + [Indexed(DistanceMetric = DistanceMetric.COSINE, Algorithm = VectorAlgorithm.HNSW, M = 16)] [OpenAIVectorizer] public Vector Prompt { get; set; } diff --git a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/SemanticCachingTests.cs b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/SemanticCachingTests.cs index 680cd368..8b61203d 100644 --- a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/SemanticCachingTests.cs +++ b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/SemanticCachingTests.cs @@ -20,7 +20,7 @@ public void OpenAISemanticCache() { var token = Environment.GetEnvironmentVariable("REDIS_OM_OAI_TOKEN"); Assert.NotNull(token); - var cache = _provider.OpenAISemanticCache(token); + var cache = _provider.OpenAISemanticCache(token, threshold: .15); cache.Store("What is the capital of France?", "Paris"); var res = cache.GetSimilar("What really is the capital of France?").First(); Assert.Equal("Paris",res.Response); diff --git a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorFunctionalTests.cs b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorFunctionalTests.cs index 9661bb50..1a497587 100644 --- a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorFunctionalTests.cs +++ b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorFunctionalTests.cs @@ -335,11 +335,14 @@ public void Insert() [SkipIfMissingEnvVar("REDIS_OM_OAI_TOKEN")] public void OpenAIQueryTest() { - _connection.DropIndexAndAssociatedRecords(typeof(OpenAIQuery)); - _connection.CreateIndex(typeof(OpenAIQuery)); + var provider = new RedisConnectionProvider(""); + + provider.RedisCollection(); + _connection.DropIndexAndAssociatedRecords(typeof(OpenAICompletionResponse)); + _connection.CreateIndex(typeof(OpenAICompletionResponse)); - var collection = new RedisCollection(_connection); - var query = new OpenAIQuery + var collection = new RedisCollection(_connection); + var query = new OpenAICompletionResponse { Language = "en_us", Prompt = Vector.Of("What is the Capital of France?"), From efa7d339e3b5ce5281c05ddad4e8021778faf85f Mon Sep 17 00:00:00 2001 From: slorello89 Date: Fri, 3 Nov 2023 16:06:32 -0400 Subject: [PATCH 22/36] changing query -> CompletionResult and queryPrompt -> prompt --- README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 9b87fa7b..ec497b6f 100644 --- a/README.md +++ b/README.md @@ -327,14 +327,14 @@ With the vector defined in our model, all we need to do is create Vectors of the ```cs var collection = _provider.RedisCollection(); -var query = new OpenAICompletionResponse +var completionResult = new OpenAICompletionResponse { Language = "en_us", Prompt = Vector.Of("What is the Capital of France?"), Response = "Paris", TimeStamp = DateTime.Now - TimeSpan.FromHours(3) }; -collection.Insert(query); +collection.Insert(completionResult); ``` The Vectorizer will manage the embedding generation for you without you having to intervene. @@ -344,20 +344,20 @@ The Vectorizer will manage the embedding generation for you without you having t To query vector fields in Redis, all you need to do is use the `VectorRange` method on a vector within our normal LINQ queries, and/or use the `NearestNeighbors` with whatever other filters you want to use, here's some examples: ```cs -var queryPrompt = "What really is the Capital of France?"; +var prompt = "What really is the Capital of France?"; // simple vector range, find first within .15 -var result = collection.First(x => x.Prompt.VectorRange(queryPrompt, .15)); +var result = collection.First(x => x.Prompt.VectorRange(prompt, .15)); // simple nearest neighbors query, finds first nearest neighbor -result = collection.NearestNeighbors(x => x.Prompt, 1, queryPrompt).First(); +result = collection.NearestNeighbors(x => x.Prompt, 1, prompt).First(); // hybrid query, pre-filters result set for english responses, then runs a nearest neighbors search. -result = collection.Where(x=>x.Language == "en_us").NearestNeighbors(x => x.Prompt, 1, queryPrompt).First(); +result = collection.Where(x=>x.Language == "en_us").NearestNeighbors(x => x.Prompt, 1, prompt).First(); // hybrid query, pre-filters responses newer than 4 hours, and finds first result within .15 var ts = DateTimeOffset.Now - TimeSpan.FromHours(4); -result = collection.First(x=>x.TimeStamp > ts && x.Prompt.VectorRange(queryPrompt, .15)); +result = collection.First(x=>x.TimeStamp > ts && x.Prompt.VectorRange(prompt, .15)); ``` #### What Happens to the Embeddings? From d33e93555a45b99fedca3449aae0fcbff90d649e Mon Sep 17 00:00:00 2001 From: slorello89 Date: Wed, 22 Nov 2023 14:53:11 -0500 Subject: [PATCH 23/36] adding setter for embedding --- src/Redis.OM/Vector.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Redis.OM/Vector.cs b/src/Redis.OM/Vector.cs index ea27c4fc..9e268a12 100644 --- a/src/Redis.OM/Vector.cs +++ b/src/Redis.OM/Vector.cs @@ -9,9 +9,9 @@ namespace Redis.OM public abstract class Vector { /// - /// Gets the Embedding. + /// Gets or sets the Embedding. You may set the embedding yourself, if it's not set when Redis OM inserts the vector, it will generate it for you. /// - public byte[]? Embedding { get; internal set; } + public byte[]? Embedding { get; set; } /// /// Gets the embedding represented as an array of floats. From d7cdae8c5d53261e0d70c48ed67a651239acc7ff Mon Sep 17 00:00:00 2001 From: slorello89 Date: Mon, 27 Nov 2023 09:08:45 -0500 Subject: [PATCH 24/36] Resnet18 and AllMiniLML6V2 vectorizers --- .gitattributes | 2 + Redis.OM.sln | 44 +++++ .../AllMiniLML6V2Tokenizer.cs | 37 ++++ .../Redis.OM.Vectorizers.AllMiniLML6V2.csproj | 25 +++ .../Resources/model.onnx | 3 + .../Resources/vocab.txt | 3 + .../SentenceVectorizer.cs | 178 ++++++++++++++++++ .../SentenceVectorizerAttribute.cs | 13 ++ .../Tokenizers/CasedTokenizer.cs | 14 ++ .../Tokenizers/StringExtensions.cs | 25 +++ .../Tokenizers/TokenizerBase.cs | 117 ++++++++++++ .../Tokenizers/Tokens.cs | 10 + .../Tokenizers/UncasedTokenizer.cs | 15 ++ .../FilePathImageVectorizer.cs | 46 +++++ .../FilePathImageVectorizerAttribute.cs | 12 ++ .../ImageModelObjects.cs | 9 + .../Redis.OM.Vectorizers.Resnet18.csproj | 20 ++ .../UriImageVectorizer.cs | 61 ++++++ .../UriImageVectorizerAttribute.cs | 12 ++ .../DocWithVector.cs | 20 ++ .../Redis.OM.Vectorizer.Tests/GlobalUsings.cs | 1 + .../Redis.OM.Vectorizer.Tests.csproj | 32 ++++ .../SentenceVectorizerTests.cs | 24 +++ 23 files changed, 723 insertions(+) create mode 100644 .gitattributes create mode 100644 src/Redis.OM.Vectorizers.AllMiniLML6V2/AllMiniLML6V2Tokenizer.cs create mode 100644 src/Redis.OM.Vectorizers.AllMiniLML6V2/Redis.OM.Vectorizers.AllMiniLML6V2.csproj create mode 100644 src/Redis.OM.Vectorizers.AllMiniLML6V2/Resources/model.onnx create mode 100644 src/Redis.OM.Vectorizers.AllMiniLML6V2/Resources/vocab.txt create mode 100644 src/Redis.OM.Vectorizers.AllMiniLML6V2/SentenceVectorizer.cs create mode 100644 src/Redis.OM.Vectorizers.AllMiniLML6V2/SentenceVectorizerAttribute.cs create mode 100644 src/Redis.OM.Vectorizers.AllMiniLML6V2/Tokenizers/CasedTokenizer.cs create mode 100644 src/Redis.OM.Vectorizers.AllMiniLML6V2/Tokenizers/StringExtensions.cs create mode 100644 src/Redis.OM.Vectorizers.AllMiniLML6V2/Tokenizers/TokenizerBase.cs create mode 100644 src/Redis.OM.Vectorizers.AllMiniLML6V2/Tokenizers/Tokens.cs create mode 100644 src/Redis.OM.Vectorizers.AllMiniLML6V2/Tokenizers/UncasedTokenizer.cs create mode 100644 src/Redis.OM.Vectorizers.Resnet18/FilePathImageVectorizer.cs create mode 100644 src/Redis.OM.Vectorizers.Resnet18/FilePathImageVectorizerAttribute.cs create mode 100644 src/Redis.OM.Vectorizers.Resnet18/ImageModelObjects.cs create mode 100644 src/Redis.OM.Vectorizers.Resnet18/Redis.OM.Vectorizers.Resnet18.csproj create mode 100644 src/Redis.OM.Vectorizers.Resnet18/UriImageVectorizer.cs create mode 100644 src/Redis.OM.Vectorizers.Resnet18/UriImageVectorizerAttribute.cs create mode 100644 test/Redis.OM.Vectorizer.Tests/DocWithVector.cs create mode 100644 test/Redis.OM.Vectorizer.Tests/GlobalUsings.cs create mode 100644 test/Redis.OM.Vectorizer.Tests/Redis.OM.Vectorizer.Tests.csproj create mode 100644 test/Redis.OM.Vectorizer.Tests/SentenceVectorizerTests.cs diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..c3339f17 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +*.onnx filter=lfs diff=lfs merge=lfs -text +src/Redis.OM.Vectorizers.AllMiniLML6V2/Resources/vocab.txt filter=lfs diff=lfs merge=lfs -text diff --git a/Redis.OM.sln b/Redis.OM.sln index 8fa8067b..266ef612 100644 --- a/Redis.OM.sln +++ b/Redis.OM.sln @@ -14,6 +14,12 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Redis.OM.Vectorizers", "src EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Vectorizers", "Vectorizers", "{452DC80B-8195-44E8-A376-C246619492A8}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Redis.OM.Vectorizers.AllMiniLML6V2", "src\Redis.OM.Vectorizers.AllMiniLML6V2\Redis.OM.Vectorizers.AllMiniLML6V2.csproj", "{081DEE32-9B26-44C6-B377-456E862D3813}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Redis.OM.Vectorizer.Tests", "test\Redis.OM.Vectorizer.Tests\Redis.OM.Vectorizer.Tests.csproj", "{7C3E1D79-408C-45E9-931C-12195DFA268D}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Redis.OM.Vectorizers.Resnet18", "src\Redis.OM.Vectorizers.Resnet18\Redis.OM.Vectorizers.Resnet18.csproj", "{FE22706B-9A28-4045-9581-2058F32C4193}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -72,6 +78,42 @@ Global {4B9F4623-3126-48B7-B690-F28F702A4717}.Release|x64.Build.0 = Release|Any CPU {4B9F4623-3126-48B7-B690-F28F702A4717}.Release|x86.ActiveCfg = Release|Any CPU {4B9F4623-3126-48B7-B690-F28F702A4717}.Release|x86.Build.0 = Release|Any CPU + {081DEE32-9B26-44C6-B377-456E862D3813}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {081DEE32-9B26-44C6-B377-456E862D3813}.Debug|Any CPU.Build.0 = Debug|Any CPU + {081DEE32-9B26-44C6-B377-456E862D3813}.Debug|x64.ActiveCfg = Debug|Any CPU + {081DEE32-9B26-44C6-B377-456E862D3813}.Debug|x64.Build.0 = Debug|Any CPU + {081DEE32-9B26-44C6-B377-456E862D3813}.Debug|x86.ActiveCfg = Debug|Any CPU + {081DEE32-9B26-44C6-B377-456E862D3813}.Debug|x86.Build.0 = Debug|Any CPU + {081DEE32-9B26-44C6-B377-456E862D3813}.Release|Any CPU.ActiveCfg = Release|Any CPU + {081DEE32-9B26-44C6-B377-456E862D3813}.Release|Any CPU.Build.0 = Release|Any CPU + {081DEE32-9B26-44C6-B377-456E862D3813}.Release|x64.ActiveCfg = Release|Any CPU + {081DEE32-9B26-44C6-B377-456E862D3813}.Release|x64.Build.0 = Release|Any CPU + {081DEE32-9B26-44C6-B377-456E862D3813}.Release|x86.ActiveCfg = Release|Any CPU + {081DEE32-9B26-44C6-B377-456E862D3813}.Release|x86.Build.0 = Release|Any CPU + {7C3E1D79-408C-45E9-931C-12195DFA268D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {7C3E1D79-408C-45E9-931C-12195DFA268D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {7C3E1D79-408C-45E9-931C-12195DFA268D}.Debug|x64.ActiveCfg = Debug|Any CPU + {7C3E1D79-408C-45E9-931C-12195DFA268D}.Debug|x64.Build.0 = Debug|Any CPU + {7C3E1D79-408C-45E9-931C-12195DFA268D}.Debug|x86.ActiveCfg = Debug|Any CPU + {7C3E1D79-408C-45E9-931C-12195DFA268D}.Debug|x86.Build.0 = Debug|Any CPU + {7C3E1D79-408C-45E9-931C-12195DFA268D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {7C3E1D79-408C-45E9-931C-12195DFA268D}.Release|Any CPU.Build.0 = Release|Any CPU + {7C3E1D79-408C-45E9-931C-12195DFA268D}.Release|x64.ActiveCfg = Release|Any CPU + {7C3E1D79-408C-45E9-931C-12195DFA268D}.Release|x64.Build.0 = Release|Any CPU + {7C3E1D79-408C-45E9-931C-12195DFA268D}.Release|x86.ActiveCfg = Release|Any CPU + {7C3E1D79-408C-45E9-931C-12195DFA268D}.Release|x86.Build.0 = Release|Any CPU + {FE22706B-9A28-4045-9581-2058F32C4193}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {FE22706B-9A28-4045-9581-2058F32C4193}.Debug|Any CPU.Build.0 = Debug|Any CPU + {FE22706B-9A28-4045-9581-2058F32C4193}.Debug|x64.ActiveCfg = Debug|Any CPU + {FE22706B-9A28-4045-9581-2058F32C4193}.Debug|x64.Build.0 = Debug|Any CPU + {FE22706B-9A28-4045-9581-2058F32C4193}.Debug|x86.ActiveCfg = Debug|Any CPU + {FE22706B-9A28-4045-9581-2058F32C4193}.Debug|x86.Build.0 = Debug|Any CPU + {FE22706B-9A28-4045-9581-2058F32C4193}.Release|Any CPU.ActiveCfg = Release|Any CPU + {FE22706B-9A28-4045-9581-2058F32C4193}.Release|Any CPU.Build.0 = Release|Any CPU + {FE22706B-9A28-4045-9581-2058F32C4193}.Release|x64.ActiveCfg = Release|Any CPU + {FE22706B-9A28-4045-9581-2058F32C4193}.Release|x64.Build.0 = Release|Any CPU + {FE22706B-9A28-4045-9581-2058F32C4193}.Release|x86.ActiveCfg = Release|Any CPU + {FE22706B-9A28-4045-9581-2058F32C4193}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -81,6 +123,8 @@ Global {E3A31119-E4F1-4793-B5C2-ED2D51502B01} = {8D9ECCFF-E022-4B68-BB43-228CEA248DEC} {452DC80B-8195-44E8-A376-C246619492A8} = {8D9ECCFF-E022-4B68-BB43-228CEA248DEC} {4B9F4623-3126-48B7-B690-F28F702A4717} = {452DC80B-8195-44E8-A376-C246619492A8} + {081DEE32-9B26-44C6-B377-456E862D3813} = {452DC80B-8195-44E8-A376-C246619492A8} + {FE22706B-9A28-4045-9581-2058F32C4193} = {452DC80B-8195-44E8-A376-C246619492A8} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {E5752441-184B-4F17-BAD0-93823AC68607} diff --git a/src/Redis.OM.Vectorizers.AllMiniLML6V2/AllMiniLML6V2Tokenizer.cs b/src/Redis.OM.Vectorizers.AllMiniLML6V2/AllMiniLML6V2Tokenizer.cs new file mode 100644 index 00000000..06b7f7a8 --- /dev/null +++ b/src/Redis.OM.Vectorizers.AllMiniLML6V2/AllMiniLML6V2Tokenizer.cs @@ -0,0 +1,37 @@ +using System.Reflection; +using Redis.OM.Vectorizers.AllMiniLML6V2.Tokenizers; + +namespace Redis.OM.Vectorizers.AllMiniLML6V2; + +internal class AllMiniLML6V2Tokenizer : UncasedTokenizer +{ + private AllMiniLML6V2Tokenizer(string[] vocabulary) : base(vocabulary) + { + } + + internal static AllMiniLML6V2Tokenizer Create() + { + var assembly = Assembly.GetExecutingAssembly(); + const string fileName = "Redis.OM.Vectorizers.AllMiniLML6V2.Resources.vocab.txt"; + using var stream = assembly.GetManifestResourceStream(fileName); + if (stream is null) + { + throw new FileNotFoundException("Could not find embedded resource file Resources.vocab.txt"); + } + using var reader = new StreamReader(stream); + + if (stream is null) + { + throw new Exception("Could not open stream reader."); + } + + var vocab = new List(); + string? line; + while ((line = reader.ReadLine()) is not null) + { + vocab.Add(line); + } + + return new AllMiniLML6V2Tokenizer(vocab.ToArray()); + } +} \ No newline at end of file diff --git a/src/Redis.OM.Vectorizers.AllMiniLML6V2/Redis.OM.Vectorizers.AllMiniLML6V2.csproj b/src/Redis.OM.Vectorizers.AllMiniLML6V2/Redis.OM.Vectorizers.AllMiniLML6V2.csproj new file mode 100644 index 00000000..82796b81 --- /dev/null +++ b/src/Redis.OM.Vectorizers.AllMiniLML6V2/Redis.OM.Vectorizers.AllMiniLML6V2.csproj @@ -0,0 +1,25 @@ + + + + net7.0 + enable + enable + + + + + + + + + + + + + + + + + + + diff --git a/src/Redis.OM.Vectorizers.AllMiniLML6V2/Resources/model.onnx b/src/Redis.OM.Vectorizers.AllMiniLML6V2/Resources/model.onnx new file mode 100644 index 00000000..49af6b9d --- /dev/null +++ b/src/Redis.OM.Vectorizers.AllMiniLML6V2/Resources/model.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a64cee3d4134bbdc86eed96e1a660efec58975417204ecfcf134140edb6e0e2 +size 90893304 diff --git a/src/Redis.OM.Vectorizers.AllMiniLML6V2/Resources/vocab.txt b/src/Redis.OM.Vectorizers.AllMiniLML6V2/Resources/vocab.txt new file mode 100644 index 00000000..cada3e34 --- /dev/null +++ b/src/Redis.OM.Vectorizers.AllMiniLML6V2/Resources/vocab.txt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:07eced375cec144d27c900241f3e339478dec958f92fddbc551f295c992038a3 +size 231508 diff --git a/src/Redis.OM.Vectorizers.AllMiniLML6V2/SentenceVectorizer.cs b/src/Redis.OM.Vectorizers.AllMiniLML6V2/SentenceVectorizer.cs new file mode 100644 index 00000000..0f3385f4 --- /dev/null +++ b/src/Redis.OM.Vectorizers.AllMiniLML6V2/SentenceVectorizer.cs @@ -0,0 +1,178 @@ +using System.Reflection; +using Microsoft.ML.OnnxRuntime; +using Microsoft.ML.OnnxRuntime.Tensors; +using Redis.OM.Contracts; +using Redis.OM.Modeling; +using Redis.OM.Vectorizers.AllMiniLML6V2.Tokenizers; + +namespace Redis.OM.Vectorizers.AllMiniLML6V2; + +public class SentenceVectorizer : IVectorizer +{ + public VectorType VectorType => VectorType.FLOAT32; + public int Dim => 384; + private static Lazy Tokenizer => new Lazy(AllMiniLML6V2Tokenizer.Create); + private static Lazy InferenceSession => new Lazy(LoadInferenceSession); + + private static InferenceSession LoadInferenceSession() + { + var file = "Redis.OM.Vectorizers.AllMiniLML6V2.Resources.model.onnx"; + using var stream = Assembly.GetExecutingAssembly().GetManifestResourceStream(file); + if (stream is null) + { + throw new InvalidOperationException("Could not find Model resource"); + } + + var resourceBytes = new byte[stream.Length]; + _ = stream.Read(resourceBytes, 0, resourceBytes.Length); + return new InferenceSession(resourceBytes); + } + + public byte[] Vectorize(string obj) + { + return Encode(new[] { obj })[0].SelectMany(BitConverter.GetBytes).ToArray(); + } + + private static Lazy OutputNames => new (() => InferenceSession.Value.OutputMetadata.Keys.ToArray()); + + public static float[][] Encode(string[] sentences) + { + const int MaxTokens = 512; + var numSentences = sentences.Length; + + var tokenized = sentences.Select(x=>Tokenizer.Value.Tokenize(x)).ToArray(); + + var seqLen = tokenized.Max(t => Math.Min(MaxTokens, t.Count)); + + List<(long[] InputIds, long[] TokenTypeIds, long[] AttentionMask)> encoded = tokenized.Select(tokens => + { + var padding = Enumerable.Repeat(0L, seqLen - Math.Min(MaxTokens, tokens.Count)).ToList(); + + var tokenIndexes = tokens.Take(MaxTokens).Select(token => (long)token.VocabularyIndex).Concat(padding).ToArray(); + var segmentIndexes = tokens.Take(MaxTokens).Select(token => token.SegmentIndex).Concat(padding).ToArray(); + var inputMask = tokens.Take(MaxTokens).Select(o => 1L).Concat(padding).ToArray(); + return (tokenIndexes, TokenTypeIds: segmentIndexes, inputMask); + }).ToList(); + var tokenCount = encoded.First().InputIds.Length; + + long[] flattenIDs = new long[encoded.Sum(s => s.InputIds.Length)]; + long[] flattenAttentionMask = new long[encoded.Sum(s => s.AttentionMask.Length)]; + long[] flattenTokenTypeIds = new long[encoded.Sum(s => s.TokenTypeIds.Length)]; + + var flattenIDsSpan = flattenIDs.AsSpan(); + var flattenAttentionMaskSpan = flattenAttentionMask.AsSpan(); + var flattenTokenTypeIdsSpan = flattenTokenTypeIds.AsSpan(); + + foreach (var (InputIds, TokenTypeIds, AttentionMask) in encoded) + { + InputIds.AsSpan().CopyTo(flattenIDsSpan); + flattenIDsSpan = flattenIDsSpan.Slice(InputIds.Length); + + AttentionMask.AsSpan().CopyTo(flattenAttentionMaskSpan); + flattenAttentionMaskSpan = flattenAttentionMaskSpan.Slice(AttentionMask.Length); + + TokenTypeIds.AsSpan().CopyTo(flattenTokenTypeIdsSpan); + flattenTokenTypeIdsSpan = flattenTokenTypeIdsSpan.Slice(TokenTypeIds.Length); + } + + var dimensions = new[] { numSentences, tokenCount }; + + var input = new NamedOnnxValue[3] + { + NamedOnnxValue.CreateFromTensor("input_ids", new DenseTensor(flattenIDs, dimensions)), + NamedOnnxValue.CreateFromTensor("attention_mask", new DenseTensor(flattenAttentionMask,dimensions)), + NamedOnnxValue.CreateFromTensor("token_type_ids", new DenseTensor(flattenTokenTypeIds, dimensions)) + }; + + using var runOptions = new RunOptions(); + + using var output = InferenceSession.Value.Run(input, OutputNames.Value, runOptions); + + var output_pooled = MeanPooling((DenseTensor)output.First().Value, encoded); + var output_pooled_normalized = Normalize(output_pooled); + + const int embDim = 384; + + var outputFlatten = new float[sentences.Length][]; + + for(int s = 0; s < sentences.Length; s++) + { + var emb = new float[embDim]; + outputFlatten[s] = emb; + + for (int i = 0; i < embDim; i++) + { + emb[i] = output_pooled_normalized[s, i]; + } + } + + return outputFlatten; + } + + public static DenseTensor Normalize(DenseTensor input_dense, float eps = 1e-12f) + { + //Computes sum(abs(x)^2)^(1/2) + + var sentencesCount = input_dense.Dimensions[0]; + var hiddenStates = input_dense.Dimensions[1]; + + var denom_dense = new float [sentencesCount]; + + for (int s = 0; s < sentencesCount; s++) + { + for (int i = 0; i < hiddenStates; i++) + { + denom_dense[s] += input_dense[s, i] * input_dense[s, i]; + } + + denom_dense[s] = MathF.Max(MathF.Sqrt(denom_dense[s]), eps); + } + + for (int s = 0; s < sentencesCount; s++) + { + var invNorm = 1 / denom_dense[s]; + + for (int i = 0; i < hiddenStates; i++) + { + input_dense[s, i] *= invNorm; + } + } + + return input_dense; + } + + + public static DenseTensor MeanPooling(DenseTensor token_embeddings_dense, List<(long[] InputIds, long[] TokenTypeIds, long[] AttentionMask)> encodedSentences, float eps = 1e-9f) + { + var sentencesCount = token_embeddings_dense.Dimensions[0]; + var sentenceLength = token_embeddings_dense.Dimensions[1]; + var hiddenStates = token_embeddings_dense.Dimensions[2]; + + var result = new DenseTensor(new[] { sentencesCount, hiddenStates }); + + for (int s = 0; s < sentencesCount; s++) + { + var maskSum = 0f; + + var attentionMask = encodedSentences[s].AttentionMask; + + for (int t = 0; t < sentenceLength; t++) + { + maskSum += attentionMask[t]; + + for (int i = 0; i < hiddenStates; i++) + { + result[s, i] += token_embeddings_dense[s, t, i] * attentionMask[t]; + } + } + + var invSum = 1f / MathF.Max(maskSum, eps); + for (int i = 0; i < hiddenStates; i++) + { + result[s, i] *= invSum; + } + } + + return result; + } +} \ No newline at end of file diff --git a/src/Redis.OM.Vectorizers.AllMiniLML6V2/SentenceVectorizerAttribute.cs b/src/Redis.OM.Vectorizers.AllMiniLML6V2/SentenceVectorizerAttribute.cs new file mode 100644 index 00000000..138ed6fc --- /dev/null +++ b/src/Redis.OM.Vectorizers.AllMiniLML6V2/SentenceVectorizerAttribute.cs @@ -0,0 +1,13 @@ +using Redis.OM.Contracts; +using Redis.OM.Modeling; + +namespace Redis.OM.Vectorizers.AllMiniLML6V2; + +public class SentenceVectorizerAttribute : VectorizerAttribute +{ + public override VectorType VectorType => Vectorizer.VectorType; + public override int Dim => Vectorizer.Dim; + public override byte[] Vectorize(object obj) => Vectorizer.Vectorize((string)obj); + + public override IVectorizer Vectorizer => new SentenceVectorizer(); +} \ No newline at end of file diff --git a/src/Redis.OM.Vectorizers.AllMiniLML6V2/Tokenizers/CasedTokenizer.cs b/src/Redis.OM.Vectorizers.AllMiniLML6V2/Tokenizers/CasedTokenizer.cs new file mode 100644 index 00000000..2a4f67cd --- /dev/null +++ b/src/Redis.OM.Vectorizers.AllMiniLML6V2/Tokenizers/CasedTokenizer.cs @@ -0,0 +1,14 @@ +namespace Redis.OM.Vectorizers.AllMiniLML6V2.Tokenizers; + +internal abstract class CasedTokenizer : TokenizerBase +{ + protected CasedTokenizer(string[] vocabulary) : base(vocabulary) + { + } + + protected override IEnumerable TokenizeSentence(string text) + { + return text.Split(new string[] { " ", " ", "\r\n" }, StringSplitOptions.None) + .SelectMany(o => o.SplitAndKeep(".,;:\\/?!#$%()=+-*\"'–_`<>&^@{}[]|~'".ToArray())); + } +} \ No newline at end of file diff --git a/src/Redis.OM.Vectorizers.AllMiniLML6V2/Tokenizers/StringExtensions.cs b/src/Redis.OM.Vectorizers.AllMiniLML6V2/Tokenizers/StringExtensions.cs new file mode 100644 index 00000000..956caee4 --- /dev/null +++ b/src/Redis.OM.Vectorizers.AllMiniLML6V2/Tokenizers/StringExtensions.cs @@ -0,0 +1,25 @@ +namespace Redis.OM.Vectorizers.AllMiniLML6V2.Tokenizers; + +internal static class StringExtension +{ + public static IEnumerable SplitAndKeep( + this string inputString, params char[] delimiters) + { + int start = 0, index; + + while ((index = inputString.IndexOfAny(delimiters, start)) != -1) + { + if (index - start > 0) + yield return inputString.Substring(start, index - start); + + yield return inputString.Substring(index, 1); + + start = index + 1; + } + + if (start < inputString.Length) + { + yield return inputString.Substring(start); + } + } +} \ No newline at end of file diff --git a/src/Redis.OM.Vectorizers.AllMiniLML6V2/Tokenizers/TokenizerBase.cs b/src/Redis.OM.Vectorizers.AllMiniLML6V2/Tokenizers/TokenizerBase.cs new file mode 100644 index 00000000..ef092d76 --- /dev/null +++ b/src/Redis.OM.Vectorizers.AllMiniLML6V2/Tokenizers/TokenizerBase.cs @@ -0,0 +1,117 @@ +using System.Text.RegularExpressions; + +namespace Redis.OM.Vectorizers.AllMiniLML6V2.Tokenizers; + +internal abstract class TokenizerBase +{ + protected readonly string[] _vocabulary; + protected readonly Dictionary _vocabularyDict; + + public TokenizerBase(string[] vocabulary) + { + _vocabulary = vocabulary; + _vocabularyDict = new Dictionary(); + + for (int i = 0; i < _vocabulary.Length; i++) + { + _vocabularyDict[_vocabulary[i]] = i; + } + } + + public List<(string Token, int VocabularyIndex, long SegmentIndex)> Tokenize(params string[] texts) + { + IEnumerable tokens = new[] { Tokens.Classification }; + + foreach (var text in texts) + { + tokens = tokens.Concat(TokenizeSentence(text)); + tokens = tokens.Concat(new[] { Tokens.Separation }); + } + + var tokenAndIndex = tokens.SelectMany(TokenizeSubWords).ToArray(); + + var segmentIndexes = SegmentIndex(tokenAndIndex); + return tokenAndIndex.Zip(segmentIndexes, (tokenIndex, segmentIndex) => (tokenIndex.Token, tokenIndex.VocabularyIndex, segmentIndex)).ToList(); + } + + public List<(long InputIds, long TokenTypeIds, long AttentionMask)> Encode(int sequenceLength, params string[] texts) + { + var tokens = Tokenize(texts); + + var padding = Enumerable.Repeat(0L, sequenceLength - tokens.Count).ToArray(); + var tokenIndexes = tokens.Select(token => (long)token.VocabularyIndex).Concat(padding).ToArray(); + var segmentIndexes = tokens.Select(token => token.SegmentIndex).Concat(padding).ToArray(); + var inputMask = tokens.Select(o => 1L).Concat(padding).ToArray(); + + var output = tokenIndexes.Zip(segmentIndexes, Tuple.Create) + .Zip(inputMask, (t, z) => Tuple.Create(t.Item1, t.Item2, z)); + + return output.Select(x => (InputIds: x.Item1, TokenTypeIds: x.Item2, AttentionMask: x.Item3)).ToList(); + } + + private IEnumerable SegmentIndex(IEnumerable<(string token, int index)> tokens) + { + var segmentIndex = 0; + var segmentIndexes = new List(); + + foreach (var (token, index) in tokens) + { + segmentIndexes.Add(segmentIndex); + + if (token == Tokens.Separation) + { + segmentIndex++; + } + } + + return segmentIndexes; + } + + private IEnumerable<(string Token, int VocabularyIndex)> TokenizeSubWords(string word) + { + if (_vocabularyDict.ContainsKey(word)) + { + return new (string, int)[] { (word, _vocabularyDict[word]) }; + } + + var tokens = new List<(string, int)>(); + var remaining = word; + + while (!string.IsNullOrEmpty(remaining) && remaining.Length > 2) + { + string? prefix = null; + int subWordLength = remaining.Length; + while (subWordLength >= 1) + { + string subWord = remaining.Substring(0, subWordLength); + if (!_vocabularyDict.ContainsKey(subWord)) + { + subWordLength--; + continue; + } + + prefix = subWord; + break; + } + + if (prefix == null) + { + tokens.Add((Tokens.Unknown, _vocabularyDict[Tokens.Unknown])); + return tokens; + } + + var regex = new Regex(prefix); + remaining = regex.Replace(remaining, "##", 1); + + tokens.Add((prefix, _vocabularyDict[prefix])); + } + + if (!string.IsNullOrEmpty(word) && !tokens.Any()) + { + tokens.Add((Tokens.Unknown, _vocabularyDict[Tokens.Unknown])); + } + + return tokens; + } + protected abstract IEnumerable TokenizeSentence(string text); +} \ No newline at end of file diff --git a/src/Redis.OM.Vectorizers.AllMiniLML6V2/Tokenizers/Tokens.cs b/src/Redis.OM.Vectorizers.AllMiniLML6V2/Tokenizers/Tokens.cs new file mode 100644 index 00000000..2963258e --- /dev/null +++ b/src/Redis.OM.Vectorizers.AllMiniLML6V2/Tokenizers/Tokens.cs @@ -0,0 +1,10 @@ +namespace Redis.OM.Vectorizers.AllMiniLML6V2.Tokenizers; + +public class Tokens +{ + public const string Padding = ""; + public const string Unknown = "[UNK]"; + public const string Classification = "[CLS]"; + public const string Separation = "[SEP]"; + public const string Mask = "[MASK]"; +} \ No newline at end of file diff --git a/src/Redis.OM.Vectorizers.AllMiniLML6V2/Tokenizers/UncasedTokenizer.cs b/src/Redis.OM.Vectorizers.AllMiniLML6V2/Tokenizers/UncasedTokenizer.cs new file mode 100644 index 00000000..59f109a8 --- /dev/null +++ b/src/Redis.OM.Vectorizers.AllMiniLML6V2/Tokenizers/UncasedTokenizer.cs @@ -0,0 +1,15 @@ +namespace Redis.OM.Vectorizers.AllMiniLML6V2.Tokenizers; + +internal abstract class UncasedTokenizer : TokenizerBase +{ + public UncasedTokenizer(string[] vocabulary) : base(vocabulary) + { + } + + protected override IEnumerable TokenizeSentence(string text) + { + return text.Split(new string[] { " ", " ", "\r\n" }, StringSplitOptions.None) + .SelectMany(o => o.SplitAndKeep(".,;:\\/?!#$%()=+-*\"'–_`<>&^@{}[]|~'".ToArray())) + .Select(o => o.ToLower()); + } +} \ No newline at end of file diff --git a/src/Redis.OM.Vectorizers.Resnet18/FilePathImageVectorizer.cs b/src/Redis.OM.Vectorizers.Resnet18/FilePathImageVectorizer.cs new file mode 100644 index 00000000..943ec15d --- /dev/null +++ b/src/Redis.OM.Vectorizers.Resnet18/FilePathImageVectorizer.cs @@ -0,0 +1,46 @@ +using Microsoft.ML; +using Microsoft.ML.Data; +using Microsoft.ML.Transforms; +using Redis.OM.Contracts; +using Redis.OM.Modeling; + +namespace Redis.OM.Vectorizers.Resnet18; + +public class FilePathImageVectorizer : IVectorizer +{ + public byte[] Vectorize(string obj) + { + var input = new ImageInput() { ImageSource = obj }; + return Vectorize(new [] { input })[0].SelectMany(BitConverter.GetBytes).ToArray(); + } + + public VectorType VectorType => VectorType.FLOAT32; + public int Dim => 512; + + public static Lazy>> Pipeline = new(CreatePipeline); + + private static readonly Lazy MlContext = new(()=>new MLContext()); + + private static EstimatorChain> CreatePipeline() + { + var mlContext = MlContext.Value; + var pipeline = mlContext.Transforms + .LoadImages("ImageObject", "", "ImageSource") + .Append(mlContext.Transforms.ResizeImages("ImageObject", 224, 224)) + .Append(mlContext.Transforms.ExtractPixels("Pixels", "ImageObject")) + .Append(mlContext.Transforms.DnnFeaturizeImage("Features", + m => m.ModelSelector.ResNet18(mlContext, m.OutputColumn, m.InputColumn), "Pixels")); + + return pipeline; + } + + public static float[][] Vectorize(IEnumerable images) + { + var mlContext = MlContext.Value; + var dataView = mlContext.Data.LoadFromEnumerable(images); + + var transformedData = Pipeline.Value.Fit(dataView).Transform(dataView); + var vector = transformedData.GetColumn("Features").ToArray(); + return vector; + } +} \ No newline at end of file diff --git a/src/Redis.OM.Vectorizers.Resnet18/FilePathImageVectorizerAttribute.cs b/src/Redis.OM.Vectorizers.Resnet18/FilePathImageVectorizerAttribute.cs new file mode 100644 index 00000000..d3e48a0a --- /dev/null +++ b/src/Redis.OM.Vectorizers.Resnet18/FilePathImageVectorizerAttribute.cs @@ -0,0 +1,12 @@ +using Redis.OM.Contracts; +using Redis.OM.Modeling; + +namespace Redis.OM.Vectorizers.Resnet18; + +public class FilePathImageVectorizerAttribute : VectorizerAttribute +{ + public override byte[] Vectorize(object obj) => Vectorizer.Vectorize((string)obj); + public override VectorType VectorType => Vectorizer.VectorType; + public override int Dim => Vectorizer.Dim; + public override IVectorizer Vectorizer => new FilePathImageVectorizer(); +} \ No newline at end of file diff --git a/src/Redis.OM.Vectorizers.Resnet18/ImageModelObjects.cs b/src/Redis.OM.Vectorizers.Resnet18/ImageModelObjects.cs new file mode 100644 index 00000000..c22c572a --- /dev/null +++ b/src/Redis.OM.Vectorizers.Resnet18/ImageModelObjects.cs @@ -0,0 +1,9 @@ +using Microsoft.ML.Data; + +namespace Redis.OM.Vectorizers.Resnet18; + +public class ImageInput +{ + [ColumnName(@"ImageSource")] + public string ImageSource { get; set; } +} \ No newline at end of file diff --git a/src/Redis.OM.Vectorizers.Resnet18/Redis.OM.Vectorizers.Resnet18.csproj b/src/Redis.OM.Vectorizers.Resnet18/Redis.OM.Vectorizers.Resnet18.csproj new file mode 100644 index 00000000..e41139e3 --- /dev/null +++ b/src/Redis.OM.Vectorizers.Resnet18/Redis.OM.Vectorizers.Resnet18.csproj @@ -0,0 +1,20 @@ + + + + net7.0 + enable + enable + + + + + + + + + + + + + + diff --git a/src/Redis.OM.Vectorizers.Resnet18/UriImageVectorizer.cs b/src/Redis.OM.Vectorizers.Resnet18/UriImageVectorizer.cs new file mode 100644 index 00000000..15404c79 --- /dev/null +++ b/src/Redis.OM.Vectorizers.Resnet18/UriImageVectorizer.cs @@ -0,0 +1,61 @@ +using System.Drawing; +using Microsoft.ML; +using Microsoft.ML.Data; +using Microsoft.ML.Transforms; +using Microsoft.ML.Transforms.Image; +using Redis.OM.Contracts; +using Redis.OM.Modeling; + +namespace Redis.OM.Vectorizers.Resnet18; + +public class UriImageVectorizer : IVectorizer +{ + public byte[] Vectorize(string obj) + { + var imageStream = Configuration.Instance.Client.GetAsync(obj).Result.Content.ReadAsStream(); + var image = Image.FromStream(imageStream); + var resized = new Bitmap(image, new Size(224, 224)); + var input = new InMemoryImageData() { Image = resized}; + var vector = Vectorize(new [] { input })[0].SelectMany(BitConverter.GetBytes).ToArray(); + return vector; + } + + public VectorType VectorType => VectorType.FLOAT32; + public int Dim => 512; + + public static Lazy>> Pipeline = new(CreatePipeline); + + private static readonly Lazy MlContext = new(()=>new MLContext()); + + private static EstimatorChain> CreatePipeline() + { + var mlContext = MlContext.Value; + var pipeline = mlContext.Transforms.ExtractPixels("Pixels", "Image") + .Append(mlContext.Transforms.DnnFeaturizeImage("Features", + m => m.ModelSelector.ResNet18(mlContext, m.OutputColumn, m.InputColumn), "Pixels")); + + return pipeline; + } + + public static float[][] Vectorize(IEnumerable images) + { + var mlContext = MlContext.Value; + var dataView = mlContext.Data.LoadFromEnumerable(images); + var transformedData = Pipeline.Value.Fit(dataView).Transform(dataView); + var vector = transformedData.GetColumn("Features").ToArray(); + return vector; + } + + public class InMemoryImageData + { + [ImageType(224,224)] + public Bitmap Image; + } + + public class InMemoryImageDataOutput + { + [ColumnName("Output")] + public float[] Output { get; set; } + } + +} \ No newline at end of file diff --git a/src/Redis.OM.Vectorizers.Resnet18/UriImageVectorizerAttribute.cs b/src/Redis.OM.Vectorizers.Resnet18/UriImageVectorizerAttribute.cs new file mode 100644 index 00000000..e06ae3f5 --- /dev/null +++ b/src/Redis.OM.Vectorizers.Resnet18/UriImageVectorizerAttribute.cs @@ -0,0 +1,12 @@ +using Redis.OM.Contracts; +using Redis.OM.Modeling; + +namespace Redis.OM.Vectorizers.Resnet18; + +public class UriImageVectorizerAttribute : VectorizerAttribute +{ + public override byte[] Vectorize(object obj) => Vectorizer.Vectorize((string)obj); + public override VectorType VectorType => Vectorizer.VectorType; + public override int Dim => Vectorizer.Dim; + public override IVectorizer Vectorizer => new UriImageVectorizer(); +} \ No newline at end of file diff --git a/test/Redis.OM.Vectorizer.Tests/DocWithVector.cs b/test/Redis.OM.Vectorizer.Tests/DocWithVector.cs new file mode 100644 index 00000000..42eddf8f --- /dev/null +++ b/test/Redis.OM.Vectorizer.Tests/DocWithVector.cs @@ -0,0 +1,20 @@ +using Redis.OM.Modeling; +using Redis.OM.Vectorizers.AllMiniLML6V2; +using Redis.OM.Vectorizers.Resnet18; + +namespace Redis.OM.Vectorizer.Tests; + +[Document(StorageType = StorageType.Json)] +public class DocWithVector +{ + [RedisIdField] + public string Id { get; set; } + + [Indexed] + [SentenceVectorizer] + public Vector Sentence { get; set; } + + [Indexed] + [UriImageVectorizer] + public Vector ImageUri { get; set; } +} \ No newline at end of file diff --git a/test/Redis.OM.Vectorizer.Tests/GlobalUsings.cs b/test/Redis.OM.Vectorizer.Tests/GlobalUsings.cs new file mode 100644 index 00000000..8c927eb7 --- /dev/null +++ b/test/Redis.OM.Vectorizer.Tests/GlobalUsings.cs @@ -0,0 +1 @@ +global using Xunit; \ No newline at end of file diff --git a/test/Redis.OM.Vectorizer.Tests/Redis.OM.Vectorizer.Tests.csproj b/test/Redis.OM.Vectorizer.Tests/Redis.OM.Vectorizer.Tests.csproj new file mode 100644 index 00000000..7e9e97e6 --- /dev/null +++ b/test/Redis.OM.Vectorizer.Tests/Redis.OM.Vectorizer.Tests.csproj @@ -0,0 +1,32 @@ + + + + net7.0 + enable + enable + latest + + false + true + + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + + + + + + diff --git a/test/Redis.OM.Vectorizer.Tests/SentenceVectorizerTests.cs b/test/Redis.OM.Vectorizer.Tests/SentenceVectorizerTests.cs new file mode 100644 index 00000000..49ba4981 --- /dev/null +++ b/test/Redis.OM.Vectorizer.Tests/SentenceVectorizerTests.cs @@ -0,0 +1,24 @@ +using Redis.OM.Contracts; +using Redis.OM.Unit.Tests; + +namespace Redis.OM.Vectorizer.Tests; + +public class SentenceVectorizerTests +{ + private readonly IRedisConnectionProvider _provider; + public SentenceVectorizerTests() + { + _provider = new RedisConnectionProvider("redis://localhost:6379"); + } + + [Fact] + public void Test() + { + var connection = _provider.Connection; + connection.Set(new DocWithVector + { + Sentence = Vector.Of("Hello world this is Hal."), + ImageUri = Vector.Of("https://triviahappy.com/wp-content/uploads/2014/05/05282014hal.jpg") + }); + } +} \ No newline at end of file From 81db70d2161c8cbcb928d4d3780e4b5b30457431 Mon Sep 17 00:00:00 2001 From: slorello89 Date: Wed, 29 Nov 2023 09:44:21 -0500 Subject: [PATCH 25/36] removing dependency on ML Resnet project (to support transitive distribution of the model files) --- .../Redis.OM.Vectorizers.AllMiniLML6V2.csproj | 25 +++++++++- .../DnnImageModelSelectorExtensions.cs | 47 +++++++++++++++++++ .../FilePathImageVectorizer.cs | 2 +- .../Redis.OM.Vectorizers.Resnet18.csproj | 47 +++++++++++++++++-- .../Redis.OM.Vectorizers.Resnet18.props | 8 ++++ .../Resources/ResNet18Onnx/ResNet18.onnx | 3 ++ .../ResNetPrepOnnx/ResNetPreprocess.onnx | 3 ++ .../UriImageVectorizer.cs | 2 +- 8 files changed, 130 insertions(+), 7 deletions(-) create mode 100644 src/Redis.OM.Vectorizers.Resnet18/DnnImageModelSelectorExtensions.cs create mode 100644 src/Redis.OM.Vectorizers.Resnet18/Redis.OM.Vectorizers.Resnet18.props create mode 100755 src/Redis.OM.Vectorizers.Resnet18/Resources/ResNet18Onnx/ResNet18.onnx create mode 100755 src/Redis.OM.Vectorizers.Resnet18/Resources/ResNetPrepOnnx/ResNetPreprocess.onnx diff --git a/src/Redis.OM.Vectorizers.AllMiniLML6V2/Redis.OM.Vectorizers.AllMiniLML6V2.csproj b/src/Redis.OM.Vectorizers.AllMiniLML6V2/Redis.OM.Vectorizers.AllMiniLML6V2.csproj index 82796b81..568d617e 100644 --- a/src/Redis.OM.Vectorizers.AllMiniLML6V2/Redis.OM.Vectorizers.AllMiniLML6V2.csproj +++ b/src/Redis.OM.Vectorizers.AllMiniLML6V2/Redis.OM.Vectorizers.AllMiniLML6V2.csproj @@ -1,9 +1,23 @@ - net7.0 + net6.0;net7.0 enable enable + Redis.OM.Vectorizers.AllMiniLML6V2 + 0.6.0 + 0.6.0 + https://github.com/redis/redis-om-dotnet/releases/tag/v0.6.0 + Sentence Vectorizer for Redis OM .NET using all-MiniLM-L6-v2 + Redis OM all-MiniLM-L6-v2 Vectorizers + Steve Lorello + Redis Inc + https://github.com/redis/redis-om-dotnet + https://github.com/redis/redis-om-dotnet + Github + redis redisearch AI Vectors + icon-square.png + true @@ -22,4 +36,13 @@ + + + + + + + bin\$(Configuration)\$(TargetFramework)\$(AssemblyName).xml + + diff --git a/src/Redis.OM.Vectorizers.Resnet18/DnnImageModelSelectorExtensions.cs b/src/Redis.OM.Vectorizers.Resnet18/DnnImageModelSelectorExtensions.cs new file mode 100644 index 00000000..19f3e7fe --- /dev/null +++ b/src/Redis.OM.Vectorizers.Resnet18/DnnImageModelSelectorExtensions.cs @@ -0,0 +1,47 @@ +using Microsoft.ML; +using Microsoft.ML.Data; +using Microsoft.ML.Runtime; +using Microsoft.ML.Transforms; +using Microsoft.ML.Transforms.Onnx; +using TransformExtensionsCatalog = Microsoft.ML.TransformExtensionsCatalog; + +namespace Redis.OM.Vectorizers.Resnet18; + +public static class DnnImageModelSelectorExtensions +{ + /// + /// Returns an estimator chain with the two corresponding models (a preprocessing one and a main one) required for the ResNet pipeline. + /// Also includes the renaming ColumnsCopyingTransforms required to be able to use arbitrary input and output column names. + /// This assumes both of the models are in the same location as the file containing this method, which they will be if used through the NuGet. + /// This should be the default way to use ResNet18 if importing the model from a NuGet. + /// + public static EstimatorChain ResNet18(this DnnImageModelSelector dnnModelContext, IHostEnvironment env, string outputColumnName, string inputColumnName, MLContext context) + { + return ResNet18(dnnModelContext, env, outputColumnName, inputColumnName, Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Resources"), context); + } + + /// + /// This allows a custom model location to be specified. This is useful is a custom model is specified, + /// or if the model is desired to be placed or shipped separately in a different folder from the main application. Note that because ONNX models + /// must be in a directory all by themselves for the OnnxTransformer to work, this method appends a ResNet18Onnx/ResNetPrepOnnx subdirectory + /// to the passed in directory to prevent having to make that directory manually each time. + /// + public static EstimatorChain ResNet18(this DnnImageModelSelector dnnModelContext, IHostEnvironment env, string outputColumnName, string inputColumnName, string modelDir, MLContext context) + { + var modelChain = new EstimatorChain(); + + var inputRename = context.Transforms.CopyColumns("OriginalInput", inputColumnName); + var midRename = context.Transforms.CopyColumns("Input247", "PreprocessedInput"); + var endRename = context.Transforms.CopyColumns(outputColumnName, "Pooling395_Output_0"); + + // There are two estimators created below. The first one is for image preprocessing and the second one is the actual DNN model. + var prepEstimator = context.Transforms.ApplyOnnxModel("PreprocessedInput", "OriginalInput", Path.Combine(modelDir, "ResNetPrepOnnx", "ResNetPreprocess.onnx")); + var mainEstimator = context.Transforms.ApplyOnnxModel("Pooling395_Output_0", "Input247", Path.Combine(modelDir, "ResNet18Onnx", "ResNet18.onnx")); + modelChain = modelChain.Append(inputRename); + var modelChain2 = modelChain.Append(prepEstimator); + modelChain = modelChain2.Append(midRename); + modelChain2 = modelChain.Append(mainEstimator); + modelChain = modelChain2.Append(endRename); + return modelChain; + } +} \ No newline at end of file diff --git a/src/Redis.OM.Vectorizers.Resnet18/FilePathImageVectorizer.cs b/src/Redis.OM.Vectorizers.Resnet18/FilePathImageVectorizer.cs index 943ec15d..6fe8d3d2 100644 --- a/src/Redis.OM.Vectorizers.Resnet18/FilePathImageVectorizer.cs +++ b/src/Redis.OM.Vectorizers.Resnet18/FilePathImageVectorizer.cs @@ -29,7 +29,7 @@ private static EstimatorChain> Create .Append(mlContext.Transforms.ResizeImages("ImageObject", 224, 224)) .Append(mlContext.Transforms.ExtractPixels("Pixels", "ImageObject")) .Append(mlContext.Transforms.DnnFeaturizeImage("Features", - m => m.ModelSelector.ResNet18(mlContext, m.OutputColumn, m.InputColumn), "Pixels")); + m => m.ModelSelector.ResNet18(mlContext, m.OutputColumn, m.InputColumn, mlContext), "Pixels")); return pipeline; } diff --git a/src/Redis.OM.Vectorizers.Resnet18/Redis.OM.Vectorizers.Resnet18.csproj b/src/Redis.OM.Vectorizers.Resnet18/Redis.OM.Vectorizers.Resnet18.csproj index e41139e3..f152ce6d 100644 --- a/src/Redis.OM.Vectorizers.Resnet18/Redis.OM.Vectorizers.Resnet18.csproj +++ b/src/Redis.OM.Vectorizers.Resnet18/Redis.OM.Vectorizers.Resnet18.csproj @@ -1,13 +1,27 @@ - - + - net7.0 + net6.0;net7.0 enable enable + Redis.OM.Vectorizers.Resnet18 + 0.6.0 + 0.6.0 + https://github.com/redis/redis-om-dotnet/releases/tag/v0.6.0 + Resnet 18 Vectorizers for Redis OM .NET. + Redis OM Resnet 18 Vectorizers + Steve Lorello + Redis Inc + https://github.com/redis/redis-om-dotnet + icon-square.png + MIT + https://github.com/redis/redis-om-dotnet + Github + redis redisearch indexing databases + true - + @@ -17,4 +31,29 @@ + + + + + + + + + + + + + + + + + + + + + + + bin\$(Configuration)\$(TargetFramework)\$(AssemblyName).xml + + diff --git a/src/Redis.OM.Vectorizers.Resnet18/Redis.OM.Vectorizers.Resnet18.props b/src/Redis.OM.Vectorizers.Resnet18/Redis.OM.Vectorizers.Resnet18.props new file mode 100644 index 00000000..7f412f54 --- /dev/null +++ b/src/Redis.OM.Vectorizers.Resnet18/Redis.OM.Vectorizers.Resnet18.props @@ -0,0 +1,8 @@ + + + + Resources\%(RecursiveDir)%(Filename)%(Extension) + PreserveNewest + + + \ No newline at end of file diff --git a/src/Redis.OM.Vectorizers.Resnet18/Resources/ResNet18Onnx/ResNet18.onnx b/src/Redis.OM.Vectorizers.Resnet18/Resources/ResNet18Onnx/ResNet18.onnx new file mode 100755 index 00000000..930f9590 --- /dev/null +++ b/src/Redis.OM.Vectorizers.Resnet18/Resources/ResNet18Onnx/ResNet18.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e2552f19db2dfdb01b3669537b6a0f1b44da38d18279535492ed5dfc2ef1ff8f +size 63653533 diff --git a/src/Redis.OM.Vectorizers.Resnet18/Resources/ResNetPrepOnnx/ResNetPreprocess.onnx b/src/Redis.OM.Vectorizers.Resnet18/Resources/ResNetPrepOnnx/ResNetPreprocess.onnx new file mode 100755 index 00000000..88790121 --- /dev/null +++ b/src/Redis.OM.Vectorizers.Resnet18/Resources/ResNetPrepOnnx/ResNetPreprocess.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:84bf8def78d0befaa3f6e73745d57d7016c6a192ead76f79686bd438c1047004 +size 602584 diff --git a/src/Redis.OM.Vectorizers.Resnet18/UriImageVectorizer.cs b/src/Redis.OM.Vectorizers.Resnet18/UriImageVectorizer.cs index 15404c79..e4deec00 100644 --- a/src/Redis.OM.Vectorizers.Resnet18/UriImageVectorizer.cs +++ b/src/Redis.OM.Vectorizers.Resnet18/UriImageVectorizer.cs @@ -32,7 +32,7 @@ private static EstimatorChain> Create var mlContext = MlContext.Value; var pipeline = mlContext.Transforms.ExtractPixels("Pixels", "Image") .Append(mlContext.Transforms.DnnFeaturizeImage("Features", - m => m.ModelSelector.ResNet18(mlContext, m.OutputColumn, m.InputColumn), "Pixels")); + m => m.ModelSelector.ResNet18(mlContext, m.OutputColumn, m.InputColumn, mlContext), "Pixels")); return pipeline; } From 8d6852881713ca9913f17921e427ce170524d854 Mon Sep 17 00:00:00 2001 From: slorello89 Date: Wed, 29 Nov 2023 12:59:16 -0500 Subject: [PATCH 26/36] moving to file based validation --- .../Redis.OM.Vectorizers.Resnet18.csproj | 4 +++- test/Redis.OM.Vectorizer.Tests/DocWithVector.cs | 4 ++-- .../Redis.OM.Vectorizer.Tests.csproj | 10 ++++++++-- .../SentenceVectorizerTests.cs | 2 +- test/Redis.OM.Vectorizer.Tests/hal.jpg | Bin 0 -> 33140 bytes 5 files changed, 14 insertions(+), 6 deletions(-) create mode 100644 test/Redis.OM.Vectorizer.Tests/hal.jpg diff --git a/src/Redis.OM.Vectorizers.Resnet18/Redis.OM.Vectorizers.Resnet18.csproj b/src/Redis.OM.Vectorizers.Resnet18/Redis.OM.Vectorizers.Resnet18.csproj index f152ce6d..fa1a81f9 100644 --- a/src/Redis.OM.Vectorizers.Resnet18/Redis.OM.Vectorizers.Resnet18.csproj +++ b/src/Redis.OM.Vectorizers.Resnet18/Redis.OM.Vectorizers.Resnet18.csproj @@ -47,8 +47,10 @@ - + + PreserveNewest + diff --git a/test/Redis.OM.Vectorizer.Tests/DocWithVector.cs b/test/Redis.OM.Vectorizer.Tests/DocWithVector.cs index 42eddf8f..e758bc57 100644 --- a/test/Redis.OM.Vectorizer.Tests/DocWithVector.cs +++ b/test/Redis.OM.Vectorizer.Tests/DocWithVector.cs @@ -15,6 +15,6 @@ public class DocWithVector public Vector Sentence { get; set; } [Indexed] - [UriImageVectorizer] - public Vector ImageUri { get; set; } + [FilePathImageVectorizer] + public Vector ImagePath { get; set; } } \ No newline at end of file diff --git a/test/Redis.OM.Vectorizer.Tests/Redis.OM.Vectorizer.Tests.csproj b/test/Redis.OM.Vectorizer.Tests/Redis.OM.Vectorizer.Tests.csproj index 7e9e97e6..0880d001 100644 --- a/test/Redis.OM.Vectorizer.Tests/Redis.OM.Vectorizer.Tests.csproj +++ b/test/Redis.OM.Vectorizer.Tests/Redis.OM.Vectorizer.Tests.csproj @@ -1,5 +1,4 @@ - net7.0 enable @@ -25,8 +24,15 @@ - + + + + + PreserveNewest + + + diff --git a/test/Redis.OM.Vectorizer.Tests/SentenceVectorizerTests.cs b/test/Redis.OM.Vectorizer.Tests/SentenceVectorizerTests.cs index 49ba4981..7e313b13 100644 --- a/test/Redis.OM.Vectorizer.Tests/SentenceVectorizerTests.cs +++ b/test/Redis.OM.Vectorizer.Tests/SentenceVectorizerTests.cs @@ -18,7 +18,7 @@ public void Test() connection.Set(new DocWithVector { Sentence = Vector.Of("Hello world this is Hal."), - ImageUri = Vector.Of("https://triviahappy.com/wp-content/uploads/2014/05/05282014hal.jpg") + ImagePath = Vector.Of("hal.jpg") }); } } \ No newline at end of file diff --git a/test/Redis.OM.Vectorizer.Tests/hal.jpg b/test/Redis.OM.Vectorizer.Tests/hal.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d581779f572737118018ef9a8820f864e6ff0747 GIT binary patch literal 33140 zcmeFZbwE_#)-Zf#7`lcIiJ?Pk1_lO@ZlpoVAcvMtkr0uV7(znn6pIi$0SRdlL9qZ4 z5ET^=5aHc3Sik$+``qWd-}}D*yuS@+pS9OsYwfky+Gp3^NDTS&uH_!!ibq*^1%w8Kcm)KabgUInrY>&Y z{=_tdkwYt}$YE75I22l51%p<>$UzWUIqbJ?La^-*5CbU% zxyKVZx1T|Y5FHg26*UzdH8mX*Ee$Ob8zUVZBO50R3mXdyClehh{J8BT{xOEp)6&v2 z&?6ZbkgN<046MY6fpv$4>3!O4DRA|V(8Mg~W~$tfwQ$jRvBK_(L!LVy`fu4U=M zA{cZGL&2I;(WEUT%w~1ZRbC-DVOWP!gkZh&`jcA-mc6q1xTtQfjl16cs!&Cak$qG6 z&n=1VkHV(E=!@l5pJ>hR7@b+Jx!n2q&5Er@cv3-a+k>&$RV4#E&xqtxbyvEcy#2}q z!Qnt#1W6}K3UVw-1pzcO0%#B<$U=@emcvTa;GWg+lDx3%>)-@}jy0QGh{7ik3Zep( zqS*T*K#3|2cfEazq4=p^D*S5=#D1%={S=}jBbCVnX+WPJ-Pe-oLF9F_$tAJgH#TIw zr%i6cG9e;N*<-?=z$s}YaqVPgfG@etvzScZ($@zwoM9KS-#LZ@wvO6T?(N?bx|zET z9lE#;MOg&4NVgxh9pyf-@S^=DGwSK|dQI8Y*4-OV9>2Rd98fjyLys>h)>1t^!m|wx zSn0Z6v^>SO{IqZ;_|p+x6Wyh`I#4c3`@WzK1*Y(AB8~XA-4RhuJI%vMbPx-Q@M%l6jBydFRzdFTlSkh)ph#q8yEODtwDh+ z^?tsKvAs{<@r8}wJ@E01gN(t2+Gl3SO9i97(#>x(b#@)T5I(W_HBQNCQ)n#f*_nu% zln~$f7|Dh2mRU(%2N?q9#wUB{A#aN4#no4J%76q?eF>V zNsj3@GS|5GDYOFu`ix*qvZqankSg%l*&E z$G`cM?JkLa$3NzKXvRP1%&JM!0Uk$tEpJ2$DmnVV$P@Xw?<(7nVT0||nC)y~ATwiO zNDgh#;#cf80z`razndg$%9wK5?@!{G2T#qvhf_9{NF$pI^!Xf8A9rz-9kF%mO+Vgy;`{V1x zJhJyYc|XQ9_|H3uPn49$K#EC`M*I(n^rR%QtVj|i*RM1mN$M9;nwk`Y2u9KajQ7Mq zLmE4Y5S&E2LIgp%vK}5F#}C9{lm-Fz15;5{{0oC2;T=KzlLkw|NdLx=WUhfU1OEKi(UL8RHq4mUS3NoEeB9g7f?5OL-Mj|TufWC3|W0Z=HoY#=;z7}&%Q zasg>Pkqlh_82-Q!CH$I0{i6GW;1}+Ps>HN^n4cfG5vxt%8W8619%An5;|5Yc_^jMQ z(I7=?rZ!-z=NI7b3DTrsg7*mhGp!vOaCj%ZqlT5I*N=Qsz+}0L=_8N>ai|_Yc{Wp1}cOfqy|!1O$6~di&%3!wv%;Iz4NmbaN68r0HG4LIVu& z{`g>*P`o=pKsHf<_#ebHJH$i`F~{hzCkovDPuHI;EZEOR&)xkQFVx1x^Dj6CH$33OM}``O7}}VcxCR9H5d~8HME`}D+AARVh?bwX=TA#9?o{6J zCmL{4y5l`u!u&!3NF9z34*e(Owm;E-C8u@u)Cuql2>!#kOgr+m4Sr<+7RW5XpJ;39 z(11Ymu+R|vpACn`4>Y9zf~9i}2n7x5zu;+si@pAlob()u6AOaqzy;S>BEdV)9C1yv z(}XEVbVLe>A`e7Fvz3Dj^iZ%@5LAB`;3FYUFz5e!mGJi}VRrTJRl?t^guhn_f3Fe< z*4zJgRtco>NdSyU5VQ|mV9p6ajF1-O4km=IU=oP>V}ck$nkohY3YLFEfr4}(Ye<18 zaA!oM03aClAu3P+_}_lb+UOM;8mJ;G>mMTHLfqYxaSJ#s8|e}#ic`-ceql%R_baSQei4D}B1M-e5ufO&DK znjp}1rxJ$)e~JC4*7`~4?)Hl=5KR3@g}b}S;{EW4@&2J9z_Mtf0e%WKHT^~YPsJWS z{EIpyR4*K8^xwHGgg7IUwZex4gay0d^}<1kLO%-&3AMuiHqAdR?62&A{1Zu!=Y|8Yic%ucxbsR?@v<2}FMu`ODh_N!z6T~DMWh{V|Fhsl( zZ~};ds}#`6KqfIJ@+#wqyvjHtjWUi%qpVD#QAU$sqBLcaG-YLCiOM9I%0x}T)2%24 z+$jZ`fRv(?k`zu#Sqhj2jg~@Vq|owGXayfTTcY7wYzBCPR2d4&KD@i&5486`={x@Y^S>i9!V3?c+W)3E+#w4I@Cc1?3C3%Af=B(|b+)qq zO79St@PDJV_FsD~WvsrQmcF8#wvIkpUQb^6XN~{R8Z9S{LEC`$jc6q>7x>vN{|8$C zFs7TAi@zrx%xh!?|MPlaK?R@u<)M)E{HnzciRTJ{a`czZ&eH#r)C%^}n;gf7{aj z{n+|{+e5O%ndZ(zFZ+-E!A@Ft$BOzy$B<@lD*x=s|DUuyFr8Cz|Bq%3JLT?dy#9x? z24X8ARrZf*_D>X;yA#1*%LetIll#B#e_P;h3;b-fkZsaZ?bMmb1u_4ZMJ@@w6aBZDf6)DRAknuw`ClwaK7+CM{l@*~{EZ8! z1jjMJ!4kmDoz&ZgRj0fA2 zC|l6Vy!}0ayZzn0NgLzd{(srV_(Q^f$o7W>BDmA80l-qd1+hBGK*&K>i0pkIgkYhC z$PSc(9M~_vnNZt;_aYEv$NzSx-2)iplj6Tb@GOvohj^3Ddq6r?HYm5S;P4$11o1`= zjyo|ztPmGC6fFdaL(&iiQi4<=O-K(ihRngC90%wSB0 zDu&9SYN!Fa05wBbpqtP==mGQ)9G-p*J%e6BZ=iS360{0^0|$vHVe~K-7#EBmCIZ_H z!@zJbb(kK^1ZD+ufH}iFVg9f%*iqPVSUQXVD~46T8emPZYp{E;9@sE!0`?j<5Bm(; zfRn-L;H+?7xF}o}j)QB#_rh)92jQOZAovk@B0Ljb2(N^ngSWx&!5_jO!(YMQ!B@b! zYFaXOG66DaG8~x>nK_vwnFm=gSu9x^Spiuk*#)xeWL;#B$X<~xlC2{s5G)83LK>lp zFhuM_xFLcNv4{*rF`^#PhUh>%LcB&SA-2fr$$7|kldF&$k=v7dl1Gpykr$BHl3yn8 zB!5i)hJ2NRf`Xkwj6#XRkivn&ha#FHgW?RuMT)x=!xYmLE0mO!T$H;h)hNv=T_{5- zlPHTQ&r#l?9HD$exkg1x#ZQHy(x-By@~29mDxf+`b(?C0>MhkbY9zG?wFyk&>NnKiXqafkXw+zIXnbgn(G=2Lpy{NUr1?TiODjmLOlwK&O&d>JNPCI4 zoAx#BIvq2eB%Kc30lF}{Ou9O{yL1zDU+C%SMd>x^9qB{qGwB=X+v#7>uQ9MN$S@c& zxHH5tlrXe0j4*s)q-GRh)M7luc!aTlv4wGvaS=(46h-PFU68TJ667`H6XXgL3lo~j zoar!A2Gd!l9;P{FN@h`JJ!W_2MCK~ycIIgoG8Q2g9Tqp1<1E!Ioh)xyDOkl=jaYqH z(^=264zMnQRQRZH zqwrG^Y7v}>m&hrRE|IUIVxo4U$)Z<8=fpV0jKq$JHHuA&(~E0}2Z>jTk4jKT;3Rw{ z&PWVN!X>eiUXmq}gS+9o6?c2@KC^pRiX80W`b$+xJ&~r9){+jFZj^p4!!BbYlOS_N z=7X%TtfOp>Y?tho99GUxu3GLHni*}3PC#Erf5u2++%P4WQF(fKefc=~tMZ=|b}Qf& zN)^VjELc-)GWI_9yP}d}u;K;9c_m>bXQk6hPjIX_3tR@STbW#0TRBeort-RqqDrVr zv&u(RX;ojPdDWvJHS*!V0 zOG3*>>zvjHZ8_~=?aSIbnfbs=^E&!=?>^I>+RDk)_b8ZsPC!YsJ~=@HHbF2 zYe;Um*D%NMi4m`nn^A+&lChF;jB&?ay1mwWi}y~MNSXwjTsMWA8k^>tPMV3B`I}ua zhs=%4bIqSyh*<<#+_0puw6H9(oV7w*9kuGRX0dj%K5M;dqivIA^K75^zOa4wZ5eG3 z*f!XHwbQjD*uAoswU4oX=)mpZ?QqqR($Utj)^TOO-v0dkZw_D&Bpnz#D0VRNV9z1$ zLw<*DIWao9IJG#FJKH&*b>4I_cd2w)aW!&1?YiWq<95pJox7%cp8Gsr4Nt(o^-%L5 zc+7dKdFFb~duezTcrAMCco%zr@-g%&^ZDv)?px=(?PuqA@i5h4=fl_hnf-nJI|BFu zA_Im4cLyc~z6w$a$`4u!-WyyS0t-19ay66{Os4w6#KTU6y$)9oF9~0Zu#ISmWQz2U z?2D3&N{xDZME^)lG$Pt9`u$;ic5@p6K@b-e~jjs@3H;_ zxdcMO@^QQ4*AsaY;}fS(7@TNKVn_;3dXlV~T$X}J@lF{?l}{~7-AZ#$>rIzSFG%0a zaLedDi9T6)ay!!_b0AAGt2CP;+dq3OMJLUHjC=`^P zqB#|I>Q$jh;ngDHqU@s0Vz1)ICE6tyPxG8kJ^l5J`xoNChLw=Yxl3KUcY=p=0@F3ftw|_*luOrrn#MXd;89jJ1chs?=If+yf=N{>Hf2J`}WZe z%Z`E0y`9|;^dGc$X?5M{R_ngmqtbJ&7uS2`q2j}~K5XCRer*5c0mXr~L8ZZ~L&`(f zhxZKM9?=-N_el5AgHfZ=hmXx350C8|8-H@($*Xbq@wo}ViBC_%pMHB5KS@5B_8j@V z;04c%%9j!^FTPTIb?de6>%J-Lsb|w})9+?NW;Wj>&eG2oyybt}Fo&7DIj=uIw6K5S z&AY&N-xibJGrd3kLHtAON6n9YOZH1|J_Uc;{+#iJ>r36T!t(tUijFL@(jgupCjOIwe-Oa%N(_|b6yy}tRMgbq`y%kM4m~ZnXsD>^kaYA6j3Cg^F)=eC znHi8sMxqcRFZfV~5->8-P|*-6{=bFo``|Mom^VD03H||pO;-={J8aB4d zW92Kp#(ZA4?fy8DhOAHD(EWtCz;R+sM-t|(o0q5#3x%qz*nB1sDn*GPHQrg_yU zc-2T4TkD@Bql`GW961C|w`i(~zAft?)=D4~HOY>zHPq%3AaDuf3OB%Qr))AMhv`_$ zfx_$x)_Mv+3b=+%k#(>I|7gaz{^f1R?hVWP>AB89zLuF-ji;}>bdA4^d=HuodH)7K zK||$^OcbDUzn1`RxuPb7YRO&4c4w4#i%S?=6-e6;R~Acve^W9&gv9aD179`vPe*Iq zSgQCCD?9lDI~slT(C(?256@2d%JP;Sn~r&Ft`RkOhj1#p*3i9&hDuXM7~|eUAQLsR z644P>cJ{cMW}N|aHTD8^t&ynAV;yeM6NjeM4(2|Z{-Ac7la+Dzm)ob<6`|S8)zHt( zi}8u#2n=0zPp+sI6%d5Xr_v#?F$m^O6V=PBk*F^pt^_sq8a9O(gdR_fy547n$7@;- z`$x`4K>}1Txs5R9L9bU0xr8Yz(BO=WICG9B zL^kpvaw3W@uLtlT1gJm-bWn*H9W32(f`|@O$0-3CLmqI`C5f}6R=_aG8i@x-o~cNU z!oWP+u731W@Ej6;T+}3i$nBnFWF>FWL&cztoTkW>1Dve2C*cJWm5a-{N#%!}5|^_|NC4!w+y7P0V*4C^m&q~?C;K=wMLmgq1z;59bZD3z*WU>IPgPxO|93qxjz*bqH0IXG;Vx96; zG3wj|5($sDi00kHiJ;Je1gNbr)S&SKuJ}Yn&7g!radC3s8t}ZFum{CznX*o+Juh`) zrP73HX)T5>p&)G}k_#bXm{@O^4M5i8iPJ)@h^B|xXYYjQmS$GQYzw_6XnI%Y5JDw)m6r9>Z zLGe8_+7^JGE&)8V+$cFvCs1itz#%T4ZgZ)Tz1o^`&LF2Z0SQ@HL?6`HgRvm@fbs<` zxK3yS#p1gWC`WhFF3em6erBSC34w*8 zIB-!pJfT3|sj};2cR1yH?3el13Tq4xoSHqmr&csq9b@AT1R}}>G@(QUN&@hL3dPrI ziW8a1#mS*z8Y*kCx#8*NZ;dO5HN*y;(gLQtY?|Me9NmWWUosq*ugE$#7p|R%5C_%- zAEathLF6cuf;(7Tf=bc=?~o$|HWVT^Zwao>BpBcs5Z81k(iP)@g`JjCR~uTi=Mj5W5X2wNI=rZP@xw zEgyaMr2SLqr`0oiUQ~Yij9Gq`9S;YOHw1hOK`3zqMMgZ3077{+#L6wec=*)IhUse? z>anpG9%eoIu*a$V`>w&S*E7{N<1W4md#Up!Mk%1E>yB*Yld@?Q)4n{=S`r{Dl0uM$ zCO!C^h(;4U#l%H;Iqv}9BlE#e&mT@Kf8J0}o{eo8J34sd)^u<5ZM%|QgOV>i>VA8rbMW!|f6WQV@0^AEGs zt!^@uyj&kFxtN?48=(LC^BG1XLN;hAMDviukfTU9;*J0WeF%!OQX2Cd?jL-z9Q-xr zK+LMT*%l*UPL2(@b13fi=hvso=TRsLK&ol6qXw`sDXY876S9^npUfr;Ki*u) zPan)WAbKsDH|R^ocdKPpu!#hMHn1%*u*Q>aU;_c{HC^v~-F#+hS-!4EX}`XIl)u`E zw?p!u%lvLJ%xZ#s0j(WcP}LtmHSj6JJC0-Is@LE6o-n|k?pq&!Efv!^^uZm;iY$-+ zDIHk;SD*n5Z<`_OP&D!q+v*v1&?Cfy%<3J2U!oykQ--&Ur63Pk@s$^l{SpQ!|Bwf% z@$DzJ_VQhO1`0tI(Jni|tNSMekgD~sBsD=Xlw-Ge-GAW$z)k$*>u2~Vl++6$BxqnO zdxvZ_e`3KMR)zYVV)!D%s|i(~&`v0`xkdgLX@4ro@Bt@9rpYBiV-aJujLrVd|5yE0 zudXka17Rr^{O-5t|FyIQ^=K(^7!5=@!OM2-Phr1p27GuUtBB}J%E{2(|GLEb^>gqP z8>pP$dss&E4^c3~KN8F65~v(mEl!D^eS-P}ztiY{ASg#b8-lct&;Ettw-5g0`w5de zwV#RqdI$42CGe3q_`Vter$)fRht9+RV*(R6Bsm2$N&p;%WYwf(VPogOC}5q16t%Pk zg+)Xybiw3~bUc_G-WsFqc+yztw5gyFgAbE_OqP2PletZ$c9)Ljxs~Io9&J??AHo!h z9S3j&k`f(Zyn3moK0ZAAbgXE(xzAOl;cn`+^@hoHVWy9ID@L%k!;D*xZ#Z_8>mA={RvRwn7iy>*Na_Ul~hn8HRaV^S*YDf(1RKIu`>b-jN2eNfla^mf;yH|zM`yYm=c7~3my znY*WHsn5wZ=HnI7m}bvW(3AsPGJV`E>mP^BsDGUHT>0#Wos1UD^bzf9$HXKT5pGW? zEgTs2VOI^Bxw_eP#^#BlUs4}9r~=MF8+|kEq>R~ zH@9A%p7Cv0*m@q5saEE%|IKD)8}b?O!7`{12rUM!2i#H#c^N|Abgtu|yxsf$3%GB= zN>A%9e$xSdi`52E(0b3K?9I14F-@KA*w|;l9w9FUzn`sINFJZgiqV970^Yd*n&mh#=H)qzAYe&d$X~CyR^{b zB)Qq1b-C+C*5%D_i!mS6JFT&@-vh)gA{TvCrVdHokeUv39v=%@KDEEJB45Z-CO$Kj z5T-~t#awp6z*5=Mtk?cxOVFSZYXh%t7D2YCi*G>fGIw%IROtNdfoQpU4?0uHz>M78 z`urIKE90#^#*TB}(>1`Y*5X?_(BY5n8!`(4!sB8YF3>qdWm{yY-*4==)a^mpb+{&=D$Cn~~xYZ|Py7+zlI!34 zc$<%#+OXv{S=tC|>-(xXH>D#}FGz%@7RaUMY;?~>TdST5P(QP${X_kGWyBHo16H3F zj+#@KB=!e&@zm60UXNkmkQIN}fhNCLYOxB$-Z^7ddZez!%XO%gD+k}3MOEI>*^Pc6 zRyE(fw|6;Ic(PxXF;~6D$MrI!hiAw(M6D9FP}+GqNN6PTgieQe3%kIoMYso82do*r zU5MtXyG#klmbfDQ`t;3juu+o!#cr_e{QpA|AQ8Bukw;ln%hoK`*Sl|@$BL&ir{sVr zwLy2ZZk-<4fE=c0+R~$wH2DxqCf|AnsilHSIJmYAiCe7BMrgo_CbXGBpuf|j_pE4g zlOh1mszs%1Ny*%;Os(~lgmGmPuwv#+aTQ1#z)fA`)nnueQ7wxf*a)2y$<|V=NCsBW zfSp;MLnR=yKem6fCS|%H1dw}0JLI-T2j0Gt8}76-E0h|RGpcmX)xGCYPJ|umnnX)G zilZ647mezgJOF4^k(Ox#$ko+_x*kNXW$`mjPwhEs&?=}_!StRg%S%66Tj(hh-%yH) zzQR5`r@o3!RWMnR1=~`@AEW_G9vI6O;sJ!&>p1KpHQomJv}Ir?Q+<~FoIn|}WuS%{GO8PxTgJ|6V}c`;Ql z4q8JkVQ=RpC_`;!qQ2MB-{^aNC2IAw;z#vIiO=6YwUqusp?q&V;izY2>``wUElbw= zfy_I{B?9x=N3UQsxG=Nnw-khiW1ExLtQW=VlIu?JPi{joDI#wQZ$K;==AwtzXCA)X zQxyF+SN7;<{Kr(IVeO>(t=i%2QtH+7}g2_e}rwg=tspp+9ycp8C0T-$P&{V3SEHZGWnvrf{?bC$0q}D)F=F${bh|?^8}U{_GQ3$r ze2QHTW)|c9HGwIi=?IAlXg07SCP~!I}5bCNsF^RhL9W zP?1BkTj9_mK5dxK>YS~}tk5BXh+}6}n;U1>T|Z0K&dAGR%s~Q?{nUwX-O0#It!;L>1RIG_HZ)h5@dX`}s>h#K${_NQa()&=ebK5DJ1tDEfUp}tywif=T;pmjFOdyrw&WAD7TkAcfA#^c)% z>t?W9+5CXf_1Qz&tUi8(KS7m#F7WjJ*SowA zmIleR4~UFMb=+XYopO20yeLwD$<60kT2-E24Cxwwo7JTJ<;fzDuhKLgV0JEi*P}+> zq~(RKKM-fzc0KwR*LwZMB0r_i7I)l6S7aw^K1(NUVilx9{AQ(^4h>Vh)feh>p|9n8n&{_y26SDv&hXth>6ihBCCvRct>~buDkk|B@ zRoj(9NErNf6Q#nvPi*WWI@Tc-KNc4h8t~WC8Dj+zHwW3xrjYD?pTG|EXGX9qR5t0L zbScPLO29ejl~DX@si5HS2dR~!M{$R->lc{%qG<1Z!L?`llC}2jBp1r}LT(Ek%s0U#1vz7>zjSJ~3N+FQpUkc{8E#F;44~8CF`}@A%%)owAqV_GCMwTEO|DX z(dB*3O-%>==o3S)KS(n&@;$QWTBX=X3#WGE8e3CWyf(7Gy`>PHeZ|MUw$H4|V&Cqw zBQ-V?dI;f07n_j*Gu`tWe&qu&sfdGg>;pR2jr~s?b+wl`qhRH0^ulbFJ(aa8XxAl= zRKE9D@)dg(_R8yExlK-K-JZYdFLhhjayu>p8uu)|%gXb{&;q-D}b2xgwe|c4swC^H$Z2 z{G;)R7%`KwKnIBy{51*0qqRnfwIV&%O`g+>(p6h`XI{4R4!Vg69;b-XTP>`0lIRJO zu}b7)&AOV;y^5Ex==rc=F@fIe&ba*Xt`bhID6OPEP2zfG@MW_n;&He`!=Jh47gcZP zisYrvewmdCY|xmX!LO5^7V4(nKEzd zgC!>=kA%U`hK(y+E?%Ft6g$m_zj+f>3(Md)+NfZM4xww@zh7bM;g-Ue_nd3DMqF$C z%p@ZlR&(0)ir8c;ONX^@b5+i56u988j<~;3w>EVjk|vj!yj;_4iaja3WGwWpqeIeE zHvRSq@|Jx2YF-M4Xcega#vDuD^}$%zi)wZC0ynRf+82E!WL|}vEr=63#f;>2TD)@< z9I>3(hS|`>d(u*Jy%t^H@1H&Ie2-Yu>_UrO7evE!blz3^oI5)dXN1^>Qf+%4G`8Ec zX(?%+H>z%^Iy-zIC)WV$y0ty6RA}zKx=yZyGI@`Dol%k*#y9GSxAR zhfyI%$mqNQoXaSxrur;(0<1)*aD}MOBejI;tEr6|y%i=!mDRnos)n%)mSPTvE!4la zo)Kx2c&==DdBXvz80R^`!u)nUtJP{OWQ?)gsJ}Tn#N1x}nAiz5=M97tmCf*&3`fd| z`=5|k4+(P6c4)tjOgSLSM0;XNAn>N{U$asjEIh{;vj@#hbvUw^B0v2RWjI!K9NMEd z-EjyV`H8|;?r74HBhi!}8|ByWviZwL_7! z9H@A<4ZNHUI%YwWdY(Pqqg66H25#Wox?inw*tywP^b)7~F`-LHi8R+i3jnp80I?W~ z$3B@|pfe@N^}hm)Q-VO#Y(5b93zyc*9$C*tfZr(cq^K zbC!aRRiK*gz1AH5BQdIIHyJyaws%2@$5X+g4i*S=o zYWZmD$UBtQp(0$1s<(!U(>u_LlIM)rgo4hQEjqxii2JhaPi33GM)7e{>p;K#8r_V* z6Tpwj$1=~mdFqF&X*3{xDfQwoh_5OLjzaj&m!d$!4o23onD_?vzPO_K!ZYi zdS{LA)hWq>X_0M6*n?me#FBCH&c%v@A7v|AA4kO|U9DuHr!-IW4Is|~<8H$G1S3d9M@3DPtKsf1oDCgeG`o4BEWv?l^ z^CRbu6LbeEp9fK$BB(sgP40Wd>^ynQbJFLOlG|ywaHE4CMc889KP;jfoMu&IKZ~X7 zn7)=*;(NYjF*Hx1<)`?7(u#NJ{_XZ5HM$s`gJ$oOV?|eY=ZGwZi`1tqTj(Mssr08l zh!1vGZ78VSCzoH95q7(#*5kG-x5pprEa4=$sN3R?cJ8=xGliLO8}W|XoHwO`Cx+%) zSh#BPd~c0FQt>6B+Sp$97fAm;%>dCBqkV=CMaDRiJy?@tObpiWr-W#jQXaCJZ>~wr z&bgPrsTb70Wl|4HJbU zsJ+DCLwYobgol<9Kepi_4l9uh8){?vCLR$X9#ie>*u~$1r{VfLrA5R+D zF_C>ke{Ygp*W=HVxTmOUt`cL|H@hj((s`$w{Pg=99IDg$W{9#{w9hV|62g5xDodY?y4QI|!D7jZEDhqa( zp6ahZ#;Yq3!Jgm{or(^-RN*d-fCcKlyL51uy#*^;>csAAGY(IbWZK7*kN2u5G28JO z<38+aGZJ<08*Ex%ipOs2JTgDkU4cqR|ag1VW_;@K3 zIm6wjm*JPp&DuKW8Tc(LpI=hFO6H@ilP4QnZp#-Rwp=b|!`3G1^nIrHhr|e~AZpEXG;RON5w|C6TbN z%)XmRBR56%9J-H7&b?c(G-)m?eCAz`6x~HK8)OSNy|e)r=J>Hj-sG3V!RpiD-<%XH z?9bxd8O2IQH+`1ROKoCf<+<4iWx%lgv?1w%sfNBvC`8)1Sr)4^O# zl3kfIgON@BVVNBSBgOs(HKk^W)~$A&5t?V_LvrI&b*kdKXYU3pl^mjyvQwo;Uv%Zs zW}DVaSz(@1RDO{@E#;T;(lV9vft*VUAzaPcHIzxvwMh*zRP3r7MgyOlbR<8pC*Qey zL5y<^JJuOlp?vLp%45I9xv8rJznmMwCBg)~(vk_D?uId)^L<9I6d#UTyEMffqBKdx zA5z!9yg!FCbyTOI{qbU0an-0&!@~hdxn+jV3G)h7BiT8YB1HkJ97o#*WPMBHy%z_n z)53(%(nqPqHkmbE<%+aOBp|1`gH9fx>(Rn4sGu+*r4&k5Mhkdvimqc4oYf z;)PJvhMU9Wn`y7wQcAAMqKrU=OXD8|3zXcr6jc`u3v-_PEFIn0*Swo8?Ou>r_^Lo~ zLh{mV(jB)eF3KCs50pe6?3INDi5J9LtZb&m(22G;X`xTJIGHb|j>HWm7f+ll*w3K! zpo4d-Lw}4PF;lqAj8{JMCMqv~8#yE=E`9!bCH8u1?An?mLL3j83_w28azMf$W-u%j8?s9$XC~YzQ zlKZnpF&qyylM?%zdshgml>3z<$EOlo$74<=yM^;MjJvp>y!>SzxVB7B)hPXZs?}$q zjo{D0XF6WoNHudu7U-n#qNb+!HH}0^d|i#MaGaXps8ckV4UO&ONDFW~wR9(`)4^|V zhq7dlm7#YtKbNem2$#RyOVI~zQ7^ju#>?zdzq}*(X)iCQ*W`8!SqfbVk}1hEUo_LG ze)G{Kl(mH&?IV{4*o2<8`J zBs(WxbSp=e2``9R67m=GYr9xwneUaf=GwxY+=Z_We0(ah)VfwwejwuZL{rnP>U(tyXX8xo3!ZXUDF zc*keNMdheON;G$jOsDZ!H!FEqdX$#I^J2=TvD29jgXp z-=%ld`k7>xD!O^fsO({5sefL?GS(w@K2>b2XQi>s+!m3x4VjFalwkLRPb1!X3!bJ< z@5vD`biu1~L~+)I2Bf6Fs8-qYo{2|OVJ~$bK}A9>X^KIT&S$2hO;Es0J1sLznOE%i zWcqyN%#E%lJBzAw{bbE&%)CAPRO->pM!e_4%`I<;_H0AO8&2vCyPAP%#hVLsbKqE4 zJy}&_%1^s=HG)lV_oZ1b+g$Rt6=+kNI2!e?)k-8p%i-!1?b#~Wz`)e_ZEAT0HFe+J zuZHzW5xF@~hi2oss0=TgtHq_9n`J!D)42r&B&M_{@3ek6*{SQCYiW}2vUv4wG$7;~ z4C>BjwwX9+91jAAEDeQ*g^$Pz4H;Ao@^K-P)U>Wnpc11gr1g)#sJ@cR;M3HXt;04^ z_V~mM<`DV88O*nyWsH$}k=W7&^|E7%yJdt4G1S5(YNx+x+0crJAL(U`4d1*f+<)9xPC;$_a3VuvyL z;LIGUI%K}`;{}ZTDQri{uI|`Yf|y&ggnRB89fX)by{uO(Iagc5-f4G6?)tRk#J;$D zS0e7_+qq*KGr5eNS2tqyW^(;F_(pnaa_4hY>HG6;WDVWY}H-mll2*%HK<#kASX3&%|oGy1!|0}%Tzyc6P^^YOlH+|n1fWMUs_c6oZn zw@zc|tC{a~{x0_1>3rEI*d>?Kqn|!2o!Aqc5E8E2Ibo%)e&X1(vWkP5lixXoBqQI@ z@p*mW-{iayGO?`4$a_sEX|f_``ieN-JHp>$P3C>X&~tvhPjTaCzF+&ktLk*4hKAd% zFL5oY>)+U~J3ZZoTDlERZ$k-Qg+?r8{KE~>>dKZM-hb-NVp49Hp6Tm98eC;*<`cQj z*4CO31T(whu))L(5~a23#*!U0Vrn-3NWU`b7BLK;aJH9)ag`Xlf=gDhSE;2?TRAlk>Gvb^x-kwncE5I_sdMF&*luGW5}l`@?u}=tbHfX zI-a<-$LNl~-I{CibdHgPc(PdGHuNep19yTIyQ^j24eWkm^sHsyhkm;o*h7nB{{7*L zkE2@--Vmn#@W1-{5^$)yxBr8#?4TkI?Sy~x`v5&ED zk2ML|lA_3-wU9iNr;vmsQ6fw4x953(@7wSFU%&rcvz&82=f2Om?{lBI&UNnldrp7T z7vMO0d0-3S@2d{}zFq?X1g-IV6~SOU0{t%KugG#~UOI8GMg}wj4oNjGN8r&>E@%)K z4dG{}(W?6u(DARlSPoE{iv}6i1c9SbJY}h1WuY8PJHLk4*vRr%RF*LsY(*nb=rm(2 z3);|!N3(3a3TP_wqsY2EFcSm?LuqMXCJO{b5p}>|ENv3an}pQlwS^d?QDCiH)1ndM zbS56l0&C=XVrkvbfn2!?MOz3Ajevm|vQ1;42o@d-0!ce!vAK9?HV++1l4h}E2ZTXv zzBCjF%_4&!WKbLrlmW&j0K3K5MnDP%8R1CHG{%>KVCV=iG=U-~>V=NiY3kPn!ppEs zcoQ2{0A&n9z_AcxC^sykogExW0kL2eX)rn&#&HZ9!2*L=By0YN#ymese=o&gQpgy9wWA^VI z-e1uSh#_|bwZs}*Vy%!4$ZXO6^7{jd_loOFU7Z)X9<>i5D?iN`u(K7tRUrkEF>U@` z^DlUX<}GaF_Ob}I3GFX#UNCvHCcAkfR%`$|9W)Z!zIx&NPf!OorHJ*xup)b`%gE+E zVNRgAjBwiq`6_O-qq(gAtYjRoo;ZnaX@0jvq9^jKy47+1OXsG-wF|P+tbreiEY=N` zJ~NVOTcJnZpQnf)?-+5`l;{0vy}GWGhY#VXZGFUcymnUD<@}Y;kyW3afr2p6H;;MG zBTk3Wo;*{_XfVP(9WXj@l*%PxyNL>VCYn6@2{M}5|2%EuzAdeH7G7WP1Pwtg3(WZ*YZ~z(m?(LhC z$oJR{cE1V}cL47{IskQQe|Or2;{RHd&J)c@n7@Ru8$F}jT`9mJ>>$?E8w<6Q7PSz) zFAeiLoaE6zz!rvi?Pk`KK&g3Yb95>KL5rjhj8{ z^AuZ^W#5W2u~xE?wJ?oCVM!Sm?6y8fJ-{{g4U>Uvs`THb%uiKo^_>Zx)Wsw|LwkMs z{^cENsnOQx>uXQ$HEr+kCb0}|DLfB!N6D&Q=o=REQ+0d(u))vit^S1Cw%h1czYlNs;KDZxXIQ1*_u)ow*TOGs>>h-iWNFG77A)QO4O@_&=O^9? zYGQjK>K0vB<(=+|x`EQl+CwTA;E#zY^us)_hX1o}QCE^7+XELC&IbkL+xM zOT6BsK(7cgD(9zhLS>6>dCZWnj>?196Vq2Yjv#@>&TtUsh2cW**}RFK=3_z{UamfV zPE4Za;w4KP?mvP%f_8mGC+lJHXPU}N#)GU*m$Ybbl5|V|baGaXH)9ysdqHiENM82H zJMDJxpePwT2G<>h{-91MiW+3&9wsv_MamF|Pc7?~zr_9_GK!#fV41`VFWs!MX;l!I zTDWxPx8D2GbtfiSlN+}WM7F9`>!=1K&6q8;k*1_%?5JXOS}$Vlt(~&I?JA^snvmTB zOF{)qC!;mp$~(PBrL|NdI~nEfaqg@>GlD%jt%2Zrcbb{+Z`Sn$Kcjrl^ZuX#4YB~$ zQICW46w3<1gE`GePB(F&=BfuoqyUv=Io_nAs_XQ89*Qa=<|I}ux?xtV=s z2-#Ia8PZjXfX%}ofHeTa6hT1Kk48Dpy&v&0JpI3)_|GFgWkOf*H-Rvpe_U?ByINbsxQ`gYN9i#QLUx3N=o68N6_@~d|&a4dHlS8Eor5g$yQCJa=a@C;u zFDK^gZ1!~$|Duzjkz~gZ zJ6jp{DLmB<((5xF!2)m;_sNp(3Eg@wRftUU$)rjJ1U|afVi^omy z;RF}S$~1ljTRCM@(b>awTy`bNw=Y4Ct<{&T1cJ}E-$oKE4q^YYY})VLiw}(Jt904z zzBFEkPi!JTN!&|o1+DFk&%3CnC z#C!ZhT?&&K$wIi0zCqs5T3Zk}=`5FPJyeO_;W*3#JenyeW)w+7aeU!PsEdIZ?@Xx} zxx@WGOr~e^P1{^-#yXJ{t-ob!=L3J$vgK;sY0>5hpeUJdnSnb|-#J(Lno>rjii}Ke zOzx%ra7Eb+wt?^gd1&DQ^?*aDaY-7EINdR?0Pf=MwWm&NoV<?2Ht&Qn54Lyjyq=Im54kwGRC@H~<5Q2wmIdpwZ`1k79_tUtNCa3g zi}91CZs7Qa{SN0HnIsZ(zD5Rd7ZycV)i=Y`5REF)(N%Q;21{C##y!MikZ0(rtkDM# zDZV?r-5x7GlXF6Lv}`r5A3{SmuQ`=;s>)38!ZIV>i1CwiH_SQj?=4wTa>gbevt^iT z;?`Lq(r1}gJ@xPmoubbggOOs5w~dho&-h!$?p#DVAW{{hOiGp_Fc1TOmvhJ$@jUlC z;D_Wr&(XfnJ;g2bS}hGmF5O5LbX&)EdK9{>3b(0W+qaL)X7}_+7_mf`cvYThB>si) zca^Y9!{+g@VCQr)XE4}FD%1Co2>Y^rC$nKhzycCzlCUgm-xJueBjh_PWzrZjZ{bES zWZYua9?vbl{#B(ye?iMb!&c@e2xiqC8UD=Ec_a7WWd6YO^RHA@wM>Q2Ka>h8P7)t4 zF0k_De4n^0cmy74V`eYDdoE>ycThXgjCb_n~pBUw+hmKnCB=gheg)cbwwqK#FXm$jlu)y8^H%zt4B>V1694% zG-TF%Wxpe??r+3KC8!n_sX+P!yN8J?%2LG+#F1qhM%xq7)a8zfS4F?uj>hM{5t2KT z*w9!0LC$LwZ)Km@@3!R5v}Up#GjtT&D|H1U#uTU7d$JYrRk?+(J1V9)_{wI)as5@J z+qkEDG3gU~>KQQC9b|a=SNMTwZ||U+#maJRne1_;(K27V7NbEGhh+prqQwL)Ix!+S zKOl#d94?#NhUhNw4%@yAk$}a4P{3)v?mm+MB9`VI}h`gf@3~ zRsYGvK&j2X-q4{&`@sFIkhxla$;}eCp`+0j zkYja`MweJ_`{Elp1Ffiu)uj(cvWmoTmD`BwBZ&S7E7z)9+cT?dwWEV3Goq!6`UDu4 zK_$3AHnDSNA3jco-f!+Uzi>xfVu;R_$>G?icfF=o-hih!kLx=0dZgx?Y3)axJ7@ZB z8TO{mPjsyx-*gaq>#Li;BPiHiME~BJefY`bo(m)M9qxp=elg)C=UihtEvNrhzIa6C z^mCV@C4tz<@nN43A4*584*o^M!6$)Hyk_yO;=P+l0fLTl@62P;3RTv;Zb}hdGs|cC z#|N8lx+h#!7;UE)7B#=o8b+lsr4n>OU3ketnL2hdI~sa%!!ROmk=fMAdM+&3rZ1oM z+Ng}ua^Yyg<)sa9riQ7qq+LBm#<8;Vy#V65Yn&C0<=4QyF=De;bk7F%8ObL#A}D@n zoLhD1@g`;yT}D#Y))IGF>=01nvT9abH{jUOy&lD3yf7=Wf`5EG@__4w{S*! zc<*Zj>1zaWu8EZj)V#}*OKguV8hq$zS!b$!Vm+8o{OHJ+JqfRd-S}QtL@N-Y?asLe z4l9zNyzNuk75%Oak3y}UtmA9v3bbev(?rL9Q~ndSJpL@I+-EFkPKa(f+gaP!&V)l@ z9Qez+;>Zpr(Lrc!ZXfb`T(G*2dsA_wsG(mim&^;tgs%Xe-I_c<4anN#Iz<%Soky~q|b=|{m5Qrq6 zV{g%j-<`Gl(Xusgkbbz0b2Se?@E(SmTk%>|h>=?y`Wn zY&JJ!PlIK*wEuwemiU>wCnT}yKS2)h%7S5bdnMoQUcHGveOu|7%Am7;2yEuOD@mmz z`w{0mjaW`+U-^WRK-uY{>C1Dxb)sBioZ|xod-r{$uir&fg>mfH@rgm-_hK$;CSV>8 zGZBWzqztBuyKVj35TUbVxZ%6+Y3dgj1ogEQJk=NV6iG&oSQhM5-9tJzRKw=bGOWBd z%W}owa`#Xd@D(qO;IAVxP8UT#EDK5=d36VA)!uQ&J9~&XUvs5ISP9X}d_R3#(2v74 zvONmxAYl+v6QkqrZ&Dg0A-1i4yc~PwucHwGU*2OSWbk9&78p2B2PIq3US6e&AMz6< zCHWmr-@Er9zY!mH`J8y~`aJagF#lE0>X&3~X8uzO`oP1(2erhhDhV3&27F7zxwm|u zug6pFet}NB+fKUPH(eVPd9p%Ab|_2&>wuG@c-CEC=KMmua@goWX5m^~tz%a*+J77@;E&d%ty{_lbY#k^k_##b`hGh4R@#}e2r zBr5!NrZw5u1g|1lD<01C*o2ZhaA}^1B~*t<(NadNrCCdk!KBCewa<{&Q&o=H6(^rOTFP0+EqO9XrG(E*r zNa){9yX*T(N{x3l{ao)!CAJV3YL%lUr4Cbq#fV~f;;*th~p%#Dm zMstOh&dJ5(?L$XelBLZiq>n6LV-rsl0H@c$#m&=3j#`pTqX+2m?T5}jJ60OSLu4sh z-%RyBB82p0wv^U&=6WiX+L}f)TPgT8$L~(EM|NOV{n1uD-JXI1!#1qu^pRm#dN&o= z*}x?NH;Q&WE!`^C;K%r!I%S<5lg`U~*V{XWC=+Sk0>;Upb5&q5>Sv3Ny7lq1PsSAU zD>tgpo}2Z4I}gfaFyGK^DUuJRi(9b0&@`)BM#gbpz?-`Pzkb7IcL1aer_4J1}f z9~%>sj@|konX>$OE*;Q9$MlP4I6?;(H%Tv-F1gKPk$qKLWW&E(D#S%-;_S*6{Zz&7 ztrf=oaN>+@jYZWJ8~Gu`m$A}2^=bEG!#fyrQX_HV9LGe@Xzm&={{+$983@d|XEJW; zIlN|wJ6tRN&J_X&YUURWxN82o8Is;B@X+l4SN(|jk=!IZfOML;;P*<#quWm-}K&EeZuzGx{>gKWP4@_FDxCe z$gI9_D9v8$8bsPv!9*R4Z*PSG z*TV`O6GJW|Wwd0pwDmZI>`YMRB9*yHUIC*;n)t-CMGGLqY1#dft#s72^J>$9`Y{bTpFHxx@A zBTlhSOL3Xi7qhP(uReBcAL;!kd>);j`<0RJD#2+Qd;-pVMx;ofJiPyate$x_pXxo`s3pIaqweRsPW8Z{* z@*_9LpG;S(d7S8xSZSf4?xe+N9fj0Zu1|=2hH{x|tM8r-=ger{{}R3Ru+?(_Ohb>^F` z?W6kX1n!cp2gQv!_c|VHpM5xw{UfI5Qr8t*r;%#jD%~)pxmhB|g2$F!-`a@i$2w2n zz!!ic&308$;SG-tO7-PL7j>vox2G*lvF;-GbvYa7=WH$w?zU-TUDSRKneB(|Ma|Fu zQ0--3J3;VQQ&|4t`VK6loVx$)HFxFL=9jjgP<`j#e<^q~-}gUOGYmW{U3ctM;Dj7KTD8`MH-vbWAlZe&9>PPmsP8THatg?sCJH+EB|v zpvRXxF?oLo+fmm#u7A`HxyWaoNRKl2*u0ZQ@Tno2xFr|v^4E%JNZ~JCa|lKFuIfg8 z{CbgN#A8c0s@d&Im_fJ4)&mAIaWQn;sCcrpE$j(v-RHTA!GkAR>)m##61z__OS96X ziMKw#ub8uwkrLfueNX-%VYHPziWeRdN|uid_0A0fMRb9lG@yW|@u z8?tKa#3H6fE*wl@!I%uLL z*D{}eK>y|%xujgc(j>YOM^qVjB`%#!Sqw@IRuHcDXIf-9E-Ni594UD)Q0JDN;1tm+gZMc Date: Wed, 29 Nov 2023 17:08:36 -0500 Subject: [PATCH 27/36] semantic caching for native vectorizers --- .../RedisConnectionProviderExtensions.cs | 32 ++++++ .../DnnImageModelSelectorExtensions.cs | 7 +- .../FilePathImageVectorizer.cs | 46 -------- .../FilePathImageVectorizerAttribute.cs | 12 -- .../ImageModelObjects.cs | 8 ++ .../ImageVectorizer.cs | 103 ++++++++++++++++++ ...tribute.cs => ImageVectorizerAttribute.cs} | 7 +- .../UriImageVectorizer.cs | 61 ----------- .../AzureOpenAIVectorizer.cs | 2 +- .../RedisConnectionProviderExtensions.cs | 16 +++ .../{DocWithVector.cs => DocWithVectors.cs} | 6 +- .../Redis.OM.Vectorizer.Tests.csproj | 2 +- .../SentenceVectorizerTests.cs | 24 ---- .../VectorizerFunctionalTests.cs | 35 ++++++ 14 files changed, 208 insertions(+), 153 deletions(-) create mode 100644 src/Redis.OM.Vectorizers.AllMiniLML6V2/RedisConnectionProviderExtensions.cs delete mode 100644 src/Redis.OM.Vectorizers.Resnet18/FilePathImageVectorizer.cs delete mode 100644 src/Redis.OM.Vectorizers.Resnet18/FilePathImageVectorizerAttribute.cs create mode 100644 src/Redis.OM.Vectorizers.Resnet18/ImageVectorizer.cs rename src/Redis.OM.Vectorizers.Resnet18/{UriImageVectorizerAttribute.cs => ImageVectorizerAttribute.cs} (66%) delete mode 100644 src/Redis.OM.Vectorizers.Resnet18/UriImageVectorizer.cs rename test/Redis.OM.Vectorizer.Tests/{DocWithVector.cs => DocWithVectors.cs} (79%) delete mode 100644 test/Redis.OM.Vectorizer.Tests/SentenceVectorizerTests.cs create mode 100644 test/Redis.OM.Vectorizer.Tests/VectorizerFunctionalTests.cs diff --git a/src/Redis.OM.Vectorizers.AllMiniLML6V2/RedisConnectionProviderExtensions.cs b/src/Redis.OM.Vectorizers.AllMiniLML6V2/RedisConnectionProviderExtensions.cs new file mode 100644 index 00000000..a0ea8395 --- /dev/null +++ b/src/Redis.OM.Vectorizers.AllMiniLML6V2/RedisConnectionProviderExtensions.cs @@ -0,0 +1,32 @@ +using Redis.OM.Contracts; + +namespace Redis.OM.Vectorizers.AllMiniLML6V2; + +/// +/// Static extensions for The RedisConnectionProvider. +/// +public static class RedisConnectionProviderExtensions +{ + /// + /// Creates a Semantic Cache using the All-MiniLM-L6-v2 Vectorizer + /// + /// The connection provider. + /// The Index that the cache will be stored in. + /// The threshold that will be considered a match + /// The Prefix. + /// The Time to Live for a record stored in Redis. + /// + public static ISemanticCache AllMiniLML6V2SemanticCache(this IRedisConnectionProvider provider, string indexName="AllMiniLML6V2SemanticCache", double threshold = .15, string? prefix = null, long? ttl = null) + { + var vectorizer = new SentenceVectorizer(); + var connection = provider.Connection; + var info = connection.GetIndexInfo(indexName); + var cache = new SemanticCache(indexName, prefix ?? indexName, threshold, ttl, vectorizer, connection); + if (info is null) + { + cache.CreateIndex(); + } + + return cache; + } +} \ No newline at end of file diff --git a/src/Redis.OM.Vectorizers.Resnet18/DnnImageModelSelectorExtensions.cs b/src/Redis.OM.Vectorizers.Resnet18/DnnImageModelSelectorExtensions.cs index 19f3e7fe..9b0fcb9f 100644 --- a/src/Redis.OM.Vectorizers.Resnet18/DnnImageModelSelectorExtensions.cs +++ b/src/Redis.OM.Vectorizers.Resnet18/DnnImageModelSelectorExtensions.cs @@ -3,11 +3,14 @@ using Microsoft.ML.Runtime; using Microsoft.ML.Transforms; using Microsoft.ML.Transforms.Onnx; -using TransformExtensionsCatalog = Microsoft.ML.TransformExtensionsCatalog; namespace Redis.OM.Vectorizers.Resnet18; -public static class DnnImageModelSelectorExtensions +/// +/// Extensions pulled and slightly modified from from ML.NET to service this package as the content files cannot be +/// reliably copied from transitive dependencies. +/// +internal static class DnnImageModelSelectorExtensions { /// /// Returns an estimator chain with the two corresponding models (a preprocessing one and a main one) required for the ResNet pipeline. diff --git a/src/Redis.OM.Vectorizers.Resnet18/FilePathImageVectorizer.cs b/src/Redis.OM.Vectorizers.Resnet18/FilePathImageVectorizer.cs deleted file mode 100644 index 6fe8d3d2..00000000 --- a/src/Redis.OM.Vectorizers.Resnet18/FilePathImageVectorizer.cs +++ /dev/null @@ -1,46 +0,0 @@ -using Microsoft.ML; -using Microsoft.ML.Data; -using Microsoft.ML.Transforms; -using Redis.OM.Contracts; -using Redis.OM.Modeling; - -namespace Redis.OM.Vectorizers.Resnet18; - -public class FilePathImageVectorizer : IVectorizer -{ - public byte[] Vectorize(string obj) - { - var input = new ImageInput() { ImageSource = obj }; - return Vectorize(new [] { input })[0].SelectMany(BitConverter.GetBytes).ToArray(); - } - - public VectorType VectorType => VectorType.FLOAT32; - public int Dim => 512; - - public static Lazy>> Pipeline = new(CreatePipeline); - - private static readonly Lazy MlContext = new(()=>new MLContext()); - - private static EstimatorChain> CreatePipeline() - { - var mlContext = MlContext.Value; - var pipeline = mlContext.Transforms - .LoadImages("ImageObject", "", "ImageSource") - .Append(mlContext.Transforms.ResizeImages("ImageObject", 224, 224)) - .Append(mlContext.Transforms.ExtractPixels("Pixels", "ImageObject")) - .Append(mlContext.Transforms.DnnFeaturizeImage("Features", - m => m.ModelSelector.ResNet18(mlContext, m.OutputColumn, m.InputColumn, mlContext), "Pixels")); - - return pipeline; - } - - public static float[][] Vectorize(IEnumerable images) - { - var mlContext = MlContext.Value; - var dataView = mlContext.Data.LoadFromEnumerable(images); - - var transformedData = Pipeline.Value.Fit(dataView).Transform(dataView); - var vector = transformedData.GetColumn("Features").ToArray(); - return vector; - } -} \ No newline at end of file diff --git a/src/Redis.OM.Vectorizers.Resnet18/FilePathImageVectorizerAttribute.cs b/src/Redis.OM.Vectorizers.Resnet18/FilePathImageVectorizerAttribute.cs deleted file mode 100644 index d3e48a0a..00000000 --- a/src/Redis.OM.Vectorizers.Resnet18/FilePathImageVectorizerAttribute.cs +++ /dev/null @@ -1,12 +0,0 @@ -using Redis.OM.Contracts; -using Redis.OM.Modeling; - -namespace Redis.OM.Vectorizers.Resnet18; - -public class FilePathImageVectorizerAttribute : VectorizerAttribute -{ - public override byte[] Vectorize(object obj) => Vectorizer.Vectorize((string)obj); - public override VectorType VectorType => Vectorizer.VectorType; - public override int Dim => Vectorizer.Dim; - public override IVectorizer Vectorizer => new FilePathImageVectorizer(); -} \ No newline at end of file diff --git a/src/Redis.OM.Vectorizers.Resnet18/ImageModelObjects.cs b/src/Redis.OM.Vectorizers.Resnet18/ImageModelObjects.cs index c22c572a..e8fb2cde 100644 --- a/src/Redis.OM.Vectorizers.Resnet18/ImageModelObjects.cs +++ b/src/Redis.OM.Vectorizers.Resnet18/ImageModelObjects.cs @@ -1,4 +1,6 @@ +using System.Drawing; using Microsoft.ML.Data; +using Microsoft.ML.Transforms.Image; namespace Redis.OM.Vectorizers.Resnet18; @@ -6,4 +8,10 @@ public class ImageInput { [ColumnName(@"ImageSource")] public string ImageSource { get; set; } +} + +public class InMemoryImageData +{ + [ImageType(224,224)] + public Bitmap Image; } \ No newline at end of file diff --git a/src/Redis.OM.Vectorizers.Resnet18/ImageVectorizer.cs b/src/Redis.OM.Vectorizers.Resnet18/ImageVectorizer.cs new file mode 100644 index 00000000..21cf6c23 --- /dev/null +++ b/src/Redis.OM.Vectorizers.Resnet18/ImageVectorizer.cs @@ -0,0 +1,103 @@ +using System.Drawing; +using Microsoft.ML; +using Microsoft.ML.Data; +using Microsoft.ML.Transforms; +using Redis.OM.Contracts; +using Redis.OM.Modeling; + +namespace Redis.OM.Vectorizers.Resnet18; + +/// +/// A Vectorizer that uses Resnet 18 to perform vectorization. It accepts either a file path or full URI to an image as +/// input and vectorizers the inputs returning a Float32 vector with a dimensionality of 512 +/// +public class ImageVectorizer : IVectorizer +{ + /// + public VectorType VectorType => VectorType.FLOAT32; + + /// + public int Dim => 512; + + /// + public byte[] Vectorize(string obj) + { + var isUri = Uri.TryCreate(obj, UriKind.Absolute, out var uri) && (uri.Scheme == Uri.UriSchemeHttp || uri.Scheme == Uri.UriSchemeHttps); + if (isUri) + { + var request = new HttpRequestMessage() + { + Method = HttpMethod.Get, + RequestUri = uri, + }; + var imageStream = Configuration.Instance.Client.Send(request).Content.ReadAsStream(); + var image = Image.FromStream(imageStream); + var resized = new Bitmap(image, new Size(224, 224)); + var vector = VectorizeBitMaps(new [] { resized })[0].SelectMany(BitConverter.GetBytes).ToArray(); + return vector; + } + + if (!File.Exists(obj)) + { + throw new ArgumentException( + $"Input {obj} was not a well formed URI, and was not a file path that exists on this system.", nameof(obj)); + } + + return VectorizeFiles(new[] { obj })[0].SelectMany(BitConverter.GetBytes).ToArray(); + } + + private static Lazy>> FilePipeline = new(CreateFilePipeline); + + private static readonly Lazy MlContext = new(()=>new MLContext()); + + private static EstimatorChain> CreateFilePipeline() + { + var mlContext = MlContext.Value; + var pipeline = mlContext.Transforms + .LoadImages("ImageObject", "", "ImageSource") + .Append(mlContext.Transforms.ResizeImages("ImageObject", 224, 224)) + .Append(mlContext.Transforms.ExtractPixels("Pixels", "ImageObject")) + .Append(mlContext.Transforms.DnnFeaturizeImage("Features", + m => m.ModelSelector.ResNet18(mlContext, m.OutputColumn, m.InputColumn, mlContext), "Pixels")); + + return pipeline; + } + + /// + /// Vectorizers a series of image file paths. + /// + /// + /// + public static float[][] VectorizeFiles(IEnumerable imagePaths) + { + var images = imagePaths.Select(x => new ImageInput { ImageSource = x }); + var mlContext = MlContext.Value; + var dataView = mlContext.Data.LoadFromEnumerable(images); + + var transformedData = FilePipeline.Value.Fit(dataView).Transform(dataView); + var vector = transformedData.GetColumn("Features").ToArray(); + return vector; + } + + public static Lazy>> BitmapPipeline = new(CreateBitmapPipeline); + + private static EstimatorChain> CreateBitmapPipeline() + { + var mlContext = MlContext.Value; + var pipeline = mlContext.Transforms.ExtractPixels("Pixels", "Image") + .Append(mlContext.Transforms.DnnFeaturizeImage("Features", + m => m.ModelSelector.ResNet18(mlContext, m.OutputColumn, m.InputColumn, mlContext), "Pixels")); + + return pipeline; + } + + public static float[][] VectorizeBitMaps(IEnumerable bitmaps) + { + var images = bitmaps.Select(x => new InMemoryImageData { Image = x }); + var mlContext = MlContext.Value; + var dataView = mlContext.Data.LoadFromEnumerable(images); + var transformedData = BitmapPipeline.Value.Fit(dataView).Transform(dataView); + var vector = transformedData.GetColumn("Features").ToArray(); + return vector; + } +} \ No newline at end of file diff --git a/src/Redis.OM.Vectorizers.Resnet18/UriImageVectorizerAttribute.cs b/src/Redis.OM.Vectorizers.Resnet18/ImageVectorizerAttribute.cs similarity index 66% rename from src/Redis.OM.Vectorizers.Resnet18/UriImageVectorizerAttribute.cs rename to src/Redis.OM.Vectorizers.Resnet18/ImageVectorizerAttribute.cs index e06ae3f5..a4690e1e 100644 --- a/src/Redis.OM.Vectorizers.Resnet18/UriImageVectorizerAttribute.cs +++ b/src/Redis.OM.Vectorizers.Resnet18/ImageVectorizerAttribute.cs @@ -3,10 +3,11 @@ namespace Redis.OM.Vectorizers.Resnet18; -public class UriImageVectorizerAttribute : VectorizerAttribute +public class ImageVectorizerAttribute : VectorizerAttribute { - public override byte[] Vectorize(object obj) => Vectorizer.Vectorize((string)obj); public override VectorType VectorType => Vectorizer.VectorType; public override int Dim => Vectorizer.Dim; - public override IVectorizer Vectorizer => new UriImageVectorizer(); + public override byte[] Vectorize(object obj) => Vectorizer.Vectorize((string)obj); + + public override IVectorizer Vectorizer { get; } = new ImageVectorizer(); } \ No newline at end of file diff --git a/src/Redis.OM.Vectorizers.Resnet18/UriImageVectorizer.cs b/src/Redis.OM.Vectorizers.Resnet18/UriImageVectorizer.cs deleted file mode 100644 index e4deec00..00000000 --- a/src/Redis.OM.Vectorizers.Resnet18/UriImageVectorizer.cs +++ /dev/null @@ -1,61 +0,0 @@ -using System.Drawing; -using Microsoft.ML; -using Microsoft.ML.Data; -using Microsoft.ML.Transforms; -using Microsoft.ML.Transforms.Image; -using Redis.OM.Contracts; -using Redis.OM.Modeling; - -namespace Redis.OM.Vectorizers.Resnet18; - -public class UriImageVectorizer : IVectorizer -{ - public byte[] Vectorize(string obj) - { - var imageStream = Configuration.Instance.Client.GetAsync(obj).Result.Content.ReadAsStream(); - var image = Image.FromStream(imageStream); - var resized = new Bitmap(image, new Size(224, 224)); - var input = new InMemoryImageData() { Image = resized}; - var vector = Vectorize(new [] { input })[0].SelectMany(BitConverter.GetBytes).ToArray(); - return vector; - } - - public VectorType VectorType => VectorType.FLOAT32; - public int Dim => 512; - - public static Lazy>> Pipeline = new(CreatePipeline); - - private static readonly Lazy MlContext = new(()=>new MLContext()); - - private static EstimatorChain> CreatePipeline() - { - var mlContext = MlContext.Value; - var pipeline = mlContext.Transforms.ExtractPixels("Pixels", "Image") - .Append(mlContext.Transforms.DnnFeaturizeImage("Features", - m => m.ModelSelector.ResNet18(mlContext, m.OutputColumn, m.InputColumn, mlContext), "Pixels")); - - return pipeline; - } - - public static float[][] Vectorize(IEnumerable images) - { - var mlContext = MlContext.Value; - var dataView = mlContext.Data.LoadFromEnumerable(images); - var transformedData = Pipeline.Value.Fit(dataView).Transform(dataView); - var vector = transformedData.GetColumn("Features").ToArray(); - return vector; - } - - public class InMemoryImageData - { - [ImageType(224,224)] - public Bitmap Image; - } - - public class InMemoryImageDataOutput - { - [ColumnName("Output")] - public float[] Output { get; set; } - } - -} \ No newline at end of file diff --git a/src/Redis.OM.Vectorizers/AzureOpenAIVectorizer.cs b/src/Redis.OM.Vectorizers/AzureOpenAIVectorizer.cs index 4ec34545..b3e4aecd 100644 --- a/src/Redis.OM.Vectorizers/AzureOpenAIVectorizer.cs +++ b/src/Redis.OM.Vectorizers/AzureOpenAIVectorizer.cs @@ -38,7 +38,7 @@ internal static float[] GetFloats(string s, string resourceName, string deployme Headers = { { "api-key", apiKey } } }; - var res = client.SendAsync(request).Result; + var res = client.Send(request); if (!res.IsSuccessStatusCode) { throw new HttpRequestException( diff --git a/src/Redis.OM.Vectorizers/RedisConnectionProviderExtensions.cs b/src/Redis.OM.Vectorizers/RedisConnectionProviderExtensions.cs index 4540b42d..25bd01db 100644 --- a/src/Redis.OM.Vectorizers/RedisConnectionProviderExtensions.cs +++ b/src/Redis.OM.Vectorizers/RedisConnectionProviderExtensions.cs @@ -2,8 +2,24 @@ namespace Redis.OM.Vectorizers; +/// +/// Static extensions for The RedisConnectionProvider. +/// public static class RedisConnectionProviderExtensions { + + /// + /// Creates a Semantic Cache using the Hugging face model API + /// + /// The Connection Provider. + /// The API token for Hugging face. + /// The activation threshold. + /// The Model Id to use. + /// The dimensionality of the tensors. + /// The Index name. + /// The prefix. + /// The TTL + /// public static ISemanticCache HuggingFaceSemanticCache(this IRedisConnectionProvider provider, string huggingFaceAuthToken, double threshold = .15, string modelId = "sentence-transformers/all-mpnet-base-v2", int dim = 768, string indexName = "HuggingFaceSemanticCache", string? prefix = null, long? ttl = null) { var vectorizer = new HuggingFaceVectorizer(huggingFaceAuthToken, modelId, dim); diff --git a/test/Redis.OM.Vectorizer.Tests/DocWithVector.cs b/test/Redis.OM.Vectorizer.Tests/DocWithVectors.cs similarity index 79% rename from test/Redis.OM.Vectorizer.Tests/DocWithVector.cs rename to test/Redis.OM.Vectorizer.Tests/DocWithVectors.cs index e758bc57..b7ef50a1 100644 --- a/test/Redis.OM.Vectorizer.Tests/DocWithVector.cs +++ b/test/Redis.OM.Vectorizer.Tests/DocWithVectors.cs @@ -5,16 +5,16 @@ namespace Redis.OM.Vectorizer.Tests; [Document(StorageType = StorageType.Json)] -public class DocWithVector +public class DocWithVectors { [RedisIdField] public string Id { get; set; } - [Indexed] + [Indexed(Algorithm = VectorAlgorithm.HNSW)] [SentenceVectorizer] public Vector Sentence { get; set; } [Indexed] - [FilePathImageVectorizer] + [ImageVectorizer] public Vector ImagePath { get; set; } } \ No newline at end of file diff --git a/test/Redis.OM.Vectorizer.Tests/Redis.OM.Vectorizer.Tests.csproj b/test/Redis.OM.Vectorizer.Tests/Redis.OM.Vectorizer.Tests.csproj index 0880d001..64e95962 100644 --- a/test/Redis.OM.Vectorizer.Tests/Redis.OM.Vectorizer.Tests.csproj +++ b/test/Redis.OM.Vectorizer.Tests/Redis.OM.Vectorizer.Tests.csproj @@ -24,7 +24,7 @@ - + diff --git a/test/Redis.OM.Vectorizer.Tests/SentenceVectorizerTests.cs b/test/Redis.OM.Vectorizer.Tests/SentenceVectorizerTests.cs deleted file mode 100644 index 7e313b13..00000000 --- a/test/Redis.OM.Vectorizer.Tests/SentenceVectorizerTests.cs +++ /dev/null @@ -1,24 +0,0 @@ -using Redis.OM.Contracts; -using Redis.OM.Unit.Tests; - -namespace Redis.OM.Vectorizer.Tests; - -public class SentenceVectorizerTests -{ - private readonly IRedisConnectionProvider _provider; - public SentenceVectorizerTests() - { - _provider = new RedisConnectionProvider("redis://localhost:6379"); - } - - [Fact] - public void Test() - { - var connection = _provider.Connection; - connection.Set(new DocWithVector - { - Sentence = Vector.Of("Hello world this is Hal."), - ImagePath = Vector.Of("hal.jpg") - }); - } -} \ No newline at end of file diff --git a/test/Redis.OM.Vectorizer.Tests/VectorizerFunctionalTests.cs b/test/Redis.OM.Vectorizer.Tests/VectorizerFunctionalTests.cs new file mode 100644 index 00000000..3fa66b2d --- /dev/null +++ b/test/Redis.OM.Vectorizer.Tests/VectorizerFunctionalTests.cs @@ -0,0 +1,35 @@ +using Redis.OM.Contracts; +using Redis.OM.Unit.Tests; +using Redis.OM.Vectorizers.AllMiniLML6V2; + +namespace Redis.OM.Vectorizer.Tests; + +public class VectorizerFunctionalTests +{ + private readonly IRedisConnectionProvider _provider; + public VectorizerFunctionalTests() + { + _provider = new RedisConnectionProvider("redis://localhost:6379"); + } + + [Fact] + public void Test() + { + var connection = _provider.Connection; + connection.Set(new DocWithVectors + { + Sentence = Vector.Of("Hello world this is Hal."), + ImagePath = Vector.Of("hal.jpg") + }); + } + + [Fact] + public void SemanticCaching() + { + var cache = _provider.AllMiniLML6V2SemanticCache(); + cache.Store("What is the Capital of France?", "Paris"); + var res = cache.GetSimilar("What really is the capital of France?"); + Assert.NotEmpty(res); + Assert.Equal("Paris", res.First().Response); + } +} \ No newline at end of file From 22e497810238c9435904174e505e4d7c78e14e06 Mon Sep 17 00:00:00 2001 From: slorello89 Date: Fri, 1 Dec 2023 12:39:16 -0500 Subject: [PATCH 28/36] normalizing vectorization pipelines --- src/Redis.OM.Vectorizers.Resnet18/ImageVectorizer.cs | 8 +++++--- test/Redis.OM.Vectorizer.Tests/DocWithVectors.cs | 3 +++ .../VectorizerFunctionalTests.cs | 10 ++++++++++ 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/src/Redis.OM.Vectorizers.Resnet18/ImageVectorizer.cs b/src/Redis.OM.Vectorizers.Resnet18/ImageVectorizer.cs index 21cf6c23..374055e6 100644 --- a/src/Redis.OM.Vectorizers.Resnet18/ImageVectorizer.cs +++ b/src/Redis.OM.Vectorizers.Resnet18/ImageVectorizer.cs @@ -32,8 +32,8 @@ public byte[] Vectorize(string obj) }; var imageStream = Configuration.Instance.Client.Send(request).Content.ReadAsStream(); var image = Image.FromStream(imageStream); - var resized = new Bitmap(image, new Size(224, 224)); - var vector = VectorizeBitMaps(new [] { resized })[0].SelectMany(BitConverter.GetBytes).ToArray(); + var bitmap = new Bitmap(image); + var vector = VectorizeBitMaps(new [] { bitmap })[0].SelectMany(BitConverter.GetBytes).ToArray(); return vector; } @@ -84,7 +84,9 @@ public static float[][] VectorizeFiles(IEnumerable imagePaths) private static EstimatorChain> CreateBitmapPipeline() { var mlContext = MlContext.Value; - var pipeline = mlContext.Transforms.ExtractPixels("Pixels", "Image") + var pipeline = mlContext.Transforms + .ResizeImages("Image", 224,224) + .Append(mlContext.Transforms.ExtractPixels("Pixels", "Image")) .Append(mlContext.Transforms.DnnFeaturizeImage("Features", m => m.ModelSelector.ResNet18(mlContext, m.OutputColumn, m.InputColumn, mlContext), "Pixels")); diff --git a/test/Redis.OM.Vectorizer.Tests/DocWithVectors.cs b/test/Redis.OM.Vectorizer.Tests/DocWithVectors.cs index b7ef50a1..e6248cfa 100644 --- a/test/Redis.OM.Vectorizer.Tests/DocWithVectors.cs +++ b/test/Redis.OM.Vectorizer.Tests/DocWithVectors.cs @@ -1,4 +1,5 @@ using Redis.OM.Modeling; +using Redis.OM.Modeling.Vectors; using Redis.OM.Vectorizers.AllMiniLML6V2; using Redis.OM.Vectorizers.Resnet18; @@ -17,4 +18,6 @@ public class DocWithVectors [Indexed] [ImageVectorizer] public Vector ImagePath { get; set; } + + public VectorScores? Scores { get; set; } } \ No newline at end of file diff --git a/test/Redis.OM.Vectorizer.Tests/VectorizerFunctionalTests.cs b/test/Redis.OM.Vectorizer.Tests/VectorizerFunctionalTests.cs index 3fa66b2d..685763aa 100644 --- a/test/Redis.OM.Vectorizer.Tests/VectorizerFunctionalTests.cs +++ b/test/Redis.OM.Vectorizer.Tests/VectorizerFunctionalTests.cs @@ -1,4 +1,5 @@ using Redis.OM.Contracts; +using Redis.OM.Searching; using Redis.OM.Unit.Tests; using Redis.OM.Vectorizers.AllMiniLML6V2; @@ -16,11 +17,20 @@ public VectorizerFunctionalTests() public void Test() { var connection = _provider.Connection; + connection.CreateIndex(typeof(DocWithVectors)); connection.Set(new DocWithVectors { Sentence = Vector.Of("Hello world this is Hal."), ImagePath = Vector.Of("hal.jpg") }); + + var collection = new RedisCollection(connection); + + // images + var res = collection.NearestNeighbors(x => x.ImagePath, 5, "hal.jpg"); + Assert.Equal(0, res.First().Scores.NearestNeighborsScore); + // sentences + collection.NearestNeighbors(x => x.Sentence, 5, "Hello world this really is Hal."); } [Fact] From 7b8f9c9c5fe41c8bff0d2af58138d892eaba6bac Mon Sep 17 00:00:00 2001 From: slorello89 Date: Fri, 1 Dec 2023 12:55:24 -0500 Subject: [PATCH 29/36] fixing csproj file --- src/Redis.OM/Redis.OM.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Redis.OM/Redis.OM.csproj b/src/Redis.OM/Redis.OM.csproj index e635e05f..93e68ddc 100644 --- a/src/Redis.OM/Redis.OM.csproj +++ b/src/Redis.OM/Redis.OM.csproj @@ -8,7 +8,7 @@ true 0.6.0 0.6.0 - https://github.com/redis/redis-om-dotnet/releases/tag/v0.6.0 Object Mapping and More for Redis Redis OM Steve Lorello From 1528684f58547724cbb4c7243fa0dfd6e2672aa7 Mon Sep 17 00:00:00 2001 From: slorello89 Date: Mon, 4 Dec 2023 07:36:52 -0500 Subject: [PATCH 30/36] fixing test host for Vectorizer tests --- src/Redis.OM.Vectorizers.Resnet18/ImageVectorizerAttribute.cs | 1 - test/Redis.OM.Vectorizer.Tests/VectorizerFunctionalTests.cs | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Redis.OM.Vectorizers.Resnet18/ImageVectorizerAttribute.cs b/src/Redis.OM.Vectorizers.Resnet18/ImageVectorizerAttribute.cs index a4690e1e..a2c46d9a 100644 --- a/src/Redis.OM.Vectorizers.Resnet18/ImageVectorizerAttribute.cs +++ b/src/Redis.OM.Vectorizers.Resnet18/ImageVectorizerAttribute.cs @@ -8,6 +8,5 @@ public class ImageVectorizerAttribute : VectorizerAttribute public override VectorType VectorType => Vectorizer.VectorType; public override int Dim => Vectorizer.Dim; public override byte[] Vectorize(object obj) => Vectorizer.Vectorize((string)obj); - public override IVectorizer Vectorizer { get; } = new ImageVectorizer(); } \ No newline at end of file diff --git a/test/Redis.OM.Vectorizer.Tests/VectorizerFunctionalTests.cs b/test/Redis.OM.Vectorizer.Tests/VectorizerFunctionalTests.cs index 685763aa..bc1370c2 100644 --- a/test/Redis.OM.Vectorizer.Tests/VectorizerFunctionalTests.cs +++ b/test/Redis.OM.Vectorizer.Tests/VectorizerFunctionalTests.cs @@ -10,7 +10,8 @@ public class VectorizerFunctionalTests private readonly IRedisConnectionProvider _provider; public VectorizerFunctionalTests() { - _provider = new RedisConnectionProvider("redis://localhost:6379"); + var host = Environment.GetEnvironmentVariable("STANDALONE_HOST_PORT") ?? "localhost"; + _provider = new RedisConnectionProvider($"redis://{host}"); } [Fact] From e5aab0ebf834a17042afae031228aaa473058f2f Mon Sep 17 00:00:00 2001 From: slorello89 Date: Mon, 4 Dec 2023 08:24:15 -0500 Subject: [PATCH 31/36] lfs --- .github/workflows/dotnet-core.yml | 2 ++ test/Redis.OM.Vectorizer.Tests/VectorizerFunctionalTests.cs | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/dotnet-core.yml b/.github/workflows/dotnet-core.yml index a1a8acca..731e3199 100644 --- a/.github/workflows/dotnet-core.yml +++ b/.github/workflows/dotnet-core.yml @@ -11,5 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 + with: + lfs: true - name: execute run: docker-compose -f ./docker/docker-compose.yaml run dotnet \ No newline at end of file diff --git a/test/Redis.OM.Vectorizer.Tests/VectorizerFunctionalTests.cs b/test/Redis.OM.Vectorizer.Tests/VectorizerFunctionalTests.cs index bc1370c2..4ff63d5f 100644 --- a/test/Redis.OM.Vectorizer.Tests/VectorizerFunctionalTests.cs +++ b/test/Redis.OM.Vectorizer.Tests/VectorizerFunctionalTests.cs @@ -1,6 +1,5 @@ using Redis.OM.Contracts; using Redis.OM.Searching; -using Redis.OM.Unit.Tests; using Redis.OM.Vectorizers.AllMiniLML6V2; namespace Redis.OM.Vectorizer.Tests; From 59b2b13592bad80c75c1eb52cacf7021bf907625 Mon Sep 17 00:00:00 2001 From: slorello89 Date: Mon, 4 Dec 2023 08:48:06 -0500 Subject: [PATCH 32/36] removing System.Drawing dep, adding onnxruntime --- .../Redis.OM.Vectorizers.AllMiniLML6V2.csproj | 5 +++-- src/Redis.OM.Vectorizers.Resnet18/ImageModelObjects.cs | 3 +-- src/Redis.OM.Vectorizers.Resnet18/ImageVectorizer.cs | 10 ++++------ .../Redis.OM.Vectorizers.Resnet18.csproj | 8 ++++---- 4 files changed, 12 insertions(+), 14 deletions(-) diff --git a/src/Redis.OM.Vectorizers.AllMiniLML6V2/Redis.OM.Vectorizers.AllMiniLML6V2.csproj b/src/Redis.OM.Vectorizers.AllMiniLML6V2/Redis.OM.Vectorizers.AllMiniLML6V2.csproj index 568d617e..30badfd0 100644 --- a/src/Redis.OM.Vectorizers.AllMiniLML6V2/Redis.OM.Vectorizers.AllMiniLML6V2.csproj +++ b/src/Redis.OM.Vectorizers.AllMiniLML6V2/Redis.OM.Vectorizers.AllMiniLML6V2.csproj @@ -21,8 +21,9 @@ - - + + + diff --git a/src/Redis.OM.Vectorizers.Resnet18/ImageModelObjects.cs b/src/Redis.OM.Vectorizers.Resnet18/ImageModelObjects.cs index e8fb2cde..5ccf756c 100644 --- a/src/Redis.OM.Vectorizers.Resnet18/ImageModelObjects.cs +++ b/src/Redis.OM.Vectorizers.Resnet18/ImageModelObjects.cs @@ -1,4 +1,3 @@ -using System.Drawing; using Microsoft.ML.Data; using Microsoft.ML.Transforms.Image; @@ -13,5 +12,5 @@ public class ImageInput public class InMemoryImageData { [ImageType(224,224)] - public Bitmap Image; + public MLImage Image; } \ No newline at end of file diff --git a/src/Redis.OM.Vectorizers.Resnet18/ImageVectorizer.cs b/src/Redis.OM.Vectorizers.Resnet18/ImageVectorizer.cs index 374055e6..61781216 100644 --- a/src/Redis.OM.Vectorizers.Resnet18/ImageVectorizer.cs +++ b/src/Redis.OM.Vectorizers.Resnet18/ImageVectorizer.cs @@ -1,4 +1,3 @@ -using System.Drawing; using Microsoft.ML; using Microsoft.ML.Data; using Microsoft.ML.Transforms; @@ -31,9 +30,8 @@ public byte[] Vectorize(string obj) RequestUri = uri, }; var imageStream = Configuration.Instance.Client.Send(request).Content.ReadAsStream(); - var image = Image.FromStream(imageStream); - var bitmap = new Bitmap(image); - var vector = VectorizeBitMaps(new [] { bitmap })[0].SelectMany(BitConverter.GetBytes).ToArray(); + var image = MLImage.CreateFromStream(imageStream); + var vector = VectorizeBitMaps(new [] { image })[0].SelectMany(BitConverter.GetBytes).ToArray(); return vector; } @@ -93,9 +91,9 @@ private static EstimatorChain> Create return pipeline; } - public static float[][] VectorizeBitMaps(IEnumerable bitmaps) + public static float[][] VectorizeBitMaps(IEnumerable mlImages) { - var images = bitmaps.Select(x => new InMemoryImageData { Image = x }); + var images = mlImages.Select(x => new InMemoryImageData { Image = x }); var mlContext = MlContext.Value; var dataView = mlContext.Data.LoadFromEnumerable(images); var transformedData = BitmapPipeline.Value.Fit(dataView).Transform(dataView); diff --git a/src/Redis.OM.Vectorizers.Resnet18/Redis.OM.Vectorizers.Resnet18.csproj b/src/Redis.OM.Vectorizers.Resnet18/Redis.OM.Vectorizers.Resnet18.csproj index fa1a81f9..58d9d1e3 100644 --- a/src/Redis.OM.Vectorizers.Resnet18/Redis.OM.Vectorizers.Resnet18.csproj +++ b/src/Redis.OM.Vectorizers.Resnet18/Redis.OM.Vectorizers.Resnet18.csproj @@ -21,10 +21,10 @@ - - - - + + + + From 843a81d83f560c8cdcbac402b1db11b36d17cb0b Mon Sep 17 00:00:00 2001 From: slorello89 Date: Mon, 4 Dec 2023 09:36:26 -0500 Subject: [PATCH 33/36] removing warnings --- .../SentenceVectorizer.cs | 22 +++++++--- .../SentenceVectorizerAttribute.cs | 9 ++++ .../Tokenizers/CasedTokenizer.cs | 2 +- .../Tokenizers/Tokens.cs | 2 +- .../Tokenizers/UncasedTokenizer.cs | 2 +- .../ImageModelObjects.cs | 14 ++++++- .../ImageVectorizer.cs | 17 +++++--- .../ImageVectorizerAttribute.cs | 10 +++++ .../AzureOpenAIVectorizer.cs | 15 +++++++ src/Redis.OM.Vectorizers/Configuration.cs | 42 ++++++++++++++++++- .../HuggingFaceVectorizer.cs | 30 ++++++++++++- .../HuggingFaceVectorizerAttribute.cs | 9 ++-- src/Redis.OM.Vectorizers/OpenAIVectorizer.cs | 16 ++++++- .../RedisConnectionProviderExtensions.cs | 23 ++++++++++ src/Redis.OM.Vectorizers/RedisOMHttpUtil.cs | 2 +- .../DocWithVectors.cs | 6 +-- .../VectorizerFunctionalTests.cs | 6 +-- 17 files changed, 196 insertions(+), 31 deletions(-) diff --git a/src/Redis.OM.Vectorizers.AllMiniLML6V2/SentenceVectorizer.cs b/src/Redis.OM.Vectorizers.AllMiniLML6V2/SentenceVectorizer.cs index 0f3385f4..0c759403 100644 --- a/src/Redis.OM.Vectorizers.AllMiniLML6V2/SentenceVectorizer.cs +++ b/src/Redis.OM.Vectorizers.AllMiniLML6V2/SentenceVectorizer.cs @@ -7,9 +7,15 @@ namespace Redis.OM.Vectorizers.AllMiniLML6V2; +/// +/// A vectorizer to Vectorize sentences using ALl Mini LM L6 V2 Model. +/// public class SentenceVectorizer : IVectorizer { + /// public VectorType VectorType => VectorType.FLOAT32; + + /// public int Dim => 384; private static Lazy Tokenizer => new Lazy(AllMiniLML6V2Tokenizer.Create); private static Lazy InferenceSession => new Lazy(LoadInferenceSession); @@ -27,7 +33,8 @@ private static InferenceSession LoadInferenceSession() _ = stream.Read(resourceBytes, 0, resourceBytes.Length); return new InferenceSession(resourceBytes); } - + + /// public byte[] Vectorize(string obj) { return Encode(new[] { obj })[0].SelectMany(BitConverter.GetBytes).ToArray(); @@ -35,6 +42,11 @@ public byte[] Vectorize(string obj) private static Lazy OutputNames => new (() => InferenceSession.Value.OutputMetadata.Keys.ToArray()); + /// + /// Vectorizers an array of sentences (which are vectorized individually). + /// + /// The Sentences + /// public static float[][] Encode(string[] sentences) { const int MaxTokens = 512; @@ -50,7 +62,7 @@ public static float[][] Encode(string[] sentences) var tokenIndexes = tokens.Take(MaxTokens).Select(token => (long)token.VocabularyIndex).Concat(padding).ToArray(); var segmentIndexes = tokens.Take(MaxTokens).Select(token => token.SegmentIndex).Concat(padding).ToArray(); - var inputMask = tokens.Take(MaxTokens).Select(o => 1L).Concat(padding).ToArray(); + var inputMask = tokens.Take(MaxTokens).Select(_ => 1L).Concat(padding).ToArray(); return (tokenIndexes, TokenTypeIds: segmentIndexes, inputMask); }).ToList(); var tokenCount = encoded.First().InputIds.Length; @@ -77,7 +89,7 @@ public static float[][] Encode(string[] sentences) var dimensions = new[] { numSentences, tokenCount }; - var input = new NamedOnnxValue[3] + var input = new [] { NamedOnnxValue.CreateFromTensor("input_ids", new DenseTensor(flattenIDs, dimensions)), NamedOnnxValue.CreateFromTensor("attention_mask", new DenseTensor(flattenAttentionMask,dimensions)), @@ -109,7 +121,7 @@ public static float[][] Encode(string[] sentences) return outputFlatten; } - public static DenseTensor Normalize(DenseTensor input_dense, float eps = 1e-12f) + internal static DenseTensor Normalize(DenseTensor input_dense, float eps = 1e-12f) { //Computes sum(abs(x)^2)^(1/2) @@ -142,7 +154,7 @@ public static DenseTensor Normalize(DenseTensor input_dense, float } - public static DenseTensor MeanPooling(DenseTensor token_embeddings_dense, List<(long[] InputIds, long[] TokenTypeIds, long[] AttentionMask)> encodedSentences, float eps = 1e-9f) + internal static DenseTensor MeanPooling(DenseTensor token_embeddings_dense, List<(long[] InputIds, long[] TokenTypeIds, long[] AttentionMask)> encodedSentences, float eps = 1e-9f) { var sentencesCount = token_embeddings_dense.Dimensions[0]; var sentenceLength = token_embeddings_dense.Dimensions[1]; diff --git a/src/Redis.OM.Vectorizers.AllMiniLML6V2/SentenceVectorizerAttribute.cs b/src/Redis.OM.Vectorizers.AllMiniLML6V2/SentenceVectorizerAttribute.cs index 138ed6fc..39d89677 100644 --- a/src/Redis.OM.Vectorizers.AllMiniLML6V2/SentenceVectorizerAttribute.cs +++ b/src/Redis.OM.Vectorizers.AllMiniLML6V2/SentenceVectorizerAttribute.cs @@ -3,11 +3,20 @@ namespace Redis.OM.Vectorizers.AllMiniLML6V2; +/// +/// +/// public class SentenceVectorizerAttribute : VectorizerAttribute { + /// public override VectorType VectorType => Vectorizer.VectorType; + + /// public override int Dim => Vectorizer.Dim; + + /// public override byte[] Vectorize(object obj) => Vectorizer.Vectorize((string)obj); + /// public override IVectorizer Vectorizer => new SentenceVectorizer(); } \ No newline at end of file diff --git a/src/Redis.OM.Vectorizers.AllMiniLML6V2/Tokenizers/CasedTokenizer.cs b/src/Redis.OM.Vectorizers.AllMiniLML6V2/Tokenizers/CasedTokenizer.cs index 2a4f67cd..3181eddc 100644 --- a/src/Redis.OM.Vectorizers.AllMiniLML6V2/Tokenizers/CasedTokenizer.cs +++ b/src/Redis.OM.Vectorizers.AllMiniLML6V2/Tokenizers/CasedTokenizer.cs @@ -8,7 +8,7 @@ protected CasedTokenizer(string[] vocabulary) : base(vocabulary) protected override IEnumerable TokenizeSentence(string text) { - return text.Split(new string[] { " ", " ", "\r\n" }, StringSplitOptions.None) + return text.Split(new [] { " ", " ", "\r\n" }, StringSplitOptions.None) .SelectMany(o => o.SplitAndKeep(".,;:\\/?!#$%()=+-*\"'–_`<>&^@{}[]|~'".ToArray())); } } \ No newline at end of file diff --git a/src/Redis.OM.Vectorizers.AllMiniLML6V2/Tokenizers/Tokens.cs b/src/Redis.OM.Vectorizers.AllMiniLML6V2/Tokenizers/Tokens.cs index 2963258e..5f7ea2ab 100644 --- a/src/Redis.OM.Vectorizers.AllMiniLML6V2/Tokenizers/Tokens.cs +++ b/src/Redis.OM.Vectorizers.AllMiniLML6V2/Tokenizers/Tokens.cs @@ -1,6 +1,6 @@ namespace Redis.OM.Vectorizers.AllMiniLML6V2.Tokenizers; -public class Tokens +internal class Tokens { public const string Padding = ""; public const string Unknown = "[UNK]"; diff --git a/src/Redis.OM.Vectorizers.AllMiniLML6V2/Tokenizers/UncasedTokenizer.cs b/src/Redis.OM.Vectorizers.AllMiniLML6V2/Tokenizers/UncasedTokenizer.cs index 59f109a8..d3581d2f 100644 --- a/src/Redis.OM.Vectorizers.AllMiniLML6V2/Tokenizers/UncasedTokenizer.cs +++ b/src/Redis.OM.Vectorizers.AllMiniLML6V2/Tokenizers/UncasedTokenizer.cs @@ -8,7 +8,7 @@ public UncasedTokenizer(string[] vocabulary) : base(vocabulary) protected override IEnumerable TokenizeSentence(string text) { - return text.Split(new string[] { " ", " ", "\r\n" }, StringSplitOptions.None) + return text.Split(new [] { " ", " ", "\r\n" }, StringSplitOptions.None) .SelectMany(o => o.SplitAndKeep(".,;:\\/?!#$%()=+-*\"'–_`<>&^@{}[]|~'".ToArray())) .Select(o => o.ToLower()); } diff --git a/src/Redis.OM.Vectorizers.Resnet18/ImageModelObjects.cs b/src/Redis.OM.Vectorizers.Resnet18/ImageModelObjects.cs index 5ccf756c..efe59d67 100644 --- a/src/Redis.OM.Vectorizers.Resnet18/ImageModelObjects.cs +++ b/src/Redis.OM.Vectorizers.Resnet18/ImageModelObjects.cs @@ -3,14 +3,24 @@ namespace Redis.OM.Vectorizers.Resnet18; -public class ImageInput +internal class ImageInput { [ColumnName(@"ImageSource")] public string ImageSource { get; set; } + + public ImageInput(string imageSource) + { + ImageSource = imageSource; + } } -public class InMemoryImageData +internal class InMemoryImageData { [ImageType(224,224)] public MLImage Image; + + public InMemoryImageData(MLImage image) + { + Image = image; + } } \ No newline at end of file diff --git a/src/Redis.OM.Vectorizers.Resnet18/ImageVectorizer.cs b/src/Redis.OM.Vectorizers.Resnet18/ImageVectorizer.cs index 61781216..3ef48c14 100644 --- a/src/Redis.OM.Vectorizers.Resnet18/ImageVectorizer.cs +++ b/src/Redis.OM.Vectorizers.Resnet18/ImageVectorizer.cs @@ -31,7 +31,7 @@ public byte[] Vectorize(string obj) }; var imageStream = Configuration.Instance.Client.Send(request).Content.ReadAsStream(); var image = MLImage.CreateFromStream(imageStream); - var vector = VectorizeBitMaps(new [] { image })[0].SelectMany(BitConverter.GetBytes).ToArray(); + var vector = VectorizeImages(new [] { image })[0].SelectMany(BitConverter.GetBytes).ToArray(); return vector; } @@ -44,7 +44,7 @@ public byte[] Vectorize(string obj) return VectorizeFiles(new[] { obj })[0].SelectMany(BitConverter.GetBytes).ToArray(); } - private static Lazy>> FilePipeline = new(CreateFilePipeline); + private static readonly Lazy>> FilePipeline = new(CreateFilePipeline); private static readonly Lazy MlContext = new(()=>new MLContext()); @@ -68,7 +68,7 @@ private static EstimatorChain> Create /// public static float[][] VectorizeFiles(IEnumerable imagePaths) { - var images = imagePaths.Select(x => new ImageInput { ImageSource = x }); + var images = imagePaths.Select(x => new ImageInput(x)); var mlContext = MlContext.Value; var dataView = mlContext.Data.LoadFromEnumerable(images); @@ -77,7 +77,7 @@ public static float[][] VectorizeFiles(IEnumerable imagePaths) return vector; } - public static Lazy>> BitmapPipeline = new(CreateBitmapPipeline); + private static readonly Lazy>> BitmapPipeline = new(CreateBitmapPipeline); private static EstimatorChain> CreateBitmapPipeline() { @@ -91,9 +91,14 @@ private static EstimatorChain> Create return pipeline; } - public static float[][] VectorizeBitMaps(IEnumerable mlImages) + /// + /// Encodes a collection of images. + /// + /// + /// + public static float[][] VectorizeImages(IEnumerable mlImages) { - var images = mlImages.Select(x => new InMemoryImageData { Image = x }); + var images = mlImages.Select(x => new InMemoryImageData(x)); var mlContext = MlContext.Value; var dataView = mlContext.Data.LoadFromEnumerable(images); var transformedData = BitmapPipeline.Value.Fit(dataView).Transform(dataView); diff --git a/src/Redis.OM.Vectorizers.Resnet18/ImageVectorizerAttribute.cs b/src/Redis.OM.Vectorizers.Resnet18/ImageVectorizerAttribute.cs index a2c46d9a..29b58492 100644 --- a/src/Redis.OM.Vectorizers.Resnet18/ImageVectorizerAttribute.cs +++ b/src/Redis.OM.Vectorizers.Resnet18/ImageVectorizerAttribute.cs @@ -3,10 +3,20 @@ namespace Redis.OM.Vectorizers.Resnet18; +/// +/// A Vectorizer Attribute for encoding images +/// public class ImageVectorizerAttribute : VectorizerAttribute { + /// public override VectorType VectorType => Vectorizer.VectorType; + + /// public override int Dim => Vectorizer.Dim; + + /// public override byte[] Vectorize(object obj) => Vectorizer.Vectorize((string)obj); + + /// public override IVectorizer Vectorizer { get; } = new ImageVectorizer(); } \ No newline at end of file diff --git a/src/Redis.OM.Vectorizers/AzureOpenAIVectorizer.cs b/src/Redis.OM.Vectorizers/AzureOpenAIVectorizer.cs index b3e4aecd..62b558e4 100644 --- a/src/Redis.OM.Vectorizers/AzureOpenAIVectorizer.cs +++ b/src/Redis.OM.Vectorizers/AzureOpenAIVectorizer.cs @@ -6,12 +6,22 @@ namespace Redis.OM.Vectorizers; +/// +/// Vectorizer for Azure's OpenAI REST API +/// public class AzureOpenAIVectorizer : IVectorizer { private readonly string _apiKey; private readonly string _resourceName; private readonly string _deploymentName; + /// + /// Initializes vectorizer + /// + /// The Vectorizers API Key + /// The Azure Resource Name. + /// The Azure Deployment Name. + /// The dimensions of the model addressed by this resource/deployment. public AzureOpenAIVectorizer(string apiKey, string resourceName, string deploymentName, int dim) { _apiKey = apiKey; @@ -20,8 +30,13 @@ public AzureOpenAIVectorizer(string apiKey, string resourceName, string deployme Dim = dim; } + /// public VectorType VectorType => VectorType.FLOAT32; + + /// public int Dim { get; } + + /// public byte[] Vectorize(string str) => GetFloats(str, _resourceName, _deploymentName, _apiKey).SelectMany(BitConverter.GetBytes).ToArray(); internal static float[] GetFloats(string s, string resourceName, string deploymentName, string apiKey) diff --git a/src/Redis.OM.Vectorizers/Configuration.cs b/src/Redis.OM.Vectorizers/Configuration.cs index 69c574bc..7810fa3f 100644 --- a/src/Redis.OM.Vectorizers/Configuration.cs +++ b/src/Redis.OM.Vectorizers/Configuration.cs @@ -1,20 +1,52 @@ using System.Net.Http.Headers; +using System.Runtime.CompilerServices; using Microsoft.Extensions.Configuration; - +[assembly: InternalsVisibleTo("Redis.OM.Vectorizers.Resnet18")] namespace Redis.OM; -public class Configuration +/// +/// Some Configuration Items. +/// +internal class Configuration { + /// + /// Gets the configuration item at the given key. + /// + /// public string? this[string str] => _settings[str]; + + /// + /// The bearer authorization token for Hugging Face's model API. + /// public string HuggingFaceAuthorizationToken => _settings["REDIS_OM_HF_TOKEN"] ?? string.Empty; + + /// + /// Bearer token for Open AI's API. + /// public string OpenAiAuthorizationToken => _settings["REDIS_OM_OAI_TOKEN"] ?? string.Empty; + + /// + /// Azure OpenAI Api Key. + /// public string AzureOpenAIApiKey => _settings["REDIS_OM_AZURE_OAI_TOKEN"] ?? string.Empty; + + /// + /// Hugging Face Model Id + /// public string ModelId => _settings["REDIS_OM_HF_MODEL_ID"] ?? string.Empty; + + /// + /// Base Address for Hugging Face Feature Extraction API + /// public string HuggingFaceBaseAddress => _settings["REDIS_OM_HF_FEATURE_EXTRACTION_URL"] ?? string.Empty; private const string DefaultHuggingFaceApiUrl = "https://api-inference.huggingface.co"; private const string DefaultOpenAiApiUrl = "https://api.openai.com"; + + /// + /// URL for OpenAI API. + /// public string OpenAiApiUrl => _settings["REDIS_OM_OAI_API_URL"] ?? String.Empty; private readonly IConfiguration _settings; @@ -22,8 +54,14 @@ public class Configuration private static readonly object LockObject = new (); private static Configuration? _instance; + /// + /// Common HTTP Client. + /// public readonly HttpClient Client; + /// + /// Singleton Instance. + /// public static Configuration Instance { get diff --git a/src/Redis.OM.Vectorizers/HuggingFaceVectorizer.cs b/src/Redis.OM.Vectorizers/HuggingFaceVectorizer.cs index c07224c3..988fe7be 100644 --- a/src/Redis.OM.Vectorizers/HuggingFaceVectorizer.cs +++ b/src/Redis.OM.Vectorizers/HuggingFaceVectorizer.cs @@ -5,8 +5,17 @@ namespace Redis.OM.Vectorizers; +/// +/// Vectorizer for HuggingFace API. +/// public class HuggingFaceVectorizer : IVectorizer { + /// + /// Initializes the Vectorizer. + /// + /// Auth token. + /// Model Id. + /// Dimensions for the output tensors of the model. public HuggingFaceVectorizer(string authToken, string modelId, int dim) { _huggingFaceAuthToken = authToken; @@ -15,16 +24,33 @@ public HuggingFaceVectorizer(string authToken, string modelId, int dim) } private readonly string _huggingFaceAuthToken; + + /// + /// The Model Id. + /// public string ModelId { get; } + + /// public VectorType VectorType => VectorType.FLOAT32; - + + /// public int Dim { get; } + + /// public byte[] Vectorize(string str) { return GetFloats(str, ModelId, _huggingFaceAuthToken).SelectMany(BitConverter.GetBytes).ToArray(); } - public static float[] GetFloats(string s, string modelId, string huggingFaceAuthToken) + /// + /// Gets the floats for the sentence. + /// + /// the string. + /// The Model Id. + /// The HF token. + /// + /// + internal static float[] GetFloats(string s, string modelId, string huggingFaceAuthToken) { var client = Configuration.Instance.Client; var requestContent = JsonContent.Create(new diff --git a/src/Redis.OM.Vectorizers/HuggingFaceVectorizerAttribute.cs b/src/Redis.OM.Vectorizers/HuggingFaceVectorizerAttribute.cs index 85a42af7..ba1905c4 100644 --- a/src/Redis.OM.Vectorizers/HuggingFaceVectorizerAttribute.cs +++ b/src/Redis.OM.Vectorizers/HuggingFaceVectorizerAttribute.cs @@ -1,5 +1,4 @@ -using System.Net.Http.Json; -using Redis.OM.Contracts; +using Redis.OM.Contracts; using Redis.OM.Modeling; namespace Redis.OM.Vectorizers; @@ -9,6 +8,9 @@ namespace Redis.OM.Vectorizers; /// public class HuggingFaceVectorizerAttribute : VectorizerAttribute { + /// + /// The Model Id. + /// public string? ModelId { get; set; } private IVectorizer? _vectorizer; @@ -26,7 +28,7 @@ public override IVectorizer Vectorizer throw new InvalidOperationException("Need a Model ID in order to process vector"); } - _vectorizer = new HuggingFaceVectorizer(Configuration.Instance.HuggingFaceAuthorizationToken, ModelId, Dim); + _vectorizer = new HuggingFaceVectorizer(Configuration.Instance.HuggingFaceAuthorizationToken, modelId, Dim); } return _vectorizer; @@ -38,6 +40,7 @@ public override IVectorizer Vectorizer public override VectorType VectorType => VectorType.FLOAT32; private int? _dim; + /// public override int Dim { get diff --git a/src/Redis.OM.Vectorizers/OpenAIVectorizer.cs b/src/Redis.OM.Vectorizers/OpenAIVectorizer.cs index 5851fe82..a6294a05 100644 --- a/src/Redis.OM.Vectorizers/OpenAIVectorizer.cs +++ b/src/Redis.OM.Vectorizers/OpenAIVectorizer.cs @@ -5,11 +5,20 @@ namespace Redis.OM.Vectorizers; +/// +/// A Vectorizer leveraging Open AI's REST API +/// public class OpenAIVectorizer : IVectorizer { private readonly string _openAIAuthToken; private readonly string _model; + /// + /// Initializes the vectorizer. + /// + /// The Authorization Token. + /// The Model ID + /// The dimension of the model's output tensor. public OpenAIVectorizer(string openAIAuthToken, string model = "text-embedding-ada-002", int dim = 1536) { _openAIAuthToken = openAIAuthToken; @@ -17,8 +26,13 @@ public OpenAIVectorizer(string openAIAuthToken, string model = "text-embedding-a Dim = dim; } + /// public VectorType VectorType => VectorType.FLOAT32; + + /// public int Dim { get; } + + /// public byte[] Vectorize(string str) { var floats = GetFloats(str, _model, _openAIAuthToken); @@ -28,7 +42,7 @@ public byte[] Vectorize(string str) internal static float[] GetFloats(string s, string model, string openAIAuthToken) { var client = Configuration.Instance.Client; - var requestContent = JsonContent.Create(new { input = s, model = model }); + var requestContent = JsonContent.Create(new { input = s, model }); var request = new HttpRequestMessage { diff --git a/src/Redis.OM.Vectorizers/RedisConnectionProviderExtensions.cs b/src/Redis.OM.Vectorizers/RedisConnectionProviderExtensions.cs index 25bd01db..4f2d017c 100644 --- a/src/Redis.OM.Vectorizers/RedisConnectionProviderExtensions.cs +++ b/src/Redis.OM.Vectorizers/RedisConnectionProviderExtensions.cs @@ -34,6 +34,16 @@ public static ISemanticCache HuggingFaceSemanticCache(this IRedisConnectionProvi return cache; } + /// + /// Creates a Semantic Cache that leverages OpenAI's REST API. + /// + /// The Provider. + /// The OpenAI bearer token. + /// The activation threshold for acceptable distance. + /// The index name to create. + /// The Prefix to use for the semantic cache. + /// The Time to Live for Items in the Semantic Cache. + /// public static ISemanticCache OpenAISemanticCache(this IRedisConnectionProvider provider, string openAIAuthToken, double threshold = .15, string indexName = "OpenAISemanticCache", string? prefix = null, long? ttl = null) { var vectorizer = new OpenAIVectorizer(openAIAuthToken); @@ -48,6 +58,19 @@ public static ISemanticCache OpenAISemanticCache(this IRedisConnectionProvider p return cache; } + /// + /// Creates a Semantic Cache leveraging Azure's Open AI REST Api. + /// + /// The RedisConnectionProvider. + /// The API Key for Azure. + /// The Resource Name. + /// The Deployment ID + /// The dimension of the model at the given Resource/Deployment. + /// The Activation Threshold. + /// The Index name. + /// The Prefix. + /// The Time to Live for a record inserted into the cache. + /// public static ISemanticCache AzureOpenAISemanticCache(this IRedisConnectionProvider provider, string apiKey, string resourceName, string deploymentId, int dim, double threshold = .15, string indexName = "AzureOpenAISemanticCache", string? prefix = null, long? ttl = null) { var vectorizer = new AzureOpenAIVectorizer(apiKey, resourceName, deploymentId, dim); diff --git a/src/Redis.OM.Vectorizers/RedisOMHttpUtil.cs b/src/Redis.OM.Vectorizers/RedisOMHttpUtil.cs index 2b2eba46..cefbb826 100644 --- a/src/Redis.OM.Vectorizers/RedisOMHttpUtil.cs +++ b/src/Redis.OM.Vectorizers/RedisOMHttpUtil.cs @@ -1,6 +1,6 @@ namespace Redis.OM; -internal class RedisOMHttpUtil +internal static class RedisOMHttpUtil { public static string ReadJsonSync(HttpResponseMessage msg) { diff --git a/test/Redis.OM.Vectorizer.Tests/DocWithVectors.cs b/test/Redis.OM.Vectorizer.Tests/DocWithVectors.cs index e6248cfa..417a2264 100644 --- a/test/Redis.OM.Vectorizer.Tests/DocWithVectors.cs +++ b/test/Redis.OM.Vectorizer.Tests/DocWithVectors.cs @@ -9,15 +9,15 @@ namespace Redis.OM.Vectorizer.Tests; public class DocWithVectors { [RedisIdField] - public string Id { get; set; } + public string? Id { get; set; } [Indexed(Algorithm = VectorAlgorithm.HNSW)] [SentenceVectorizer] - public Vector Sentence { get; set; } + public Vector? Sentence { get; set; } [Indexed] [ImageVectorizer] - public Vector ImagePath { get; set; } + public Vector? ImagePath { get; set; } public VectorScores? Scores { get; set; } } \ No newline at end of file diff --git a/test/Redis.OM.Vectorizer.Tests/VectorizerFunctionalTests.cs b/test/Redis.OM.Vectorizer.Tests/VectorizerFunctionalTests.cs index 4ff63d5f..97dd6aa4 100644 --- a/test/Redis.OM.Vectorizer.Tests/VectorizerFunctionalTests.cs +++ b/test/Redis.OM.Vectorizer.Tests/VectorizerFunctionalTests.cs @@ -27,10 +27,10 @@ public void Test() var collection = new RedisCollection(connection); // images - var res = collection.NearestNeighbors(x => x.ImagePath, 5, "hal.jpg"); - Assert.Equal(0, res.First().Scores.NearestNeighborsScore); + var res = collection.NearestNeighbors(x => x.ImagePath!, 5, "hal.jpg"); + Assert.Equal(0, res.First().Scores!.NearestNeighborsScore); // sentences - collection.NearestNeighbors(x => x.Sentence, 5, "Hello world this really is Hal."); + collection.NearestNeighbors(x => x.Sentence!, 5, "Hello world this really is Hal."); } [Fact] From 3fc563f2b7472b15f9c5c4f115c7ca52bd83d19d Mon Sep 17 00:00:00 2001 From: slorello89 Date: Mon, 4 Dec 2023 16:27:53 -0500 Subject: [PATCH 34/36] adding MIT licenses for sources --- .../LICENSE | 35 +++++++++++++++++++ .../SentenceVectorizer.cs | 2 -- 2 files changed, 35 insertions(+), 2 deletions(-) create mode 100644 src/Redis.OM.Vectorizers.AllMiniLML6V2/LICENSE diff --git a/src/Redis.OM.Vectorizers.AllMiniLML6V2/LICENSE b/src/Redis.OM.Vectorizers.AllMiniLML6V2/LICENSE new file mode 100644 index 00000000..3939dec8 --- /dev/null +++ b/src/Redis.OM.Vectorizers.AllMiniLML6V2/LICENSE @@ -0,0 +1,35 @@ +Copyright (c) 2023 Redis Inc. + + Permission is hereby granted, free of charge, to any person + obtaining a copy of this software and associated documentation + files (the "Software"), to deal in the Software without + restriction, including without limitation the rights to use, + copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following + conditions: + + The above copyright notice and this permission notice shall be + included in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + OTHER DEALINGS IN THE SOFTWARE. + + =============================================== + + Third Party Licenses: + + The BERT tokenization code heavily influenced by the BertTokenizers + Project: https://github.com/NMZivkovic/BertTokenizers which is + licensed under an MIT license: + https://github.com/NMZivkovic/BertTokenizers/blob/master/LICENSE.txt + + Some parts of the pre/post processing pipeline were adapted + from Curiosity AI's MiniLM project, which uses an MIT license: + https://github.com/curiosity-ai/MiniLM/blob/4a7c629c223b6244cb8a394f17920ea1de363dce/MiniLM/MiniLM.csproj#L13 \ No newline at end of file diff --git a/src/Redis.OM.Vectorizers.AllMiniLML6V2/SentenceVectorizer.cs b/src/Redis.OM.Vectorizers.AllMiniLML6V2/SentenceVectorizer.cs index 0c759403..ed65276c 100644 --- a/src/Redis.OM.Vectorizers.AllMiniLML6V2/SentenceVectorizer.cs +++ b/src/Redis.OM.Vectorizers.AllMiniLML6V2/SentenceVectorizer.cs @@ -123,8 +123,6 @@ public static float[][] Encode(string[] sentences) internal static DenseTensor Normalize(DenseTensor input_dense, float eps = 1e-12f) { - //Computes sum(abs(x)^2)^(1/2) - var sentencesCount = input_dense.Dimensions[0]; var hiddenStates = input_dense.Dimensions[1]; From 4411d026277adceadcf48e75b0f11b52005ed872 Mon Sep 17 00:00:00 2001 From: slorello89 Date: Mon, 4 Dec 2023 16:51:15 -0500 Subject: [PATCH 35/36] readme updates --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 591ec4db..97ff9dc3 100644 --- a/README.md +++ b/README.md @@ -387,6 +387,11 @@ cache.Store("What is the capital of France?", "Paris"); var res = cache.GetSimilar("What really is the capital of France?").First(); ``` +### ML.NET Based Vectorizers + +We also provide the packages `Redis.OM.Vectorizers.ResNet18` and `Redis.OM.Vectorizers.AllMiniLML6V2` which have embedded models / ML Pipelines in them to +allow you to easily Vectorize Images and Sentences respectively without the need to depend on an external API. + ### 🖩 Aggregations We can also run aggregations on the customer object, again using expressions in LINQ: From 96eae51f4f4fb7be176febd982638845fa89c43f Mon Sep 17 00:00:00 2001 From: slorello89 Date: Tue, 5 Dec 2023 08:01:24 -0500 Subject: [PATCH 36/36] Encode -> Vectorize --- src/Redis.OM.Vectorizers.AllMiniLML6V2/SentenceVectorizer.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Redis.OM.Vectorizers.AllMiniLML6V2/SentenceVectorizer.cs b/src/Redis.OM.Vectorizers.AllMiniLML6V2/SentenceVectorizer.cs index ed65276c..92dd96e7 100644 --- a/src/Redis.OM.Vectorizers.AllMiniLML6V2/SentenceVectorizer.cs +++ b/src/Redis.OM.Vectorizers.AllMiniLML6V2/SentenceVectorizer.cs @@ -37,7 +37,7 @@ private static InferenceSession LoadInferenceSession() /// public byte[] Vectorize(string obj) { - return Encode(new[] { obj })[0].SelectMany(BitConverter.GetBytes).ToArray(); + return Vectorize(new[] { obj })[0].SelectMany(BitConverter.GetBytes).ToArray(); } private static Lazy OutputNames => new (() => InferenceSession.Value.OutputMetadata.Keys.ToArray()); @@ -47,7 +47,7 @@ public byte[] Vectorize(string obj) /// /// The Sentences /// - public static float[][] Encode(string[] sentences) + public static float[][] Vectorize(string[] sentences) { const int MaxTokens = 512; var numSentences = sentences.Length;