Skip to content

.Net: Fix timezone bug in qdrant datetime unit tests. #11407

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Apr 15, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -40,11 +40,9 @@ internal static class QdrantVectorStoreCollectionCreateMapping
{ typeof(decimal?), PayloadSchemaType.Float },

{ typeof(string), PayloadSchemaType.Keyword },
{ typeof(DateTime), PayloadSchemaType.Datetime },
{ typeof(DateTimeOffset), PayloadSchemaType.Datetime },
{ typeof(bool), PayloadSchemaType.Bool },

{ typeof(DateTime?), PayloadSchemaType.Datetime },
{ typeof(DateTimeOffset?), PayloadSchemaType.Datetime },
{ typeof(bool?), PayloadSchemaType.Bool },
};
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -52,13 +52,9 @@ public static Filter BuildFromLegacyFilter(VectorSearchFilter basicVectorSearchF
throw new InvalidOperationException($"Property name '{fieldName}' provided as part of the filter clause is not a valid property name.");
}

// Map datetime equality.
if (filterValue is DateTime or DateTimeOffset)
// Map DateTimeOffset equality.
if (filterValue is DateTimeOffset dateTimeOffset)
{
var dateTimeOffset = filterValue is DateTime dateTime
? new DateTimeOffset(dateTime, TimeSpan.Zero)
: (DateTimeOffset)filterValue;

var range = new global::Qdrant.Client.Grpc.DatetimeRange
{
Gte = Google.Protobuf.WellKnownTypes.Timestamp.FromDateTimeOffset(dateTimeOffset),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using Microsoft.Extensions.VectorData;
using Qdrant.Client.Grpc;
Expand All @@ -23,14 +24,12 @@ internal static class QdrantVectorStoreRecordFieldMapping
typeof(double),
typeof(float),
typeof(bool),
typeof(DateTime),
typeof(DateTimeOffset),
typeof(int?),
typeof(long?),
typeof(double?),
typeof(float?),
typeof(bool?),
typeof(DateTime?),
typeof(DateTimeOffset?),
];

Expand Down Expand Up @@ -79,8 +78,7 @@ object ConvertStringValue(string stringValue)
{
return targetType switch
{
Type t when t == typeof(DateTime) || t == typeof(DateTime?) => DateTime.Parse(payloadValue.StringValue),
Type t when t == typeof(DateTimeOffset) || t == typeof(DateTimeOffset?) => DateTimeOffset.Parse(payloadValue.StringValue),
Type t when t == typeof(DateTimeOffset) || t == typeof(DateTimeOffset?) => DateTimeOffset.Parse(stringValue, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind),
_ => stringValue,
};
}
Expand Down Expand Up @@ -123,10 +121,6 @@ public static Value ConvertToGrpcFieldValue(object? sourceValue)
{
value.BoolValue = boolValue;
}
else if (sourceValue is DateTime datetimeValue)
{
value.StringValue = datetimeValue.ToString("O");
}
else if (sourceValue is DateTimeOffset dateTimeOffsetValue)
{
value.StringValue = dateTimeOffsetValue.ToString("O");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -139,14 +139,13 @@ public void MapsMultiPropsFromDataToStorageModelWithUlong()
// Assert.
Assert.NotNull(actual);
Assert.Equal(5ul, actual.Id.Num);
Assert.Equal(9, actual.Payload.Count);
Assert.Equal(8, actual.Payload.Count);
Assert.Equal("data 1", actual.Payload["dataString"].StringValue);
Assert.Equal(5, actual.Payload["dataInt"].IntegerValue);
Assert.Equal(5, actual.Payload["dataLong"].IntegerValue);
Assert.Equal(5.5f, actual.Payload["dataFloat"].DoubleValue);
Assert.Equal(5.5d, actual.Payload["dataDouble"].DoubleValue);
Assert.True(actual.Payload["dataBool"].BoolValue);
Assert.Equal("2025-02-10T05:10:15.0000000Z", actual.Payload["dataDateTime"].StringValue);
Assert.Equal("2025-02-10T05:10:15.0000000+01:00", actual.Payload["dataDateTimeOffset"].StringValue);
Assert.Equal(new int[] { 1, 2, 3, 4 }, actual.Payload["dataArrayInt"].ListValue.Values.Select(x => (int)x.IntegerValue).ToArray());
Assert.Equal(new float[] { 1, 2, 3, 4 }, actual.Vectors.Vectors_.Vectors["vector1"].Data.ToArray());
Expand All @@ -167,14 +166,13 @@ public void MapsMultiPropsFromDataToStorageModelWithGuid()
// Assert.
Assert.NotNull(actual);
Assert.Equal(Guid.Parse("11111111-1111-1111-1111-111111111111"), Guid.Parse(actual.Id.Uuid));
Assert.Equal(9, actual.Payload.Count);
Assert.Equal(8, actual.Payload.Count);
Assert.Equal("data 1", actual.Payload["dataString"].StringValue);
Assert.Equal(5, actual.Payload["dataInt"].IntegerValue);
Assert.Equal(5, actual.Payload["dataLong"].IntegerValue);
Assert.Equal(5.5f, actual.Payload["dataFloat"].DoubleValue);
Assert.Equal(5.5d, actual.Payload["dataDouble"].DoubleValue);
Assert.True(actual.Payload["dataBool"].BoolValue);
Assert.Equal("2025-02-10T05:10:15.0000000Z", actual.Payload["dataDateTime"].StringValue);
Assert.Equal("2025-02-10T05:10:15.0000000+01:00", actual.Payload["dataDateTimeOffset"].StringValue);
Assert.Equal(new int[] { 1, 2, 3, 4 }, actual.Payload["dataArrayInt"].ListValue.Values.Select(x => (int)x.IntegerValue).ToArray());
Assert.Equal(new float[] { 1, 2, 3, 4 }, actual.Vectors.Vectors_.Vectors["vector1"].Data.ToArray());
Expand Down Expand Up @@ -203,7 +201,6 @@ public void MapsMultiPropsFromStorageToDataModelWithUlong(bool includeVectors)
Assert.Equal(5.5f, actual.DataFloat);
Assert.Equal(5.5d, actual.DataDouble);
Assert.True(actual.DataBool);
Assert.Equal(new DateTime(2025, 2, 10, 5, 10, 15, DateTimeKind.Utc), actual.DataDateTime);
Assert.Equal(new DateTimeOffset(2025, 2, 10, 5, 10, 15, TimeSpan.FromHours(1)), actual.DataDateTimeOffset);
Assert.Equal(new int[] { 1, 2, 3, 4 }, actual.DataArrayInt);

Expand Down Expand Up @@ -241,7 +238,6 @@ public void MapsMultiPropsFromStorageToDataModelWithGuid(bool includeVectors)
Assert.Equal(5.5f, actual.DataFloat);
Assert.Equal(5.5d, actual.DataDouble);
Assert.True(actual.DataBool);
Assert.Equal(new DateTime(2025, 2, 10, 5, 10, 15, DateTimeKind.Utc), actual.DataDateTime);
Assert.Equal(new DateTimeOffset(2025, 2, 10, 5, 10, 15, TimeSpan.FromHours(1)), actual.DataDateTimeOffset);
Assert.Equal(new int[] { 1, 2, 3, 4 }, actual.DataArrayInt);

Expand Down Expand Up @@ -279,7 +275,6 @@ private static MultiPropsModel<TKey> CreateMultiPropsModel<TKey>(TKey key)
DataFloat = 5.5f,
DataDouble = 5.5d,
DataBool = true,
DataDateTime = new DateTime(2025, 2, 10, 5, 10, 15, DateTimeKind.Utc),
DataDateTimeOffset = new DateTimeOffset(2025, 2, 10, 5, 10, 15, TimeSpan.FromHours(1)),
DataArrayInt = new List<int> { 1, 2, 3, 4 },
Vector1 = new float[] { 1, 2, 3, 4 },
Expand Down Expand Up @@ -344,7 +339,6 @@ private static void AddDataToMultiPropsPointStruct(PointStruct pointStruct)
pointStruct.Payload.Add("dataFloat", 5.5f);
pointStruct.Payload.Add("dataDouble", 5.5d);
pointStruct.Payload.Add("dataBool", true);
pointStruct.Payload.Add("dataDateTime", "2025-02-10T05:10:15.0000000Z");
pointStruct.Payload.Add("dataDateTimeOffset", "2025-02-10T05:10:15.0000000+01:00");

var dataIntArray = new ListValue();
Expand Down Expand Up @@ -395,7 +389,6 @@ private sealed class SinglePropsModel<TKey>
new VectorStoreRecordDataProperty("DataFloat", typeof(float)) { StoragePropertyName = "dataFloat" },
new VectorStoreRecordDataProperty("DataDouble", typeof(double)) { StoragePropertyName = "dataDouble" },
new VectorStoreRecordDataProperty("DataBool", typeof(bool)) { StoragePropertyName = "dataBool" },
new VectorStoreRecordDataProperty("DataDateTime", typeof(DateTime)) { StoragePropertyName = "dataDateTime" },
new VectorStoreRecordDataProperty("DataDateTimeOffset", typeof(DateTimeOffset)) { StoragePropertyName = "dataDateTimeOffset" },
new VectorStoreRecordDataProperty("DataArrayInt", typeof(List<int>)) { StoragePropertyName = "dataArrayInt" },
new VectorStoreRecordVectorProperty("Vector1", typeof(ReadOnlyMemory<float>)) { StoragePropertyName = "vector1" },
Expand Down Expand Up @@ -427,9 +420,6 @@ private sealed class MultiPropsModel<TKey>
[VectorStoreRecordData(StoragePropertyName = "dataBool")]
public bool DataBool { get; set; } = false;

[VectorStoreRecordData(StoragePropertyName = "dataDateTime")]
public DateTime DataDateTime { get; set; }

[VectorStoreRecordData(StoragePropertyName = "dataDateTimeOffset")]
public DateTimeOffset DataDateTimeOffset { get; set; }

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -57,8 +57,7 @@ public QdrantVectorStoreFixture()
new VectorStoreRecordDataProperty("HotelCode", typeof(int)) { IsFilterable = true },
new VectorStoreRecordDataProperty("ParkingIncluded", typeof(bool)) { IsFilterable = true, StoragePropertyName = "parking_is_included" },
new VectorStoreRecordDataProperty("HotelRating", typeof(float)) { IsFilterable = true },
new VectorStoreRecordDataProperty("LastRenovationDate", typeof(DateTime)) { IsFilterable = true },
new VectorStoreRecordDataProperty("OpeningDate", typeof(DateTimeOffset)) { IsFilterable = true },
new VectorStoreRecordDataProperty("LastRenovationDate", typeof(DateTimeOffset)) { IsFilterable = true },
new VectorStoreRecordDataProperty("Tags", typeof(List<string>)) { IsFilterable = true },
new VectorStoreRecordDataProperty("Description", typeof(string)),
new VectorStoreRecordVectorProperty("DescriptionEmbedding", typeof(ReadOnlyMemory<float>?)) { Dimensions = VectorDimensions, DistanceFunction = DistanceFunction.ManhattanDistance }
Expand Down Expand Up @@ -179,7 +178,7 @@ await this.QdrantClient.CreateCollectionAsync(
{
Id = 11,
Vectors = new Vectors { Vectors_ = namedVectors1 },
Payload = { ["HotelName"] = "My Hotel 11", ["HotelCode"] = 11, ["parking_is_included"] = true, ["Tags"] = tagsValue, ["HotelRating"] = 4.5f, ["Description"] = "This is a great hotel.", ["LastRenovationDate"] = "2025-02-10T05:10:15.0000000Z", ["OpeningDate"] = "2025-02-10T05:10:15.0000000+01:00" }
Payload = { ["HotelName"] = "My Hotel 11", ["HotelCode"] = 11, ["parking_is_included"] = true, ["Tags"] = tagsValue, ["HotelRating"] = 4.5f, ["Description"] = "This is a great hotel.", ["LastRenovationDate"] = "2025-02-10T05:10:15.0000000Z" }
},
new PointStruct
{
Expand Down Expand Up @@ -210,7 +209,7 @@ await this.QdrantClient.CreateCollectionAsync(
{
Id = 11,
Vectors = embeddingArray,
Payload = { ["HotelName"] = "My Hotel 11", ["HotelCode"] = 11, ["parking_is_included"] = true, ["Tags"] = tagsValue, ["HotelRating"] = 4.5f, ["Description"] = "This is a great hotel.", ["LastRenovationDate"] = "2025-02-10T05:10:15.0000000Z", ["OpeningDate"] = "2025-02-10T05:10:15.0000000+01:00" }
Payload = { ["HotelName"] = "My Hotel 11", ["HotelCode"] = 11, ["parking_is_included"] = true, ["Tags"] = tagsValue, ["HotelRating"] = 4.5f, ["Description"] = "This is a great hotel.", ["LastRenovationDate"] = "2025-02-10T05:10:15.0000000Z" }
},
new PointStruct
{
Expand Down Expand Up @@ -338,13 +337,9 @@ public record HotelInfo()
[VectorStoreRecordData(IsFilterable = true)]
public List<string> Tags { get; set; } = new List<string>();

/// <summary>A datetime metadata field.</summary>
/// <summary>A DateTimeOffset metadata field.</summary>
[VectorStoreRecordData(IsFilterable = true)]
public DateTime? LastRenovationDate { get; set; }

/// <summary>A datetimeoffset metadata field.</summary>
[VectorStoreRecordData(IsFilterable = true)]
public DateTimeOffset? OpeningDate { get; set; }
public DateTimeOffset? LastRenovationDate { get; set; }

/// <summary>A data field.</summary>
[VectorStoreRecordData]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,6 @@ public async Task ItCanCreateACollectionUpsertGetAndSearchAsync(bool hasNamedVec
Assert.Equal(record.HotelRating, getResult?.HotelRating);
Assert.Equal(record.ParkingIncluded, getResult?.ParkingIncluded);
Assert.Equal(record.LastRenovationDate, getResult?.LastRenovationDate);
Assert.Equal(record.OpeningDate, getResult?.OpeningDate);
Assert.Equal(record.Tags.ToArray(), getResult?.Tags.ToArray());
Assert.Equal(record.Description, getResult?.Description);

Expand All @@ -95,7 +94,6 @@ public async Task ItCanCreateACollectionUpsertGetAndSearchAsync(bool hasNamedVec
Assert.Equal(record.HotelRating, searchResultRecord?.HotelRating);
Assert.Equal(record.ParkingIncluded, searchResultRecord?.ParkingIncluded);
Assert.Equal(record.LastRenovationDate, searchResultRecord?.LastRenovationDate);
Assert.Equal(record.OpeningDate, searchResultRecord?.OpeningDate);
Assert.Equal(record.Tags.ToArray(), searchResultRecord?.Tags.ToArray());
Assert.Equal(record.Description, searchResultRecord?.Description);

Expand Down Expand Up @@ -226,8 +224,7 @@ public async Task ItCanGetDocumentFromVectorStoreAsync(bool useRecordDefinition,
Assert.Equal(11, getResult?.HotelCode);
Assert.True(getResult?.ParkingIncluded);
Assert.Equal(4.5f, getResult?.HotelRating);
Assert.Equal(new DateTime(2025, 2, 10, 5, 10, 15, DateTimeKind.Utc), getResult?.LastRenovationDate);
Assert.Equal(new DateTimeOffset(2025, 2, 10, 5, 10, 15, TimeSpan.FromHours(1)), getResult?.OpeningDate);
Assert.Equal(new DateTimeOffset(2025, 2, 10, 5, 10, 15, TimeSpan.Zero), getResult?.LastRenovationDate);
Assert.Equal(2, getResult?.Tags.Count);
Assert.Equal("t11.1", getResult?.Tags[0]);
Assert.Equal("t11.2", getResult?.Tags[1]);
Expand Down Expand Up @@ -483,8 +480,7 @@ private async Task<HotelInfo> CreateTestHotelAsync(uint hotelId, ITextEmbeddingG
HotelCode = (int)hotelId,
HotelRating = 4.5f,
ParkingIncluded = true,
LastRenovationDate = new DateTime(2025, 2, 10, 5, 10, 15, DateTimeKind.Utc),
OpeningDate = new DateTimeOffset(2025, 2, 10, 5, 10, 15, TimeSpan.FromHours(1)),
LastRenovationDate = new DateTimeOffset(2025, 2, 10, 5, 10, 15, TimeSpan.Zero),
Tags = { "t1", "t2" },
Description = "This is a great hotel.",
DescriptionEmbedding = await embeddingGenerator.GenerateEmbeddingAsync("This is a great hotel."),
Expand Down
Loading