[ENH] Add root collection id and lineage file name in collection table

go/pkg/sysdb/metastore/db/dao/collection.go

@@ -64,8 +64,9 @@ func (s *collectionDb) ListCollectionsToGc(cutoffTimeSecs *uint64, limit *uint64
 		Joins("INNER JOIN databases ON collections.database_id = databases.id").
 		Where("version > 0").
 		Where("version_file_name IS NOT NULL").
-		Where("version_file_name != ''")
-
+		Where("version_file_name != ''").
+		Where("root_collection_id IS NULL OR root_collection_id == ''").
+		Where("lineage_file_name IS NULL OR lineage_file_name == ''")
 	// Apply cutoff time filter only if provided
 	if cutoffTimeSecs != nil {
 		cutoffTime := time.Unix(int64(*cutoffTimeSecs), 0)
@@ -102,6 +103,8 @@ func (s *collectionDb) getCollections(id *string, name *string, tenantID string,
 		LogPosition                int64      `gorm:"column:log_position"`
 		Version                    int32      `gorm:"column:version"`
 		VersionFileName            string     `gorm:"column:version_file_name"`
+		RootCollectionId           string     `gorm:"column:root_collection_id"`
+		LineageFileName            string     `gorm:"column:lineage_file_name"`
 		TotalRecordsPostCompaction uint64     `gorm:"column:total_records_post_compaction"`
 		SizeBytesPostCompaction    uint64     `gorm:"column:size_bytes_post_compaction"`
 		LastCompactionTimeSecs     uint64     `gorm:"column:last_compaction_time_secs"`
@@ -120,7 +123,7 @@ func (s *collectionDb) getCollections(id *string, name *string, tenantID string,
 	}
 
 	query := s.db.Table("collections").
-		Select("collections.id as collection_id, collections.name as collection_name, collections.configuration_json_str, collections.dimension, collections.database_id, collections.ts as collection_ts, collections.is_deleted, collections.created_at as collection_created_at, collections.updated_at as collection_updated_at, collections.log_position, collections.version, collections.version_file_name, collections.total_records_post_compaction, collections.size_bytes_post_compaction, collections.last_compaction_time_secs, databases.name as database_name, databases.tenant_id as db_tenant_id, collections.tenant as tenant").
+		Select("collections.id as collection_id, collections.name as collection_name, collections.configuration_json_str, collections.dimension, collections.database_id, collections.ts as collection_ts, collections.is_deleted, collections.created_at as collection_created_at, collections.updated_at as collection_updated_at, collections.log_position, collections.version, collections.version_file_name, collections.root_collection_id, collections.lineage_file_name, collections.total_records_post_compaction, collections.size_bytes_post_compaction, collections.last_compaction_time_secs, databases.name as database_name, databases.tenant_id as db_tenant_id, collections.tenant as tenant").
 		Joins("INNER JOIN databases ON collections.database_id = databases.id").
 		Order("collections.created_at ASC")
 
@@ -181,6 +184,8 @@ func (s *collectionDb) getCollections(id *string, name *string, tenantID string,
 				LogPosition:                r.LogPosition,
 				Version:                    r.Version,
 				VersionFileName:            r.VersionFileName,
+				RootCollectionId:           r.RootCollectionId,
+				LineageFileName:            r.LineageFileName,
 				TotalRecordsPostCompaction: r.TotalRecordsPostCompaction,
 				SizeBytesPostCompaction:    r.SizeBytesPostCompaction,
 				LastCompactionTimeSecs:     r.LastCompactionTimeSecs,

go/pkg/sysdb/metastore/db/dbmodel/collection.go

@@ -19,6 +19,8 @@ type Collection struct {
 	LogPosition                int64           `gorm:"log_position;default:0"`
 	Version                    int32           `gorm:"version;default:0"`
 	VersionFileName            string          `gorm:"version_file_name"`
+	RootCollectionId           string          `gorm:"column:root_collection_id"`
+	LineageFileName            string          `gorm:"column:lineage_file_name"`
 	TotalRecordsPostCompaction uint64          `gorm:"total_records_post_compaction;default:0"`
 	SizeBytesPostCompaction    uint64          `gorm:"size_bytes_post_compaction;default:0"`
 	LastCompactionTimeSecs     uint64          `gorm:"last_compaction_time_secs;default:0"`

go/pkg/sysdb/metastore/db/migrations/20250415160000.sql

@@ -0,0 +1,2 @@
+ALTER TABLE "public"."collections" ADD COLUMN "lineage_file_name" text NULL;
+ALTER TABLE "public"."collections" ADD COLUMN "root_collection_id" text NULL;

go/pkg/sysdb/metastore/db/migrations/atlas.sum

@@ -1,4 +1,4 @@
-h1:uxEQy73b71+QtlD+KEOVaNXN+o6zW68RxPT4f/zGmZk=
+h1:qSAPpKj97DIUtMarnXIR5T3aD9SKcv+fTuTS+hEwP+0=
 20240313233558.sql h1:Gv0TiSYsqGoOZ2T2IWvX4BOasauxool8PrBOIjmmIdg=
 20240321194713.sql h1:kVkNpqSFhrXGVGFFvL7JdK3Bw31twFcEhI6A0oCFCkg=
 20240327075032.sql h1:nlr2J74XRU8erzHnKJgMr/tKqJxw9+R6RiiEBuvuzgo=
@@ -13,3 +13,4 @@ h1:uxEQy73b71+QtlD+KEOVaNXN+o6zW68RxPT4f/zGmZk=
 20250303211712.sql h1:0InApNBRlxhGoA59Wd5v4loKD2ZJv+j1gA+koZaVz2w=
 20250307212458.sql h1:cETui+Y7vylQzMf7JdCTNyBxpePKMgcGzwSaMxwGlMs=
 20250326224328.sql h1:A4f6Hl7ghJCrL4rgLx5EDEO0mHc0wpHiyM5hG6FnEQU=
+20250415160000.sql h1:iEpmmkOUps8VvOgS2RuLjfF/zhBwA5o0l/aacXWU+P8=