Backport/2412: Expose the unstable metadata v16 (#6676)

This PR is a backport for:
- #5732

cc @paritytech/subxt-team

---------

Signed-off-by: Alexandru Vasile <[email protected]>
Co-authored-by: Egor_P <[email protected]>

Author: lexnv

Cargo.lock

@@ -779,7 +779,7 @@ frame-benchmarking-pallet-pov = { default-features = false, path = "substrate/fr
 frame-election-provider-solution-type = { path = "substrate/frame/election-provider-support/solution-type", default-features = false }
 frame-election-provider-support = { path = "substrate/frame/election-provider-support", default-features = false }
 frame-executive = { path = "substrate/frame/executive", default-features = false }
-frame-metadata = { version = "16.0.0", default-features = false }
+frame-metadata = { version = "18.0.0", default-features = false }
 frame-metadata-hash-extension = { path = "substrate/frame/metadata-hash-extension", default-features = false }
 frame-support = { path = "substrate/frame/support", default-features = false }
 frame-support-procedural = { path = "substrate/frame/support/procedural", default-features = false }
@@ -854,7 +854,7 @@ macro_magic = { version = "0.5.1" }
 maplit = { version = "1.0.2" }
 memmap2 = { version = "0.9.3" }
 memory-db = { version = "0.32.0", default-features = false }
-merkleized-metadata = { version = "0.1.0" }
+merkleized-metadata = { version = "0.1.2" }
 merlin = { version = "3.0", default-features = false }
 messages-relay = { path = "bridges/relays/messages" }
 metered = { version = "0.6.1", default-features = false, package = "prioritized-metered-channel" }

Cargo.toml

@@ -0,0 +1,29 @@
+title: Expose the unstable metadata v16
+doc:
+- audience: Node Dev
+  description: |
+    This PR exposes the *unstable* metadata V16. The metadata is exposed under the unstable u32::MAX number.
+    Developers can start experimenting with the new features of the metadata v16. *Please note that this metadata is under development and expect breaking changes until stabilization.*
+    The `ExtrinsicMetadata` trait receives a breaking change. Its associated type `VERSION` is rename to `VERSIONS` and now supports a constant static list of metadata versions.
+    The versions implemented for `UncheckedExtrinsic` are v4 (legacy version) and v5 (new version).
+    For metadata collection, it is assumed that all `TransactionExtensions` are under version 0.
+
+crates:
+  - name: sp-metadata-ir
+    bump: major
+  - name: frame-support-procedural
+    bump: patch
+  - name: frame-support
+    bump: minor
+  - name: frame-support-test
+    bump: major
+  - name: frame-metadata-hash-extension
+    bump: patch
+  - name: substrate-wasm-builder
+    bump: minor
+  - name: pallet-revive
+    bump: minor
+  - name: sp-runtime
+    bump: major
+  - name: frame-benchmarking-cli
+    bump: patch

prdoc/pr_5732.prdoc

@@ -0,0 +1,29 @@
+title: Expose the unstable metadata v16
+doc:
+- audience: Node Dev
+  description: |
+    This PR exposes the *unstable* metadata V16. The metadata is exposed under the unstable u32::MAX number.
+    Developers can start experimenting with the new features of the metadata v16. *Please note that this metadata is under development and expect breaking changes until stabilization.*
+    The `ExtrinsicMetadata` trait receives a breaking change. Its associated type `VERSION` is rename to `VERSIONS` and now supports a constant static list of metadata versions.
+    The versions implemented for `UncheckedExtrinsic` are v4 (legacy version) and v5 (new version).
+    For metadata collection, it is assumed that all `TransactionExtensions` are under version 0.
+
+crates:
+  - name: sp-metadata-ir
+    bump: major
+  - name: frame-support-procedural
+    bump: patch
+  - name: frame-support
+    bump: minor
+  - name: frame-support-test
+    bump: major
+  - name: frame-metadata-hash-extension
+    bump: patch
+  - name: substrate-wasm-builder
+    bump: minor
+  - name: pallet-revive
+    bump: minor
+  - name: sp-runtime
+    bump: major
+  - name: frame-benchmarking-cli
+    bump: patch

prdoc/pr_6676.prdoc

@@ -25,7 +25,7 @@ substrate-test-runtime-client = { workspace = true }
 sp-api = { workspace = true, default-features = true }
 sp-transaction-pool = { workspace = true, default-features = true }
 merkleized-metadata = { workspace = true }
-frame-metadata = { features = ["current"], workspace = true, default-features = true }
+frame-metadata = { features = ["current", "unstable"], workspace = true, default-features = true }
 sp-tracing = { workspace = true, default-features = true }
 
 [features]

substrate/frame/metadata-hash-extension/Cargo.toml

@@ -92,8 +92,12 @@ impl<Address: TypeInfo, Signature: TypeInfo, E: EthExtra> ExtrinsicLike
 impl<Address, Signature, E: EthExtra> ExtrinsicMetadata
 	for UncheckedExtrinsic<Address, Signature, E>
 {
-	const VERSION: u8 =
-		generic::UncheckedExtrinsic::<Address, CallOf<E::Config>, Signature, E::Extension>::VERSION;
+	const VERSIONS: &'static [u8] = generic::UncheckedExtrinsic::<
+		Address,
+		CallOf<E::Config>,
+		Signature,
+		E::Extension,
+	>::VERSIONS;
 	type TransactionExtensions = E::Extension;
 }
 

substrate/frame/revive/src/evm/runtime.rs

@@ -28,6 +28,7 @@ scale-info = { features = [
 ], workspace = true }
 frame-metadata = { features = [
 	"current",
+	"unstable",
 ], workspace = true }
 sp-api = { features = [
 	"frame-metadata",

substrate/frame/support/Cargo.toml

@@ -117,7 +117,7 @@ pub fn expand_runtime_metadata(
 					pallets: #scrate::__private::vec![ #(#pallets),* ],
 					extrinsic: #scrate::__private::metadata_ir::ExtrinsicMetadataIR {
 						ty,
-						version: <#extrinsic as #scrate::sp_runtime::traits::ExtrinsicMetadata>::VERSION,
+						versions: <#extrinsic as #scrate::sp_runtime::traits::ExtrinsicMetadata>::VERSIONS.into_iter().map(|ref_version| *ref_version).collect(),
 						address_ty,
 						call_ty,
 						signature_ty,

substrate/frame/support/procedural/src/construct_runtime/expand/metadata.rs

@@ -19,7 +19,7 @@ static_assertions = { workspace = true, default-features = true }
 serde = { features = ["derive"], workspace = true }
 codec = { features = ["derive"], workspace = true }
 scale-info = { features = ["derive"], workspace = true }
-frame-metadata = { features = ["current"], workspace = true }
+frame-metadata = { features = ["current", "unstable"], workspace = true }
 sp-api = { workspace = true }
 sp-arithmetic = { workspace = true }
 sp-io = { workspace = true }

substrate/frame/support/test/Cargo.toml

@@ -53,6 +53,9 @@ parameter_types! {
 /// Latest stable metadata version used for testing.
 const LATEST_METADATA_VERSION: u32 = 15;
 
+/// Unstable metadata version.
+const UNSTABLE_METADATA_VERSION: u32 = u32::MAX;
+
 pub struct SomeType1;
 impl From<SomeType1> for u64 {
 	fn from(_t: SomeType1) -> Self {
@@ -1944,7 +1947,10 @@ fn metadata_at_version() {
 
 #[test]
 fn metadata_versions() {
-	assert_eq!(vec![14, LATEST_METADATA_VERSION], Runtime::metadata_versions());
+	assert_eq!(
+		vec![14, LATEST_METADATA_VERSION, UNSTABLE_METADATA_VERSION],
+		Runtime::metadata_versions()
+	);
 }
 
 #[test]

substrate/frame/support/test/tests/pallet.rs

@@ -17,7 +17,7 @@ targets = ["x86_64-unknown-linux-gnu"]
 
 [dependencies]
 codec = { workspace = true }
-frame-metadata = { features = ["current"], workspace = true }
+frame-metadata = { features = ["current", "unstable"], workspace = true }
 scale-info = { features = ["derive"], workspace = true }
 
 [features]

substrate/primitives/metadata-ir/Cargo.toml

@@ -30,6 +30,7 @@ mod types;
 use frame_metadata::RuntimeMetadataPrefixed;
 pub use types::*;
 
+mod unstable;
 mod v14;
 mod v15;
 
@@ -39,23 +40,33 @@ const V14: u32 = 14;
 /// Metadata V15.
 const V15: u32 = 15;
 
+/// Unstable metadata V16.
+const UNSTABLE_V16: u32 = u32::MAX;
+
 /// Transform the IR to the specified version.
 ///
 /// Use [`supported_versions`] to find supported versions.
 pub fn into_version(metadata: MetadataIR, version: u32) -> Option<RuntimeMetadataPrefixed> {
 	// Note: Unstable metadata version is `u32::MAX` until stabilized.
 	match version {
-		// Latest stable version.
+		// Version V14. This needs to be around until the
+		// deprecation of the `Metadata_metadata` runtime call in favor of
+		// `Metadata_metadata_at_version.
 		V14 => Some(into_v14(metadata)),
-		// Unstable metadata.
+
+		// Version V15 - latest stable.
 		V15 => Some(into_latest(metadata)),
+
+		// Unstable metadata under `u32::MAX`.
+		UNSTABLE_V16 => Some(into_unstable(metadata)),
+
 		_ => None,
 	}
 }
 
 /// Returns the supported metadata versions.
 pub fn supported_versions() -> alloc::vec::Vec<u32> {
-	alloc::vec![V14, V15]
+	alloc::vec![V14, V15, UNSTABLE_V16]
 }
 
 /// Transform the IR to the latest stable metadata version.
@@ -70,6 +81,12 @@ pub fn into_v14(metadata: MetadataIR) -> RuntimeMetadataPrefixed {
 	latest.into()
 }
 
+/// Transform the IR to unstable metadata version 16.
+pub fn into_unstable(metadata: MetadataIR) -> RuntimeMetadataPrefixed {
+	let latest: frame_metadata::v16::RuntimeMetadataV16 = metadata.into();
+	latest.into()
+}
+
 #[cfg(test)]
 mod test {
 	use super::*;
@@ -81,7 +98,7 @@ mod test {
 			pallets: vec![],
 			extrinsic: ExtrinsicMetadataIR {
 				ty: meta_type::<()>(),
-				version: 0,
+				versions: vec![0],
 				address_ty: meta_type::<()>(),
 				call_ty: meta_type::<()>(),
 				signature_ty: meta_type::<()>(),

substrate/primitives/metadata-ir/src/lib.rs

@@ -170,8 +170,8 @@ pub struct ExtrinsicMetadataIR<T: Form = MetaForm> {
 	///
 	/// Note: Field used for metadata V14 only.
 	pub ty: T::Type,
-	/// Extrinsic version.
-	pub version: u8,
+	/// Extrinsic versions.
+	pub versions: Vec<u8>,
 	/// The type of the address that signs the extrinsic
 	pub address_ty: T::Type,
 	/// The type of the outermost Call enum.
@@ -191,7 +191,7 @@ impl IntoPortable for ExtrinsicMetadataIR {
 	fn into_portable(self, registry: &mut Registry) -> Self::Output {
 		ExtrinsicMetadataIR {
 			ty: registry.register_type(&self.ty),
-			version: self.version,
+			versions: self.versions,
 			address_ty: registry.register_type(&self.address_ty),
 			call_ty: registry.register_type(&self.call_ty),
 			signature_ty: registry.register_type(&self.signature_ty),