From cdc68ee6670e2b03360c11980526b75a672e6d36 Mon Sep 17 00:00:00 2001 From: Eduardo Souza Date: Thu, 11 Jul 2024 14:05:55 +1000 Subject: [PATCH] Support VO bit (#158) This PR introduces the `is_mmtk_object` feature supporting a valid object (VO) bit for conservative stack scanning. It also sets this feature as default. NB: merge with https://github.com/mmtk/julia/pull/59. ~NB2: it requires a change in `mmtk-core` to expose an api function to bulk set the VO bit. (see https://github.com/mmtk/mmtk-core/pull/1157)~ --- .github/scripts/Make.user | 1 + mmtk/Cargo.toml | 6 +- mmtk/api/mmtk.h | 1 + mmtk/src/api.rs | 19 +++++ mmtk/src/util.rs | 159 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 184 insertions(+), 2 deletions(-) diff --git a/.github/scripts/Make.user b/.github/scripts/Make.user index f815ce1a..8cec2b5d 100644 --- a/.github/scripts/Make.user +++ b/.github/scripts/Make.user @@ -1,3 +1,4 @@ WITH_MMTK=1 +MMTK_CONSERVATIVE=1 FORCE_ASSERTIONS=1 LLVM_ASSERTIONS=1 \ No newline at end of file diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 4843e16d..c72c0db4 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -10,7 +10,7 @@ edition = "2018" [package.metadata.julia] # Our CI matches the following line and extract mmtk/julia. If this line is updated, please check ci yaml files and make sure it works. julia_repo = "https://github.com/mmtk/julia.git" -julia_version = "5c9b37044fd6e446141d29111fb6c894ba0a42ff" +julia_version = "084d8a08f0cfc2b1d7acae2e000c165e8b11de5b" [lib] crate-type = ["cdylib"] @@ -44,7 +44,7 @@ memoffset = "*" # ykstackmaps = { git = "https://github.com/udesou/ykstackmaps.git", branch = "udesou-master", version = "*" } [features] -default = ["mmtk/vm_space", "julia_copy_stack", "object_pinning"] +default = ["mmtk/vm_space", "julia_copy_stack", "object_pinning", "is_mmtk_object"] # Plans nogc = [] @@ -56,3 +56,5 @@ object_pinning = ["mmtk/object_pinning"] # This feature disables moving non_moving = ["mmtk/immix_non_moving", "mmtk/immix_smaller_block"] julia_copy_stack = [] +# Feature necessary to support conservative stack scanning +is_mmtk_object = ["mmtk/is_mmtk_object"] diff --git a/mmtk/api/mmtk.h b/mmtk/api/mmtk.h index b45549d5..f3ff4fd5 100644 --- a/mmtk/api/mmtk.h +++ b/mmtk/api/mmtk.h @@ -61,6 +61,7 @@ extern void mmtk_memory_region_copy(MMTk_Mutator mutator, void* src_obj, void* s extern void mmtk_object_reference_write_post(MMTk_Mutator mutator, const void* src, const void* target); extern void mmtk_object_reference_write_slow(MMTk_Mutator mutator, const void* src, const void* target); extern const void* MMTK_SIDE_LOG_BIT_BASE_ADDRESS; +extern const void* MMTK_SIDE_VO_BIT_BASE_ADDRESS; extern uintptr_t JULIA_MALLOC_BYTES; diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index ca8bd07f..4415a5d1 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -390,6 +390,8 @@ pub extern "C" fn mmtk_memory_region_copy( pub extern "C" fn mmtk_immortal_region_post_alloc(start: Address, size: usize) { #[cfg(feature = "stickyimmix")] set_side_log_bit_for_region(start, size); + #[cfg(feature = "is_mmtk_object")] + set_side_vo_bit_for_region(start, size); } #[cfg(feature = "stickyimmix")] @@ -402,6 +404,18 @@ fn set_side_log_bit_for_region(start: Address, size: usize) { } } +#[cfg(feature = "is_mmtk_object")] +fn set_side_vo_bit_for_region(start: Address, size: usize) { + debug!( + "Bulk set VO bit {} to {} ({} bytes)", + start, + start + size, + size + ); + + crate::util::bulk_update_vo_bit(start, size, &crate::util::set_meta_bits) +} + #[no_mangle] pub extern "C" fn mmtk_object_reference_write_post( mutator: *mut Mutator, @@ -436,6 +450,11 @@ pub extern "C" fn mmtk_object_reference_write_slow( pub static MMTK_SIDE_LOG_BIT_BASE_ADDRESS: Address = mmtk::util::metadata::side_metadata::GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS; +/// VO bit base address +#[no_mangle] +pub static MMTK_SIDE_VO_BIT_BASE_ADDRESS: Address = + mmtk::util::metadata::side_metadata::VO_BIT_SIDE_METADATA_ADDR; + #[no_mangle] pub extern "C" fn mmtk_object_is_managed_by_mmtk(addr: usize) -> bool { crate::api::mmtk_is_mapped_address(unsafe { Address::from_usize(addr) }) diff --git a/mmtk/src/util.rs b/mmtk/src/util.rs index 6a18ee08..7d3368e8 100644 --- a/mmtk/src/util.rs +++ b/mmtk/src/util.rs @@ -1,6 +1,10 @@ +use crate::api::MMTK_SIDE_VO_BIT_BASE_ADDRESS; use crate::JuliaVM; +use core::sync::atomic::Ordering; use enum_map::Enum; +use mmtk::util::Address; use mmtk::util::ObjectReference; +use std::sync::atomic::AtomicU8; #[repr(i32)] #[derive(Clone, Copy, Debug, Enum, PartialEq, Hash, Eq)] @@ -152,3 +156,158 @@ pub extern "C" fn mmtk_get_possibly_forwared(object: ObjectReference) -> ObjectR None => object, } } + +// Functions to set the side metadata for the VO bit (copied from mmtk-core) +pub const VO_BIT_LOG_NUM_OF_BITS: i32 = 0; +pub const VO_BIT_LOG_BYTES_PER_REGION: usize = mmtk::util::constants::LOG_MIN_OBJECT_SIZE as usize; + +pub fn bulk_update_vo_bit( + start: Address, + size: usize, + update_meta_bits: &impl Fn(Address, u8, Address, u8), +) { + // Update bits for a contiguous side metadata spec. We can simply calculate the data end address, and + // calculate the metadata address for the data end. + let update_contiguous = |data_start: Address, data_bytes: usize| { + if data_bytes == 0 { + return; + } + let meta_start = address_to_meta_address(data_start); + let meta_start_shift = meta_byte_lshift(data_start); + let meta_end = address_to_meta_address(data_start + data_bytes); + let meta_end_shift = meta_byte_lshift(data_start + data_bytes); + update_meta_bits(meta_start, meta_start_shift, meta_end, meta_end_shift); + }; + + // VO bit is global + update_contiguous(start, size); +} + +/// Performs the translation of data address (`data_addr`) to metadata address for the specified metadata (`metadata_spec`). +pub fn address_to_meta_address(data_addr: Address) -> Address { + #[cfg(target_pointer_width = "32")] + let res = { + if metadata_spec.is_global { + address_to_contiguous_meta_address(metadata_spec, data_addr) + } else { + address_to_chunked_meta_address(metadata_spec, data_addr) + } + }; + #[cfg(target_pointer_width = "64")] + let res = { address_to_contiguous_meta_address(data_addr) }; + + res +} + +/// Performs address translation in contiguous metadata spaces (e.g. global and policy-specific in 64-bits, and global in 32-bits) +pub fn address_to_contiguous_meta_address(data_addr: Address) -> Address { + let rshift = (mmtk::util::constants::LOG_BITS_IN_BYTE as i32) - VO_BIT_LOG_NUM_OF_BITS; + + if rshift >= 0 { + MMTK_SIDE_VO_BIT_BASE_ADDRESS + ((data_addr >> VO_BIT_LOG_BYTES_PER_REGION) >> rshift) + } else { + MMTK_SIDE_VO_BIT_BASE_ADDRESS + ((data_addr >> VO_BIT_LOG_BYTES_PER_REGION) << (-rshift)) + } +} + +pub fn meta_byte_lshift(data_addr: Address) -> u8 { + if VO_BIT_LOG_NUM_OF_BITS >= 3 { + return 0; + } + let rem_shift = mmtk::util::constants::BITS_IN_WORD as i32 + - ((mmtk::util::constants::LOG_BITS_IN_BYTE as i32) - VO_BIT_LOG_NUM_OF_BITS); + ((((data_addr >> VO_BIT_LOG_BYTES_PER_REGION) << rem_shift) >> rem_shift) + << VO_BIT_LOG_NUM_OF_BITS) as u8 +} + +/// This method is used for bulk updating side metadata for a data address range. As we cannot guarantee +/// that the data address range can be mapped to whole metadata bytes, we have to deal with cases that +/// we need to mask and zero certain bits in a metadata byte. The end address and the end bit are exclusive. +/// The end bit for update_bits could be 8, so overflowing needs to be taken care of. +pub fn update_meta_bits( + meta_start_addr: Address, + meta_start_bit: u8, + meta_end_addr: Address, + meta_end_bit: u8, + update_bytes: &impl Fn(Address, Address), + update_bits: &impl Fn(Address, u8, u8), +) { + // Start/end is the same, we don't need to do anything. + if meta_start_addr == meta_end_addr && meta_start_bit == meta_end_bit { + return; + } + + // zeroing bytes + if meta_start_bit == 0 && meta_end_bit == 0 { + update_bytes(meta_start_addr, meta_end_addr); + return; + } + + if meta_start_addr == meta_end_addr { + // Update bits in the same byte between start and end bit + update_bits(meta_start_addr, meta_start_bit, meta_end_bit); + } else if meta_start_addr + 1usize == meta_end_addr && meta_end_bit == 0 { + // Update bits in the same byte after the start bit (between start bit and 8) + update_bits(meta_start_addr, meta_start_bit, 8); + } else { + // update bits in the first byte + update_meta_bits( + meta_start_addr, + meta_start_bit, + meta_start_addr + 1usize, + 0, + update_bytes, + update_bits, + ); + // update bytes in the middle + update_meta_bits( + meta_start_addr + 1usize, + 0, + meta_end_addr, + 0, + update_bytes, + update_bits, + ); + // update bits in the last byte + update_meta_bits( + meta_end_addr, + 0, + meta_end_addr, + meta_end_bit, + update_bytes, + update_bits, + ); + } +} + +/// This method is used for bulk setting side metadata for a data address range. +pub fn set_meta_bits( + meta_start_addr: Address, + meta_start_bit: u8, + meta_end_addr: Address, + meta_end_bit: u8, +) { + let set_bytes = |start: Address, end: Address| { + set(start, 0xff, end - start); + }; + let set_bits = |addr: Address, start_bit: u8, end_bit: u8| { + // we are setting selected bits in one byte + let mask: u8 = !(u8::MAX.checked_shl(end_bit.into()).unwrap_or(0)) & (u8::MAX << start_bit); // Get a mask that the bits we need to set are 1, and the other bits are 0. + unsafe { addr.as_ref::() }.fetch_or(mask, Ordering::SeqCst); + }; + update_meta_bits( + meta_start_addr, + meta_start_bit, + meta_end_addr, + meta_end_bit, + &set_bytes, + &set_bits, + ); +} + +/// Set a range of memory to the given value. Similar to memset. +pub fn set(start: Address, val: u8, len: usize) { + unsafe { + std::ptr::write_bytes::(start.to_mut_ptr(), val, len); + } +}