Support VO bit

.github/scripts/Make.user

@@ -1,3 +1,4 @@
 WITH_MMTK=1
+MMTK_CONSERVATIVE=1
 FORCE_ASSERTIONS=1
 LLVM_ASSERTIONS=1

mmtk/Cargo.toml

@@ -9,8 +9,8 @@ edition = "2018"
 # Metadata for the Julia repository
 [package.metadata.julia]
 # Our CI matches the following line and extract mmtk/julia. If this line is updated, please check ci yaml files and make sure it works.
-julia_repo = "https://github.com/mmtk/julia.git"
-julia_version = "5c9b37044fd6e446141d29111fb6c894ba0a42ff"
+julia_repo = "https://github.com/udesou/julia.git"
+julia_version = "5c82cc96396f5b266a25c6019567a3539636197c"
 
 [lib]
 crate-type = ["cdylib"]
@@ -44,7 +44,7 @@ memoffset = "*"
 # ykstackmaps = { git = "https://github.com/udesou/ykstackmaps.git", branch = "udesou-master", version = "*" }
 
 [features]
-default = ["mmtk/vm_space", "julia_copy_stack", "object_pinning"]
+default = ["mmtk/vm_space", "julia_copy_stack", "object_pinning", "is_mmtk_object"]
 
 # Plans
 nogc = []
@@ -56,3 +56,5 @@ object_pinning = ["mmtk/object_pinning"]
 # This feature disables moving
 non_moving = ["mmtk/immix_non_moving", "mmtk/immix_smaller_block"]
 julia_copy_stack = []
+# Feature necessary to support conservative stack scanning
+is_mmtk_object = ["mmtk/is_mmtk_object"]

mmtk/api/mmtk.h

@@ -61,6 +61,7 @@ extern void mmtk_memory_region_copy(MMTk_Mutator mutator, void* src_obj, void* s
 extern void mmtk_object_reference_write_post(MMTk_Mutator mutator, const void* src, const void* target);
 extern void mmtk_object_reference_write_slow(MMTk_Mutator mutator, const void* src, const void* target);
 extern const void* MMTK_SIDE_LOG_BIT_BASE_ADDRESS;
+extern const void* MMTK_SIDE_VO_BIT_BASE_ADDRESS;
 
 extern uintptr_t JULIA_MALLOC_BYTES;
 

mmtk/src/api.rs

@@ -226,6 +226,16 @@ pub extern "C" fn mmtk_post_alloc(
     bytes: usize,
     semantics: AllocationSemantics,
 ) {
+    #[cfg(all(feature = "object_pinning", not(feature = "non_moving")))]
+    if semantics == AllocationSemantics::Default {
+        // clearing pinning bits
+        use crate::mmtk::vm::ObjectModel;
+        use crate::object_model::VMObjectModel;
+        use mmtk::util::metadata::MetadataSpec;
+        if let MetadataSpec::OnSide(side) = *VMObjectModel::LOCAL_PINNING_BIT_SPEC {
+            side.bzero_metadata(refer.to_raw_address(), bytes);
+        }
+    }
     memory_manager::post_alloc::<JuliaVM>(unsafe { &mut *mutator }, refer, bytes, semantics)
 }
 
@@ -390,6 +400,8 @@ pub extern "C" fn mmtk_memory_region_copy(
 pub extern "C" fn mmtk_immortal_region_post_alloc(start: Address, size: usize) {
     #[cfg(feature = "stickyimmix")]
     set_side_log_bit_for_region(start, size);
+    #[cfg(feature = "is_mmtk_object")]
+    set_side_vo_bit_for_region(start, size);
 }
 
 #[cfg(feature = "stickyimmix")]
@@ -402,6 +414,18 @@ fn set_side_log_bit_for_region(start: Address, size: usize) {
     }
 }
 
+#[cfg(feature = "is_mmtk_object")]
+fn set_side_vo_bit_for_region(start: Address, size: usize) {
+    debug!(
+        "Bulk set VO bit {} to {} ({} bytes)",
+        start,
+        start + size,
+        size
+    );
+
+    crate::util::bulk_update_vo_bit(start, size, &crate::util::set_meta_bits)
+}
+
 #[no_mangle]
 pub extern "C" fn mmtk_object_reference_write_post(
     mutator: *mut Mutator<JuliaVM>,
@@ -436,6 +460,11 @@ pub extern "C" fn mmtk_object_reference_write_slow(
 pub static MMTK_SIDE_LOG_BIT_BASE_ADDRESS: Address =
     mmtk::util::metadata::side_metadata::GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS;
 
+/// VO bit base address
+#[no_mangle]
+pub static MMTK_SIDE_VO_BIT_BASE_ADDRESS: Address =
+    mmtk::util::metadata::side_metadata::VO_BIT_SIDE_METADATA_ADDR;
+
 #[no_mangle]
 pub extern "C" fn mmtk_object_is_managed_by_mmtk(addr: usize) -> bool {
     crate::api::mmtk_is_mapped_address(unsafe { Address::from_usize(addr) })

mmtk/src/util.rs

@@ -1,6 +1,10 @@
+use crate::api::MMTK_SIDE_VO_BIT_BASE_ADDRESS;
 use crate::JuliaVM;
+use core::sync::atomic::Ordering;
 use enum_map::Enum;
+use mmtk::util::Address;
 use mmtk::util::ObjectReference;
+use std::sync::atomic::AtomicU8;
 
 #[repr(i32)]
 #[derive(Clone, Copy, Debug, Enum, PartialEq, Hash, Eq)]
@@ -152,3 +156,159 @@ pub extern "C" fn mmtk_get_possibly_forwared(object: ObjectReference) -> ObjectR
         None => object,
     }
 }
+
+// Functions to set the side metadata for the VO bit (copied from mmtk-core)
+pub fn bulk_update_vo_bit(
+    start: Address,
+    size: usize,
+    update_meta_bits: &impl Fn(Address, u8, Address, u8),
+) {
+    // Update bits for a contiguous side metadata spec. We can simply calculate the data end address, and
+    // calculate the metadata address for the data end.
+    let update_contiguous = |data_start: Address, data_bytes: usize| {
+        if data_bytes == 0 {
+            return;
+        }
+        let meta_start = address_to_meta_address(data_start);
+        let meta_start_shift = meta_byte_lshift(data_start);
+        let meta_end = address_to_meta_address(data_start + data_bytes);
+        let meta_end_shift = meta_byte_lshift(data_start + data_bytes);
+        update_meta_bits(meta_start, meta_start_shift, meta_end, meta_end_shift);
+    };
+
+    // VO bit is global
+    update_contiguous(start, size);
+}
+
+/// Performs the translation of data address (`data_addr`) to metadata address for the specified metadata (`metadata_spec`).
+pub fn address_to_meta_address(data_addr: Address) -> Address {
+    #[cfg(target_pointer_width = "32")]
+    let res = {
+        if metadata_spec.is_global {
+            address_to_contiguous_meta_address(metadata_spec, data_addr)
+        } else {
+            address_to_chunked_meta_address(metadata_spec, data_addr)
+        }
+    };
+    #[cfg(target_pointer_width = "64")]
+    let res = { address_to_contiguous_meta_address(data_addr) };
+
+    res
+}
+
+/// Performs address translation in contiguous metadata spaces (e.g. global and policy-specific in 64-bits, and global in 32-bits)
+pub fn address_to_contiguous_meta_address(data_addr: Address) -> Address {
+    let log_bits_num = 0 as i32; // VO_BIT log_num_of_bits
+    let log_bytes_in_region = mmtk::util::constants::LOG_MIN_OBJECT_SIZE as usize; // VO_BIT log_bytes_in_region
+
+    let rshift = (mmtk::util::constants::LOG_BITS_IN_BYTE as i32) - log_bits_num;
+
+    if rshift >= 0 {
+        MMTK_SIDE_VO_BIT_BASE_ADDRESS + ((data_addr >> log_bytes_in_region) >> rshift)
+    } else {
+        MMTK_SIDE_VO_BIT_BASE_ADDRESS + ((data_addr >> log_bytes_in_region) << (-rshift))
+    }
+}
+
+pub fn meta_byte_lshift(data_addr: Address) -> u8 {
+    let log_bytes_in_region = mmtk::util::constants::LOG_MIN_OBJECT_SIZE as usize; // VO_BIT log_bytes_in_region
+    let bits_num_log = 0 as i32; // VO_BIT log_num_of_bits
+    if bits_num_log >= 3 {
+        return 0;
+    }
+    let rem_shift = mmtk::util::constants::BITS_IN_WORD as i32
+        - ((mmtk::util::constants::LOG_BITS_IN_BYTE as i32) - bits_num_log);
+    ((((data_addr >> log_bytes_in_region) << rem_shift) >> rem_shift) << bits_num_log) as u8
+}
+
+/// This method is used for bulk updating side metadata for a data address range. As we cannot guarantee
+/// that the data address range can be mapped to whole metadata bytes, we have to deal with cases that
+/// we need to mask and zero certain bits in a metadata byte. The end address and the end bit are exclusive.
+/// The end bit for update_bits could be 8, so overflowing needs to be taken care of.
+pub fn update_meta_bits(
+    meta_start_addr: Address,
+    meta_start_bit: u8,
+    meta_end_addr: Address,
+    meta_end_bit: u8,
+    update_bytes: &impl Fn(Address, Address),
+    update_bits: &impl Fn(Address, u8, u8),
+) {
+    // Start/end is the same, we don't need to do anything.
+    if meta_start_addr == meta_end_addr && meta_start_bit == meta_end_bit {
+        return;
+    }
+
+    // zeroing bytes
+    if meta_start_bit == 0 && meta_end_bit == 0 {
+        update_bytes(meta_start_addr, meta_end_addr);
+        return;
+    }
+
+    if meta_start_addr == meta_end_addr {
+        // Update bits in the same byte between start and end bit
+        update_bits(meta_start_addr, meta_start_bit, meta_end_bit);
+    } else if meta_start_addr + 1usize == meta_end_addr && meta_end_bit == 0 {
+        // Update bits in the same byte after the start bit (between start bit and 8)
+        update_bits(meta_start_addr, meta_start_bit, 8);
+    } else {
+        // update bits in the first byte
+        update_meta_bits(
+            meta_start_addr,
+            meta_start_bit,
+            meta_start_addr + 1usize,
+            0,
+            update_bytes,
+            update_bits,
+        );
+        // update bytes in the middle
+        update_meta_bits(
+            meta_start_addr + 1usize,
+            0,
+            meta_end_addr,
+            0,
+            update_bytes,
+            update_bits,
+        );
+        // update bits in the last byte
+        update_meta_bits(
+            meta_end_addr,
+            0,
+            meta_end_addr,
+            meta_end_bit,
+            update_bytes,
+            update_bits,
+        );
+    }
+}
+
+/// This method is used for bulk setting side metadata for a data address range.
+pub fn set_meta_bits(
+    meta_start_addr: Address,
+    meta_start_bit: u8,
+    meta_end_addr: Address,
+    meta_end_bit: u8,
+) {
+    let set_bytes = |start: Address, end: Address| {
+        set(start, 0xff, end - start);
+    };
+    let set_bits = |addr: Address, start_bit: u8, end_bit: u8| {
+        // we are setting selected bits in one byte
+        let mask: u8 = !(u8::MAX.checked_shl(end_bit.into()).unwrap_or(0)) & (u8::MAX << start_bit); // Get a mask that the bits we need to set are 1, and the other bits are 0.
+        unsafe { addr.as_ref::<AtomicU8>() }.fetch_or(mask, Ordering::SeqCst);
+    };
+    update_meta_bits(
+        meta_start_addr,
+        meta_start_bit,
+        meta_end_addr,
+        meta_end_bit,
+        &set_bytes,
+        &set_bits,
+    );
+}
+
+/// Set a range of memory to the given value. Similar to memset.
+pub fn set(start: Address, val: u8, len: usize) {
+    unsafe {
+        std::ptr::write_bytes::<u8>(start.to_mut_ptr(), val, len);
+    }
+}