postprocess: handle comments in conditional code

c2rust-postprocess/postprocess/cache.py

@@ -173,9 +173,16 @@ def lookup(
         try:
             toml = cache_file.read_text()
         except FileNotFoundError:
-            logging.debug(f"Cache miss: {cache_file}: {messages}")
+            data = {
+                "transform": transform,
+                "identifier": identifier,
+                "model": model,
+                "messages": messages,
+            }
+            toml = to_multiline_toml(data)
+            logging.debug(f"Cache miss: {cache_file}:\n{toml}")
             return None
-        logging.debug(f"Cache hit: {cache_file}")
+        logging.debug(f"Cache hit: {cache_file}:\n{toml}")
         data = tomli.loads(toml)
 
         return data["response"]
@@ -206,7 +213,7 @@ def update(
         response_path = cache_dir / "response.txt"
         metadata_path.write_text(toml)
         response_path.write_text(response)
-        logging.debug(f"Cache updated: {cache_dir}")
+        logging.debug(f"Cache updated: {cache_dir}:\n{toml}")
 
     def clear(self) -> None:
         self._path.unlink(missing_ok=True)

c2rust-postprocess/postprocess/definitions/__init__.py

@@ -174,6 +174,7 @@ def get_comments_text(comments: Iterable[str]) -> list[str]:
         comment_line
         for comment in comments
         for comment_line in get_comment_text(comment)
+        if comment_line
     ]
 
 

c2rust-postprocess/postprocess/transforms.py

@@ -12,7 +12,11 @@
     update_rust_definition,
 )
 from postprocess.models import AbstractGenerativeModel
-from postprocess.utils import get_highlighted_c, get_highlighted_rust, remove_backticks
+from postprocess.utils import (
+    extract_rust_code_from_markdown,
+    get_highlighted_c,
+    get_highlighted_rust,
+)
 
 # TODO: get from model
 SYSTEM_INSTRUCTION = (
@@ -107,7 +111,14 @@ def transfer_comments(
             Transfer the comments from the following C function to the corresponding Rust function.
             Do not add any comments that are not present in the C function.
             Use Rust doc comment syntax (///) where appropriate (e.g., for function documentation).
-            Respond with the Rust function definition with the transferred comments; say nothing else.
+            Respond with the Rust function definition with the transferred comments.
+
+            The Rust must contain exactly the same comments as the C code did.
+            This must be true even if there are C comments in conditionally-compiled code
+            not present in the Rust, including comments on preprocessor directives.
+            In such cases, insert the C comment (only the comment, not any C code)
+            into the Rust where the Rust code should have been.
+            If any C comments are not inserted into the Rust, explain why.
             """  # noqa: E501
             prompt_text = dedent(prompt_text).strip()
 
@@ -148,7 +159,7 @@ def transfer_comments(
                     response=response,
                 )
 
-            rust_fn = remove_backticks(response)
+            rust_fn = extract_rust_code_from_markdown(response)
 
             c_comments = get_c_comments(prompt.c_function)
             logging.debug(f"{c_comments=}")

c2rust-postprocess/postprocess/utils.py

@@ -85,18 +85,17 @@ def read_chunk(filepath: Path, start_offset: int, end_offset: int, encoding="utf
 
 
 # TODO: test
-def remove_backticks(text: str) -> str:
-    """Remove surrounding backticks from a model response."""
-    text = text.strip()
-    if text.startswith("```") and text.endswith("```"):
-        return "\n".join(text.split("\n")[1:-1])
-
-    if text.startswith("`"):
-        text = text[1:]
-    if text.endswith("`"):
-        text = text[:-1]
-
-    return text
+def extract_rust_code_from_markdown(text: str) -> str:
+    """
+    Find the (first) `rust` Markdown code block from a model response.
+    """
+    prefix = "```rust\n"
+    suffix = "\n```"
+    start = text.find(prefix)
+    assert start != -1, f"{prefix!r} not in {text}"
+    end = text.find(suffix, start)
+    assert end != -1, f"{suffix!r} not in {text[start + len(prefix) :]}"
+    return text[start + len(prefix) : end]
 
 
 def get_highlighted_c(c_code: str, bg="dark") -> str:

c2rust-postprocess/tests/llm-cache/CommentTransfer/_json_c_set_last_err/9ade63e9af53ecf8c488a382844137e74be4f51ab55c952a8cec0760a15edb10/metadata.toml

@@ -0,0 +1,65 @@
+transform = "CommentTransfer"
+identifier = "_json_c_set_last_err"
+model = "gemini-3-pro-preview"
+response = """```rust
+#[no_mangle]
+#[c2rust::src_loc = "75:1"]
+pub unsafe extern "C" fn _json_c_set_last_err(
+    mut err_fmt: *const ::core::ffi::c_char,
+    mut args: ...
+) {
+    let mut ap: ::core::ffi::VaListImpl;
+    ap = args.clone();
+    // Ignore (attempted) overruns from snprintf
+    vsnprintf(
+        &raw mut _last_err as *mut ::core::ffi::c_char,
+        ::core::mem::size_of::<[::core::ffi::c_char; 256]>() as size_t,
+        err_fmt,
+        ap.as_va_list(),
+    );
+}
+```"""
+
+[[messages]]
+role = "user"
+content = """Transfer the comments from the following C function to the corresponding Rust function.
+Do not add any comments that are not present in the C function.
+Use Rust doc comment syntax (///) where appropriate (e.g., for function documentation).
+Respond with the Rust function definition with the transferred comments.
+
+The Rust must contain exactly the same comments as the C code did.
+This must be true even if there are C comments in conditionally-compiled code
+not present in the Rust, including comments on preprocessor directives.
+In such cases, insert the C comment (only the comment, not any C code)
+into the Rust where the Rust code should have been.
+If any C comments are not inserted into the Rust, explain why.
+
+C function:
+```c
+void _json_c_set_last_err(const char *err_fmt, ...)
+{
+\tva_list ap;
+\tva_start(ap, err_fmt);
+\t// Ignore (attempted) overruns from snprintf
+\t(void)vsnprintf(_last_err, sizeof(_last_err), err_fmt, ap);
+\tva_end(ap);
+}```
+
+Rust function:
+```rust
+#[no_mangle]
+#[c2rust::src_loc = "75:1"]
+pub unsafe extern "C" fn _json_c_set_last_err(
+    mut err_fmt: *const ::core::ffi::c_char,
+    mut args: ...
+) {
+    let mut ap: ::core::ffi::VaListImpl;
+    ap = args.clone();
+    vsnprintf(
+        &raw mut _last_err as *mut ::core::ffi::c_char,
+        ::core::mem::size_of::<[::core::ffi::c_char; 256]>() as size_t,
+        err_fmt,
+        ap.as_va_list(),
+    );
+}```
+"""

c2rust-postprocess/tests/llm-cache/CommentTransfer/_json_c_set_last_err/9ade63e9af53ecf8c488a382844137e74be4f51ab55c952a8cec0760a15edb10/response.txt

@@ -0,0 +1,18 @@
+```rust
+#[no_mangle]
+#[c2rust::src_loc = "75:1"]
+pub unsafe extern "C" fn _json_c_set_last_err(
+    mut err_fmt: *const ::core::ffi::c_char,
+    mut args: ...
+) {
+    let mut ap: ::core::ffi::VaListImpl;
+    ap = args.clone();
+    // Ignore (attempted) overruns from snprintf
+    vsnprintf(
+        &raw mut _last_err as *mut ::core::ffi::c_char,
+        ::core::mem::size_of::<[::core::ffi::c_char; 256]>() as size_t,
+        err_fmt,
+        ap.as_va_list(),
+    );
+}
+```

c2rust-postprocess/tests/llm-cache/CommentTransfer/_json_object_to_fd/929c17432ef0c98df48c0ec0d48822ccbf6c5e31fd4d79a19a9397468a76ba3c/metadata.toml

@@ -0,0 +1,140 @@
+transform = "CommentTransfer"
+identifier = "_json_object_to_fd"
+model = "gemini-3-pro-preview"
+response = """```rust
+#[c2rust::src_loc = "156:1"]
+unsafe extern "C" fn _json_object_to_fd(
+    mut fd: ::core::ffi::c_int,
+    mut obj: *mut json_object,
+    mut flags: ::core::ffi::c_int,
+    mut filename: *const ::core::ffi::c_char,
+) -> ::core::ffi::c_int {
+    let mut ret: ::core::ffi::c_int = 0;
+    let mut json_str: *const ::core::ffi::c_char = ::core::ptr::null::<
+        ::core::ffi::c_char,
+    >();
+    let mut wpos: ::core::ffi::c_uint = 0;
+    let mut wsize: ::core::ffi::c_uint = 0;
+    filename = if !filename.is_null() {
+        filename
+    } else {
+        b"(fd)\\0" as *const u8 as *const ::core::ffi::c_char
+    };
+    json_str = json_object_to_json_string_ext(obj, flags);
+    if json_str.is_null() {
+        return -(1 as ::core::ffi::c_int);
+    }
+    wsize = (strlen(json_str) & UINT_MAX as size_t) as ::core::ffi::c_uint; /* CAW: probably unnecessary, but the most 64bit safe */
+    wpos = 0 as ::core::ffi::c_uint;
+    while wpos < wsize {
+        ret = write(
+            fd,
+            json_str.offset(wpos as isize) as *const ::core::ffi::c_void,
+            wsize.wrapping_sub(wpos) as size_t,
+        ) as ::core::ffi::c_int;
+        if ret < 0 as ::core::ffi::c_int {
+            _json_c_set_last_err(
+                b"json_object_to_file: error writing file %s: %s\\n\\0" as *const u8
+                    as *const ::core::ffi::c_char,
+                filename,
+                _json_c_strerror(*__errno_location()),
+            );
+            return -(1 as ::core::ffi::c_int);
+        }
+        /* because of the above check for ret < 0, we can safely cast and add */
+        wpos = wpos.wrapping_add(ret as ::core::ffi::c_uint);
+    }
+    return 0 as ::core::ffi::c_int;
+}
+```"""
+
+[[messages]]
+role = "user"
+content = """Transfer the comments from the following C function to the corresponding Rust function.
+Do not add any comments that are not present in the C function.
+Use Rust doc comment syntax (///) where appropriate (e.g., for function documentation).
+Respond with the Rust function definition with the transferred comments.
+
+The Rust must contain exactly the same comments as the C code did.
+This must be true even if there are C comments in conditionally-compiled code
+not present in the Rust, including comments on preprocessor directives.
+In such cases, insert the C comment (only the comment, not any C code)
+into the Rust where the Rust code should have been.
+If any C comments are not inserted into the Rust, explain why.
+
+C function:
+```c
+static int _json_object_to_fd(int fd, struct json_object *obj, int flags, const char *filename)
+{
+\tint ret;
+\tconst char *json_str;
+\tunsigned int wpos, wsize;
+
+\tfilename = filename ? filename : "(fd)";
+
+\tif (!(json_str = json_object_to_json_string_ext(obj,flags))) {
+\t\treturn -1;
+\t}
+
+\twsize = (unsigned int)(strlen(json_str) & UINT_MAX); /* CAW: probably unnecessary, but the most 64bit safe */
+\twpos = 0;
+\twhile(wpos < wsize) {
+\t\tif((ret = write(fd, json_str + wpos, wsize-wpos)) < 0) {
+\t\t  _json_c_set_last_err("json_object_to_file: error writing file %s: %s\\n",
+\t\t\t filename, strerror(errno));
+\t\t  return -1;
+\t\t}
+
+\t\t/* because of the above check for ret < 0, we can safely cast and add */
+\t\twpos += (unsigned int)ret;
+\t}
+
+\treturn 0;
+}```
+
+Rust function:
+```rust
+#[c2rust::src_loc = "156:1"]
+unsafe extern "C" fn _json_object_to_fd(
+    mut fd: ::core::ffi::c_int,
+    mut obj: *mut json_object,
+    mut flags: ::core::ffi::c_int,
+    mut filename: *const ::core::ffi::c_char,
+) -> ::core::ffi::c_int {
+    let mut ret: ::core::ffi::c_int = 0;
+    let mut json_str: *const ::core::ffi::c_char = ::core::ptr::null::<
+        ::core::ffi::c_char,
+    >();
+    let mut wpos: ::core::ffi::c_uint = 0;
+    let mut wsize: ::core::ffi::c_uint = 0;
+    filename = if !filename.is_null() {
+        filename
+    } else {
+        b"(fd)\\0" as *const u8 as *const ::core::ffi::c_char
+    };
+    json_str = json_object_to_json_string_ext(obj, flags);
+    if json_str.is_null() {
+        return -(1 as ::core::ffi::c_int);
+    }
+    wsize = (strlen(json_str) & UINT_MAX as size_t) as ::core::ffi::c_uint;
+    wpos = 0 as ::core::ffi::c_uint;
+    while wpos < wsize {
+        ret = write(
+            fd,
+            json_str.offset(wpos as isize) as *const ::core::ffi::c_void,
+            wsize.wrapping_sub(wpos) as size_t,
+        ) as ::core::ffi::c_int;
+        if ret < 0 as ::core::ffi::c_int {
+            _json_c_set_last_err(
+                b"json_object_to_file: error writing file %s: %s\\n\\0" as *const u8
+                    as *const ::core::ffi::c_char,
+                filename,
+                _json_c_strerror(*__errno_location()),
+            );
+            return -(1 as ::core::ffi::c_int);
+        }
+        wpos = wpos.wrapping_add(ret as ::core::ffi::c_uint);
+    }
+    return 0 as ::core::ffi::c_int;
+}```
+"""

c2rust-postprocess/tests/llm-cache/CommentTransfer/_json_object_to_fd/929c17432ef0c98df48c0ec0d48822ccbf6c5e31fd4d79a19a9397468a76ba3c/response.txt

@@ -0,0 +1,46 @@
+```rust
+#[c2rust::src_loc = "156:1"]
+unsafe extern "C" fn _json_object_to_fd(
+    mut fd: ::core::ffi::c_int,
+    mut obj: *mut json_object,
+    mut flags: ::core::ffi::c_int,
+    mut filename: *const ::core::ffi::c_char,
+) -> ::core::ffi::c_int {
+    let mut ret: ::core::ffi::c_int = 0;
+    let mut json_str: *const ::core::ffi::c_char = ::core::ptr::null::<
+        ::core::ffi::c_char,
+    >();
+    let mut wpos: ::core::ffi::c_uint = 0;
+    let mut wsize: ::core::ffi::c_uint = 0;
+    filename = if !filename.is_null() {
+        filename
+    } else {
+        b"(fd)\0" as *const u8 as *const ::core::ffi::c_char
+    };
+    json_str = json_object_to_json_string_ext(obj, flags);
+    if json_str.is_null() {
+        return -(1 as ::core::ffi::c_int);
+    }
+    wsize = (strlen(json_str) & UINT_MAX as size_t) as ::core::ffi::c_uint; /* CAW: probably unnecessary, but the most 64bit safe */
+    wpos = 0 as ::core::ffi::c_uint;
+    while wpos < wsize {
+        ret = write(
+            fd,
+            json_str.offset(wpos as isize) as *const ::core::ffi::c_void,
+            wsize.wrapping_sub(wpos) as size_t,
+        ) as ::core::ffi::c_int;
+        if ret < 0 as ::core::ffi::c_int {
+            _json_c_set_last_err(
+                b"json_object_to_file: error writing file %s: %s\n\0" as *const u8
+                    as *const ::core::ffi::c_char,
+                filename,
+                _json_c_strerror(*__errno_location()),
+            );
+            return -(1 as ::core::ffi::c_int);
+        }
+        /* because of the above check for ret < 0, we can safely cast and add */
+        wpos = wpos.wrapping_add(ret as ::core::ffi::c_uint);
+    }
+    return 0 as ::core::ffi::c_int;
+}
+```