From ca165a6dca2e8947abc3deb9ed8312d6c62338a2 Mon Sep 17 00:00:00 2001 From: Nadir Ali <36660597+Xnadir@users.noreply.github.com> Date: Wed, 18 Nov 2026 19:39:31 +0500 Subject: [PATCH] uniq: skip characters, not bytes, for -s in a UTF-8 locale Closes #16661 --- src/uu/uniq/src/uniq.rs | 15 +++++++++++++-- tests/by-util/test_uniq.rs | 19 +++++++++++++++++++ 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/src/uu/uniq/src/uniq.rs b/src/uu/uniq/src/uniq.rs index e34255ca792..ffd3995bbc0 100644 --- a/src/uu/uniq/src/uniq.rs +++ b/src/uu/uniq/src/uniq.rs @@ -178,8 +178,8 @@ impl Uniq { fn key_bounds(&self, line: &[u8]) -> (usize, usize) { let mut start = self.skip_fields_offset(line); - if let Some(skip_bytes) = self.slice_start { - start = start.saturating_add(skip_bytes).min(line.len()); + if let Some(skip_chars) = self.slice_start { + start = self.char_offset(line, start, skip_chars); } let end = self.key_end_index(line, start); @@ -242,6 +242,17 @@ impl Uniq { } } + /// The index `count` characters after `start`: bytes in the C locale or + /// for invalid UTF-8, characters otherwise, as for `-w`. + fn char_offset(&self, line: &[u8], start: usize, count: usize) -> usize { + if !self.is_c_locale + && let Ok(valid) = std::str::from_utf8(&line[start..]) + { + return start + Self::char_prefix_len(valid, count); + } + start.saturating_add(count).min(line.len()) + } + fn char_prefix_len(text: &str, limit: usize) -> usize { for (count, (idx, _)) in text.char_indices().enumerate() { if count == limit { diff --git a/tests/by-util/test_uniq.rs b/tests/by-util/test_uniq.rs index a66f30fc551..f686b2a5ba1 100644 --- a/tests/by-util/test_uniq.rs +++ b/tests/by-util/test_uniq.rs @@ -1214,6 +1214,25 @@ fn test_stdin_w1_multibyte() { .stdout_is("à\ná\n"); } +#[uutests::wasi_ignore(NoLocaleData)] +#[test] +fn test_skip_chars_multibyte() { + // Skipping two characters leaves "y" and "xy", which differ. + new_ucmd!() + .args(&["-s", "2", "-c"]) + .env("LC_ALL", "en_US.UTF-8") + .pipe_in("éxy\nABxy\n") + .succeeds() + .stdout_is(" 1 éxy\n 1 ABxy\n"); + // In the C locale, two bytes are skipped and both keys are "xy". + new_ucmd!() + .args(&["-s", "2", "-c"]) + .env("LC_ALL", "C") + .pipe_in("éxy\nABxy\n") + .succeeds() + .stdout_is(" 2 éxy\n"); +} + #[test] fn test_c_locale_counts_bytes() { let input = "가나다라마\n가나다바사\n";