+260

crates/core/src/cloner/detector.rs

reviewed

··· 1 1 + use crate::Result; 2 2 + use crate::cloner::rolling_hash::{RollingHash, token_hash}; 3 3 + use crate::tokenizer::{Language, Token, Tokenizer}; 4 4 + use serde::{Deserialize, Serialize}; 5 5 + use std::collections::HashMap; 6 6 + use std::path::PathBuf; 7 7 + 8 8 + /// A detected code clone 9 9 + #[derive(Debug, Clone, Serialize, Deserialize)] 10 10 + pub struct Clone { 11 11 + /// Unique ID for this clone group 12 12 + pub id: usize, 13 13 + /// Number of tokens in the cloned sequence 14 14 + pub length: usize, 15 15 + /// All locations where this clone appears 16 16 + pub locations: Vec<CloneLocation>, 17 17 + /// Hash value of the clone (for deduplication) 18 18 + #[serde(skip)] 19 19 + pub hash: u64, 20 20 + } 21 21 + 22 22 + /// Location of a code clone 23 23 + #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] 24 24 + pub struct CloneLocation { 25 25 + /// File path 26 26 + pub file: PathBuf, 27 27 + /// Starting line number 28 28 + pub start_line: usize, 29 29 + /// Ending line number 30 30 + pub end_line: usize, 31 31 + } 32 32 + 33 33 + pub struct CloneDetector { 34 34 + /// Minimum number of tokens to consider as a clone 35 35 + _min_tokens: usize, 36 36 + /// Window size for rolling hash 37 37 + window_size: usize, 38 38 + } 39 39 + 40 40 + impl Default for CloneDetector { 41 41 + fn default() -> Self { 42 42 + Self { _min_tokens: 30, window_size: 30 } 43 43 + } 44 44 + } 45 45 + 46 46 + impl CloneDetector { 47 47 + pub fn new(min_tokens: usize) -> Self { 48 48 + Self { _min_tokens: min_tokens, window_size: min_tokens } 49 49 + } 50 50 + 51 51 + /// Detect clones in a single file 52 52 + pub fn detect_in_file(&self, source: &str, language: Language, file_path: PathBuf) -> Result<Vec<Clone>> { 53 53 + let tokens = Tokenizer::new(source, language).tokenize()?; 54 54 + let significant_tokens: Vec<&Token> = tokens.iter().filter(|t| t.token_type.is_significant()).collect(); 55 55 + 56 56 + if significant_tokens.len() < self.window_size { 57 57 + return Ok(Vec::new()); 58 58 + } 59 59 + 60 60 + let mut hash_map: HashMap<u64, Vec<(usize, usize)>> = HashMap::new(); 61 61 + let mut rh = RollingHash::new(self.window_size); 62 62 + 63 63 + let token_hashes: Vec<u64> = significant_tokens.iter().map(|t| token_hash(&t.text)).collect(); 64 64 + 65 65 + rh.init(&token_hashes[0..self.window_size]); 66 66 + let start_line = significant_tokens[0].line; 67 67 + let end_line = significant_tokens[self.window_size - 1].line; 68 68 + hash_map.entry(rh.get()).or_default().push((start_line, end_line)); 69 69 + 70 70 + for i in self.window_size..token_hashes.len() { 71 71 + let hash = rh.roll(token_hashes[i - self.window_size], token_hashes[i]); 72 72 + let start_line = significant_tokens[i - self.window_size + 1].line; 73 73 + let end_line = significant_tokens[i].line; 74 74 + hash_map.entry(hash).or_default().push((start_line, end_line)); 75 75 + } 76 76 + 77 77 + let mut clones = Vec::new(); 78 78 + let mut clone_id = 0; 79 79 + 80 80 + for (hash, locations) in hash_map { 81 81 + if locations.len() > 1 { 82 82 + clone_id += 1; 83 83 + clones.push(Clone { 84 84 + id: clone_id, 85 85 + length: self.window_size, 86 86 + locations: locations 87 87 + .into_iter() 88 88 + .map(|(start, end)| CloneLocation { file: file_path.clone(), start_line: start, end_line: end }) 89 89 + .collect(), 90 90 + hash, 91 91 + }); 92 92 + } 93 93 + } 94 94 + 95 95 + Ok(clones) 96 96 + } 97 97 + 98 98 + /// Detect clones across multiple files 99 99 + pub fn detect_across_files(&self, files: &[(PathBuf, String, Language)]) -> Result<Vec<Clone>> { 100 100 + let mut global_hash_map: HashMap<u64, Vec<CloneLocation>> = HashMap::new(); 101 101 + 102 102 + for (file_path, source, language) in files { 103 103 + let tokens = Tokenizer::new(source, *language).tokenize()?; 104 104 + let significant_tokens: Vec<&Token> = tokens.iter().filter(|t| t.token_type.is_significant()).collect(); 105 105 + 106 106 + if significant_tokens.len() < self.window_size { 107 107 + continue; 108 108 + } 109 109 + 110 110 + let mut rh = RollingHash::new(self.window_size); 111 111 + 112 112 + let token_hashes: Vec<u64> = significant_tokens.iter().map(|t| token_hash(&t.text)).collect(); 113 113 + 114 114 + rh.init(&token_hashes[0..self.window_size]); 115 115 + let start_line = significant_tokens[0].line; 116 116 + let end_line = significant_tokens[self.window_size - 1].line; 117 117 + global_hash_map.entry(rh.get()).or_default().push(CloneLocation { 118 118 + file: file_path.clone(), 119 119 + start_line, 120 120 + end_line, 121 121 + }); 122 122 + 123 123 + for i in self.window_size..token_hashes.len() { 124 124 + let hash = rh.roll(token_hashes[i - self.window_size], token_hashes[i]); 125 125 + let start_line = significant_tokens[i - self.window_size + 1].line; 126 126 + let end_line = significant_tokens[i].line; 127 127 + global_hash_map.entry(hash).or_default().push(CloneLocation { 128 128 + file: file_path.clone(), 129 129 + start_line, 130 130 + end_line, 131 131 + }); 132 132 + } 133 133 + } 134 134 + 135 135 + let mut clones = Vec::new(); 136 136 + let mut clone_id = 0; 137 137 + 138 138 + for (hash, mut locations) in global_hash_map { 139 139 + if locations.len() > 1 { 140 140 + locations.sort_by(|a, b| { 141 141 + a.file 142 142 + .cmp(&b.file) 143 143 + .then(a.start_line.cmp(&b.start_line)) 144 144 + .then(a.end_line.cmp(&b.end_line)) 145 145 + }); 146 146 + locations.dedup(); 147 147 + 148 148 + if locations.len() > 1 { 149 149 + clone_id += 1; 150 150 + clones.push(Clone { id: clone_id, length: self.window_size, locations, hash }); 151 151 + } 152 152 + } 153 153 + } 154 154 + 155 155 + clones.sort_by(|a, b| b.locations.len().cmp(&a.locations.len())); 156 156 + Ok(clones) 157 157 + } 158 158 + } 159 159 + 160 160 + #[cfg(test)] 161 161 + mod tests { 162 162 + use super::*; 163 163 + 164 164 + #[test] 165 165 + fn test_no_clones_in_simple_file() { 166 166 + let source = r#" 167 167 + fn simple() { 168 168 + let x = 5; 169 169 + let y = 10; 170 170 + return x + y; 171 171 + } 172 172 + "#; 173 173 + let detector = CloneDetector::new(10); 174 174 + let clones = detector 175 175 + .detect_in_file(source, Language::Rust, PathBuf::from("test.rs")) 176 176 + .unwrap(); 177 177 + 178 178 + assert_eq!(clones.len(), 0); 179 179 + } 180 180 + 181 181 + #[test] 182 182 + fn test_detect_simple_clone() { 183 183 + let source = r#" 184 184 + fn process_a() { 185 185 + let x = input.get(); 186 186 + let y = x * 2; 187 187 + let z = y + 5; 188 188 + return z; 189 189 + } 190 190 + 191 191 + fn process_b() { 192 192 + let x = input.get(); 193 193 + let y = x * 2; 194 194 + let z = y + 5; 195 195 + return z; 196 196 + } 197 197 + "#; 198 198 + let detector = CloneDetector::new(5); 199 199 + let clones = detector 200 200 + .detect_in_file(source, Language::Rust, PathBuf::from("test.rs")) 201 201 + .unwrap(); 202 202 + 203 203 + assert!(!clones.is_empty()); 204 204 + 205 205 + for clone in &clones { 206 206 + assert!(clone.locations.len() >= 2); 207 207 + } 208 208 + } 209 209 + 210 210 + #[test] 211 211 + fn test_across_files() { 212 212 + let file1 = r#" 213 213 + fn helper() { 214 214 + for i in 0..10 { 215 215 + println!("{}", i); 216 216 + } 217 217 + } 218 218 + "#; 219 219 + let file2 = r#" 220 220 + fn another() { 221 221 + for i in 0..10 { 222 222 + println!("{}", i); 223 223 + } 224 224 + } 225 225 + "#; 226 226 + 227 227 + let files = vec![ 228 228 + (PathBuf::from("file1.rs"), file1.to_string(), Language::Rust), 229 229 + (PathBuf::from("file2.rs"), file2.to_string(), Language::Rust), 230 230 + ]; 231 231 + 232 232 + let detector = CloneDetector::new(5); 233 233 + let clones = detector.detect_across_files(&files).unwrap(); 234 234 + 235 235 + if !clones.is_empty() { 236 236 + let has_cross_file = clones.iter().any(|clone| { 237 237 + let files: std::collections::HashSet<_> = clone.locations.iter().map(|l| &l.file).collect(); 238 238 + files.len() > 1 239 239 + }); 240 240 + assert!(has_cross_file, "Should detect clones across different files"); 241 241 + } 242 242 + } 243 243 + 244 244 + #[test] 245 245 + fn test_min_tokens_threshold() { 246 246 + let source = "let x = 5; let y = 10; let x = 5; let y = 10;"; 247 247 + 248 248 + let detector1 = CloneDetector::new(3); 249 249 + let clones1 = detector1 250 250 + .detect_in_file(source, Language::Rust, PathBuf::from("test.rs")) 251 251 + .unwrap(); 252 252 + 253 253 + let detector2 = CloneDetector::new(100); 254 254 + let clones2 = detector2 255 255 + .detect_in_file(source, Language::Rust, PathBuf::from("test.rs")) 256 256 + .unwrap(); 257 257 + 258 258 + assert!(clones2.len() <= clones1.len()); 259 259 + } 260 260 + }

+5

crates/core/src/cloner/mod.rs

reviewed

··· 1 1 + pub mod detector; 2 2 + pub mod rolling_hash; 3 3 + 4 4 + pub use detector::{Clone, CloneDetector, CloneLocation}; 5 5 + pub use rolling_hash::RollingHash;

+155

crates/core/src/cloner/rolling_hash.rs

reviewed

··· 1 1 + /// Rabin-Karp rolling hash implementation for efficient string/token matching 2 2 + /// 3 3 + /// This uses polynomial rolling hash with a large prime modulus to minimize collisions 4 4 + /// while allowing O(1) hash updates when the window slides. 5 5 + pub struct RollingHash { 6 6 + /// Base for polynomial hash (using 257 for byte values + 1) 7 7 + base: u64, 8 8 + /// Large prime modulus to reduce collisions 9 9 + modulus: u64, 10 10 + /// Current hash value 11 11 + hash: u64, 12 12 + /// Window size 13 13 + window_size: usize, 14 14 + /// Precomputed base^(window_size-1) mod modulus for removing leftmost element 15 15 + base_power: u64, 16 16 + } 17 17 + 18 18 + impl RollingHash { 19 19 + /// Create a new rolling hash with the specified window size 20 20 + pub fn new(window_size: usize) -> Self { 21 21 + let base = 257u64; 22 22 + let modulus = 1_000_000_007u64; 23 23 + 24 24 + let mut base_power = 1u64; 25 25 + for _ in 0..window_size.saturating_sub(1) { 26 26 + base_power = Self::mul_mod(base_power, base, modulus); 27 27 + } 28 28 + 29 29 + Self { base, modulus, hash: 0, window_size, base_power } 30 30 + } 31 31 + 32 32 + /// Initialize hash with a sequence of values 33 33 + pub fn init(&mut self, values: &[u64]) { 34 34 + self.hash = 0; 35 35 + for &val in values.iter().take(self.window_size) { 36 36 + self.hash = Self::mul_mod(self.hash, self.base, self.modulus); 37 37 + self.hash = Self::add_mod(self.hash, val, self.modulus); 38 38 + } 39 39 + } 40 40 + 41 41 + /// Roll the window: remove the leftmost value and add a new rightmost value 42 42 + /// Returns the new hash value 43 43 + pub fn roll(&mut self, old_value: u64, new_value: u64) -> u64 { 44 44 + let old_contrib = Self::mul_mod(old_value, self.base_power, self.modulus); 45 45 + self.hash = Self::sub_mod(self.hash, old_contrib, self.modulus); 46 46 + 47 47 + self.hash = Self::mul_mod(self.hash, self.base, self.modulus); 48 48 + self.hash = Self::add_mod(self.hash, new_value, self.modulus); 49 49 + 50 50 + self.hash 51 51 + } 52 52 + 53 53 + /// Get current hash value 54 54 + pub fn get(&self) -> u64 { 55 55 + self.hash 56 56 + } 57 57 + 58 58 + /// Multiply with modular arithmetic to prevent overflow 59 59 + fn mul_mod(a: u64, b: u64, modulus: u64) -> u64 { 60 60 + ((a as u128 * b as u128) % modulus as u128) as u64 61 61 + } 62 62 + 63 63 + /// Add with modular arithmetic 64 64 + fn add_mod(a: u64, b: u64, modulus: u64) -> u64 { 65 65 + ((a as u128 + b as u128) % modulus as u128) as u64 66 66 + } 67 67 + 68 68 + /// Subtract with modular arithmetic (handles underflow) 69 69 + fn sub_mod(a: u64, b: u64, modulus: u64) -> u64 { 70 70 + if a >= b { (a - b) % modulus } else { (modulus - (b - a) % modulus) % modulus } 71 71 + } 72 72 + } 73 73 + 74 74 + /// Compute a hash value for a token string (for use in rolling hash) 75 75 + pub fn token_hash(token: &str) -> u64 { 76 76 + let mut hash = 5381u64; 77 77 + for byte in token.bytes() { 78 78 + hash = hash.wrapping_mul(33).wrapping_add(byte as u64); 79 79 + } 80 80 + hash 81 81 + } 82 82 + 83 83 + #[cfg(test)] 84 84 + mod tests { 85 85 + use super::*; 86 86 + 87 87 + #[test] 88 88 + fn test_rolling_hash_basic() { 89 89 + let window_size = 3; 90 90 + let mut rh = RollingHash::new(window_size); 91 91 + 92 92 + let values = [1, 2, 3, 4, 5]; 93 93 + rh.init(&values[0..3]); 94 94 + 95 95 + let hash1 = rh.get(); 96 96 + assert_ne!(hash1, 0); 97 97 + 98 98 + let hash2 = rh.roll(1, 4); 99 99 + assert_ne!(hash2, 0); 100 100 + assert_ne!(hash1, hash2); 101 101 + 102 102 + let hash3 = rh.roll(2, 5); 103 103 + assert_ne!(hash3, 0); 104 104 + assert_ne!(hash2, hash3); 105 105 + } 106 106 + 107 107 + #[test] 108 108 + fn test_same_sequence_same_hash() { 109 109 + let mut rh1 = RollingHash::new(4); 110 110 + let mut rh2 = RollingHash::new(4); 111 111 + 112 112 + let seq = vec![10, 20, 30, 40]; 113 113 + rh1.init(&seq); 114 114 + rh2.init(&seq); 115 115 + 116 116 + assert_eq!(rh1.get(), rh2.get()); 117 117 + } 118 118 + 119 119 + #[test] 120 120 + fn test_different_sequences_different_hash() { 121 121 + let mut rh1 = RollingHash::new(4); 122 122 + let mut rh2 = RollingHash::new(4); 123 123 + 124 124 + rh1.init(&[1, 2, 3, 4]); 125 125 + rh2.init(&[1, 2, 3, 5]); 126 126 + 127 127 + assert_ne!(rh1.get(), rh2.get()); 128 128 + } 129 129 + 130 130 + #[test] 131 131 + fn test_token_hash() { 132 132 + let h1 = token_hash("if"); 133 133 + let h2 = token_hash("else"); 134 134 + let h3 = token_hash("if"); 135 135 + 136 136 + assert_ne!(h1, h2); 137 137 + assert_eq!(h1, h3); 138 138 + } 139 139 + 140 140 + #[test] 141 141 + fn test_rolling_preserves_pattern() { 142 142 + let mut rh = RollingHash::new(3); 143 143 + let values = vec![1, 2, 3, 4, 5, 6, 1, 2, 3]; 144 144 + 145 145 + rh.init(&values[0..3]); 146 146 + let first_hash = rh.get(); 147 147 + 148 148 + let mut current_hash = first_hash; 149 149 + for i in 3..9 { 150 150 + current_hash = rh.roll(values[i - 3], values[i]); 151 151 + } 152 152 + 153 153 + assert_eq!(current_hash, first_hash); 154 154 + } 155 155 + }