docs: c_lex: properly implement a sub() method for CMatch

+259 -13

1 changed file

expand all

tools

lib

python

kdoc

c_lex.py

+259 -13

tools/lib/python/kdoc/c_lex.py

··· 16 16 import logging 17 17 import re 18 18 19 + from copy import copy 20 + 19 21 from .kdoc_re import KernRe 20 22 21 23 log = logging.getLogger(__name__) ··· 286 284 return out 287 285 288 286 287 + class CTokenArgs: 288 + """ 289 + Ancillary class to help using backrefs from sub matches. 290 + 291 + If the highest backref contain a "+" at the last element, 292 + the logic will be greedy, picking all other delims. 293 + 294 + This is needed to parse struct_group macros with end with ``MEMBERS...``. 295 + """ 296 + def __init__(self, sub_str): 297 + self.sub_groups = set() 298 + self.max_group = -1 299 + self.greedy = None 300 + 301 + for m in KernRe(r'\\(\d+)([+]?)').finditer(sub_str): 302 + group = int(m.group(1)) 303 + if m.group(2) == "+": 304 + if self.greedy and self.greedy != group: 305 + raise ValueError("There are multiple greedy patterns!") 306 + self.greedy = group 307 + 308 + self.sub_groups.add(group) 309 + self.max_group = max(self.max_group, group) 310 + 311 + if self.greedy: 312 + if self.greedy != self.max_group: 313 + raise ValueError("Greedy pattern is not the last one!") 314 + 315 + sub_str = KernRe(r'(\\\d+)[+]').sub(r"\1", sub_str) 316 + 317 + self.sub_str = sub_str 318 + self.sub_tokeninzer = CTokenizer(sub_str) 319 + 320 + def groups(self, new_tokenizer): 321 + """ 322 + Create replacement arguments for backrefs like: 323 + 324 + ``\0``, ``\1``, ``\2``, ...``\n`` 325 + 326 + It also accepts a ``+`` character to the highest backref. When used, 327 + it means in practice to ignore delimins after it, being greedy. 328 + 329 + The logic is smart enough to only go up to the maximum required 330 + argument, even if there are more. 331 + 332 + If there is a backref for an argument above the limit, it will 333 + raise an exception. Please notice that, on C, square brackets 334 + don't have any separator on it. Trying to use ``\1``..``\n`` for 335 + brackets also raise an exception. 336 + """ 337 + 338 + level = (0, 0, 0) 339 + 340 + if self.max_group < 0: 341 + return level, [] 342 + 343 + tokens = new_tokenizer.tokens 344 + 345 + # 346 + # Fill \0 with the full token contents 347 + # 348 + groups_list = [ [] ] 349 + 350 + if 0 in self.sub_groups: 351 + inner_level = 0 352 + 353 + for i in range(0, len(tokens)): 354 + tok = tokens[i] 355 + 356 + if tok.kind == CToken.BEGIN: 357 + inner_level += 1 358 + 359 + # 360 + # Discard first begin 361 + # 362 + if not groups_list[0]: 363 + continue 364 + elif tok.kind == CToken.END: 365 + inner_level -= 1 366 + if inner_level < 0: 367 + break 368 + 369 + if inner_level: 370 + groups_list[0].append(tok) 371 + 372 + if not self.max_group: 373 + return level, groups_list 374 + 375 + delim = None 376 + 377 + # 378 + # Ignore everything before BEGIN. The value of begin gives the 379 + # delimiter to be used for the matches 380 + # 381 + for i in range(0, len(tokens)): 382 + tok = tokens[i] 383 + if tok.kind == CToken.BEGIN: 384 + if tok.value == "{": 385 + delim = ";" 386 + elif tok.value == "(": 387 + delim = "," 388 + else: 389 + self.log.error(fr"Can't handle \1..\n on {sub_str}") 390 + 391 + level = tok.level 392 + break 393 + 394 + pos = 1 395 + groups_list.append([]) 396 + 397 + inner_level = 0 398 + for i in range(i + 1, len(tokens)): 399 + tok = tokens[i] 400 + 401 + if tok.kind == CToken.BEGIN: 402 + inner_level += 1 403 + if tok.kind == CToken.END: 404 + inner_level -= 1 405 + if inner_level < 0: 406 + break 407 + 408 + if tok.kind in [CToken.PUNC, CToken.ENDSTMT] and delim == tok.value: 409 + pos += 1 410 + if self.greedy and pos > self.max_group: 411 + pos -= 1 412 + else: 413 + groups_list.append([]) 414 + 415 + if pos > self.max_group: 416 + break 417 + 418 + continue 419 + 420 + groups_list[pos].append(tok) 421 + 422 + if pos < self.max_group: 423 + log.error(fr"{self.sub_str} groups are up to {pos} instead of {self.max_group}") 424 + 425 + return level, groups_list 426 + 427 + def tokens(self, new_tokenizer): 428 + level, groups = self.groups(new_tokenizer) 429 + 430 + new = CTokenizer() 431 + 432 + for tok in self.sub_tokeninzer.tokens: 433 + if tok.kind == CToken.BACKREF: 434 + group = int(tok.value[1:]) 435 + 436 + for group_tok in groups[group]: 437 + new_tok = copy(group_tok) 438 + 439 + new_level = [0, 0, 0] 440 + 441 + for i in range(0, len(level)): 442 + new_level[i] = new_tok.level[i] + level[i] 443 + 444 + new_tok.level = tuple(new_level) 445 + 446 + new.tokens += [ new_tok ] 447 + else: 448 + new.tokens += [ tok ] 449 + 450 + return new.tokens 451 + 452 + 289 453 class CMatch: 290 454 """ 291 455 Finding nested delimiters is hard with regular expressions. It is ··· 477 309 will ignore the search string. 478 310 """ 479 311 480 - # TODO: add a sub method 481 312 482 - def __init__(self, regex): 483 - self.regex = KernRe(regex) 313 + def __init__(self, regex, delim="("): 314 + self.regex = KernRe("^" + regex + r"\b") 315 + self.start_delim = delim 484 316 485 317 def _search(self, tokenizer): 486 318 """ ··· 503 335 """ 504 336 505 337 start = None 506 - offset = -1 507 338 started = False 508 339 509 340 import sys ··· 518 351 519 352 continue 520 353 521 - if not started and tok.kind == CToken.BEGIN: 522 - started = True 523 - continue 354 + if not started: 355 + if tok.kind == CToken.SPACE: 356 + continue 357 + 358 + if tok.kind == CToken.BEGIN and tok.value == self.start_delim: 359 + started = True 360 + continue 361 + 362 + # Name only token without BEGIN/END 363 + if i > start: 364 + i -= 1 365 + yield start, i 366 + start = None 524 367 525 368 if tok.kind == CToken.END and tok.level == stack[-1][1]: 526 369 start, level = stack.pop() 527 - offset = i 528 370 529 - yield CTokenizer(tokenizer.tokens[start:offset + 1]) 371 + yield start, i 530 372 start = None 531 373 532 374 # ··· 543 367 # This is meant to solve cases where the caller logic might be 544 368 # picking an incomplete block. 545 369 # 546 - if start and offset < 0: 547 - print("WARNING: can't find an end", file=sys.stderr) 548 - yield CTokenizer(tokenizer.tokens[start:]) 370 + if start and stack: 371 + if started: 372 + s = str(tokenizer) 373 + log.warning(f"can't find a final end at {s}") 374 + 375 + yield start, len(tokenizer.tokens) 549 376 550 377 def search(self, source): 551 378 """ ··· 565 386 tokenizer = CTokenizer(source) 566 387 is_token = False 567 388 568 - for new_tokenizer in self._search(tokenizer): 389 + for start, end in self._search(tokenizer): 390 + new_tokenizer = CTokenizer(tokenizer.tokens[start:end + 1]) 391 + 569 392 if is_token: 570 393 yield new_tokenizer 571 394 else: 572 395 yield str(new_tokenizer) 396 + 397 + def sub(self, sub_str, source, count=0): 398 + """ 399 + This is similar to re.sub: 400 + 401 + It matches a regex that it is followed by a delimiter, 402 + replacing occurrences only if all delimiters are paired. 403 + 404 + if the sub argument contains:: 405 + 406 + r'\0' 407 + 408 + it will work just like re: it places there the matched paired data 409 + with the delimiter stripped. 410 + 411 + If count is different than zero, it will replace at most count 412 + items. 413 + """ 414 + if isinstance(source, CTokenizer): 415 + is_token = True 416 + tokenizer = source 417 + else: 418 + is_token = False 419 + tokenizer = CTokenizer(source) 420 + 421 + # Detect if sub_str contains sub arguments 422 + 423 + args_match = CTokenArgs(sub_str) 424 + 425 + new_tokenizer = CTokenizer() 426 + pos = 0 427 + n = 0 428 + 429 + # 430 + # NOTE: the code below doesn't consider overlays at sub. 431 + # We may need to add some extra unit tests to check if those 432 + # would cause problems. When replacing by "", this should not 433 + # be a problem, but other transformations could be problematic 434 + # 435 + for start, end in self._search(tokenizer): 436 + new_tokenizer.tokens += tokenizer.tokens[pos:start] 437 + 438 + new = CTokenizer(tokenizer.tokens[start:end + 1]) 439 + 440 + new_tokenizer.tokens += args_match.tokens(new) 441 + 442 + pos = end + 1 443 + 444 + n += 1 445 + if count and n >= count: 446 + break 447 + 448 + new_tokenizer.tokens += tokenizer.tokens[pos:] 449 + 450 + if not is_token: 451 + return str(new_tokenizer) 452 + 453 + return new_tokenizer 454 + 455 + def __repr__(self): 456 + """ 457 + Returns a displayable version of the class init. 458 + """ 459 + 460 + return f'CMatch("{self.regex.regex.pattern}")'

Configure Feed

Configure Feed