remove some unnecessary sink cli commands (#17)

+1 -341

1 changed file

expand all

osprey_worker

src

osprey

worker

cli

sinks.py

+1 -341

osprey_worker/src/osprey/worker/cli/sinks.py

··· 8 8 9 9 10 10 import signal 11 - import time 12 11 from uuid import uuid1 13 12 14 13 # this is required to avoid memory leaks with gRPC ··· 21 20 import os 22 21 from concurrent.futures import ProcessPoolExecutor 23 22 from pathlib import Path 24 - from typing import Any, Dict, Optional, Set, TextIO, cast 23 + from typing import Optional, Set, TextIO, cast 25 24 26 25 import click 27 26 import gevent 28 27 import kafka 29 - import msgpack 30 28 import sentry_sdk 31 29 from google.api_core.exceptions import AlreadyExists 32 30 from google.cloud import pubsub_v1 ··· 38 36 39 37 from osprey.worker.lib.bulk_label import TaskStatus 40 38 from osprey.worker.lib.config import Config 41 - from osprey.worker.lib.encryption.envelope import Envelope 42 39 from osprey.worker.lib.osprey_engine import OspreyEngine, get_sources_provider 43 40 from osprey.worker.lib.osprey_shared.logging import get_logger 44 41 from osprey.worker.lib.publisher import PubSubPublisher ··· 384 381 subscriber = pubsub_v1.SubscriberClient() 385 382 386 383 create_pubsub_topic_and_subscription('osprey-dev', 'osprey-actions', 'osprey-dev', 'rules-sink') 387 - create_pubsub_topic_and_subscription('osprey-dev', 'osprey-actions', 'osprey-dev', 'attachments-sent') 388 - create_pubsub_topic_and_subscription('osprey-dev', 'media-hasher-output', 'osprey-dev', 'media-match-sink') 389 - create_pubsub_topic_and_subscription('osprey-dev', 'osprey-analytics', 'osprey-dev', 'osprey-analytics-sub') 390 - create_pubsub_topic_and_subscription('osprey-dev', 'media-uploaded', 'osprey-dev', 'media-uploaded') 391 - create_pubsub_topic_and_subscription('osprey-dev', 'osprey-safety-signals', 'osprey-dev', 'safety-dispatch') 392 - create_pubsub_topic_and_subscription('osprey-dev', 'ihd-agent-decisions', 'osprey-dev', 'ihd-agent-decisions-sub') 393 - create_pubsub_topic_and_subscription('osprey-dev', 'osprey-evidence', 'osprey-dev', 'osprey-evidence-subscription') 394 - create_pubsub_topic_and_subscription('osprey-dev', 'osprey-webhooks', 'osprey-dev', 'osprey-webhooks-subscription') 395 - create_pubsub_topic_and_subscription('osprey-dev', 'panther-osprey-events', 'osprey-dev', 'panther-osprey-events') 396 - 397 - 398 - # NOTE(austin) in reality this shouldn't live in the osprey cli but the infrastructure already exists 399 - def _bootstrap_safety_record_pubsub() -> None: 400 - os.environ['PUBSUB_EMULATOR_HOST'] = '127.0.0.1:8085' 401 - 402 - def create_pubsub_topic_and_subscription( 403 - topic_project_id: str, topic_id: str, subscription_project_id: str, subscription_id: str 404 - ) -> None: 405 - _create_pubsub_topic_and_subscription( 406 - publisher, subscriber, topic_project_id, topic_id, subscription_project_id, subscription_id 407 - ) 408 - 409 - publisher = pubsub_v1.PublisherClient() 410 - subscriber = pubsub_v1.SubscriberClient() 411 - 412 - create_pubsub_topic_and_subscription( 413 - 'osprey-dev', 'safety-record-write-topic', 'osprey-dev', 'safety-record-write-topic-subscription' 414 - ) 415 - create_pubsub_topic_and_subscription( 416 - 'osprey-dev', 'safety-classification-emit-topic', 'osprey-dev', 'safety-classification-emit-subscription' 417 - ) 418 - create_pubsub_topic_and_subscription( 419 - 'osprey-dev', 420 - 'safety-classification-removal-emit-topic', 421 - 'osprey-dev', 422 - 'safety-classification-removal-emit-subscription', 423 - ) 424 - 425 - create_pubsub_topic_and_subscription( 426 - 'osprey-dev', 'interventions-request', 'osprey-dev', 'interventions-request-subscription' 427 - ) 428 - create_pubsub_topic_and_subscription( 429 - 'osprey-dev', 430 - 'safety-record-evidence-write-topic', 431 - 'osprey-dev', 432 - 'safety-record-evidence-write-topic-subscription', 433 - ) 434 - create_pubsub_topic_and_subscription( 435 - 'osprey-dev', 436 - 'safety-record-evidence-write-dead-letter-topic', 437 - 'osprey-dev', 438 - 'safety-record-evidence-write-dead-letter-topic-subscription', 439 - ) 440 - create_pubsub_topic_and_subscription('osprey-dev', 'generic-events', 'osprey-dev', 'generic-events-subscription') 441 - 442 - create_pubsub_topic_and_subscription( 443 - 'osprey-dev', 444 - 'safety-classification-removal-requests', 445 - 'osprey-dev', 446 - 'safety-classification-removal-requests-subscriber', 447 - ) 448 - 449 - create_pubsub_topic_and_subscription( 450 - 'osprey-dev', 451 - 'interventions-request', 452 - 'osprey-dev', 453 - 'interventions-request-subscription', 454 - ) 455 - create_pubsub_topic_and_subscription( 456 - 'osprey-dev', 457 - 'interventions-flush', 458 - 'osprey-dev', 459 - 'interventions-flush-subscription', 460 - ) 461 - create_pubsub_topic_and_subscription( 462 - 'osprey-dev', 463 - 'safety-classification-blast-radius-review', 464 - 'osprey-dev', 465 - 'safety-classification-blast-radius-review-subscription', 466 - ) 467 384 468 385 469 386 def _create_pubsub_topic_and_subscription( ··· 504 421 def bootstrap_pubsub() -> None: 505 422 init_config() 506 423 _bootstrap_pubsub() 507 - 508 - 509 - @cli.command() 510 - @click.option( 511 - '--sleep-forever/--no-sleep-forever', 512 - default=False, 513 - help='Whether to sleep forever after bootstrapping the pubsub topics/subscriptions. ' 514 - 'Intended for clyde service start so that we can do one clean deployment.', 515 - ) 516 - def bootstrap_safety_record_pubsub(sleep_forever: bool) -> None: 517 - _bootstrap_safety_record_pubsub() 518 - if sleep_forever: 519 - while True: 520 - time.sleep(1) 521 - 522 - 523 - unencrypted_action_example_json = { 524 - 'name': 'guild_joined', 525 - 'data': { 526 - 'user': { 527 - 'id': '1', 528 - 'username': 'RyanWeinstein', 529 - 'avatar': 'a_9e573467f84ae50d47ffb69f4b11e4cb', 530 - 'avatar_decoration': None, 531 - 'discriminator': '0001', 532 - 'public_flags': 129, 533 - 'mfa_enabled': True, 534 - 'bot': False, 535 - 'flags': 10857677324417, 536 - 'email_verified': True, 537 - 'required_action': None, 538 - 'premium_type': 2, 539 - 'premium_until': '2022-05-03T21:49:26.832000+00:00', 540 - 'phone': '+14045528784', 541 - 'email': 'rnw144@gmail.com', 542 - 'ip': '76.21.5.241', 543 - 'ip_is_whitelisted': False, 544 - }, 545 - 'invite': { 546 - 'code': '4ucRCcAV', 547 - 'max_age': 604800, 548 - 'created_at': '2022-04-28T19:55:05.704694+00:00', 549 - 'temporary': False, 550 - 'max_uses': 0, 551 - 'custom': False, 552 - }, 553 - 'guild': { 554 - 'id': '1', 555 - 'name': 'SF Go Club', 556 - 'description': None, 557 - 'owner': {'id': '1'}, 558 - 'member_count': 113, 559 - 'features': ['EXPOSED_TO_ACTIVITIES_WTP_EXPERIMENT'], 560 - 'premium_subscription_count': 0, 561 - 'verification_level': 1, 562 - 'is_nsfw': False, 563 - 'nsfw_level': 0, 564 - 'has_public_stage': False, 565 - 'channel_names': [], 566 - }, 567 - 'channel': {'id': '1', 'name': 'general', 'type': 0, 'is_nsfw': False}, 568 - 'inviter': { 569 - 'id': '1', 570 - 'username': 'RyanWeinstein', 571 - 'avatar': 'a_9e573467f84ae50d47ffb69f4b11e4cb', 572 - 'avatar_decoration': None, 573 - 'discriminator': '0001', 574 - 'public_flags': 129, 575 - 'mfa_enabled': True, 576 - 'bot': False, 577 - 'flags': 10857677324417, 578 - 'email_verified': True, 579 - 'required_action': None, 580 - 'premium_type': 2, 581 - 'premium_until': '2022-05-03T21:49:26.832000+00:00', 582 - 'phone': '+14045528784', 583 - 'email': 'rnw144@gmail.com', 584 - 'ip': '76.21.5.241', 585 - 'ip_is_whitelisted': False, 586 - }, 587 - 'properties': { 588 - 'os': 'Mac OS X', 589 - 'browser': 'Osprey Client', 590 - 'release_channel': 'canary', 591 - 'client_version': '0.0.284', 592 - 'os_version': '20.6.0', 593 - 'os_arch': 'x64', 594 - 'system_locale': 'en-US', 595 - 'client_build_number': 124425, 596 - 'client_event_source': None, 597 - 'location': 'Guild Events', 598 - 'cfduid': 'bb13eb5bf3ea11ebb95442010a0a0015', 599 - 'cfduid_signed': True, 600 - }, 601 - 'http_request': { 602 - 'headers': { 603 - 'Host': 'test.example.com', 604 - 'X-Forwarded-For': '76.21.5.241', 605 - 'X-Forwarded-Proto': 'https', 606 - 'Content-Length': '84', 607 - 'Accept-Encoding': 'gzip', 608 - 'Cf-Ipcountry': 'US', 609 - 'Cf-Ray': '70325a8013769858-SJC', 610 - 'Cf-Visitor': '{"scheme":"https"}', 611 - 'Content-Type': 'application/json', 612 - 'Cf-Ew-Via': '15', 613 - 'Cdn-Loop': 'cloudflare; subreqs=1', 614 - 'Accept-Language': 'en-US', 615 - 'Accept': '*/*', 616 - 'Referer': 'https://text.example.com/channels/1/1', 617 - 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) osprey/0.0.284 Chrome/91.0.4472.164 Electron/13.6.6 Safari/537.36', 618 - 'Cf-Connecting-Ip': '76.21.5.241', 619 - 'Origin': 'https://test.example.com', 620 - 'Sec-Fetch-Dest': 'empty', 621 - 'Sec-Fetch-Mode': 'cors', 622 - 'Sec-Fetch-Site': 'same-origin', 623 - 'X-Context-Properties': 'eyJsb2NhdGlvbiI6Ikd1aWxkIEV2ZW50cyJ9', 624 - 'X-Debug-Options': 'bugReporterEnabled', 625 - 'X-Super-Properties': 'eyJvcyI6Ik1hYyBPUyBYIiwiYnJvd3NlciI6IkRpc2NvcmQgQ2xpZW50IiwicmVsZWFzZV9jaGFubmVsIjoiY2FuYXJ5IiwiY2xpZW50X3ZlcnNpb24iOiIwLjAuMjg0Iiwib3NfdmVyc2lvbiI6IjIwLjYuMCIsIm9zX2FyY2giOiJ4NjQiLCJzeXN0ZW1fbG9jYWxlIjoiZW4tVVMiLCJjbGllbnRfYnVpbGRfbnVtYmVyIjoxMjQ0MjUsImNsaWVudF9ldmVudF9zb3VyY2UiOm51bGx9', 626 - 'Cf-Worker': 'example.com', 627 - 'X-Cloud-Trace-Context': '26c610f0ab9788509722bd55f34e480c/4736332299159169251', 628 - 'Cookie': '__dcfduid=5ad2d58a06e14efa95adb3ff566ae6d5; __sdcfduid=bb13eb5bf3ea11ebb95442010a0a00151f59c766ecd2e78b5037913f45cf34e462ebde0db80fdcceb2e479c04e633b3a; __stripe_mid=d0fa98c9-ea86-43c0-93d9-e65470664ea24b054b; __cfruid=85af4fba395350750926aa70ffe390723c78fccc-1650303703', 629 - 'Via': '1.1 google', 630 - 'X-Envoy-External-Address': '10.10.3.166', 631 - 'X-Request-Id': 'cd8d1e03-0763-42c7-83b2-d1f42d6a70a0', 632 - 'X-Datadog-Trace-Id': '5264020418323037859', 633 - 'X-Datadog-Parent-Id': '7050569386306678149', 634 - 'X-Datadog-Sampling-Priority': '-1', 635 - }, 636 - 'endpoint': 'invites.create_channel_invite', 637 - 'url': 'https://test.example.com/api/channels/900896022295871500/invites', 638 - 'query_string': '', 639 - 'method': 'POST', 640 - 'api_version': 9, 641 - 'remote_addr': '76.21.5.241', 642 - 'remote_addr_is_whitelisted': False, 643 - }, 644 - 'did_solve_captcha': False, 645 - }, 646 - } 647 - 648 - encrypted_action_example_json = { 649 - 'name': 'message_sent', 650 - 'secret_data': { 651 - 'message': { 652 - 'content': 'This should be encrypted message content!', 653 - }, 654 - }, 655 - 'data': { 656 - 'channel': { 657 - 'id': '1', 658 - 'name': None, 659 - 'type': 1, 660 - 'is_nsfw': False, 661 - 'message_count': 9, 662 - 'recipients': [{'id': '1'}], 663 - }, 664 - 'message': { 665 - 'id': '1/1', 666 - 'message_id': '1', 667 - 'channel_id': '1', 668 - 'edited_timestamp': None, 669 - 'flags': 0, 670 - 'mention_ids': [], 671 - 'mention_channel_ids': [], 672 - 'mention_everyone': None, 673 - 'mention_everyone_online_only': None, 674 - 'author': { 675 - 'id': '1', 676 - 'username': 'person_a', 677 - 'global_name': 'person_a', 678 - 'avatar': '393729487298e9e7b98739287923d927', 679 - 'discriminator': '0', 680 - 'public_flags': 64, 681 - 'bio': 'owo', 682 - 'mfa_enabled': True, 683 - 'avatar_decoration': None, 684 - 'bot': False, 685 - 'flags': 10445360463936, 686 - 'suspicious_activity_flags': 0, 687 - 'email_verified': True, 688 - 'required_action': None, 689 - 'premium_type': None, 690 - 'premium_until': None, 691 - 'is_unique_username': True, 692 - 'phone': '+1112223333', 693 - 'email': 'person_a@example.com', 694 - 'ip': '127.0.0.1', 695 - 'ip_is_whitelisted': False, 696 - }, 697 - }, 698 - 'user': { 699 - 'id': '1', 700 - 'username': 'person_a', 701 - 'global_name': 'person_a', 702 - 'avatar': '393729487298e9e7b98739287923d927', 703 - 'discriminator': '0', 704 - 'public_flags': 64, 705 - 'bio': 'owo', 706 - 'mfa_enabled': True, 707 - 'avatar_decoration': None, 708 - 'bot': False, 709 - 'flags': 10445360463936, 710 - 'suspicious_activity_flags': 0, 711 - 'email_verified': True, 712 - 'required_action': None, 713 - 'premium_type': None, 714 - 'premium_until': None, 715 - 'is_unique_username': True, 716 - 'phone': '+1112223333', 717 - 'email': 'person_a@example.com', 718 - 'ip': '127.0.0.1', 719 - 'ip_is_whitelisted': False, 720 - }, 721 - }, 722 - } 723 - 724 - 725 - @cli.command() 726 - @click.argument('count', type=int, default=1) 727 - @click.argument('delay', type=float, default=0.0) 728 - @click.option('--encrypted/--unencrypted', 'encrypted', default=False, help='Whether to encrypt the action') 729 - def publish_action(count: int, delay: float, encrypted: bool) -> None: 730 - os.environ['PUBSUB_EMULATOR_HOST'] = 'localhost:8085' 731 - import json 732 - import time 733 - 734 - _bootstrap_pubsub() 735 - publisher = pubsub_v1.PublisherClient() 736 - envelope: Optional[Envelope] = None 737 - 738 - if encrypted: 739 - config = init_config() 740 - envelope = Envelope( 741 - kek_uri=config.get_str('PUBSUB_ENCRYPTION_KEY_URI', ''), 742 - gcp_credential_path='', 743 - ) 744 - 745 - topic_path = publisher.topic_path('osprey-dev', 'osprey-actions') 746 - for i in range(count): 747 - # coordinator should generate the snowflake id 748 - action: Dict[str, Any] = {} 749 - # action: Dict[str, Any] = {'id': str(round(datetime.now().timestamp()) + i)} 750 - if encrypted: 751 - action.update({**encrypted_action_example_json}) 752 - else: 753 - action.update({**unencrypted_action_example_json}) 754 - attributes = {} 755 - data = msgpack.dumps(json.dumps(action)) 756 - if encrypted and envelope is not None: 757 - attributes['encrypted'] = 'true' 758 - data = envelope.encrypt(data) 759 - else: 760 - attributes['encrypted'] = 'false' 761 - future = publisher.publish(topic_path, data, **attributes) 762 - print('Published message with id:', future.result()) 763 - time.sleep(delay) 764 424 765 425 766 426 @cli.command()

Configure Feed

Configure Feed