fix: Twitter rending issue for some 404 tweets

+3 -2

Makefile

··· 13 13 fmt: ## Run go fmt and cleanup whitespace 14 14 go fmt ./... 15 15 find internal/templates -type f $ -name "*.html" -o -name "*.xml" $ -exec sed -i '' 's/[ \t]*$$//' {} + 16 + find tests -type f -name "*.sh" -exec sed -i '' 's/[ \t]*$$//' {} + 16 17 17 18 GIT_COMMIT=$(shell git rev-parse --short HEAD) 18 19 LDFLAGS=-ldflags "-X tumble/internal/version.CommitHash=$(GIT_COMMIT)" ··· 60 61 load-fixtures: ## Load test fixtures 61 62 ./tests/load_fixtures.sh 62 63 63 - run-test: build ## Run the application with the test database 64 + run-test: kill build ## Run the application with the test database 64 65 $(BUILD_DIR)/$(BINARY_NAME) conf/config-test.yaml 65 66 66 67 67 68 68 - test-db: build ## Create a fresh test database with fixtures 69 + test-db: kill build ## Create a fresh test database with fixtures 69 70 ./tests/setup_test_db.sh 70 71 71 72 help: ## Show this help message

+12

conf/config-dev-mysql.yaml

··· 1 + driver: mysql 2 + database: tumble 3 + username: tumble 4 + host: 127.0.0.1:13306 5 + baseurl: localhost:8080 6 + mode: dev 7 + logging: 8 + level: debug 9 + output: tumble.log 10 + 11 + caching: 12 + enabled: true

+1 -12

conf/config.yaml

··· 1 - driver: mysql 2 - database: tumble 3 - username: tumble 4 - host: 127.0.0.1:13306 5 - baseurl: localhost:8080 6 - mode: dev 7 - logging: 8 - level: debug 9 - output: tumble.log 10 - 11 - caching: 12 - enabled: true 1 + config-test.yaml

+1 -3

internal/handler/preview_twitter.go

··· 1 1 package handler 2 2 3 - import ( 4 - "strings" 5 - ) 3 + import "strings" 6 4 7 5 // GetTwitterPreview handles preview logic for Twitter/X. 8 6 // Currently, it leverages the default OEmbed behavior but suppresses the output

+1 -1

internal/service/content.go

··· 88 88 embed := fmt.Sprintf(`<blockquote class="twitter-tweet"><a href="%s" target="_blank">%s</a></blockquote><script async src="https://platform.twitter.com/widgets.js" charset="utf-8"></script>`, embedURL, item.Title) 89 89 d.Content = template.HTML(embed) 90 90 isTwitter = true 91 - d.SuppressOG = true 91 + d.SuppressOG = false 92 92 } 93 93 } 94 94

+3 -3

tests/api_test.sh

··· 41 41 check_200 "/index.xml?dtype=rss" 42 42 check_content_type "/index.xml?dtype=rss" "text/xml" 43 43 44 - # Optional: XML Validation if xmllint is present 44 + # Optional: XML Validation if xmllint is presen 45 45 if command -v xmllint &> /dev/null; then 46 46 echo -n "Validating RSS XML structure... " 47 47 curl -s "$BASE_URL/index.xml?dtype=rss" > rss_temp.xml ··· 88 88 89 89 # 7. Delete Link Test (Create -> Delete -> Verify) 90 90 echo -n "Testing DELETE /irclink/ flow... " 91 - # Create a link first 91 + # Create a link firs 92 92 CREATE_OUT=$(curl -s "$BASE_URL/irclink/?user=testdel&url=http://delete-test.com&source=irc") 93 93 # Check if we got an ID (numeric) 94 94 if [[ "$CREATE_OUT" =~ ^[0-9]+$ ]]; then 95 95 DEL_ID=$CREATE_OUT 96 - # Delete it 96 + # Delete i 97 97 DEL_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X DELETE "$BASE_URL/irclink/?id=$DEL_ID") 98 98 if [ "$DEL_STATUS" == "200" ]; then 99 99 # Verify it's gone

+44 -8

tests/load_fixtures.sh

··· 6 6 DB_PATH="${DB_PATH:-tumble.sqlite}" 7 7 ADD_LINK_SCRIPT="./tests/add_link.sh" 8 8 9 + # Detect Driver from Config if not se 10 + if [ -z "$DRIVER" ]; then 11 + if [ -f "conf/config.yaml" ]; then 12 + DETECTED_DRIVER=$(grep "driver:" conf/config.yaml | awk '{print $2}') 13 + if [ -n "$DETECTED_DRIVER" ]; then 14 + DRIVER=$DETECTED_DRIVER 15 + echo "Auto-detected driver: $DRIVER" 16 + fi 17 + fi 18 + fi 19 + 9 20 echo "Using BASE_URL: $BASE_URL" 10 21 echo "Using DB_PATH: $DB_PATH" 11 22 ··· 17 28 # Image 18 29 $ADD_LINK_SCRIPT "pic_poster" "https://cdn.tinnies.club/accounts/avatars/109/626/500/076/902/223/original/2a4a0d1a4ce728c4.jpg" 19 30 20 - # Mastodon Post 31 + # Mastodon Pos 21 32 $ADD_LINK_SCRIPT "social_butterfly" "https://fosstodon.org/@genebean/113945244453254504" 22 33 23 34 # Standard Link ··· 29 40 # Imgur (Animated) 30 41 $ADD_LINK_SCRIPT "meme_lord" "https://imgur.com/only-one-jack-black-0qetp3u" 31 42 32 - # Twitter Post 43 + # Twitter Pos 33 44 $ADD_LINK_SCRIPT "tweet_master" "https://x.com/jcockhren/status/1229101594505097216?s=20" 34 45 35 46 # Wikipedia (Go) ··· 82 93 # Giphy 83 94 $ADD_LINK_SCRIPT "gif_master" "https://giphy.com/gifs/cant-hardly-wait-kW8mnYSNkUYKc" 84 95 85 - echo "Loading backdated 'Hot Links' directly into DB..." 96 + # Kevin's Broken Twitter Link (Sad Path) 97 + # Use the specific broken ID if possible, but add_link auto-increments. 98 + # We will just assert on content behavior for "kevin". 99 + $ADD_LINK_SCRIPT "kevin" "https://twitter.com/darkuncle/status/1483507577174441985" 100 + 101 + # Tester's Valid Twitter Link (Happy Path) 102 + $ADD_LINK_SCRIPT "tester" "https://twitter.com/jcockhren/status/1229101594505097216?s=20" 103 + 86 104 echo "Loading backdated 'Hot Links' directly into DB..." 87 105 if [ "$DRIVER" == "mysql" ]; then 88 106 MYSQL_HOST="${MYSQL_HOST:-localhost}" 89 107 MYSQL_USER="${MYSQL_USER:-tumble}" 90 - # Defaulting to no password for local dev if not set, or prompt? Better to rely on .my.cnf or env var. 91 - # We will assume MYSQL_PASSWORD is set if needed or it's empty. 108 + # Defaulting to no password for local dev if not se 109 + 110 + # Try to detect port from config if not se 111 + if [ -z "$MYSQL_PORT" ] && [ -f "conf/config.yaml" ]; then 112 + # simple grep for host: ip:por 113 + HOST_LINE=$(grep "host:" conf/config.yaml | awk '{print $2}') 114 + if [[ "$HOST_LINE" == *":"* ]]; then 115 + MYSQL_PORT=${HOST_LINE#*:} 116 + echo "Auto-detected MySQL Port: $MYSQL_PORT" 117 + fi 118 + fi 92 119 MYSQL_PORT="${MYSQL_PORT:-3306}" 120 + 93 121 CMD="mysql -h $MYSQL_HOST -P $MYSQL_PORT -u $MYSQL_USER" 94 122 if [ -n "$MYSQL_PASSWORD" ]; then 95 123 CMD="$CMD -p$MYSQL_PASSWORD" 96 124 fi 97 125 # Use database from config or env? 98 - # We need the database name. Let's assume MYSQL_DATABASE env var or "tumble_test" 99 - DB_NAME="${MYSQL_DATABASE:-tumble_test}" 100 - $CMD "$DB_NAME" < tests/fixtures_hot_mysql.sql 126 + DB_NAME="${MYSQL_DATABASE:-tumble}" 127 + # Check config for database name too if possible 128 + if [ -z "$MYSQL_DATABASE" ] && [ -f "conf/config.yaml" ]; then 129 + DETECTED_DB=$(grep "database:" conf/config.yaml | awk '{print $2}') 130 + if [ -n "$DETECTED_DB" ]; then 131 + DB_NAME=$DETECTED_DB 132 + fi 133 + fi 134 + 135 + # Suppress password warning 136 + $CMD "$DB_NAME" < tests/fixtures_hot_mysql.sql 2>/dev/null 101 137 else 102 138 sqlite3 "$DB_PATH" < tests/fixtures_hot.sql 103 139 fi

+45

tests/poster_test.sh

··· 1 + #!/bin/bash 2 + BASE_URL="http://localhost:8080" 3 + FAIL=0 4 + 5 + echo "Starting Poster Tests against $BASE_URL..." 6 + 7 + # Fetch page conten 8 + CONTENT=$(curl -s "$BASE_URL/?poster=kevin") 9 + 10 + # Check we got conten 11 + if [[ -z "$CONTENT" ]]; then 12 + echo "FAIL: No content returned" 13 + exit 1 14 + fi 15 + 16 + # Check for Item 79480 (The broken link item) 17 + if [[ "$CONTENT" != *"data-irc-link-id=\"79480\""* ]]; then 18 + echo "FAIL: Item 79480 not found in output" 19 + FAIL=1 20 + fi 21 + 22 + # Check that 'twitter-tweet' class (Server Side Embed) IS present (We restored it) 23 + if [[ "$CONTENT" == *"class=\"twitter-tweet\""* ]]; then 24 + echo "OK: Found hardcoded 'twitter-tweet' class." 25 + else 26 + echo "FAIL: 'twitter-tweet' class NOT found (Server side embed missing)." 27 + FAIL=1 28 + fi 29 + 30 + # Check that 'og-preview' IS present (SuppressOG=false) 31 + # The template renders: <div class="og-preview" id="og-preview-{{.ID}}"></div> 32 + if [[ "$CONTENT" == *"id=\"og-preview-79480\""* ]]; then 33 + echo "OK: og-preview-79480 found." 34 + else 35 + echo "FAIL: og-preview-79480 NOT found (SuppressOG might still be true)." 36 + FAIL=1 37 + fi 38 + 39 + if [ $FAIL -eq 0 ]; then 40 + echo "Poster tests passed!" 41 + exit 0 42 + else 43 + echo "Poster tests failed!" 44 + exit 1 45 + fi

+25 -25

tests/preview_test.sh

··· 44 44 45 45 # --- REDDIT --- 46 46 # Valid: Should have rich metadata (provider_name or type) 47 - test_preview "Reddit Valid" \ 48 - "https://www.reddit.com/r/valheim/comments/leqdj6/our_first_encounter_with_the_troll/" \ 47 + test_preview "Reddit Valid" 48 + "https://www.reddit.com/r/valheim/comments/leqdj6/our_first_encounter_with_the_troll/" 49 49 '.provider_name == "Reddit" or .title != null' 50 50 51 51 # Invalid: specific non-existent post. ··· 56 56 # For now, let's assume 'Invalid' means we check for absence of specific post content if possible, 57 57 # OR just that it doesn't crash. But user requested testing 404 logic. 58 58 # If preview_reddit.go fails, it falls back. 59 - test_preview "Reddit Invalid" \ 60 - "https://www.reddit.com/r/valheim/comments/INVALID_ID_12345/" \ 59 + test_preview "Reddit Invalid" 60 + "https://www.reddit.com/r/valheim/comments/INVALID_ID_12345/" 61 61 '.error != null or (.title | contains("Page not found") or contains("Reddit"))' 62 62 63 63 # --- SPOTIFY --- 64 64 # Valid 65 - test_preview "Spotify Valid" \ 66 - "https://open.spotify.com/episode/7makk4oTQel546B0PZlDM5" \ 65 + test_preview "Spotify Valid" 66 + "https://open.spotify.com/episode/7makk4oTQel546B0PZlDM5" 67 67 '.provider_name == "Spotify" or .type == "rich"' 68 68 69 69 70 70 # Invalid 71 71 # Spotify returns a 200 OK on invalid tracks with a "Spotify - Web Player" title, 72 72 # so we expect a valid scrape fallback, not an error. 73 - test_preview "Spotify Invalid" \ 74 - "https://open.spotify.com/track/INVALID_TRACK_ID" \ 73 + test_preview "Spotify Invalid" 74 + "https://open.spotify.com/track/INVALID_TRACK_ID" 75 75 '.provider_name == "Spotify"' 76 76 77 77 # --- IMGUR --- 78 78 # Valid 79 - test_preview "Imgur Valid" \ 80 - "https://imgur.com/only-one-jack-black-0qetp3u" \ 79 + test_preview "Imgur Valid" 80 + "https://imgur.com/only-one-jack-black-0qetp3u" 81 81 '.title != null' 82 82 83 83 # Invalid 84 - test_preview "Imgur Invalid" \ 85 - "https://imgur.com/gallery/INVALID_GALLERY_ID" \ 84 + test_preview "Imgur Invalid" 85 + "https://imgur.com/gallery/INVALID_GALLERY_ID" 86 86 '.error != null or (.title | contains("Imgur"))' 87 87 88 88 # --- YOUTUBE --- 89 89 # Valid 90 - test_preview "YouTube Valid" \ 91 - "https://www.youtube.com/watch?v=dQw4w9WgXcQ" \ 90 + test_preview "YouTube Valid" 91 + "https://www.youtube.com/watch?v=dQw4w9WgXcQ" 92 92 '.provider_name == "YouTube"' 93 93 94 94 # Invalid (Video Unavailable) - Special handling in preview.go returning status 404 95 - test_preview "YouTube Invalid" \ 96 - "https://youtu.be/Ie_Wl9eNffE" \ 95 + test_preview "YouTube Invalid" 96 + "https://youtu.be/Ie_Wl9eNffE" 97 97 '.status == 404 and .error == "Video Unavailable"' 98 98 99 99 # --- TWITTER --- 100 100 # Valid - Returns empty object {} on success per preview_twitter.go 101 - test_preview "Twitter Valid" \ 102 - "https://x.com/jcockhren/status/1229101594505097216" \ 101 + test_preview "Twitter Valid" 102 + "https://x.com/jcockhren/status/1229101594505097216" 103 103 '. == {}' 104 104 105 105 # Invalid - Falls back to scrape. Twitter returns a generic page title / icon. 106 - test_preview "Twitter Invalid" \ 107 - "https://x.com/jcockhren/status/0000000000000000000" \ 108 - '.provider_name != null' 106 + test_preview "Twitter Invalid" 107 + "https://x.com/jcockhren/status/0000000000000000000" 108 + '.error == "Tweet Unavailable" and .status == 404' 109 109 110 110 # --- TIKTOK --- 111 111 # Valid 112 - test_preview "TikTok Valid" \ 113 - "https://www.tiktok.com/@tiagogreis/video/6830059644233223429" \ 112 + test_preview "TikTok Valid" 113 + "https://www.tiktok.com/@tiagogreis/video/6830059644233223429" 114 114 '.provider_name == "TikTok" or .type == "video"' 115 115 116 116 117 117 # Invalid - Falls back to scrape. 118 - test_preview "TikTok Invalid" \ 119 - "https://www.tiktok.com/@user/video/1234567890123456789" \ 118 + test_preview "TikTok Invalid" 119 + "https://www.tiktok.com/@user/video/1234567890123456789" 120 120 '.title | contains("TikTok")' 121 121 122 122

+4 -1

tests/setup_test_db.sh

··· 12 12 rm -f "$DB_PATH" 13 13 14 14 # Start server 15 + # Explicitly set driver for load_fixtures 16 + export DRIVER="sqlite" 17 + 15 18 # Start server 16 19 echo "Starting server (logging to tumble-test.log)..." 17 20 $BINARY "$CONFIG" > tumble-test.log 2>&1 & ··· 19 22 echo "Server PID: $PID" 20 23 21 24 # Ensure cleanup 22 - trap "echo 'Stopping server...'; kill $PID || true" EXIT 25 + trap "echo 'Stopping server...'; kill $PID 2>/dev/null || true" EXIT 23 26 24 27 # Wait for server to be ready 25 28 echo "Waiting for server to be ready on port $PORT..."

+83

tests/twitter_behavior_test.sh

··· 1 + #!/bin/bash 2 + BASE_URL="http://localhost:8080" 3 + FAIL=0 4 + 5 + echo "Starting Twitter Behavior Tests against $BASE_URL..." 6 + 7 + # Load Fixtures 8 + echo "Loading fixtures..." 9 + ./tests/load_fixtures.sh > /dev/null 10 + 11 + # 1. HAPPY PATH: Valid Twee 12 + # We need a user/post that has a VALID twitter link. 13 + # Based on fixtures/logs, we might look for a known valid one or insert one if we had an insert script. 14 + # For now, let's assume one exists or we check the 'tester' user mentioned by the user. 15 + # "See localhost:8080 for posts from 'tester'" 16 + 17 + echo -n "Checking for Valid Tweet Render (tester)... " 18 + CONTENT_VALID=$(curl -s "$BASE_URL/?poster=tester") 19 + 20 + # Check for server-side widget code 21 + if [[ "$CONTENT_VALID" == *"class=\"twitter-tweet\""* ]]; then 22 + echo "OK (Server-side widget found)" 23 + else 24 + # It might be that 'tester' has no tweets, but user said "posts from tester... tweets no longer rendering". 25 + # So we assume they exist. 26 + echo "FAIL (Server-side widget MISSING for valid tweet)" 27 + FAIL=1 28 + fi 29 + 30 + # Check for og-preview hook (required for 404 check even on valid ones, though it does nothing if valid) 31 + if [[ "$CONTENT_VALID" == *"class=\"og-preview\""* ]]; then 32 + echo "OK (og-preview hook found)" 33 + else 34 + echo "FAIL (og-preview hook MISSING)" 35 + FAIL=1 36 + fi 37 + 38 + 39 + # 2. SAD PATH: Broken Tweet (404) 40 + # We know 'kevin' has the broken one (ID 79480). 41 + echo -n "Checking for Broken Tweet Handling (kevin)... " 42 + CONTENT_BROKEN=$(curl -s "$BASE_URL/?poster=kevin") 43 + 44 + # It SHOULD have the widget code initially (server-side doesn't know it's broken yet) 45 + if [[ "$CONTENT_BROKEN" == *"class=\"twitter-tweet\""* ]]; then 46 + echo "OK (Server-side widget present initially)" 47 + else 48 + echo "FAIL (Widget missing on broken link - strictly speaking it should be there, then removed by JS?)" 49 + # Actually, in our logic, we ALWAYS render the widget if it looks like a tweet. 50 + # The JS then runs ogpreview, gets 404, and REPLACES the content. 51 + # So purely from curl (server response), the widget MUST be there. 52 + FAIL=1 53 + fi 54 + 55 + # It MUST have the og-preview hook 56 + if [[ "$CONTENT_BROKEN" == *"class=\"og-preview\""* ]]; then 57 + echo "OK (og-preview hook present)" 58 + else 59 + echo "FAIL (og-preview hook MISSING on broken link)" 60 + FAIL=1 61 + fi 62 + 63 + # 3. VERIFY CLIENT SIDE 404 (Simulated) 64 + # We can't easily run JS here, but we can verify the *endpoint* acting correctly. 65 + # Request ogpreview for the broken URL. 66 + BROKEN_URL="https://twitter.com/darkuncle/status/1483507577174441985" 67 + echo -n "Checking ogpreview response for broken URL... " 68 + PREVIEW_RESP=$(curl -s "$BASE_URL/ogpreview.cgi?url=$BROKEN_URL") 69 + 70 + if [[ "$PREVIEW_RESP" == *"\"status\":404"* ]] || [[ "$PREVIEW_RESP" == *"\"status\": 404"* ]]; then 71 + echo "OK (API returns 404 status)" 72 + else 73 + echo "FAIL (API did not return 404 status: $PREVIEW_RESP)" 74 + FAIL=1 75 + fi 76 + 77 + if [ $FAIL -eq 0 ]; then 78 + echo "All Twitter behavior tests passed!" 79 + exit 0 80 + else 81 + echo "Tests failed!" 82 + exit 1 83 + fi

Configure Feed

Configure Feed