+1 -2

index.html

reviewed

··· 20 20 <link href="https://fonts.googleapis.com/css2?family=Open+Sans:wght@400;500;600;700&display=swap" rel="stylesheet" /> 21 21 22 22 <script src="/config.js"></script> 23 23 - <script src="https://cdn.jsdelivr.net/gh/movie-web/6C6F6C7A@8b821f445b83d51ef1b8f42c99b7346f6b47dce5/out.js"></script> 24 23 25 24 <!-- prevent darkreader extension from messing with our already dark site --> 26 25 <meta name="darkreader-lock" /> ··· 59 58 <script type="module" src="/src/index.tsx"></script> 60 59 </body> 61 60 62 62 - </html> 61 61 + </html>

+2

package.json

reviewed

··· 45 45 "i18next": "^22.4.5", 46 46 "immer": "^10.0.2", 47 47 "iso-639-1": "^3.1.0", 48 48 + "jwt-decode": "^4.0.0", 48 49 "lodash.isequal": "^4.5.0", 49 50 "nanoid": "^5.0.4", 50 51 "node-forge": "^1.3.1", ··· 57 58 "react-i18next": "^12.1.1", 58 59 "react-router-dom": "^5.2.0", 59 60 "react-sticky-el": "^2.1.0", 61 61 + "react-turnstile": "^1.1.2", 60 62 "react-use": "^17.4.0", 61 63 "slugify": "^1.6.6", 62 64 "subsrt-ts": "^2.1.1",

+21

pnpm-lock.yaml

reviewed

··· 68 68 iso-639-1: 69 69 specifier: ^3.1.0 70 70 version: 3.1.0 71 71 + jwt-decode: 72 72 + specifier: ^4.0.0 73 73 + version: 4.0.0 71 74 lodash.isequal: 72 75 specifier: ^4.5.0 73 76 version: 4.5.0 ··· 104 107 react-sticky-el: 105 108 specifier: ^2.1.0 106 109 version: 2.1.0(react-dom@17.0.2)(react@17.0.2) 110 110 + react-turnstile: 111 111 + specifier: ^1.1.2 112 112 + version: 1.1.2(react-dom@17.0.2)(react@17.0.2) 107 113 react-use: 108 114 specifier: ^17.4.0 109 115 version: 17.4.0(react-dom@17.0.2)(react@17.0.2) ··· 4523 4529 resolution: {integrity: sha512-cxQGGUiit6CGUpuuiezY8N4m1wgF4o7127rXEXDFcxeDUFfdV7gSkwA26Fe2wWBiNQq2SZOgN4gSmMxB/StA8Q==} 4524 4530 dev: true 4525 4531 4532 4532 + /jwt-decode@4.0.0: 4533 4533 + resolution: {integrity: sha512-+KJGIyHgkGuIq3IEBNftfhW/LfWhXUIY6OmyVWjliu5KH1y0fw7VQ8YndE2O4qZdMSd9SqbnC8GOcZEy0Om7sA==} 4534 4534 + engines: {node: '>=18'} 4535 4535 + dev: false 4536 4536 + 4526 4537 /keyv@4.5.3: 4527 4538 resolution: {integrity: sha512-QCiSav9WaX1PgETJ+SpNnx2PRRapJ/oRSXM4VO5OGYGSjrxbKPVFVhB3l2OCbLCk329N8qyAtsJjSjvVBWzEug==} 4528 4539 dependencies: ··· 5316 5327 peerDependencies: 5317 5328 react: '>=16.3.0' 5318 5329 react-dom: '>=16.3.0' 5330 5330 + dependencies: 5331 5331 + react: 17.0.2 5332 5332 + react-dom: 17.0.2(react@17.0.2) 5333 5333 + dev: false 5334 5334 + 5335 5335 + /react-turnstile@1.1.2(react-dom@17.0.2)(react@17.0.2): 5336 5336 + resolution: {integrity: sha512-wfhSf4JtXlmLRkfxMryU8yEeCbh401muKoInhx+TegYwP8RprUW5XPZa8WnCNZiYpMy1i6IXAb1Ar7xj5HxJag==} 5337 5337 + peerDependencies: 5338 5338 + react: '>= 17.0.0' 5339 5339 + react-dom: '>= 17.0.0' 5319 5340 dependencies: 5320 5341 react: 17.0.2 5321 5342 react-dom: 17.0.2(react@17.0.2)

+24 -34

src/backend/helpers/fetch.ts

reviewed

··· 1 1 - import { FetchOptions, FetchResponse, ofetch } from "ofetch"; 1 1 + import { ofetch } from "ofetch"; 2 2 3 3 + import { getApiToken, setApiToken } from "@/backend/helpers/providerApi"; 3 4 import { getLoadbalancedProxyUrl } from "@/utils/providers"; 4 5 5 6 type P<T> = Parameters<typeof ofetch<T, any>>; ··· 21 22 return baseFetch<T>(url, ops); 22 23 } 23 24 24 24 - export function proxiedFetch<T>(url: string, ops: P<T>[1] = {}): R<T> { 25 25 + export async function singularProxiedFetch<T>( 26 26 + proxyUrl: string, 27 27 + url: string, 28 28 + ops: P<T>[1] = {} 29 29 + ): R<T> { 25 30 let combinedUrl = ops?.baseURL ?? ""; 26 31 if ( 27 32 combinedUrl.length > 0 && ··· 45 50 parsedUrl.searchParams.set(k, v); 46 51 }); 47 52 48 48 - return baseFetch<T>(getLoadbalancedProxyUrl(), { 53 53 + let headers = ops.headers ?? {}; 54 54 + const apiToken = await getApiToken(); 55 55 + if (apiToken) 56 56 + headers = { 57 57 + ...headers, 58 58 + "X-Token": apiToken, 59 59 + }; 60 60 + 61 61 + return baseFetch<T>(proxyUrl, { 49 62 ...ops, 50 63 baseURL: undefined, 51 64 params: { 52 65 destination: parsedUrl.toString(), 53 66 }, 54 67 query: {}, 68 68 + headers, 69 69 + onResponse(context) { 70 70 + const tokenHeader = context.response.headers.get("X-Token"); 71 71 + if (tokenHeader) setApiToken(tokenHeader); 72 72 + ops.onResponse?.(context); 73 73 + }, 55 74 }); 56 75 } 57 76 58 58 - export function rawProxiedFetch<T>( 59 59 - url: string, 60 60 - ops: FetchOptions = {} 61 61 - ): Promise<FetchResponse<T>> { 62 62 - let combinedUrl = ops?.baseURL ?? ""; 63 63 - if ( 64 64 - combinedUrl.length > 0 && 65 65 - combinedUrl.endsWith("/") && 66 66 - url.startsWith("/") 67 67 - ) 68 68 - combinedUrl += url.slice(1); 69 69 - else if ( 70 70 - combinedUrl.length > 0 && 71 71 - !combinedUrl.endsWith("/") && 72 72 - !url.startsWith("/") 73 73 - ) 74 74 - combinedUrl += `/${url}`; 75 75 - else combinedUrl += url; 76 76 - 77 77 - const parsedUrl = new URL(combinedUrl); 78 78 - Object.entries(ops?.params ?? {}).forEach(([k, v]) => { 79 79 - parsedUrl.searchParams.set(k, v); 80 80 - }); 81 81 - 82 82 - return baseFetch.raw(getLoadbalancedProxyUrl(), { 83 83 - ...ops, 84 84 - baseURL: undefined, 85 85 - params: { 86 86 - destination: parsedUrl.toString(), 87 87 - }, 88 88 - }); 77 77 + export function proxiedFetch<T>(url: string, ops: P<T>[1] = {}): R<T> { 78 78 + return singularProxiedFetch<T>(getLoadbalancedProxyUrl(), url, ops); 89 79 }

+158

src/backend/helpers/providerApi.ts

reviewed

··· 1 1 + import { MetaOutput, NotFoundError, ScrapeMedia } from "@movie-web/providers"; 2 2 + import { jwtDecode } from "jwt-decode"; 3 3 + 4 4 + import { mwFetch } from "@/backend/helpers/fetch"; 5 5 + import { getTurnstileToken, isTurnstileInitialized } from "@/stores/turnstile"; 6 6 + 7 7 + let metaDataCache: MetaOutput[] | null = null; 8 8 + let token: null | string = null; 9 9 + 10 10 + export function setCachedMetadata(data: MetaOutput[]) { 11 11 + metaDataCache = data; 12 12 + } 13 13 + 14 14 + export function getCachedMetadata(): MetaOutput[] { 15 15 + return metaDataCache ?? []; 16 16 + } 17 17 + 18 18 + export function setApiToken(newToken: string) { 19 19 + token = newToken; 20 20 + } 21 21 + 22 22 + function getTokenIfValid(): null | string { 23 23 + if (!token) return null; 24 24 + try { 25 25 + const body = jwtDecode(token); 26 26 + if (!body.exp) return `jwt|${token}`; 27 27 + if (Date.now() / 1000 < body.exp) return `jwt|${token}`; 28 28 + } catch (err) { 29 29 + // we dont care about parse errors 30 30 + } 31 31 + return null; 32 32 + } 33 33 + 34 34 + export async function fetchMetadata(base: string) { 35 35 + if (metaDataCache) return; 36 36 + const data = await mwFetch<MetaOutput[][]>(`${base}/metadata`); 37 37 + metaDataCache = data.flat(); 38 38 + } 39 39 + 40 40 + function scrapeMediaToQueryMedia(media: ScrapeMedia) { 41 41 + let extra: Record<string, string> = {}; 42 42 + if (media.type === "show") { 43 43 + extra = { 44 44 + episodeNumber: media.episode.number.toString(), 45 45 + episodeTmdbId: media.episode.tmdbId, 46 46 + seasonNumber: media.season.number.toString(), 47 47 + seasonTmdbId: media.season.tmdbId, 48 48 + }; 49 49 + } 50 50 + 51 51 + return { 52 52 + type: media.type, 53 53 + releaseYear: media.releaseYear.toString(), 54 54 + imdbId: media.imdbId, 55 55 + tmdbId: media.tmdbId, 56 56 + title: media.title, 57 57 + ...extra, 58 58 + }; 59 59 + } 60 60 + 61 61 + function addQueryDataToUrl(url: URL, data: Record<string, string | undefined>) { 62 62 + Object.entries(data).forEach((entry) => { 63 63 + if (entry[1]) url.searchParams.set(entry[0], entry[1]); 64 64 + }); 65 65 + } 66 66 + 67 67 + export function makeProviderUrl(base: string) { 68 68 + const makeUrl = (p: string) => new URL(`${base}${p}`); 69 69 + return { 70 70 + scrapeSource(sourceId: string, media: ScrapeMedia) { 71 71 + const url = makeUrl("/scrape/source"); 72 72 + addQueryDataToUrl(url, scrapeMediaToQueryMedia(media)); 73 73 + addQueryDataToUrl(url, { id: sourceId }); 74 74 + return url.toString(); 75 75 + }, 76 76 + scrapeAll(media: ScrapeMedia) { 77 77 + const url = makeUrl("/scrape"); 78 78 + addQueryDataToUrl(url, scrapeMediaToQueryMedia(media)); 79 79 + return url.toString(); 80 80 + }, 81 81 + scrapeEmbed(embedId: string, embedUrl: string) { 82 82 + const url = makeUrl("/scrape/embed"); 83 83 + addQueryDataToUrl(url, { id: embedId, url: embedUrl }); 84 84 + return url.toString(); 85 85 + }, 86 86 + }; 87 87 + } 88 88 + 89 89 + export async function getApiToken(): Promise<string | null> { 90 90 + let apiToken = getTokenIfValid(); 91 91 + if (!apiToken && isTurnstileInitialized()) { 92 92 + apiToken = `turnstile|${await getTurnstileToken()}`; 93 93 + } 94 94 + return apiToken; 95 95 + } 96 96 + 97 97 + export async function connectServerSideEvents<T>( 98 98 + url: string, 99 99 + endEvents: string[] 100 100 + ) { 101 101 + const apiToken = await getApiToken(); 102 102 + 103 103 + // insert token, if its set 104 104 + const parsedUrl = new URL(url); 105 105 + if (apiToken) parsedUrl.searchParams.set("token", apiToken); 106 106 + const eventSource = new EventSource(parsedUrl.toString()); 107 107 + 108 108 + let promReject: (reason?: any) => void; 109 109 + let promResolve: (value: T) => void; 110 110 + const promise = new Promise<T>((resolve, reject) => { 111 111 + promResolve = resolve; 112 112 + promReject = reject; 113 113 + }); 114 114 + 115 115 + endEvents.forEach((evt) => { 116 116 + eventSource.addEventListener(evt, (e) => { 117 117 + eventSource.close(); 118 118 + promResolve(JSON.parse(e.data)); 119 119 + }); 120 120 + }); 121 121 + 122 122 + eventSource.addEventListener("token", (e) => { 123 123 + setApiToken(JSON.parse(e.data)); 124 124 + }); 125 125 + 126 126 + eventSource.addEventListener("error", (err: MessageEvent<any>) => { 127 127 + eventSource.close(); 128 128 + if (err.data) { 129 129 + const data = JSON.parse(err.data); 130 130 + let errObj = new Error("scrape error"); 131 131 + if (data.name === NotFoundError.name) 132 132 + errObj = new NotFoundError("Notfound from server"); 133 133 + Object.assign(errObj, data); 134 134 + promReject(errObj); 135 135 + return; 136 136 + } 137 137 + 138 138 + console.error("Failed to connect to SSE", err); 139 139 + promReject(err); 140 140 + }); 141 141 + 142 142 + eventSource.addEventListener("message", (ev) => { 143 143 + if (!ev) { 144 144 + eventSource.close(); 145 145 + return; 146 146 + } 147 147 + setTimeout(() => { 148 148 + promReject(new Error("SSE closed improperly")); 149 149 + }, 1000); 150 150 + }); 151 151 + 152 152 + return { 153 153 + promise: () => promise, 154 154 + on<Data>(event: string, cb: (data: Data) => void) { 155 155 + eventSource.addEventListener(event, (e) => cb(JSON.parse(e.data))); 156 156 + }, 157 157 + }; 158 158 + }

+5 -2

src/components/player/atoms/settings/SettingsMenu.tsx

reviewed

··· 1 1 import { useMemo } from "react"; 2 2 import { useTranslation } from "react-i18next"; 3 3 4 4 + import { getCachedMetadata } from "@/backend/helpers/providerApi"; 4 5 import { Toggle } from "@/components/buttons/Toggle"; 5 6 import { Icon, Icons } from "@/components/Icon"; 6 7 import { useCaptions } from "@/components/player/hooks/useCaptions"; ··· 10 11 import { usePlayerStore } from "@/stores/player/store"; 11 12 import { qualityToString } from "@/stores/player/utils/qualities"; 12 13 import { useSubtitleStore } from "@/stores/subtitles"; 13 13 - import { providers } from "@/utils/providers"; 14 14 15 15 export function SettingsMenu({ id }: { id: string }) { 16 16 const { t } = useTranslation(); ··· 23 23 const currentSourceId = usePlayerStore((s) => s.sourceId); 24 24 const sourceName = useMemo(() => { 25 25 if (!currentSourceId) return "..."; 26 26 - return providers.getMetadata(currentSourceId)?.name ?? "..."; 26 26 + const source = getCachedMetadata().find( 27 27 + (src) => src.id === currentSourceId 28 28 + ); 29 29 + return source?.name ?? "..."; 27 30 }, [currentSourceId]); 28 31 const { toggleLastUsed } = useCaptions(); 29 32

+5 -5

src/components/player/atoms/settings/SourceSelectingView.tsx

reviewed

··· 1 1 import { ReactNode, useEffect, useMemo, useRef } from "react"; 2 2 import { useTranslation } from "react-i18next"; 3 3 4 4 + import { getCachedMetadata } from "@/backend/helpers/providerApi"; 4 5 import { Loading } from "@/components/layout/Loading"; 5 6 import { 6 7 useEmbedScraping, ··· 10 11 import { SelectableLink } from "@/components/player/internals/ContextMenu/Links"; 11 12 import { useOverlayRouter } from "@/hooks/useOverlayRouter"; 12 13 import { usePlayerStore } from "@/stores/player/store"; 13 13 - import { providers } from "@/utils/providers"; 14 14 15 15 export interface SourceSelectionViewProps { 16 16 id: string; ··· 33 33 34 34 const embedName = useMemo(() => { 35 35 if (!props.embedId) return unknownEmbedName; 36 36 - const sourceMeta = providers.getMetadata(props.embedId); 36 36 + const sourceMeta = getCachedMetadata().find((s) => s.id === props.embedId); 37 37 return sourceMeta?.name ?? unknownEmbedName; 38 38 }, [props.embedId, unknownEmbedName]); 39 39 ··· 61 61 62 62 const sourceName = useMemo(() => { 63 63 if (!sourceId) return "..."; 64 64 - const sourceMeta = providers.getMetadata(sourceId); 64 64 + const sourceMeta = getCachedMetadata().find((s) => s.id === sourceId); 65 65 return sourceMeta?.name ?? "..."; 66 66 }, [sourceId]); 67 67 ··· 137 137 const currentSourceId = usePlayerStore((s) => s.sourceId); 138 138 const sources = useMemo(() => { 139 139 if (!metaType) return []; 140 140 - return providers 141 141 - .listSources() 140 140 + return getCachedMetadata() 141 141 + .filter((v) => v.type === "source") 142 142 .filter((v) => v.mediaTypes?.includes(metaType)); 143 143 }, [metaType]); 144 144

+49 -13

src/components/player/hooks/useSourceSelection.ts

reviewed

··· 6 6 import { useAsyncFn } from "react-use"; 7 7 8 8 import { 9 9 + connectServerSideEvents, 10 10 + makeProviderUrl, 11 11 + } from "@/backend/helpers/providerApi"; 12 12 + import { 9 13 scrapeSourceOutputToProviderMetric, 10 14 useReportProviders, 11 15 } from "@/backend/helpers/report"; ··· 14 18 import { useOverlayRouter } from "@/hooks/useOverlayRouter"; 15 19 import { metaToScrapeMedia } from "@/stores/player/slices/source"; 16 20 import { usePlayerStore } from "@/stores/player/store"; 17 17 - import { providers } from "@/utils/providers"; 21 21 + import { getLoadbalancedProviderApiUrl, providers } from "@/utils/providers"; 18 22 19 23 export function useEmbedScraping( 20 24 routerId: string, ··· 31 35 const { report } = useReportProviders(); 32 36 33 37 const [request, run] = useAsyncFn(async () => { 38 38 + const providerApiUrl = getLoadbalancedProviderApiUrl(); 34 39 let result: EmbedOutput | undefined; 35 40 if (!meta) return; 36 41 try { 37 37 - result = await providers.runEmbedScraper({ 38 38 - id: embedId, 39 39 - url, 40 40 - }); 42 42 + if (providerApiUrl) { 43 43 + const baseUrlMaker = makeProviderUrl(providerApiUrl); 44 44 + const conn = await connectServerSideEvents<EmbedOutput>( 45 45 + baseUrlMaker.scrapeEmbed(embedId, url), 46 46 + ["completed", "noOutput"] 47 47 + ); 48 48 + result = await conn.promise(); 49 49 + } else { 50 50 + result = await providers.runEmbedScraper({ 51 51 + id: embedId, 52 52 + url, 53 53 + }); 54 54 + } 41 55 } catch (err) { 42 56 console.error(`Failed to scrape ${embedId}`, err); 43 57 const notFound = err instanceof NotFoundError; ··· 85 99 const [request, run] = useAsyncFn(async () => { 86 100 if (!sourceId || !meta) return null; 87 101 const scrapeMedia = metaToScrapeMedia(meta); 102 102 + const providerApiUrl = getLoadbalancedProviderApiUrl(); 88 103 89 104 let result: SourcererOutput | undefined; 90 105 try { 91 91 - result = await providers.runSourceScraper({ 92 92 - id: sourceId, 93 93 - media: scrapeMedia, 94 94 - }); 106 106 + if (providerApiUrl) { 107 107 + const baseUrlMaker = makeProviderUrl(providerApiUrl); 108 108 + const conn = await connectServerSideEvents<SourcererOutput>( 109 109 + baseUrlMaker.scrapeSource(sourceId, scrapeMedia), 110 110 + ["completed", "noOutput"] 111 111 + ); 112 112 + result = await conn.promise(); 113 113 + } else { 114 114 + result = await providers.runSourceScraper({ 115 115 + id: sourceId, 116 116 + media: scrapeMedia, 117 117 + }); 118 118 + } 95 119 } catch (err) { 96 120 console.error(`Failed to scrape ${sourceId}`, err); 97 121 const notFound = err instanceof NotFoundError; ··· 120 144 let embedResult: EmbedOutput | undefined; 121 145 if (!meta) return; 122 146 try { 123 123 - embedResult = await providers.runEmbedScraper({ 124 124 - id: result.embeds[0].embedId, 125 125 - url: result.embeds[0].url, 126 126 - }); 147 147 + if (providerApiUrl) { 148 148 + const baseUrlMaker = makeProviderUrl(providerApiUrl); 149 149 + const conn = await connectServerSideEvents<EmbedOutput>( 150 150 + baseUrlMaker.scrapeEmbed( 151 151 + result.embeds[0].embedId, 152 152 + result.embeds[0].url 153 153 + ), 154 154 + ["completed", "noOutput"] 155 155 + ); 156 156 + embedResult = await conn.promise(); 157 157 + } else { 158 158 + embedResult = await providers.runEmbedScraper({ 159 159 + id: result.embeds[0].embedId, 160 160 + url: result.embeds[0].url, 161 161 + }); 162 162 + } 127 163 } catch (err) { 128 164 console.error(`Failed to scrape ${result.embeds[0].embedId}`, err); 129 165 const notFound = err instanceof NotFoundError;

+161 -79

src/hooks/useProviderScrape.tsx

reviewed

··· 1 1 - import { ScrapeMedia } from "@movie-web/providers"; 1 1 + import { 2 2 + FullScraperEvents, 3 3 + RunOutput, 4 4 + ScrapeMedia, 5 5 + } from "@movie-web/providers"; 2 6 import { RefObject, useCallback, useEffect, useRef, useState } from "react"; 3 7 4 4 - import { providers } from "@/utils/providers"; 8 8 + import { 9 9 + connectServerSideEvents, 10 10 + getCachedMetadata, 11 11 + makeProviderUrl, 12 12 + } from "@/backend/helpers/providerApi"; 13 13 + import { getLoadbalancedProviderApiUrl, providers } from "@/utils/providers"; 5 14 6 15 export interface ScrapingItems { 7 16 id: string; ··· 18 27 percentage: number; 19 28 } 20 29 21 21 - export function useScrape() { 30 30 + type ScraperEvent<Event extends keyof FullScraperEvents> = Parameters< 31 31 + NonNullable<FullScraperEvents[Event]> 32 32 + >[0]; 33 33 + 34 34 + function useBaseScrape() { 22 35 const [sources, setSources] = useState<Record<string, ScrapingSegment>>({}); 23 36 const [sourceOrder, setSourceOrder] = useState<ScrapingItems[]>([]); 24 37 const [currentSource, setCurrentSource] = useState<string>(); 38 38 + const lastId = useRef<string | null>(null); 39 39 + 40 40 + const initEvent = useCallback((evt: ScraperEvent<"init">) => { 41 41 + setSources( 42 42 + evt.sourceIds 43 43 + .map((v) => { 44 44 + const source = getCachedMetadata().find((s) => s.id === v); 45 45 + if (!source) throw new Error("invalid source id"); 46 46 + const out: ScrapingSegment = { 47 47 + name: source.name, 48 48 + id: source.id, 49 49 + status: "waiting", 50 50 + percentage: 0, 51 51 + }; 52 52 + return out; 53 53 + }) 54 54 + .reduce<Record<string, ScrapingSegment>>((a, v) => { 55 55 + a[v.id] = v; 56 56 + return a; 57 57 + }, {}) 58 58 + ); 59 59 + setSourceOrder(evt.sourceIds.map((v) => ({ id: v, children: [] }))); 60 60 + }, []); 61 61 + 62 62 + const startEvent = useCallback((id: ScraperEvent<"start">) => { 63 63 + setSources((s) => { 64 64 + if (s[id]) s[id].status = "pending"; 65 65 + return { ...s }; 66 66 + }); 67 67 + setCurrentSource(id); 68 68 + lastId.current = id; 69 69 + }, []); 70 70 + 71 71 + const updateEvent = useCallback((evt: ScraperEvent<"update">) => { 72 72 + setSources((s) => { 73 73 + if (s[evt.id]) { 74 74 + s[evt.id].status = evt.status; 75 75 + s[evt.id].reason = evt.reason; 76 76 + s[evt.id].error = evt.error; 77 77 + s[evt.id].percentage = evt.percentage; 78 78 + } 79 79 + return { ...s }; 80 80 + }); 81 81 + }, []); 82 82 + 83 83 + const discoverEmbedsEvent = useCallback( 84 84 + (evt: ScraperEvent<"discoverEmbeds">) => { 85 85 + setSources((s) => { 86 86 + evt.embeds.forEach((v) => { 87 87 + const source = getCachedMetadata().find( 88 88 + (src) => src.id === v.embedScraperId 89 89 + ); 90 90 + if (!source) throw new Error("invalid source id"); 91 91 + const out: ScrapingSegment = { 92 92 + embedId: v.embedScraperId, 93 93 + name: source.name, 94 94 + id: v.id, 95 95 + status: "waiting", 96 96 + percentage: 0, 97 97 + }; 98 98 + s[v.id] = out; 99 99 + }); 100 100 + return { ...s }; 101 101 + }); 102 102 + setSourceOrder((s) => { 103 103 + const source = s.find((v) => v.id === evt.sourceId); 104 104 + if (!source) throw new Error("invalid source id"); 105 105 + source.children = evt.embeds.map((v) => v.id); 106 106 + return [...s]; 107 107 + }); 108 108 + }, 109 109 + [] 110 110 + ); 111 111 + 112 112 + const startScrape = useCallback(() => { 113 113 + lastId.current = null; 114 114 + }, []); 115 115 + 116 116 + const getResult = useCallback((output: RunOutput | null) => { 117 117 + if (output && lastId.current) { 118 118 + setSources((s) => { 119 119 + if (!lastId.current) return s; 120 120 + if (s[lastId.current]) s[lastId.current].status = "success"; 121 121 + return { ...s }; 122 122 + }); 123 123 + } 124 124 + return output; 125 125 + }, []); 126 126 + 127 127 + return { 128 128 + initEvent, 129 129 + startEvent, 130 130 + updateEvent, 131 131 + discoverEmbedsEvent, 132 132 + startScrape, 133 133 + getResult, 134 134 + sources, 135 135 + sourceOrder, 136 136 + currentSource, 137 137 + }; 138 138 + } 139 139 + 140 140 + export function useScrape() { 141 141 + const { 142 142 + sources, 143 143 + sourceOrder, 144 144 + currentSource, 145 145 + updateEvent, 146 146 + discoverEmbedsEvent, 147 147 + initEvent, 148 148 + getResult, 149 149 + startEvent, 150 150 + startScrape, 151 151 + } = useBaseScrape(); 25 152 26 153 const startScraping = useCallback( 27 154 async (media: ScrapeMedia) => { 28 28 - if (!providers) return null; 155 155 + const providerApiUrl = getLoadbalancedProviderApiUrl(); 156 156 + if (providerApiUrl) { 157 157 + startScrape(); 158 158 + const baseUrlMaker = makeProviderUrl(providerApiUrl); 159 159 + const conn = await connectServerSideEvents<RunOutput | "">( 160 160 + baseUrlMaker.scrapeAll(media), 161 161 + ["completed", "noOutput"] 162 162 + ); 163 163 + conn.on("init", initEvent); 164 164 + conn.on("start", startEvent); 165 165 + conn.on("update", updateEvent); 166 166 + conn.on("discoverEmbeds", discoverEmbedsEvent); 167 167 + const sseOutput = await conn.promise(); 29 168 30 30 - let lastId: string | null = null; 169 169 + return getResult(sseOutput === "" ? null : sseOutput); 170 170 + } 171 171 + 172 172 + if (!providers) return null; 173 173 + startScrape(); 31 174 const output = await providers.runAll({ 32 175 media, 33 176 events: { 34 34 - init(evt) { 35 35 - setSources( 36 36 - evt.sourceIds 37 37 - .map((v) => { 38 38 - const source = providers.getMetadata(v); 39 39 - if (!source) throw new Error("invalid source id"); 40 40 - const out: ScrapingSegment = { 41 41 - name: source.name, 42 42 - id: source.id, 43 43 - status: "waiting", 44 44 - percentage: 0, 45 45 - }; 46 46 - return out; 47 47 - }) 48 48 - .reduce<Record<string, ScrapingSegment>>((a, v) => { 49 49 - a[v.id] = v; 50 50 - return a; 51 51 - }, {}) 52 52 - ); 53 53 - setSourceOrder(evt.sourceIds.map((v) => ({ id: v, children: [] }))); 54 54 - }, 55 55 - start(id) { 56 56 - setSources((s) => { 57 57 - if (s[id]) s[id].status = "pending"; 58 58 - return { ...s }; 59 59 - }); 60 60 - setCurrentSource(id); 61 61 - lastId = id; 62 62 - }, 63 63 - update(evt) { 64 64 - setSources((s) => { 65 65 - if (s[evt.id]) { 66 66 - s[evt.id].status = evt.status; 67 67 - s[evt.id].reason = evt.reason; 68 68 - s[evt.id].error = evt.error; 69 69 - s[evt.id].percentage = evt.percentage; 70 70 - } 71 71 - return { ...s }; 72 72 - }); 73 73 - }, 74 74 - discoverEmbeds(evt) { 75 75 - setSources((s) => { 76 76 - evt.embeds.forEach((v) => { 77 77 - const source = providers.getMetadata(v.embedScraperId); 78 78 - if (!source) throw new Error("invalid source id"); 79 79 - const out: ScrapingSegment = { 80 80 - embedId: v.embedScraperId, 81 81 - name: source.name, 82 82 - id: v.id, 83 83 - status: "waiting", 84 84 - percentage: 0, 85 85 - }; 86 86 - s[v.id] = out; 87 87 - }); 88 88 - return { ...s }; 89 89 - }); 90 90 - setSourceOrder((s) => { 91 91 - const source = s.find((v) => v.id === evt.sourceId); 92 92 - if (!source) throw new Error("invalid source id"); 93 93 - source.children = evt.embeds.map((v) => v.id); 94 94 - return [...s]; 95 95 - }); 96 96 - }, 177 177 + init: initEvent, 178 178 + start: startEvent, 179 179 + update: updateEvent, 180 180 + discoverEmbeds: discoverEmbedsEvent, 97 181 }, 98 182 }); 99 99 - 100 100 - if (output && lastId) { 101 101 - setSources((s) => { 102 102 - if (!lastId) return s; 103 103 - if (s[lastId]) s[lastId].status = "success"; 104 104 - return { ...s }; 105 105 - }); 106 106 - } 107 107 - 108 108 - return output; 183 183 + return getResult(output); 109 184 }, 110 110 - [setSourceOrder, setSources] 185 185 + [ 186 186 + initEvent, 187 187 + startEvent, 188 188 + updateEvent, 189 189 + discoverEmbedsEvent, 190 190 + getResult, 191 191 + startScrape, 192 192 + ] 111 193 ); 112 194 113 195 return {

+3 -5

src/index.tsx

reviewed

··· 10 10 import { HelmetProvider } from "react-helmet-async"; 11 11 import { useTranslation } from "react-i18next"; 12 12 import { BrowserRouter, HashRouter } from "react-router-dom"; 13 13 + import Turnstile from "react-turnstile"; 13 14 import { useAsync } from "react-use"; 14 15 15 16 import { Button } from "@/components/buttons/Button"; ··· 30 31 import { ProgressSyncer } from "@/stores/progress/ProgressSyncer"; 31 32 import { SettingsSyncer } from "@/stores/subtitles/SettingsSyncer"; 32 33 import { ThemeProvider } from "@/stores/theme"; 34 34 + import { TurnstileProvider } from "@/stores/turnstile"; 33 35 34 36 import { initializeChromecast } from "./setup/chromecast"; 35 37 import { initializeOldStores } from "./stores/__old/migrations"; 36 38 37 39 // initialize 38 38 - const key = 39 39 - (window as any)?.__CONFIG__?.VITE_KEY ?? import.meta.env.VITE_KEY ?? null; 40 40 - if (key) { 41 41 - (window as any).initMW(conf().PROXY_URLS, key); 42 42 - } 43 40 initializeChromecast(); 44 41 45 42 function LoadingScreen(props: { type: "user" | "lazy" }) { ··· 148 145 ReactDOM.render( 149 146 <React.StrictMode> 150 147 <ErrorBoundary> 148 148 + <TurnstileProvider /> 151 149 <HelmetProvider> 152 150 <Suspense fallback={<LoadingScreen type="lazy" />}> 153 151 <ThemeProvider applyGlobal>

+7 -7

src/pages/parts/admin/WorkerTestPart.tsx

reviewed

··· 2 2 import { useMemo, useState } from "react"; 3 3 import { useAsyncFn } from "react-use"; 4 4 5 5 - import { mwFetch } from "@/backend/helpers/fetch"; 5 5 + import { singularProxiedFetch } from "@/backend/helpers/fetch"; 6 6 import { Button } from "@/components/buttons/Button"; 7 7 import { Icon, Icons } from "@/components/Icon"; 8 8 import { Box } from "@/components/layout/Box"; ··· 69 69 }); 70 70 continue; 71 71 } 72 72 - await mwFetch(worker.url, { 73 73 - query: { 74 74 - destination: "https://postman-echo.com/get", 75 75 - }, 76 76 - }); 72 72 + await singularProxiedFetch( 73 73 + worker.url, 74 74 + "https://postman-echo.com/get", 75 75 + {} 76 76 + ); 77 77 updateWorker(worker.id, { 78 78 id: worker.id, 79 79 status: "success", ··· 94 94 <p className="mb-8 mt-2">{workerList.length} worker(s) registered</p> 95 95 <Box> 96 96 {workerList.map((v, i) => { 97 97 - const s = workerState.find((segment) => segment.id); 97 97 + const s = workerState.find((segment) => segment.id === v.id); 98 98 const name = `Worker ${i + 1}`; 99 99 if (!s) return <WorkerItem name={name} key={v.id} />; 100 100 if (s.status === "error")

+15

src/pages/parts/player/MetaPart.tsx

reviewed

··· 3 3 import { useAsync } from "react-use"; 4 4 import type { AsyncReturnType } from "type-fest"; 5 5 6 6 + import { 7 7 + fetchMetadata, 8 8 + setCachedMetadata, 9 9 + } from "@/backend/helpers/providerApi"; 6 10 import { DetailedMeta, getMetaFromId } from "@/backend/metadata/getmeta"; 7 11 import { decodeTMDBId } from "@/backend/metadata/tmdb"; 8 12 import { MWMediaType } from "@/backend/metadata/types/mw"; ··· 14 18 import { Title } from "@/components/text/Title"; 15 19 import { ErrorContainer, ErrorLayout } from "@/pages/layouts/ErrorLayout"; 16 20 import { conf } from "@/setup/config"; 21 21 + import { getLoadbalancedProviderApiUrl, providers } from "@/utils/providers"; 17 22 18 23 export interface MetaPartProps { 19 24 onGetMeta?: (meta: DetailedMeta, episodeId?: string) => void; ··· 36 41 const history = useHistory(); 37 42 38 43 const { error, value, loading } = useAsync(async () => { 44 44 + const providerApiUrl = getLoadbalancedProviderApiUrl(); 45 45 + if (providerApiUrl) { 46 46 + await fetchMetadata(providerApiUrl); 47 47 + } else { 48 48 + setCachedMetadata([ 49 49 + ...providers.listSources(), 50 50 + ...providers.listEmbeds(), 51 51 + ]); 52 52 + } 53 53 + 39 54 let data: ReturnType<typeof decodeTMDBId> = null; 40 55 try { 41 56 data = decodeTMDBId(params.media);

+5

src/setup/config.ts

reviewed

··· 17 17 NORMAL_ROUTER: boolean; 18 18 BACKEND_URL: string; 19 19 DISALLOWED_IDS: string; 20 20 + TURNSTILE_KEY: string; 20 21 } 21 22 22 23 export interface RuntimeConfig { ··· 30 31 PROXY_URLS: string[]; 31 32 BACKEND_URL: string; 32 33 DISALLOWED_IDS: string[]; 34 34 + TURNSTILE_KEY: string | null; 33 35 } 34 36 35 37 const env: Record<keyof Config, undefined | string> = { ··· 43 45 NORMAL_ROUTER: import.meta.env.VITE_NORMAL_ROUTER, 44 46 BACKEND_URL: import.meta.env.VITE_BACKEND_URL, 45 47 DISALLOWED_IDS: import.meta.env.VITE_DISALLOWED_IDS, 48 48 + TURNSTILE_KEY: import.meta.env.VITE_TURNSTILE_KEY, 46 49 }; 47 50 48 51 // loads from different locations, in order: environment (VITE_{KEY}), window (public/config.js) ··· 63 66 64 67 export function conf(): RuntimeConfig { 65 68 const dmcaEmail = getKey("DMCA_EMAIL"); 69 69 + const turnstileKey = getKey("TURNSTILE_KEY"); 66 70 return { 67 71 APP_VERSION, 68 72 GITHUB_LINK, ··· 75 79 .split(",") 76 80 .map((v) => v.trim()), 77 81 NORMAL_ROUTER: getKey("NORMAL_ROUTER", "false") === "true", 82 82 + TURNSTILE_KEY: turnstileKey.length > 0 ? turnstileKey : null, 78 83 DISALLOWED_IDS: getKey("DISALLOWED_IDS", "") 79 84 .split(",") 80 85 .map((v) => v.trim())

+81

src/stores/turnstile/index.tsx

reviewed

··· 1 1 + import Turnstile, { BoundTurnstileObject } from "react-turnstile"; 2 2 + import { create } from "zustand"; 3 3 + import { immer } from "zustand/middleware/immer"; 4 4 + 5 5 + import { conf } from "@/setup/config"; 6 6 + 7 7 + export interface TurnstileStore { 8 8 + turnstile: BoundTurnstileObject | null; 9 9 + cbs: ((token: string | null) => void)[]; 10 10 + setTurnstile(v: BoundTurnstileObject | null): void; 11 11 + getToken(): Promise<string>; 12 12 + processToken(token: string | null): void; 13 13 + } 14 14 + 15 15 + export const useTurnstileStore = create( 16 16 + immer<TurnstileStore>((set, get) => ({ 17 17 + turnstile: null, 18 18 + cbs: [], 19 19 + processToken(token) { 20 20 + const cbs = get().cbs; 21 21 + cbs.forEach((fn) => fn(token)); 22 22 + set((s) => { 23 23 + s.cbs = []; 24 24 + }); 25 25 + }, 26 26 + getToken() { 27 27 + return new Promise((resolve, reject) => { 28 28 + set((s) => { 29 29 + s.cbs = [ 30 30 + ...s.cbs, 31 31 + (token) => { 32 32 + if (!token) reject(new Error("Failed to get token")); 33 33 + else resolve(token); 34 34 + }, 35 35 + ]; 36 36 + }); 37 37 + }); 38 38 + }, 39 39 + setTurnstile(v) { 40 40 + set((s) => { 41 41 + s.turnstile = v; 42 42 + }); 43 43 + }, 44 44 + })) 45 45 + ); 46 46 + 47 47 + export function getTurnstile() { 48 48 + return useTurnstileStore.getState().turnstile; 49 49 + } 50 50 + 51 51 + export function isTurnstileInitialized() { 52 52 + return !!getTurnstile(); 53 53 + } 54 54 + 55 55 + export function getTurnstileToken() { 56 56 + const turnstile = getTurnstile(); 57 57 + turnstile?.reset(); 58 58 + turnstile?.execute(); 59 59 + return useTurnstileStore.getState().getToken(); 60 60 + } 61 61 + 62 62 + export function TurnstileProvider() { 63 63 + const siteKey = conf().TURNSTILE_KEY; 64 64 + const setTurnstile = useTurnstileStore((s) => s.setTurnstile); 65 65 + const processToken = useTurnstileStore((s) => s.processToken); 66 66 + if (!siteKey) return null; 67 67 + return ( 68 68 + <Turnstile 69 69 + sitekey={siteKey} 70 70 + onLoad={(_widgetId, bound) => { 71 71 + setTurnstile(bound); 72 72 + }} 73 73 + onError={() => { 74 74 + processToken(null); 75 75 + }} 76 76 + onVerify={(token) => { 77 77 + processToken(token); 78 78 + }} 79 79 + /> 80 80 + ); 81 81 + }

+39 -14

src/utils/providers.ts

reviewed

··· 7 7 targets, 8 8 } from "@movie-web/providers"; 9 9 10 10 - import { conf } from "@/setup/config"; 11 11 - import { useAuthStore } from "@/stores/auth"; 10 10 + import { getApiToken, setApiToken } from "@/backend/helpers/providerApi"; 11 11 + import { getProviderApiUrls, getProxyUrls } from "@/utils/proxyUrls"; 12 12 + 13 13 + function makeLoadbalancedList(getter: () => string[]) { 14 14 + let listIndex = -1; 15 15 + return () => { 16 16 + const fetchers = getter(); 17 17 + if (listIndex === -1 || listIndex >= fetchers.length) { 18 18 + listIndex = Math.floor(Math.random() * fetchers.length); 19 19 + } 20 20 + const proxyUrl = fetchers[listIndex]; 21 21 + listIndex = (listIndex + 1) % fetchers.length; 22 22 + return proxyUrl; 23 23 + }; 24 24 + } 12 25 13 13 - const originalUrls = conf().PROXY_URLS; 14 14 - let fetchersIndex = -1; 26 26 + export const getLoadbalancedProxyUrl = makeLoadbalancedList(getProxyUrls); 27 27 + export const getLoadbalancedProviderApiUrl = 28 28 + makeLoadbalancedList(getProviderApiUrls); 15 29 16 16 - export function getLoadbalancedProxyUrl() { 17 17 - const fetchers = useAuthStore.getState().proxySet ?? originalUrls; 18 18 - if (fetchersIndex === -1 || fetchersIndex >= fetchers.length) { 19 19 - fetchersIndex = Math.floor(Math.random() * fetchers.length); 20 20 - } 21 21 - const proxyUrl = fetchers[fetchersIndex]; 22 22 - fetchersIndex = (fetchersIndex + 1) % fetchers.length; 23 23 - return proxyUrl; 30 30 + async function fetchButWithApiTokens( 31 31 + input: RequestInfo | URL, 32 32 + init?: RequestInit | undefined 33 33 + ): Promise<Response> { 34 34 + const apiToken = await getApiToken(); 35 35 + const headers = new Headers(init?.headers); 36 36 + if (apiToken) headers.set("X-Token", apiToken); 37 37 + const response = await fetch( 38 38 + input, 39 39 + init 40 40 + ? { 41 41 + ...init, 42 42 + headers, 43 43 + } 44 44 + : undefined 45 45 + ); 46 46 + const newApiToken = response.headers.get("X-Token"); 47 47 + if (newApiToken) setApiToken(newApiToken); 48 48 + return response; 24 49 } 25 50 26 51 function makeLoadBalancedSimpleProxyFetcher() { 27 27 - const fetcher: ProviderBuilderOptions["fetcher"] = (a, b) => { 52 52 + const fetcher: ProviderBuilderOptions["fetcher"] = async (a, b) => { 28 53 const currentFetcher = makeSimpleProxyFetcher( 29 54 getLoadbalancedProxyUrl(), 30 30 - fetch 55 55 + fetchButWithApiTokens 31 56 ); 32 57 return currentFetcher(a, b); 33 58 };

+77

src/utils/proxyUrls.ts

reviewed

··· 1 1 + import { conf } from "@/setup/config"; 2 2 + import { useAuthStore } from "@/stores/auth"; 3 3 + 4 4 + const originalUrls = conf().PROXY_URLS; 5 5 + const types = ["proxy", "api"] as const; 6 6 + 7 7 + type ParsedUrlType = (typeof types)[number]; 8 8 + 9 9 + export interface ParsedUrl { 10 10 + url: string; 11 11 + type: ParsedUrlType; 12 12 + } 13 13 + 14 14 + function canParseUrl(url: string): boolean { 15 15 + try { 16 16 + return !!new URL(url); 17 17 + } catch { 18 18 + return false; 19 19 + } 20 20 + } 21 21 + 22 22 + function isParsedUrlType(type: string): type is ParsedUrlType { 23 23 + return types.includes(type as any); 24 24 + } 25 25 + 26 26 + /** 27 27 + * Turn a string like "a=b;c=d;d=e" into a dictionary object 28 28 + */ 29 29 + function parseParams(input: string): Record<string, string> { 30 30 + const entriesParams = input 31 31 + .split(";") 32 32 + .map((param) => param.split("=", 2).filter((part) => part.length !== 0)) 33 33 + .filter((v) => v.length === 2); 34 34 + return Object.fromEntries(entriesParams); 35 35 + } 36 36 + 37 37 + export function getParsedUrls() { 38 38 + const urls = useAuthStore.getState().proxySet ?? originalUrls; 39 39 + const output: ParsedUrl[] = []; 40 40 + urls.forEach((url) => { 41 41 + if (!url.startsWith("|")) { 42 42 + if (canParseUrl(url)) { 43 43 + output.push({ 44 44 + url, 45 45 + type: "proxy", 46 46 + }); 47 47 + return; 48 48 + } 49 49 + } 50 50 + 51 51 + const match = /^\|([^|]+)\|(.*)$/g.exec(url); 52 52 + if (!match || !match[2]) return; 53 53 + if (!canParseUrl(match[2])) return; 54 54 + const params = parseParams(match[1]); 55 55 + const type = params.type ?? "proxy"; 56 56 + 57 57 + if (!isParsedUrlType(type)) return; 58 58 + output.push({ 59 59 + url: match[2], 60 60 + type, 61 61 + }); 62 62 + }); 63 63 + 64 64 + return output; 65 65 + } 66 66 + 67 67 + export function getProxyUrls() { 68 68 + return getParsedUrls() 69 69 + .filter((v) => v.type === "proxy") 70 70 + .map((v) => v.url); 71 71 + } 72 72 + 73 73 + export function getProviderApiUrls() { 74 74 + return getParsedUrls() 75 75 + .filter((v) => v.type === "api") 76 76 + .map((v) => v.url); 77 77 + }