Add support for aborting and new lines · dunkirk.sh/pstream-ng@b8a972f

+7 -6

src/stores/player/slices/source.ts

··· 464 464 return; 465 465 } 466 466 467 - let cancelled = false; 467 + const abortController = new AbortController(); 468 468 469 469 set((s) => { 470 470 s.caption.translateTask = { ··· 476 476 if (!this.done && !this.error) { 477 477 console.log("Translation task was cancelled"); 478 478 } 479 - cancelled = true; 479 + abortController.abort(); 480 480 }, 481 481 }; 482 482 }); 483 483 484 484 function handleError(err: any) { 485 - console.error("Translation task ran into an error", err); 486 - if (cancelled) { 485 + if (abortController.signal.aborted) { 487 486 return; 488 487 } 488 + console.error("Translation task ran into an error", err); 489 489 set((s) => { 490 490 if (!s.caption.translateTask) return; 491 491 s.caption.translateTask.error = true; ··· 494 494 495 495 try { 496 496 const srtData = await downloadCaption(targetCaption); 497 - if (cancelled) { 497 + if (abortController.signal.aborted) { 498 498 return; 499 499 } 500 500 if (!srtData) { ··· 519 519 store.caption.translateTask!.fetchedTargetCaption!, 520 520 targetLanguage, 521 521 googletranslate, 522 + abortController.signal, 522 523 ); 523 - if (cancelled) { 524 + if (abortController.signal.aborted) { 524 525 return; 525 526 } 526 527 if (!result) {

+19 -7

src/utils/translation/googletranslate.ts

··· 12 12 13 13 getConfig() { 14 14 return { 15 - singleBatchSize: 15, 16 - multiBatchSize: 80, 15 + single: { 16 + batchSize: 250, 17 + batchDelayMs: 1000, 18 + }, 19 + multi: { 20 + batchSize: 80, 21 + batchDelayMs: 200, 22 + }, 17 23 maxRetryCount: 3, 18 - batchSleepMs: 200, 19 24 }; 20 25 }, 21 26 22 - async translate(str, targetLang) { 27 + async translate(str, targetLang, abortSignal) { 23 28 if (!str) { 24 29 return ""; 25 30 } 31 + str = str.replaceAll("\n", "<br />"); 26 32 27 33 const response = await ( 28 34 await fetch( 29 35 `${SINGLE_API_URL}&tl=${targetLang}&q=${encodeURIComponent(str)}`, 30 36 { 31 37 method: "GET", 38 + signal: abortSignal, 32 39 headers: { 33 40 Accept: "application/json", 34 41 }, ··· 43 50 44 51 return (response.sentences as any[]) 45 52 .map((s: any) => s.trans as string) 46 - .join(""); 53 + .join("") 54 + .replaceAll("<br />", "\n"); 47 55 }, 48 56 49 - async translateMulti(batch, targetLang) { 57 + async translateMulti(batch, targetLang, abortSignal) { 50 58 if (!batch || batch.length === 0) { 51 59 return []; 52 60 } 61 + batch = batch.map((s) => s.replaceAll("\n", "<br />")); 53 62 54 63 const response = await ( 55 64 await fetch(BATCH_API_URL, { 56 65 method: "POST", 66 + signal: abortSignal, 57 67 headers: { 58 68 "Content-Type": "application/json+protobuf", 59 69 "X-goog-api-key": BATCH_API_KEY, ··· 67 77 throw new Error("Invalid response"); 68 78 } 69 79 70 - return response[0].map((s: any) => s as string); 80 + return response[0].map((s: any) => 81 + (s as string).replaceAll("<br />", "\n"), 82 + ); 71 83 }, 72 84 } satisfies TranslateService;

+90 -24

src/utils/translation/index.ts

··· 1 + /* eslint-disable no-console */ 1 2 import subsrt from "subsrt-ts"; 2 3 import { Caption, ContentCaption } from "subsrt-ts/dist/types/handler"; 3 4 ··· 7 8 8 9 const CAPTIONS_CACHE: Map<string, ArrayBuffer> = new Map<string, ArrayBuffer>(); 9 10 11 + // single will not be used if multi-line is supported 12 + export interface TranslateServiceConfig { 13 + single: { 14 + batchSize: number; 15 + batchDelayMs: number; 16 + }; 17 + multi?: { 18 + batchSize: number; 19 + batchDelayMs: number; 20 + }; 21 + maxRetryCount: number; 22 + } 23 + 10 24 export interface TranslateService { 11 25 getName(): string; 12 - getConfig(): { 13 - singleBatchSize: number; 14 - multiBatchSize: number; // -1 = unsupported 15 - maxRetryCount: number; 16 - batchSleepMs: number; 17 - }; 18 - translate(str: string, targetLang: string): Promise<string>; 19 - translateMulti(batch: string[], targetLang: string): Promise<string[]>; 26 + getConfig(): TranslateServiceConfig; 27 + translate( 28 + str: string, 29 + targetLang: string, 30 + abortSignal?: AbortSignal, 31 + ): Promise<string>; 32 + translateMulti( 33 + batch: string[], 34 + targetLang: string, 35 + abortSignal?: AbortSignal, 36 + ): Promise<string[]>; 20 37 } 21 38 22 39 class Translator { ··· 30 47 31 48 private service: TranslateService; 32 49 33 - constructor(srtData: string, targetLang: string, service: TranslateService) { 50 + private serviceCfg: TranslateServiceConfig; 51 + 52 + private abortSignal?: AbortSignal; 53 + 54 + constructor( 55 + srtData: string, 56 + targetLang: string, 57 + service: TranslateService, 58 + abortSignal?: AbortSignal, 59 + ) { 34 60 this.captions = subsrt.parse(srtData); 35 61 this.targetLang = targetLang; 36 62 this.service = service; 63 + this.serviceCfg = service.getConfig(); 64 + this.abortSignal = abortSignal; 37 65 38 66 for (const caption of this.captions) { 39 67 if (caption.type !== "caption") { ··· 64 92 65 93 while (!result && attempts < 3) { 66 94 try { 67 - result = await this.service.translate(content.text, this.targetLang); 95 + result = await this.service.translate( 96 + content.text, 97 + this.targetLang, 98 + this.abortSignal, 99 + ); 68 100 } catch (err) { 101 + if (this.abortSignal?.aborted) { 102 + break; 103 + } 69 104 console.warn("Translation attempt failed"); 70 105 errors.push(err); 71 106 await sleep(500); ··· 73 108 } 74 109 } 75 110 111 + if (this.abortSignal?.aborted) { 112 + return false; 113 + } 114 + 76 115 if (!result) { 77 116 console.warn("Translation failed", errors); 78 117 return false; ··· 88 127 const result = await this.service.translateMulti( 89 128 batch.map((content) => content.text), 90 129 this.targetLang, 130 + this.abortSignal, 91 131 ); 92 132 93 133 if (result.length !== batch.length) { ··· 106 146 107 147 return true; 108 148 } catch (err) { 149 + if (this.abortSignal?.aborted) { 150 + return false; 151 + } 109 152 console.warn("Batch translation failed", err); 110 153 return false; 111 154 } ··· 113 156 114 157 takeBatch(): ContentCaption[] { 115 158 const batch: ContentCaption[] = []; 116 - const batchSize = 117 - this.service.getConfig().multiBatchSize === -1 118 - ? this.service.getConfig().singleBatchSize 119 - : this.service.getConfig().multiBatchSize; 159 + const batchSize = !this.serviceCfg.multi 160 + ? this.serviceCfg.single.batchSize 161 + : this.serviceCfg.multi!.batchSize; 120 162 121 163 let count = 0; 122 164 while (count < batchSize && this.contentCaptions.length > 0) { ··· 132 174 } 133 175 134 176 async translate(): Promise<string | undefined> { 177 + const batchDelay = !this.serviceCfg.multi 178 + ? this.serviceCfg.single.batchDelayMs 179 + : this.serviceCfg.multi!.batchDelayMs; 180 + 181 + console.info( 182 + "Translating captions", 183 + this.service.getName(), 184 + this.contentCaptions.length, 185 + batchDelay, 186 + ); 187 + console.time("translation"); 188 + 135 189 let batch: ContentCaption[] = this.takeBatch(); 136 190 while (batch.length > 0) { 137 191 let result: boolean; 138 - console.info("Translating captions batch", batch.length, batch); 192 + console.info("Translating batch", batch.length, batch); 139 193 140 - if (this.service.getConfig().multiBatchSize === -1) { 194 + if (!this.serviceCfg.multi) { 141 195 result = ( 142 196 await Promise.all( 143 197 batch.map((content) => this.translateContent(content)), ··· 147 201 result = await this.translateContentBatch(batch); 148 202 } 149 203 204 + if (this.abortSignal?.aborted) { 205 + return undefined; 206 + } 207 + 150 208 if (!result) { 151 - console.error( 152 - "Failed to translate captions batch", 153 - batch.length, 154 - batch, 155 - ); 209 + console.error("Failed to translate batch", batch.length, batch); 156 210 return undefined; 157 211 } 158 212 159 213 batch = this.takeBatch(); 160 - await sleep(this.service.getConfig().batchSleepMs); 214 + await sleep(batchDelay); 161 215 } 216 + 217 + if (this.abortSignal?.aborted) { 218 + return undefined; 219 + } 220 + 221 + console.timeEnd("translation"); 162 222 return subsrt.build(this.captions, { format: "srt" }); 163 223 } 164 224 } ··· 167 227 caption: PlayerCaption, 168 228 targetLang: string, 169 229 service: TranslateService, 230 + abortSignal?: AbortSignal, 170 231 ): Promise<string | undefined> { 171 232 const cacheID = `${caption.id}_${targetLang}`; 172 233 ··· 175 236 return decompressStr(cachedData); 176 237 } 177 238 178 - const translator = new Translator(caption.srtData, targetLang, service); 239 + const translator = new Translator( 240 + caption.srtData, 241 + targetLang, 242 + service, 243 + abortSignal, 244 + ); 179 245 180 246 const result = await translator.translate(); 181 - if (!result) { 247 + if (!result || abortSignal?.aborted) { 182 248 return undefined; 183 249 } 184 250

Configure Feed

Configure Feed