Home Reference Source

src/controller/id3-track-controller.ts

  1. import { Events } from '../events';
  2. import {
  3. sendAddTrackEvent,
  4. clearCurrentCues,
  5. removeCuesInRange,
  6. } from '../utils/texttrack-utils';
  7. import * as ID3 from '../demux/id3';
  8. import { DateRange, DateRangeAttribute } from '../loader/date-range';
  9. import { MetadataSchema } from '../types/demuxer';
  10. import type {
  11. BufferFlushingData,
  12. FragParsingMetadataData,
  13. LevelUpdatedData,
  14. MediaAttachedData,
  15. } from '../types/events';
  16. import type { ComponentAPI } from '../types/component-api';
  17. import type Hls from '../hls';
  18.  
  19. declare global {
  20. interface Window {
  21. WebKitDataCue: VTTCue | void;
  22. }
  23. }
  24.  
  25. type Cue = VTTCue | TextTrackCue;
  26.  
  27. const MIN_CUE_DURATION = 0.25;
  28.  
  29. function getCueClass() {
  30. // Attempt to recreate Safari functionality by creating
  31. // WebKitDataCue objects when available and store the decoded
  32. // ID3 data in the value property of the cue
  33. return (self.WebKitDataCue || self.VTTCue || self.TextTrackCue) as any;
  34. }
  35.  
  36. function dateRangeDateToTimelineSeconds(date: Date, offset: number): number {
  37. return date.getTime() / 1000 - offset;
  38. }
  39.  
  40. function hexToArrayBuffer(str): ArrayBuffer {
  41. return Uint8Array.from(
  42. str
  43. .replace(/^0x/, '')
  44. .replace(/([\da-fA-F]{2}) ?/g, '0x$1 ')
  45. .replace(/ +$/, '')
  46. .split(' ')
  47. ).buffer;
  48. }
  49. class ID3TrackController implements ComponentAPI {
  50. private hls: Hls;
  51. private id3Track: TextTrack | null = null;
  52. private media: HTMLMediaElement | null = null;
  53. private dateRangeCuesAppended: Record<
  54. string,
  55. { cues: Record<string, Cue>; dateRange: DateRange; durationKnown: boolean }
  56. > = {};
  57.  
  58. constructor(hls) {
  59. this.hls = hls;
  60. this._registerListeners();
  61. }
  62.  
  63. destroy() {
  64. this._unregisterListeners();
  65. this.id3Track = null;
  66. this.media = null;
  67. this.dateRangeCuesAppended = {};
  68. // @ts-ignore
  69. this.hls = null;
  70. }
  71.  
  72. private _registerListeners() {
  73. const { hls } = this;
  74. hls.on(Events.MEDIA_ATTACHED, this.onMediaAttached, this);
  75. hls.on(Events.MEDIA_DETACHING, this.onMediaDetaching, this);
  76. hls.on(Events.MANIFEST_LOADING, this.onManifestLoading, this);
  77. hls.on(Events.FRAG_PARSING_METADATA, this.onFragParsingMetadata, this);
  78. hls.on(Events.BUFFER_FLUSHING, this.onBufferFlushing, this);
  79. hls.on(Events.LEVEL_UPDATED, this.onLevelUpdated, this);
  80. }
  81.  
  82. private _unregisterListeners() {
  83. const { hls } = this;
  84. hls.off(Events.MEDIA_ATTACHED, this.onMediaAttached, this);
  85. hls.off(Events.MEDIA_DETACHING, this.onMediaDetaching, this);
  86. hls.off(Events.MANIFEST_LOADING, this.onManifestLoading, this);
  87. hls.off(Events.FRAG_PARSING_METADATA, this.onFragParsingMetadata, this);
  88. hls.off(Events.BUFFER_FLUSHING, this.onBufferFlushing, this);
  89. hls.off(Events.LEVEL_UPDATED, this.onLevelUpdated, this);
  90. }
  91.  
  92. // Add ID3 metatadata text track.
  93. protected onMediaAttached(
  94. event: Events.MEDIA_ATTACHED,
  95. data: MediaAttachedData
  96. ): void {
  97. this.media = data.media;
  98. }
  99.  
  100. protected onMediaDetaching(): void {
  101. if (!this.id3Track) {
  102. return;
  103. }
  104. clearCurrentCues(this.id3Track);
  105. this.id3Track = null;
  106. this.media = null;
  107. this.dateRangeCuesAppended = {};
  108. }
  109.  
  110. private onManifestLoading() {
  111. this.dateRangeCuesAppended = {};
  112. }
  113.  
  114. createTrack(media: HTMLMediaElement): TextTrack {
  115. const track = this.getID3Track(media.textTracks) as TextTrack;
  116. track.mode = 'hidden';
  117. return track;
  118. }
  119.  
  120. getID3Track(textTracks: TextTrackList): TextTrack | void {
  121. if (!this.media) {
  122. return;
  123. }
  124. for (let i = 0; i < textTracks.length; i++) {
  125. const textTrack: TextTrack = textTracks[i];
  126. if (textTrack.kind === 'metadata' && textTrack.label === 'id3') {
  127. // send 'addtrack' when reusing the textTrack for metadata,
  128. // same as what we do for captions
  129. sendAddTrackEvent(textTrack, this.media);
  130.  
  131. return textTrack;
  132. }
  133. }
  134. return this.media.addTextTrack('metadata', 'id3');
  135. }
  136.  
  137. onFragParsingMetadata(
  138. event: Events.FRAG_PARSING_METADATA,
  139. data: FragParsingMetadataData
  140. ) {
  141. if (!this.media) {
  142. return;
  143. }
  144.  
  145. const {
  146. hls: {
  147. config: { enableEmsgMetadataCues, enableID3MetadataCues },
  148. },
  149. } = this;
  150. if (!enableEmsgMetadataCues && !enableID3MetadataCues) {
  151. return;
  152. }
  153.  
  154. const { frag: fragment, samples, details } = data;
  155.  
  156. // create track dynamically
  157. if (!this.id3Track) {
  158. this.id3Track = this.createTrack(this.media);
  159. }
  160.  
  161. // VTTCue end time must be finite, so use playlist edge or fragment end until next fragment with same frame type is found
  162. const maxCueTime = details.edge || fragment.end;
  163. const Cue = getCueClass();
  164. let updateCueRanges = false;
  165. const frameTypesAdded: Record<string, number | null> = {};
  166.  
  167. for (let i = 0; i < samples.length; i++) {
  168. const type = samples[i].type;
  169. if (
  170. (type === MetadataSchema.emsg && !enableEmsgMetadataCues) ||
  171. !enableID3MetadataCues
  172. ) {
  173. continue;
  174. }
  175.  
  176. const frames = ID3.getID3Frames(samples[i].data);
  177. if (frames) {
  178. const startTime = samples[i].pts;
  179. let endTime: number = maxCueTime;
  180.  
  181. const timeDiff = endTime - startTime;
  182. if (timeDiff <= 0) {
  183. endTime = startTime + MIN_CUE_DURATION;
  184. }
  185.  
  186. for (let j = 0; j < frames.length; j++) {
  187. const frame = frames[j];
  188. // Safari doesn't put the timestamp frame in the TextTrack
  189. if (!ID3.isTimeStampFrame(frame)) {
  190. const cue = new Cue(startTime, endTime, '');
  191. cue.value = frame;
  192. if (type) {
  193. cue.type = type;
  194. }
  195. this.id3Track.addCue(cue);
  196. frameTypesAdded[frame.key] = null;
  197. updateCueRanges = true;
  198. }
  199. }
  200. }
  201. }
  202. if (updateCueRanges) {
  203. this.updateId3CueEnds(frameTypesAdded);
  204. }
  205. }
  206.  
  207. updateId3CueEnds(frameTypesAdded: Record<string, number | null>) {
  208. // Update endTime of previous cue with same IDR frame.type (Ex: TXXX cue spans to next TXXX)
  209. const cues = this.id3Track?.cues;
  210. if (cues) {
  211. for (let i = cues.length; i--; ) {
  212. const cue = cues[i] as any;
  213. const frameType = cue.value?.key;
  214. if (frameType && frameType in frameTypesAdded) {
  215. const startTime = frameTypesAdded[frameType];
  216. if (startTime && cue.endTime !== startTime) {
  217. cue.endTime = startTime;
  218. }
  219. frameTypesAdded[frameType] = cue.startTime;
  220. }
  221. }
  222. }
  223. }
  224.  
  225. onBufferFlushing(
  226. event: Events.BUFFER_FLUSHING,
  227. { startOffset, endOffset, type }: BufferFlushingData
  228. ) {
  229. const { id3Track, hls } = this;
  230. if (!hls) {
  231. return;
  232. }
  233.  
  234. const {
  235. config: { enableEmsgMetadataCues, enableID3MetadataCues },
  236. } = hls;
  237. if (id3Track && (enableEmsgMetadataCues || enableID3MetadataCues)) {
  238. let predicate;
  239.  
  240. if (type === 'audio') {
  241. predicate = (cue) =>
  242. (cue as any).type === MetadataSchema.audioId3 &&
  243. enableID3MetadataCues;
  244. } else if (type === 'video') {
  245. predicate = (cue) =>
  246. (cue as any).type === MetadataSchema.emsg && enableEmsgMetadataCues;
  247. } else {
  248. predicate = (cue) =>
  249. ((cue as any).type === MetadataSchema.audioId3 &&
  250. enableID3MetadataCues) ||
  251. ((cue as any).type === MetadataSchema.emsg && enableEmsgMetadataCues);
  252. }
  253. removeCuesInRange(id3Track, startOffset, endOffset, predicate);
  254. }
  255. }
  256.  
  257. onLevelUpdated(event: Events.LEVEL_UPDATED, { details }: LevelUpdatedData) {
  258. if (
  259. !this.media ||
  260. !details.hasProgramDateTime ||
  261. !this.hls.config.enableDateRangeMetadataCues
  262. ) {
  263. return;
  264. }
  265. const { dateRangeCuesAppended, id3Track } = this;
  266. const { dateRanges } = details;
  267. const ids = Object.keys(dateRanges);
  268. // Remove cues from track not found in details.dateRanges
  269. if (id3Track) {
  270. const idsToRemove = Object.keys(dateRangeCuesAppended).filter(
  271. (id) => !ids.includes(id)
  272. );
  273. for (let i = idsToRemove.length; i--; ) {
  274. const id = idsToRemove[i];
  275. Object.keys(dateRangeCuesAppended[id].cues).forEach((key) => {
  276. id3Track.removeCue(dateRangeCuesAppended[id].cues[key]);
  277. });
  278. delete dateRangeCuesAppended[id];
  279. }
  280. }
  281. // Exit if the playlist does not have Date Ranges or does not have Program Date Time
  282. const lastFragment = details.fragments[details.fragments.length - 1];
  283. if (ids.length === 0 || !Number.isFinite(lastFragment?.programDateTime)) {
  284. return;
  285. }
  286.  
  287. if (!this.id3Track) {
  288. this.id3Track = this.createTrack(this.media);
  289. }
  290.  
  291. const dateTimeOffset =
  292. (lastFragment.programDateTime as number) / 1000 - lastFragment.start;
  293. const maxCueTime = details.edge || lastFragment.end;
  294. const Cue = getCueClass();
  295.  
  296. for (let i = 0; i < ids.length; i++) {
  297. const id = ids[i];
  298. const dateRange = dateRanges[id];
  299. const appendedDateRangeCues = dateRangeCuesAppended[id];
  300. const cues = appendedDateRangeCues?.cues || {};
  301. let durationKnown = appendedDateRangeCues?.durationKnown || false;
  302. const startTime = dateRangeDateToTimelineSeconds(
  303. dateRange.startDate,
  304. dateTimeOffset
  305. );
  306. let endTime = maxCueTime;
  307. const endDate = dateRange.endDate;
  308. if (endDate) {
  309. endTime = dateRangeDateToTimelineSeconds(endDate, dateTimeOffset);
  310. durationKnown = true;
  311. } else if (dateRange.endOnNext && !durationKnown) {
  312. const nextDateRangeWithSameClass = ids
  313. .reduce((filterMapArray, id) => {
  314. const candidate = dateRanges[id];
  315. if (
  316. candidate.class === dateRange.class &&
  317. candidate.id !== id &&
  318. candidate.startDate > dateRange.startDate
  319. ) {
  320. filterMapArray.push(candidate);
  321. }
  322. return filterMapArray;
  323. }, [] as DateRange[])
  324. .sort((a, b) => a.startDate.getTime() - b.startDate.getTime())[0];
  325. if (nextDateRangeWithSameClass) {
  326. endTime = dateRangeDateToTimelineSeconds(
  327. nextDateRangeWithSameClass.startDate,
  328. dateTimeOffset
  329. );
  330. durationKnown = true;
  331. }
  332. }
  333.  
  334. const attributes = Object.keys(dateRange.attr);
  335. for (let j = 0; j < attributes.length; j++) {
  336. const key = attributes[j];
  337. if (
  338. key === DateRangeAttribute.ID ||
  339. key === DateRangeAttribute.CLASS ||
  340. key === DateRangeAttribute.START_DATE ||
  341. key === DateRangeAttribute.DURATION ||
  342. key === DateRangeAttribute.END_DATE ||
  343. key === DateRangeAttribute.END_ON_NEXT
  344. ) {
  345. continue;
  346. }
  347. let cue = cues[key] as any;
  348. if (cue) {
  349. if (durationKnown && !appendedDateRangeCues.durationKnown) {
  350. cue.endTime = endTime;
  351. }
  352. } else {
  353. let data = dateRange.attr[key];
  354. cue = new Cue(startTime, endTime, '');
  355. if (
  356. key === DateRangeAttribute.SCTE35_OUT ||
  357. key === DateRangeAttribute.SCTE35_IN
  358. ) {
  359. data = hexToArrayBuffer(data);
  360. }
  361. cue.value = { key, data };
  362. cue.type = MetadataSchema.dateRange;
  363. this.id3Track.addCue(cue);
  364. cues[key] = cue;
  365. }
  366. }
  367. dateRangeCuesAppended[id] = {
  368. cues,
  369. dateRange,
  370. durationKnown,
  371. };
  372. }
  373. }
  374. }
  375.  
  376. export default ID3TrackController;