Source: lib/util/ts_parser.js

  1. /*! @license
  2. * Shaka Player
  3. * Copyright 2016 Google LLC
  4. * SPDX-License-Identifier: Apache-2.0
  5. */
  6. goog.provide('shaka.util.TsParser');
  7. goog.require('goog.asserts');
  8. goog.require('shaka.Deprecate');
  9. goog.require('shaka.log');
  10. goog.require('shaka.util.ExpGolomb');
  11. goog.require('shaka.util.Id3Utils');
  12. goog.require('shaka.util.StringUtils');
  13. goog.require('shaka.util.Uint8ArrayUtils');
  14. /**
  15. * @see https://en.wikipedia.org/wiki/MPEG_transport_stream
  16. * @export
  17. */
  18. shaka.util.TsParser = class {
  19. /** */
  20. constructor() {
  21. /** @private {?number} */
  22. this.pmtId_ = null;
  23. /** @private {boolean} */
  24. this.pmtParsed_ = false;
  25. /** @private {?number} */
  26. this.videoPid_ = null;
  27. /** @private {?string} */
  28. this.videoCodec_ = null;
  29. /** @private {!Array<!Array<Uint8Array>>} */
  30. this.videoData_ = [];
  31. /** @private {!Array<shaka.extern.MPEG_PES>} */
  32. this.videoPes_ = [];
  33. /** @private {?number} */
  34. this.audioPid_ = null;
  35. /** @private {?string} */
  36. this.audioCodec_ = null;
  37. /** @private {!Array<!Array<Uint8Array>>} */
  38. this.audioData_ = [];
  39. /** @private {!Array<shaka.extern.MPEG_PES>} */
  40. this.audioPes_ = [];
  41. /** @private {?number} */
  42. this.id3Pid_ = null;
  43. /** @private {!Array<!Array<Uint8Array>>} */
  44. this.id3Data_ = [];
  45. /** @private {?number} */
  46. this.referencePts_ = null;
  47. /** @private {?number} */
  48. this.referenceDts_ = null;
  49. /** @private {?shaka.util.TsParser.OpusMetadata} */
  50. this.opusMetadata_ = null;
  51. }
  52. /**
  53. * Clear previous data
  54. *
  55. * @export
  56. */
  57. clearData() {
  58. this.videoData_ = [];
  59. this.videoPes_ = [];
  60. this.audioData_ = [];
  61. this.audioPes_ = [];
  62. this.id3Data_ = [];
  63. }
  64. /**
  65. * Parse the given data
  66. *
  67. * @param {Uint8Array} data
  68. * @return {!shaka.util.TsParser}
  69. * @export
  70. */
  71. parse(data) {
  72. const packetLength = shaka.util.TsParser.PacketLength_;
  73. // A TS fragment should contain at least 3 TS packets, a PAT, a PMT, and
  74. // one PID.
  75. if (data.length < 3 * packetLength) {
  76. return this;
  77. }
  78. const syncOffset = Math.max(0, shaka.util.TsParser.syncOffset(data));
  79. const length = data.length - (data.length + syncOffset) % packetLength;
  80. let unknownPIDs = false;
  81. // loop through TS packets
  82. for (let start = syncOffset; start < length; start += packetLength) {
  83. if (data[start] == 0x47) {
  84. const payloadUnitStartIndicator = !!(data[start + 1] & 0x40);
  85. // pid is a 13-bit field starting at the last 5 bits of TS[1]
  86. const pid = ((data[start + 1] & 0x1f) << 8) + data[start + 2];
  87. const adaptationFieldControl = (data[start + 3] & 0x30) >> 4;
  88. // if an adaption field is present, its length is specified by the
  89. // fifth byte of the TS packet header.
  90. let offset;
  91. if (adaptationFieldControl > 1) {
  92. offset = start + 5 + data[start + 4];
  93. // continue if there is only adaptation field
  94. if (offset == start + packetLength) {
  95. continue;
  96. }
  97. } else {
  98. offset = start + 4;
  99. }
  100. switch (pid) {
  101. case 0:
  102. if (payloadUnitStartIndicator) {
  103. offset += data[offset] + 1;
  104. }
  105. this.pmtId_ = this.getPmtId_(data, offset);
  106. break;
  107. case 17:
  108. case 0x1fff:
  109. break;
  110. case this.pmtId_: {
  111. if (payloadUnitStartIndicator) {
  112. offset += data[offset] + 1;
  113. }
  114. const parsedPIDs = this.parsePMT_(data, offset);
  115. // only update track id if track PID found while parsing PMT
  116. // this is to avoid resetting the PID to -1 in case
  117. // track PID transiently disappears from the stream
  118. // this could happen in case of transient missing audio samples
  119. // for example
  120. // NOTE this is only the PID of the track as found in TS,
  121. // but we are not using this for MP4 track IDs.
  122. if (parsedPIDs.video != -1) {
  123. this.videoPid_ = parsedPIDs.video;
  124. this.videoCodec_ = parsedPIDs.videoCodec;
  125. }
  126. if (parsedPIDs.audio != -1) {
  127. this.audioPid_ = parsedPIDs.audio;
  128. this.audioCodec_ = parsedPIDs.audioCodec;
  129. }
  130. if (parsedPIDs.id3 != -1) {
  131. this.id3Pid_ = parsedPIDs.id3;
  132. }
  133. if (unknownPIDs && !this.pmtParsed_) {
  134. shaka.log.debug('reparse from beginning');
  135. unknownPIDs = false;
  136. // we set it to -188, the += 188 in the for loop will reset
  137. // start to 0
  138. start = syncOffset - packetLength;
  139. }
  140. this.pmtParsed_ = true;
  141. break;
  142. }
  143. case this.videoPid_: {
  144. const videoData = data.subarray(offset, start + packetLength);
  145. if (payloadUnitStartIndicator) {
  146. this.videoData_.push([videoData]);
  147. } else if (this.videoData_.length) {
  148. const prevVideoData = this.videoData_[this.videoData_.length - 1];
  149. if (prevVideoData) {
  150. this.videoData_[this.videoData_.length - 1].push(videoData);
  151. }
  152. }
  153. break;
  154. }
  155. case this.audioPid_: {
  156. const audioData = data.subarray(offset, start + packetLength);
  157. if (payloadUnitStartIndicator) {
  158. this.audioData_.push([audioData]);
  159. } else if (this.audioData_.length) {
  160. const prevAudioData = this.audioData_[this.audioData_.length - 1];
  161. if (prevAudioData) {
  162. this.audioData_[this.audioData_.length - 1].push(audioData);
  163. }
  164. }
  165. break;
  166. }
  167. case this.id3Pid_: {
  168. const id3Data = data.subarray(offset, start + packetLength);
  169. if (payloadUnitStartIndicator) {
  170. this.id3Data_.push([id3Data]);
  171. } else if (this.id3Data_.length) {
  172. const prevId3Data = this.id3Data_[this.id3Data_.length - 1];
  173. if (prevId3Data) {
  174. this.id3Data_[this.id3Data_.length - 1].push(id3Data);
  175. }
  176. }
  177. break;
  178. }
  179. default:
  180. unknownPIDs = true;
  181. break;
  182. }
  183. } else {
  184. shaka.log.warning('Found TS packet that do not start with 0x47');
  185. }
  186. }
  187. return this;
  188. }
  189. /**
  190. * Get the PMT ID from the PAT
  191. *
  192. * @param {Uint8Array} data
  193. * @param {number} offset
  194. * @return {number}
  195. * @private
  196. */
  197. getPmtId_(data, offset) {
  198. // skip the PSI header and parse the first PMT entry
  199. return ((data[offset + 10] & 0x1f) << 8) | data[offset + 11];
  200. }
  201. /**
  202. * Parse PMT
  203. *
  204. * @param {Uint8Array} data
  205. * @param {number} offset
  206. * @return {!shaka.util.TsParser.PMT}
  207. * @private
  208. */
  209. parsePMT_(data, offset) {
  210. const StringUtils = shaka.util.StringUtils;
  211. const result = {
  212. audio: -1,
  213. video: -1,
  214. id3: -1,
  215. audioCodec: '',
  216. videoCodec: '',
  217. };
  218. const sectionLength = ((data[offset + 1] & 0x0f) << 8) | data[offset + 2];
  219. const tableEnd = offset + 3 + sectionLength - 4;
  220. // to determine where the table is, we have to figure out how
  221. // long the program info descriptors are
  222. const programInfoLength =
  223. ((data[offset + 10] & 0x0f) << 8) | data[offset + 11];
  224. // advance the offset to the first entry in the mapping table
  225. offset += 12 + programInfoLength;
  226. while (offset < tableEnd) {
  227. const pid = ((data[offset + 1] & 0x1f) << 8) | data[offset + 2];
  228. const esInfoLength = ((data[offset + 3] & 0x0f) << 8) | data[offset + 4];
  229. switch (data[offset]) {
  230. case 0x06:
  231. // stream_type 6 can mean a lot of different things in case of DVB.
  232. // We need to look at the descriptors. Right now, we're only
  233. // interested in a few audio and video formats,.
  234. if (esInfoLength > 0) {
  235. let parsePos = offset + 5;
  236. let remaining = esInfoLength;
  237. // Descriptor info: https://www.streamguru.de/mpeg-analyzer/supported-descriptor-list/
  238. while (remaining > 2) {
  239. const descriptorId = data[parsePos];
  240. const descriptorLen = data[parsePos + 1] + 2;
  241. switch (descriptorId) {
  242. // Registration descriptor
  243. case 0x05: {
  244. const registrationData =
  245. data.subarray(parsePos + 2, parsePos + descriptorLen);
  246. const registration =
  247. StringUtils.fromCharCode(registrationData);
  248. if (result.audio == -1 && registration === 'Opus') {
  249. result.audio = pid;
  250. result.audioCodec = 'opus';
  251. } else if (result.video == -1 && registration === 'AV01') {
  252. result.video = pid;
  253. result.videoCodec = 'av1';
  254. }
  255. break;
  256. }
  257. // DVB Descriptor for AC-3
  258. case 0x6a:
  259. if (result.audio == -1) {
  260. result.audio = pid;
  261. result.audioCodec = 'ac3';
  262. }
  263. break;
  264. // DVB Descriptor for EC-3
  265. case 0x7a:
  266. if (result.audio == -1) {
  267. result.audio = pid;
  268. result.audioCodec = 'ec3';
  269. }
  270. break;
  271. // DVB Descriptor for AAC
  272. case 0x7c:
  273. if (result.audio == -1) {
  274. result.audio = pid;
  275. result.audioCodec = 'aac';
  276. }
  277. break;
  278. // DVB extension descriptor
  279. case 0x7f:
  280. if (result.audioCodec == 'opus') {
  281. const extensionDescriptorId = data[parsePos + 2];
  282. let channelConfigCode = null;
  283. // User defined (provisional Opus)
  284. if (extensionDescriptorId === 0x80) {
  285. channelConfigCode = data[parsePos + 3];
  286. }
  287. if (channelConfigCode == null) {
  288. // Not Supported Opus channel count.
  289. break;
  290. }
  291. const channelCount = (channelConfigCode & 0x0F) === 0 ?
  292. 2 : (channelConfigCode & 0x0F);
  293. this.opusMetadata_ = {
  294. channelCount,
  295. channelConfigCode,
  296. sampleRate: 48000,
  297. };
  298. }
  299. break;
  300. }
  301. parsePos += descriptorLen;
  302. remaining -= descriptorLen;
  303. }
  304. }
  305. break;
  306. // SAMPLE-AES AAC
  307. case 0xcf:
  308. break;
  309. // ISO/IEC 13818-7 ADTS AAC (MPEG-2 lower bit-rate audio)
  310. case 0x0f:
  311. if (result.audio == -1) {
  312. result.audio = pid;
  313. result.audioCodec = 'aac';
  314. }
  315. break;
  316. // LOAS/LATM AAC
  317. case 0x11:
  318. if (result.audio == -1) {
  319. result.audio = pid;
  320. result.audioCodec = 'aac-loas';
  321. }
  322. break;
  323. // Packetized metadata (ID3)
  324. case 0x15:
  325. if (result.id3 == -1) {
  326. result.id3 = pid;
  327. }
  328. break;
  329. // SAMPLE-AES AVC
  330. case 0xdb:
  331. break;
  332. // ITU-T Rec. H.264 and ISO/IEC 14496-10 (lower bit-rate video)
  333. case 0x1b:
  334. if (result.video == -1) {
  335. result.video = pid;
  336. result.videoCodec = 'avc';
  337. }
  338. break;
  339. // ISO/IEC 11172-3 (MPEG-1 audio)
  340. // or ISO/IEC 13818-3 (MPEG-2 halved sample rate audio)
  341. case 0x03:
  342. case 0x04:
  343. if (result.audio == -1) {
  344. result.audio = pid;
  345. result.audioCodec = 'mp3';
  346. }
  347. break;
  348. // HEVC
  349. case 0x24:
  350. if (result.video == -1) {
  351. result.video = pid;
  352. result.videoCodec = 'hvc';
  353. }
  354. break;
  355. // AC-3
  356. case 0x81:
  357. if (result.audio == -1) {
  358. result.audio = pid;
  359. result.audioCodec = 'ac3';
  360. }
  361. break;
  362. // EC-3
  363. case 0x84:
  364. case 0x87:
  365. if (result.audio == -1) {
  366. result.audio = pid;
  367. result.audioCodec = 'ec3';
  368. }
  369. break;
  370. default:
  371. // shaka.log.warning('Unknown stream type:', data[offset]);
  372. break;
  373. }
  374. // move to the next table entry
  375. // skip past the elementary stream descriptors, if present
  376. offset += esInfoLength + 5;
  377. }
  378. return result;
  379. }
  380. /**
  381. * Parse PES
  382. *
  383. * @param {Uint8Array} data
  384. * @return {?shaka.extern.MPEG_PES}
  385. * @private
  386. */
  387. parsePES_(data) {
  388. const startPrefix = (data[0] << 16) | (data[1] << 8) | data[2];
  389. // In certain live streams, the start of a TS fragment has ts packets
  390. // that are frame data that is continuing from the previous fragment. This
  391. // is to check that the pes data is the start of a new pes data
  392. if (startPrefix !== 1) {
  393. return null;
  394. }
  395. /** @type {shaka.extern.MPEG_PES} */
  396. const pes = {
  397. data: new Uint8Array(0),
  398. // get the packet length, this will be 0 for video
  399. packetLength: ((data[4] << 8) | data[5]),
  400. pts: null,
  401. dts: null,
  402. nalus: [],
  403. };
  404. // if PES parsed length is not zero and greater than total received length,
  405. // stop parsing. PES might be truncated. minus 6 : PES header size
  406. if (pes.packetLength && pes.packetLength > data.byteLength - 6) {
  407. return null;
  408. }
  409. // PES packets may be annotated with a PTS value, or a PTS value
  410. // and a DTS value. Determine what combination of values is
  411. // available to work with.
  412. const ptsDtsFlags = data[7];
  413. // PTS and DTS are normally stored as a 33-bit number. Javascript
  414. // performs all bitwise operations on 32-bit integers but javascript
  415. // supports a much greater range (52-bits) of integer using standard
  416. // mathematical operations.
  417. // We construct a 31-bit value using bitwise operators over the 31
  418. // most significant bits and then multiply by 4 (equal to a left-shift
  419. // of 2) before we add the final 2 least significant bits of the
  420. // timestamp (equal to an OR.)
  421. if (ptsDtsFlags & 0xC0) {
  422. // the PTS and DTS are not written out directly. For information
  423. // on how they are encoded, see
  424. // http://dvd.sourceforge.net/dvdinfo/pes-hdr.html
  425. const pts =
  426. (data[9] & 0x0e) * 536870912 + // 1 << 29
  427. (data[10] & 0xff) * 4194304 + // 1 << 22
  428. (data[11] & 0xfe) * 16384 + // 1 << 14
  429. (data[12] & 0xff) * 128 + // 1 << 7
  430. (data[13] & 0xfe) / 2;
  431. if (this.referencePts_ == null) {
  432. this.referencePts_ = pts;
  433. }
  434. pes.pts = this.handleRollover_(pts, this.referencePts_);
  435. this.referencePts_ = pes.pts;
  436. pes.dts = pes.pts;
  437. if (ptsDtsFlags & 0x40) {
  438. const dts =
  439. (data[14] & 0x0e) * 536870912 + // 1 << 29
  440. (data[15] & 0xff) * 4194304 + // 1 << 22
  441. (data[16] & 0xfe) * 16384 + // 1 << 14
  442. (data[17] & 0xff) * 128 + // 1 << 7
  443. (data[18] & 0xfe) / 2;
  444. if (this.referenceDts_ == null) {
  445. this.referenceDts_ = dts;
  446. }
  447. pes.dts = this.handleRollover_(dts, this.referenceDts_);
  448. }
  449. this.referenceDts_ = pes.dts;
  450. }
  451. const pesHdrLen = data[8];
  452. // 9 bytes : 6 bytes for PES header + 3 bytes for PES extension
  453. const payloadStartOffset = pesHdrLen + 9;
  454. if (data.byteLength <= payloadStartOffset) {
  455. return null;
  456. }
  457. pes.data = data.subarray(payloadStartOffset);
  458. return pes;
  459. }
  460. /**
  461. * Parse AVC Nalus
  462. *
  463. * The code is based on hls.js
  464. * Credit to https://github.com/video-dev/hls.js/blob/master/src/demux/tsdemuxer.ts
  465. *
  466. * @param {shaka.extern.MPEG_PES} pes
  467. * @param {?shaka.extern.MPEG_PES=} nextPes
  468. * @return {!Array<shaka.extern.VideoNalu>}
  469. * @export
  470. */
  471. parseAvcNalus(pes, nextPes) {
  472. shaka.Deprecate.deprecateFeature(5,
  473. 'TsParser.parseAvcNalus',
  474. 'Please use parseNalus function instead.');
  475. const lastInfo = {
  476. nalu: null,
  477. state: null,
  478. };
  479. return this.parseNalus(pes, lastInfo);
  480. }
  481. /**
  482. * Parse AVC and HVC Nalus
  483. *
  484. * The code is based on hls.js
  485. * Credit to https://github.com/video-dev/hls.js/blob/master/src/demux/tsdemuxer.ts
  486. *
  487. * @param {shaka.extern.MPEG_PES} pes
  488. * @param {{nalu: ?shaka.extern.VideoNalu, state: ?number}} lastInfo
  489. * @return {!Array<shaka.extern.VideoNalu>}
  490. * @export
  491. */
  492. parseNalus(pes, lastInfo) {
  493. const timescale = shaka.util.TsParser.Timescale;
  494. const time = pes.pts ? pes.pts / timescale : null;
  495. const data = pes.data;
  496. const len = data.byteLength;
  497. let naluHeaderSize = 1;
  498. if (this.videoCodec_ == 'hvc') {
  499. naluHeaderSize = 2;
  500. }
  501. // A NALU does not contain is its size.
  502. // The Annex B specification solves this by requiring ‘Start Codes’ to
  503. // precede each NALU. A start code is 2 or 3 0x00 bytes followed with a
  504. // 0x01 byte. e.g. 0x000001 or 0x00000001.
  505. // More info in: https://stackoverflow.com/questions/24884827/possible-locations-for-sequence-picture-parameter-sets-for-h-264-stream/24890903#24890903
  506. let numZeros = lastInfo.state || 0;
  507. const initialNumZeros = numZeros;
  508. /** @type {number} */
  509. let i = 0;
  510. /** @type {!Array<shaka.extern.VideoNalu>} */
  511. const nalus = [];
  512. // Start position includes the first byte where we read the type.
  513. // The data we extract begins at the next byte.
  514. let lastNaluStart = -1;
  515. // Extracted from the first byte.
  516. let lastNaluType = 0;
  517. const getNaluType = (offset) => {
  518. if (this.videoCodec_ == 'hvc') {
  519. return (data[offset] >> 1) & 0x3f;
  520. } else {
  521. return data[offset] & 0x1f;
  522. }
  523. };
  524. const getLastNalu = () => {
  525. if (nalus.length) {
  526. return nalus[nalus.length - 1];
  527. }
  528. return lastInfo.nalu;
  529. };
  530. if (numZeros == -1) {
  531. // special use case where we found 3 or 4-byte start codes exactly at the
  532. // end of previous PES packet
  533. lastNaluStart = 0;
  534. // NALu type is value read from offset 0
  535. lastNaluType = getNaluType(0);
  536. numZeros = 0;
  537. i = 1;
  538. }
  539. while (i < len) {
  540. const value = data[i++];
  541. // Optimization. numZeros 0 and 1 are the predominant case.
  542. if (!numZeros) {
  543. numZeros = value ? 0 : 1;
  544. continue;
  545. }
  546. if (numZeros === 1) {
  547. numZeros = value ? 0 : 2;
  548. continue;
  549. }
  550. if (!value) {
  551. numZeros = 3;
  552. } else if (value == 1) {
  553. const overflow = i - numZeros - 1;
  554. if (lastNaluStart >= 0) {
  555. /** @type {shaka.extern.VideoNalu} */
  556. const nalu = {
  557. data: data.subarray(lastNaluStart + naluHeaderSize, overflow),
  558. fullData: data.subarray(lastNaluStart, overflow),
  559. type: lastNaluType,
  560. time: time,
  561. state: null,
  562. };
  563. nalus.push(nalu);
  564. } else {
  565. const lastNalu = getLastNalu();
  566. if (lastNalu) {
  567. if (initialNumZeros && i <= 4 - initialNumZeros) {
  568. // Start delimiter overlapping between PES packets
  569. // strip start delimiter bytes from the end of last NAL unit
  570. // check if lastNalu had a state different from zero
  571. if (lastNalu.state) {
  572. // strip last bytes
  573. lastNalu.data = lastNalu.data.subarray(
  574. 0, lastNalu.data.byteLength - initialNumZeros);
  575. lastNalu.fullData = lastNalu.fullData.subarray(
  576. 0, lastNalu.fullData.byteLength - initialNumZeros);
  577. }
  578. }
  579. // If NAL units are not starting right at the beginning of the PES
  580. // packet, push preceding data into previous NAL unit.
  581. if (overflow > 0) {
  582. const prevData = data.subarray(0, overflow);
  583. lastNalu.data = shaka.util.Uint8ArrayUtils.concat(
  584. lastNalu.data, prevData);
  585. lastNalu.fullData = shaka.util.Uint8ArrayUtils.concat(
  586. lastNalu.fullData, prevData);
  587. lastNalu.state = 0;
  588. }
  589. }
  590. }
  591. // Check if we can read unit type
  592. if (i < len) {
  593. lastNaluType = getNaluType(i);
  594. lastNaluStart = i;
  595. numZeros = 0;
  596. } else {
  597. // Not enough byte to read unit type.
  598. // Let's read it on next PES parsing.
  599. numZeros = -1;
  600. }
  601. } else {
  602. numZeros = 0;
  603. }
  604. }
  605. if (lastNaluStart >= 0 && numZeros >= 0) {
  606. const nalu = {
  607. data: data.subarray(lastNaluStart + naluHeaderSize, len),
  608. fullData: data.subarray(lastNaluStart, len),
  609. type: lastNaluType,
  610. time: time,
  611. state: numZeros,
  612. };
  613. nalus.push(nalu);
  614. }
  615. if (!nalus.length && lastInfo.nalu) {
  616. const lastNalu = getLastNalu();
  617. if (lastNalu) {
  618. lastNalu.data = shaka.util.Uint8ArrayUtils.concat(
  619. lastNalu.data, data);
  620. lastNalu.fullData = shaka.util.Uint8ArrayUtils.concat(
  621. lastNalu.fullData, data);
  622. }
  623. }
  624. lastInfo.state = numZeros;
  625. return nalus;
  626. }
  627. /**
  628. * Return the ID3 metadata
  629. *
  630. * @return {!Array<shaka.extern.ID3Metadata>}
  631. * @export
  632. */
  633. getMetadata() {
  634. const timescale = shaka.util.TsParser.Timescale;
  635. const metadata = [];
  636. for (const id3DataArray of this.id3Data_) {
  637. const id3Data = shaka.util.Uint8ArrayUtils.concat(...id3DataArray);
  638. const pes = this.parsePES_(id3Data);
  639. if (pes) {
  640. metadata.push({
  641. cueTime: pes.pts ? pes.pts / timescale : null,
  642. data: pes.data,
  643. frames: shaka.util.Id3Utils.getID3Frames(pes.data),
  644. dts: pes.dts,
  645. pts: pes.pts,
  646. });
  647. }
  648. }
  649. return metadata;
  650. }
  651. /**
  652. * Return the audio data
  653. *
  654. * @return {!Array<shaka.extern.MPEG_PES>}
  655. * @export
  656. */
  657. getAudioData() {
  658. if (this.audioData_.length && !this.audioPes_.length) {
  659. let sort = false;
  660. for (const audioDataArray of this.audioData_) {
  661. const audioData = shaka.util.Uint8ArrayUtils.concat(...audioDataArray);
  662. const pes = this.parsePES_(audioData);
  663. let previousPes = this.audioPes_.length ?
  664. this.audioPes_[this.audioPes_.length - 1] : null;
  665. if (pes && pes.pts != null && pes.dts != null && (!previousPes ||
  666. (previousPes.pts != pes.pts && previousPes.dts != pes.dts))) {
  667. if (this.audioPes_.length && pes.dts < (previousPes.dts || 0)) {
  668. sort = true;
  669. }
  670. this.audioPes_.push(pes);
  671. } else if (this.audioPes_.length) {
  672. const data = pes ? pes.data : audioData;
  673. if (!data) {
  674. continue;
  675. }
  676. previousPes = this.audioPes_.pop();
  677. previousPes.data =
  678. shaka.util.Uint8ArrayUtils.concat(previousPes.data, data);
  679. this.audioPes_.push(previousPes);
  680. }
  681. }
  682. if (sort) {
  683. this.audioPes_ = this.audioPes_.sort((a, b) => {
  684. const deltaDts = (a.dts || 0) - (b.dts || 0);
  685. const deltaPts = (a.pts || 0) - (b.pts || 0);
  686. return deltaDts || deltaPts;
  687. });
  688. }
  689. }
  690. return this.audioPes_;
  691. }
  692. /**
  693. * Return the video data
  694. *
  695. * @param {boolean=} naluProcessing
  696. * @return {!Array<shaka.extern.MPEG_PES>}
  697. * @export
  698. */
  699. getVideoData(naluProcessing = true) {
  700. if (this.videoData_.length && !this.videoPes_.length) {
  701. let sort = false;
  702. for (const videoDataArray of this.videoData_) {
  703. const videoData = shaka.util.Uint8ArrayUtils.concat(...videoDataArray);
  704. const pes = this.parsePES_(videoData);
  705. let previousPes = this.videoPes_.length ?
  706. this.videoPes_[this.videoPes_.length - 1] : null;
  707. if (pes && pes.pts != null && pes.dts != null && (!previousPes ||
  708. (previousPes.pts != pes.pts && previousPes.dts != pes.dts))) {
  709. if (this.videoPes_.length && pes.dts < (previousPes.dts || 0)) {
  710. sort = true;
  711. }
  712. this.videoPes_.push(pes);
  713. } else if (this.videoPes_.length) {
  714. const data = pes ? pes.data : videoData;
  715. if (!data) {
  716. continue;
  717. }
  718. previousPes = this.videoPes_.pop();
  719. previousPes.data =
  720. shaka.util.Uint8ArrayUtils.concat(previousPes.data, data);
  721. this.videoPes_.push(previousPes);
  722. }
  723. }
  724. if (naluProcessing) {
  725. const lastInfo = {
  726. nalu: null,
  727. state: null,
  728. };
  729. const pesWithLength = [];
  730. for (const pes of this.videoPes_) {
  731. pes.nalus = this.parseNalus(pes, lastInfo);
  732. if (pes.nalus.length) {
  733. pesWithLength.push(pes);
  734. lastInfo.nalu = pes.nalus[pes.nalus.length - 1];
  735. }
  736. }
  737. this.videoPes_ = pesWithLength;
  738. }
  739. if (sort) {
  740. this.videoPes_ = this.videoPes_.sort((a, b) => {
  741. const deltaDts = (a.dts || 0) - (b.dts || 0);
  742. const deltaPts = (a.pts || 0) - (b.pts || 0);
  743. return deltaDts || deltaPts;
  744. });
  745. }
  746. }
  747. if (!naluProcessing) {
  748. const prevVideoPes = this.videoPes_;
  749. this.videoPes_ = [];
  750. return prevVideoPes;
  751. }
  752. return this.videoPes_;
  753. }
  754. /**
  755. * Return the start time for the audio and video
  756. *
  757. * @param {string} contentType
  758. * @return {?number}
  759. * @export
  760. */
  761. getStartTime(contentType) {
  762. const timescale = shaka.util.TsParser.Timescale;
  763. if (contentType == 'audio') {
  764. let audioStartTime = null;
  765. const audioData = this.getAudioData();
  766. if (audioData.length) {
  767. const pes = audioData[0];
  768. audioStartTime = Math.min(pes.dts, pes.pts) / timescale;
  769. }
  770. return audioStartTime;
  771. } else if (contentType == 'video') {
  772. let videoStartTime = null;
  773. const videoData = this.getVideoData(/* naluProcessing= */ false);
  774. if (videoData.length) {
  775. const pes = videoData[0];
  776. videoStartTime = Math.min(pes.dts, pes.pts) / timescale;
  777. }
  778. return videoStartTime;
  779. }
  780. return null;
  781. }
  782. /**
  783. * Return the audio and video codecs
  784. *
  785. * @return {{audio: ?string, video: ?string}}
  786. * @export
  787. */
  788. getCodecs() {
  789. return {
  790. audio: this.audioCodec_,
  791. video: this.videoCodec_,
  792. };
  793. }
  794. /**
  795. * Return the video data
  796. *
  797. * @return {!Array<shaka.extern.VideoNalu>}
  798. * @export
  799. */
  800. getVideoNalus() {
  801. const nalus = [];
  802. for (const pes of this.getVideoData()) {
  803. nalus.push(...pes.nalus);
  804. }
  805. return nalus;
  806. }
  807. /**
  808. * Return the video resolution
  809. *
  810. * @return {{height: ?string, width: ?string}}
  811. * @export
  812. */
  813. getVideoResolution() {
  814. shaka.Deprecate.deprecateFeature(5,
  815. 'TsParser.getVideoResolution',
  816. 'Please use getVideoInfo function instead.');
  817. const videoInfo = this.getVideoInfo();
  818. return {
  819. height: videoInfo.height,
  820. width: videoInfo.width,
  821. };
  822. }
  823. /**
  824. * Return the video information
  825. *
  826. * @return {{height: ?string, width: ?string, codec: ?string,
  827. * frameRate: ?string}}
  828. * @export
  829. */
  830. getVideoInfo() {
  831. if (this.videoCodec_ == 'hvc') {
  832. return this.getHvcInfo_();
  833. }
  834. return this.getAvcInfo_();
  835. }
  836. /**
  837. * @return {?string}
  838. * @private
  839. */
  840. getFrameRate_() {
  841. const timescale = shaka.util.TsParser.Timescale;
  842. const videoData = this.getVideoData();
  843. if (videoData.length > 1) {
  844. const firstPts = videoData[0].pts;
  845. goog.asserts.assert(typeof(firstPts) == 'number',
  846. 'Should be an number!');
  847. const secondPts = videoData[1].pts;
  848. goog.asserts.assert(typeof(secondPts) == 'number',
  849. 'Should be an number!');
  850. if (!isNaN(secondPts - firstPts)) {
  851. return String(1 / (secondPts - firstPts) * timescale);
  852. }
  853. }
  854. return null;
  855. }
  856. /**
  857. * Return the video information for AVC
  858. *
  859. * @return {{height: ?string, width: ?string, codec: ?string,
  860. * frameRate: ?string}}
  861. * @private
  862. */
  863. getAvcInfo_() {
  864. const TsParser = shaka.util.TsParser;
  865. const videoInfo = {
  866. height: null,
  867. width: null,
  868. codec: null,
  869. frameRate: null,
  870. };
  871. const videoNalus = this.getVideoNalus();
  872. if (!videoNalus.length) {
  873. return videoInfo;
  874. }
  875. const spsNalu = videoNalus.find((nalu) => {
  876. return nalu.type == TsParser.H264_NALU_TYPE_SPS_;
  877. });
  878. if (!spsNalu) {
  879. return videoInfo;
  880. }
  881. const expGolombDecoder = new shaka.util.ExpGolomb(spsNalu.data);
  882. // profile_idc
  883. const profileIdc = expGolombDecoder.readUnsignedByte();
  884. // constraint_set[0-5]_flag
  885. const profileCompatibility = expGolombDecoder.readUnsignedByte();
  886. // level_idc u(8)
  887. const levelIdc = expGolombDecoder.readUnsignedByte();
  888. // seq_parameter_set_id
  889. expGolombDecoder.skipExpGolomb();
  890. // some profiles have more optional data we don't need
  891. if (TsParser.PROFILES_WITH_OPTIONAL_SPS_DATA_.includes(profileIdc)) {
  892. const chromaFormatIdc = expGolombDecoder.readUnsignedExpGolomb();
  893. if (chromaFormatIdc === 3) {
  894. // separate_colour_plane_flag
  895. expGolombDecoder.skipBits(1);
  896. }
  897. // bit_depth_luma_minus8
  898. expGolombDecoder.skipExpGolomb();
  899. // bit_depth_chroma_minus8
  900. expGolombDecoder.skipExpGolomb();
  901. // qpprime_y_zero_transform_bypass_flag
  902. expGolombDecoder.skipBits(1);
  903. // seq_scaling_matrix_present_flag
  904. if (expGolombDecoder.readBoolean()) {
  905. const scalingListCount = (chromaFormatIdc !== 3) ? 8 : 12;
  906. for (let i = 0; i < scalingListCount; i++) {
  907. // seq_scaling_list_present_flag[ i ]
  908. if (expGolombDecoder.readBoolean()) {
  909. if (i < 6) {
  910. expGolombDecoder.skipScalingList(16);
  911. } else {
  912. expGolombDecoder.skipScalingList(64);
  913. }
  914. }
  915. }
  916. }
  917. }
  918. // log2_max_frame_num_minus4
  919. expGolombDecoder.skipExpGolomb();
  920. const picOrderCntType = expGolombDecoder.readUnsignedExpGolomb();
  921. if (picOrderCntType === 0) {
  922. // log2_max_pic_order_cnt_lsb_minus4
  923. expGolombDecoder.readUnsignedExpGolomb();
  924. } else if (picOrderCntType === 1) {
  925. // delta_pic_order_always_zero_flag
  926. expGolombDecoder.skipBits(1);
  927. // offset_for_non_ref_pic
  928. expGolombDecoder.skipExpGolomb();
  929. // offset_for_top_to_bottom_field
  930. expGolombDecoder.skipExpGolomb();
  931. const numRefFramesInPicOrderCntCycle =
  932. expGolombDecoder.readUnsignedExpGolomb();
  933. for (let i = 0; i < numRefFramesInPicOrderCntCycle; i++) {
  934. // offset_for_ref_frame[ i ]
  935. expGolombDecoder.skipExpGolomb();
  936. }
  937. }
  938. // max_num_ref_frames
  939. expGolombDecoder.skipExpGolomb();
  940. // gaps_in_frame_num_value_allowed_flag
  941. expGolombDecoder.skipBits(1);
  942. const picWidthInMbsMinus1 =
  943. expGolombDecoder.readUnsignedExpGolomb();
  944. const picHeightInMapUnitsMinus1 =
  945. expGolombDecoder.readUnsignedExpGolomb();
  946. const frameMbsOnlyFlag = expGolombDecoder.readBits(1);
  947. if (frameMbsOnlyFlag === 0) {
  948. // mb_adaptive_frame_field_flag
  949. expGolombDecoder.skipBits(1);
  950. }
  951. // direct_8x8_inference_flag
  952. expGolombDecoder.skipBits(1);
  953. let frameCropLeftOffset = 0;
  954. let frameCropRightOffset = 0;
  955. let frameCropTopOffset = 0;
  956. let frameCropBottomOffset = 0;
  957. // frame_cropping_flag
  958. if (expGolombDecoder.readBoolean()) {
  959. frameCropLeftOffset = expGolombDecoder.readUnsignedExpGolomb();
  960. frameCropRightOffset = expGolombDecoder.readUnsignedExpGolomb();
  961. frameCropTopOffset = expGolombDecoder.readUnsignedExpGolomb();
  962. frameCropBottomOffset = expGolombDecoder.readUnsignedExpGolomb();
  963. }
  964. videoInfo.height = String(((2 - frameMbsOnlyFlag) *
  965. (picHeightInMapUnitsMinus1 + 1) * 16) - (frameCropTopOffset * 2) -
  966. (frameCropBottomOffset * 2));
  967. videoInfo.width = String(((picWidthInMbsMinus1 + 1) * 16) -
  968. frameCropLeftOffset * 2 - frameCropRightOffset * 2);
  969. videoInfo.codec = 'avc1.' + this.byteToHex_(profileIdc) +
  970. this.byteToHex_(profileCompatibility) + this.byteToHex_(levelIdc);
  971. videoInfo.frameRate = this.getFrameRate_();
  972. return videoInfo;
  973. }
  974. /**
  975. * Return the video information for HVC
  976. *
  977. * @return {{height: ?string, width: ?string, codec: ?string,
  978. * frameRate: ?string}}
  979. * @private
  980. */
  981. getHvcInfo_() {
  982. const TsParser = shaka.util.TsParser;
  983. const videoInfo = {
  984. height: null,
  985. width: null,
  986. codec: null,
  987. frameRate: null,
  988. };
  989. const videoNalus = this.getVideoNalus();
  990. if (!videoNalus.length) {
  991. return videoInfo;
  992. }
  993. const spsNalu = videoNalus.find((nalu) => {
  994. return nalu.type == TsParser.H265_NALU_TYPE_SPS_;
  995. });
  996. if (!spsNalu) {
  997. return videoInfo;
  998. }
  999. const gb = new shaka.util.ExpGolomb(
  1000. spsNalu.fullData, /* convertEbsp2rbsp= */ true);
  1001. // remove NALu Header
  1002. gb.readUnsignedByte();
  1003. gb.readUnsignedByte();
  1004. // SPS
  1005. gb.readBits(4); // video_parameter_set_id
  1006. const maxSubLayersMinus1 = gb.readBits(3);
  1007. gb.readBoolean(); // temporal_id_nesting_flag
  1008. // profile_tier_level begin
  1009. const generalProfileSpace = gb.readBits(2);
  1010. const generalTierFlag = gb.readBits(1);
  1011. const generalProfileIdc = gb.readBits(5);
  1012. const generalProfileCompatibilityFlags = gb.readBits(32);
  1013. const generalConstraintIndicatorFlags1 = gb.readUnsignedByte();
  1014. const generalConstraintIndicatorFlags2 = gb.readUnsignedByte();
  1015. const generalConstraintIndicatorFlags3 = gb.readUnsignedByte();
  1016. const generalConstraintIndicatorFlags4 = gb.readUnsignedByte();
  1017. const generalConstraintIndicatorFlags5 = gb.readUnsignedByte();
  1018. const generalConstraintIndicatorFlags6 = gb.readUnsignedByte();
  1019. const generalLevelIdc = gb.readUnsignedByte();
  1020. const subLayerProfilePresentFlag = [];
  1021. const subLayerLevelPresentFlag = [];
  1022. for (let i = 0; i < maxSubLayersMinus1; i++) {
  1023. subLayerProfilePresentFlag.push(gb.readBoolean());
  1024. subLayerLevelPresentFlag.push(gb.readBoolean());
  1025. }
  1026. if (maxSubLayersMinus1 > 0) {
  1027. for (let i = maxSubLayersMinus1; i < 8; i++) {
  1028. gb.readBits(2);
  1029. }
  1030. }
  1031. for (let i = 0; i < maxSubLayersMinus1; i++) {
  1032. if (subLayerProfilePresentFlag[i]) {
  1033. gb.readBits(88);
  1034. }
  1035. if (subLayerLevelPresentFlag[i]) {
  1036. gb.readUnsignedByte();
  1037. }
  1038. }
  1039. // profile_tier_level end
  1040. gb.readUnsignedExpGolomb(); // seq_parameter_set_id
  1041. const chromaFormatIdc = gb.readUnsignedExpGolomb();
  1042. if (chromaFormatIdc == 3) {
  1043. gb.readBits(1); // separate_colour_plane_flag
  1044. }
  1045. const picWidthInLumaSamples = gb.readUnsignedExpGolomb();
  1046. const picHeightInLumaSamples = gb.readUnsignedExpGolomb();
  1047. let leftOffset = 0;
  1048. let rightOffset = 0;
  1049. let topOffset = 0;
  1050. let bottomOffset = 0;
  1051. const conformanceWindowFlag = gb.readBoolean();
  1052. if (conformanceWindowFlag) {
  1053. leftOffset += gb.readUnsignedExpGolomb();
  1054. rightOffset += gb.readUnsignedExpGolomb();
  1055. topOffset += gb.readUnsignedExpGolomb();
  1056. bottomOffset += gb.readUnsignedExpGolomb();
  1057. }
  1058. const subWc = chromaFormatIdc === 1 || chromaFormatIdc === 2 ? 2 : 1;
  1059. const subHc = chromaFormatIdc === 1 ? 2 : 1;
  1060. videoInfo.width =
  1061. String(picWidthInLumaSamples - (leftOffset + rightOffset) * subWc);
  1062. videoInfo.height =
  1063. String(picHeightInLumaSamples - (topOffset + bottomOffset) * subHc);
  1064. const reverseBits = (integer) => {
  1065. let result = 0;
  1066. for (let i = 0; i < 32; i++) {
  1067. result |= ((integer >> i) & 1) << (31 - i);
  1068. }
  1069. return result >>> 0;
  1070. };
  1071. const profileSpace = ['', 'A', 'B', 'C'][generalProfileSpace];
  1072. const profileCompatibility = reverseBits(generalProfileCompatibilityFlags);
  1073. const tierFlag = generalTierFlag == 1 ? 'H' : 'L';
  1074. let codec = 'hvc1';
  1075. codec += '.' + profileSpace + generalProfileIdc;
  1076. codec += '.' + profileCompatibility.toString(16).toUpperCase();
  1077. codec += '.' + tierFlag + generalLevelIdc;
  1078. if (generalConstraintIndicatorFlags6) {
  1079. codec += '.' +
  1080. generalConstraintIndicatorFlags6.toString(16).toUpperCase();
  1081. }
  1082. if (generalConstraintIndicatorFlags5) {
  1083. codec += '.' +
  1084. generalConstraintIndicatorFlags5.toString(16).toUpperCase();
  1085. }
  1086. if (generalConstraintIndicatorFlags4) {
  1087. codec += '.' +
  1088. generalConstraintIndicatorFlags4.toString(16).toUpperCase();
  1089. }
  1090. if (generalConstraintIndicatorFlags3) {
  1091. codec += '.' +
  1092. generalConstraintIndicatorFlags3.toString(16).toUpperCase();
  1093. }
  1094. if (generalConstraintIndicatorFlags2) {
  1095. codec += '.' +
  1096. generalConstraintIndicatorFlags2.toString(16).toUpperCase();
  1097. }
  1098. if (generalConstraintIndicatorFlags1) {
  1099. codec += '.' +
  1100. generalConstraintIndicatorFlags1.toString(16).toUpperCase();
  1101. }
  1102. videoInfo.codec = codec;
  1103. videoInfo.frameRate = this.getFrameRate_();
  1104. return videoInfo;
  1105. }
  1106. /**
  1107. * Return the Opus metadata
  1108. *
  1109. * @return {?shaka.util.TsParser.OpusMetadata}
  1110. */
  1111. getOpusMetadata() {
  1112. return this.opusMetadata_;
  1113. }
  1114. /**
  1115. * Convert a byte to 2 digits of hex. (Only handles values 0-255.)
  1116. *
  1117. * @param {number} x
  1118. * @return {string}
  1119. * @private
  1120. */
  1121. byteToHex_(x) {
  1122. return ('0' + x.toString(16).toUpperCase()).slice(-2);
  1123. }
  1124. /**
  1125. * @param {number} value
  1126. * @param {number} reference
  1127. * @return {number}
  1128. * @private
  1129. */
  1130. handleRollover_(value, reference) {
  1131. const MAX_TS = 8589934592;
  1132. const RO_THRESH = 4294967296;
  1133. let direction = 1;
  1134. if (value > reference) {
  1135. // If the current timestamp value is greater than our reference timestamp
  1136. // and we detect a timestamp rollover, this means the roll over is
  1137. // happening in the opposite direction.
  1138. // Example scenario: Enter a long stream/video just after a rollover
  1139. // occurred. The reference point will be set to a small number, e.g. 1.
  1140. // The user then seeks backwards over the rollover point. In loading this
  1141. // segment, the timestamp values will be very large, e.g. 2^33 - 1. Since
  1142. // this comes before the data we loaded previously, we want to adjust the
  1143. // time stamp to be `value - 2^33`.
  1144. direction = -1;
  1145. }
  1146. // Note: A seek forwards or back that is greater than the RO_THRESH
  1147. // (2^32, ~13 hours) will cause an incorrect adjustment.
  1148. while (Math.abs(reference - value) > RO_THRESH) {
  1149. value += (direction * MAX_TS);
  1150. }
  1151. return value;
  1152. }
  1153. /**
  1154. * Check if the passed data corresponds to an MPEG2-TS
  1155. *
  1156. * @param {Uint8Array} data
  1157. * @return {boolean}
  1158. * @export
  1159. */
  1160. static probe(data) {
  1161. const syncOffset = shaka.util.TsParser.syncOffset(data);
  1162. if (syncOffset < 0) {
  1163. return false;
  1164. } else {
  1165. if (syncOffset > 0) {
  1166. shaka.log.warning('MPEG2-TS detected but first sync word found @ ' +
  1167. 'offset ' + syncOffset + ', junk ahead ?');
  1168. }
  1169. return true;
  1170. }
  1171. }
  1172. /**
  1173. * Returns the synchronization offset
  1174. *
  1175. * @param {Uint8Array} data
  1176. * @return {number}
  1177. * @export
  1178. */
  1179. static syncOffset(data) {
  1180. const packetLength = shaka.util.TsParser.PacketLength_;
  1181. // scan 1000 first bytes
  1182. const scanWindow = Math.min(1000, data.length - 3 * packetLength);
  1183. let i = 0;
  1184. while (i < scanWindow) {
  1185. // a TS fragment should contain at least 3 TS packets, a PAT, a PMT, and
  1186. // one PID, each starting with 0x47
  1187. if (data[i] == 0x47 &&
  1188. data[i + packetLength] == 0x47 &&
  1189. data[i + 2 * packetLength] == 0x47) {
  1190. return i;
  1191. } else {
  1192. i++;
  1193. }
  1194. }
  1195. return -1;
  1196. }
  1197. };
  1198. /**
  1199. * @const {number}
  1200. * @export
  1201. */
  1202. shaka.util.TsParser.Timescale = 90000;
  1203. /**
  1204. * @const {number}
  1205. * @private
  1206. */
  1207. shaka.util.TsParser.PacketLength_ = 188;
  1208. /**
  1209. * NALU type for Sequence Parameter Set (SPS) for H.264.
  1210. * @const {number}
  1211. * @private
  1212. */
  1213. shaka.util.TsParser.H264_NALU_TYPE_SPS_ = 0x07;
  1214. /**
  1215. * NALU type for Sequence Parameter Set (SPS) for H.265.
  1216. * @const {number}
  1217. * @private
  1218. */
  1219. shaka.util.TsParser.H265_NALU_TYPE_SPS_ = 0x21;
  1220. /**
  1221. * Values of profile_idc that indicate additional fields are included in the
  1222. * SPS.
  1223. * see Recommendation ITU-T H.264 (4/2013)
  1224. * 7.3.2.1.1 Sequence parameter set data syntax
  1225. *
  1226. * @const {!Array<number>}
  1227. * @private
  1228. */
  1229. shaka.util.TsParser.PROFILES_WITH_OPTIONAL_SPS_DATA_ =
  1230. [100, 110, 122, 244, 44, 83, 86, 118, 128, 138, 139, 134];
  1231. /**
  1232. * @typedef {{
  1233. * audio: number,
  1234. * video: number,
  1235. * id3: number,
  1236. * audioCodec: string,
  1237. * videoCodec: string
  1238. * }}
  1239. *
  1240. * @summary PMT.
  1241. * @property {number} audio
  1242. * Audio PID
  1243. * @property {number} video
  1244. * Video PID
  1245. * @property {number} id3
  1246. * ID3 PID
  1247. * @property {string} audioCodec
  1248. * Audio codec
  1249. * @property {string} videoCodec
  1250. * Video codec
  1251. */
  1252. shaka.util.TsParser.PMT;
  1253. /**
  1254. * @typedef {{
  1255. * channelCount: number,
  1256. * channelConfigCode: number,
  1257. * sampleRate: number
  1258. * }}
  1259. *
  1260. * @summary PMT.
  1261. * @property {number} channelCount
  1262. * @property {number} channelConfigCode
  1263. * @property {number} sampleRate
  1264. */
  1265. shaka.util.TsParser.OpusMetadata;