api/core: remove non-printable unicode character in boot message (#1182)

all links expire apparently

.github/test.sh

@@ -18,7 +18,7 @@ test_api() {
          -X POST \
          -H "Accept: application/json" \
          -H "Content-Type: application/json" \
-         -d '{"url":"https://www.tiktok.com/@fatfatmillycat/video/7195741644585454894"}')
+         -d '{"url":"https://garfield-69.tumblr.com/post/696499862852780032","alwaysProxy":true}')
 
     echo "API_RESPONSE=$API_RESPONSE"
     STATUS=$(echo "$API_RESPONSE" | jq -r .status)
@@ -46,6 +46,7 @@ setup_api() {
 }
 
 setup_web() {
+    pnpm run --prefix web check
     pnpm run --prefix web build
 }
 

.github/workflows/codeql.yml

@@ -0,0 +1,93 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches:
+      - '**'
+  pull_request:
+    branches: [ "main", "7" ]
+  schedule:
+    - cron: '33 7 * * 5'
+
+jobs:
+  analyze:
+    name: Analyze (${{ matrix.language }})
+    # Runner size impacts CodeQL analysis time. To learn more, please see:
+    #   - https://gh.io/recommended-hardware-resources-for-running-codeql
+    #   - https://gh.io/supported-runners-and-hardware-resources
+    #   - https://gh.io/using-larger-runners (GitHub.com only)
+    # Consider using larger runners or machines with greater resources for possible analysis time improvements.
+    runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }}
+    permissions:
+      # required for all workflows
+      security-events: write
+
+      # required to fetch internal or private CodeQL packs
+      packages: read
+
+      # only required for workflows in private repositories
+      actions: read
+      contents: read
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+        - language: javascript-typescript
+          build-mode: none
+        # CodeQL supports the following values keywords for 'language': 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift'
+        # Use `c-cpp` to analyze code written in C, C++ or both
+        # Use 'java-kotlin' to analyze code written in Java, Kotlin or both
+        # Use 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both
+        # To learn more about changing the languages that are analyzed or customizing the build mode for your analysis,
+        # see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning.
+        # If you are analyzing a compiled language, you can modify the 'build-mode' for that language to customize how
+        # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v3
+      with:
+        languages: ${{ matrix.language }}
+        build-mode: ${{ matrix.build-mode }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+
+        # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+        # queries: security-extended,security-and-quality
+
+    # If the analyze step fails for one of the languages you are analyzing with
+    # "We were unable to automatically build your code", modify the matrix above
+    # to set the build mode to "manual" for that language. Then modify this step
+    # to build your code.
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+    - if: matrix.build-mode == 'manual'
+      shell: bash
+      run: |
+        echo 'If you are using a "manual" build mode for one or more of the' \
+          'languages you are analyzing, replace this with the commands to build' \
+          'your code, for example:'
+        echo '  make bootstrap'
+        echo '  make release'
+        exit 1
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v3
+      with:
+        category: "/language:${{matrix.language}}"

.github/workflows/docker.yml

@@ -51,7 +51,7 @@ jobs:
         uses: docker/build-push-action@v6
         with:
           context: .
-          platforms: linux/amd64,linux/arm64,linux/arm/v7
+          platforms: linux/amd64,linux/arm64
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}

.github/workflows/test-services.yml

@@ -17,7 +17,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: pnpm/action-setup@v4
       - id: checkServices
-        run: pnpm i --frozen-lockfile && echo "service_list=$(node api/src/util/test-ci get-services)" >> "$GITHUB_OUTPUT"
+        run: pnpm i --frozen-lockfile && echo "service_list=$(node api/src/util/test get-services)" >> "$GITHUB_OUTPUT"
 
   test-services:
     needs: check-services
@@ -30,4 +30,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - uses: pnpm/action-setup@v4
-      - run: pnpm i --frozen-lockfile && node api/src/util/test-ci run-tests-for ${{ matrix.service }}
+      - run: pnpm i --frozen-lockfile && node api/src/util/test run-tests-for ${{ matrix.service }}
+        env:
+          API_EXTERNAL_PROXY: ${{ secrets.API_EXTERNAL_PROXY }}
+          TEST_IGNORE_SERVICES: ${{ vars.TEST_IGNORE_SERVICES }}

.gitignore

@@ -13,6 +13,7 @@ build
 .env.*
 !.env.example
 cookies.json
+keys.json
 
 # docker
 docker-compose.yml

CONTRIBUTING.md

@@ -4,7 +4,23 @@ if you're reading this, you are probably interested in contributing to cobalt, w
 this document serves as a guide to help you make contributions that we can merge into the cobalt codebase.
 
 ## translations
-currently, we are **not accepting** translations of cobalt. this is because we are making significant changes to the frontend, and the currently used localization structure is being completely reworked. if this changes, this document will be updated.
+we are currently accepting translations via the [i18n platform](https://i18n.imput.net).
+
+thank you for showing interest in making cobalt more accessible around the world, we really appreciate it! here are some guidelines for how a cobalt translation should look:
+
+- cobalt's writing style is informal. please do not use formal language, unless there is no other way to express the same idea of the original text in your language.
+- all cobalt text is written in lowercase. this is a stylistic choice, please do not capitalize translated sentences.
+- do not translate the name "cobalt", or "imput"
+- you can translate "meowbalt" into whatever your language's equivalent of _meow_ is (e.g. _miaubalt_ in German)
+- **please don't translate cobalt into languages which you are not experienced in.** we can use google translate ourselves, but we would prefer cobalt to be translated by humans, not computers.
+
+if your language does not exist on the translation platform yet, you can request to add it by adding it to any of cobalt's components (e.g. [here](https://i18n.imput.net/projects/cobalt/about/)).
+
+before translating a piece of text, check that no one has made a translation yet. pending translations are displayed in the **Suggestions** tab on the translate page. if someone already made a suggestion, and you think it's correct, you can upvote it! this helps us distinguish that a translation is correct.
+
+if no one has submitted a translation, or the submitted translation seems wrong to you, you can submit your translation by clicking the **Suggest** button for each individual string, which sends it off for human review. we will then check it to to ensure no malicious translations are submitted, and add it to cobalt.
+
+if any translation string's meaning seems unclear to you, please leave a comment on the *Comments* tab, and we will either add an explanation or a screenshot.
 
 ## adding features or support for services
 before putting in the effort to implement a feature, it's worth considering whether it would be appropriate to add it to cobalt. the cobalt api is built to assist people **only with downloading freely accessible content**. other functionality, such as:
@@ -22,9 +38,9 @@ when contributing code to cobalt, there are a few guidelines in place to ensure
 ### clean commit messages
 internally, we use a format similar to [conventional commits](https://www.conventionalcommits.org/en/v1.0.0/) - the first part signifies which part of the code you are changing (the *scope*), and the second part explains the change. for inspiration on how to write appropriate commit titles, you can take a look at the [commit history](https://github.com/imputnet/cobalt/commits/).
 
-the scope is not strictly defined, you can write whatever you find most fitting for the particular change. suppose you are changing a small part of a more significant part of the codebase. in that case, you can specify both the larger and smaller scopes in the commit message for clarity (e.g., if you were changing something in internal streams, the commit could be something like `stream/internal: fix object not being handled properly`).
+the scope is not strictly defined, you can write whatever you find most fitting for the particular change. suppose you are changing a small part of a more significant part of the codebase. in that case, you can specify both the larger and smaller scopes in the commit message for clarity (e.g., if you were changing something in internal streams, the commit could be something like `api/stream: fix object not being handled properly`).
 
-if you think a change deserves further explanation, we encourage you to write a short explanation in the commit message ([example](https://github.com/imputnet/cobalt/commit/d2e5b6542f71f3809ba94d56c26f382b5cb62762)), which will save both you and us time having to enquire about the change, and you explaining the reason behind it.
+if you think a change deserves further explanation, we encourage you to write a short explanation in the commit message ([example](https://github.com/imputnet/cobalt/commit/31be60484de8eaf63bba8a4f508e16438aa7ba6e)), which will save both you and us time having to enquire about the change, and you explaining the reason behind it.
 
 if your contribution has uninformative commit titles, you may be asked to interactively rebase your branch and amend each commit to include a meaningful title.
 

Dockerfile

@@ -1,4 +1,4 @@
-FROM node:20-bullseye-slim AS base
+FROM node:23-alpine AS base
 ENV PNPM_HOME="/pnpm"
 ENV PATH="$PNPM_HOME:$PATH"
 
@@ -7,8 +7,7 @@ WORKDIR /app
 COPY . /app
 
 RUN corepack enable
-RUN apt-get update && \
-    apt-get install -y python3 build-essential
+RUN apk add --no-cache python3 alpine-sdk
 
 RUN --mount=type=cache,id=pnpm,target=/pnpm/store \
     pnpm install --prod --frozen-lockfile
@@ -18,8 +17,10 @@ RUN pnpm deploy --filter=@imput/cobalt-api --prod /prod/api
 FROM base AS api
 WORKDIR /app
 
-COPY --from=build /prod/api /app
-COPY --from=build /app/.git /app/.git
+COPY --from=build --chown=node:node /prod/api /app
+COPY --from=build --chown=node:node /app/.git /app/.git
+
+USER node
 
 EXPOSE 9000
-CMD [ "node", "src/cobalt" ]
+CMD [ "node", "src/cobalt" ]

README.md

@@ -14,109 +14,47 @@
         <a href="https://discord.gg/pQPt8HBUPu">
             💬 community discord server
         </a>
+        <br/>
         <a href="https://x.com/justusecobalt">
-            🐦 twitter/x
+            🐦 twitter
+        </a>
+        <a href="https://bsky.app/profile/cobalt.tools">
+            🦋 bluesky
         </a>
     </p>
     <br/>
 </div>
 
-cobalt is a media downloader that doesn't piss you off. it's fast, friendly, and doesn't have any bullshit that modern web is filled with: ***no ads, trackers, or paywalls***.
+cobalt is a media downloader that doesn't piss you off. it's friendly, efficient, and doesn't have ads, trackers, paywalls or other nonsense.
 
-paste the link, get the file, move on. it's that simple. just how it should be.
+paste the link, get the file, move on. that simple, just how it should be.
 
-### supported services
-this list is not final and keeps expanding over time. if support for a service you want is missing, create an issue (or a pull request 👀).
+### cobalt monorepo
+this monorepo includes source code for api, frontend, and related packages:
+- [api tree & readme](/api/)
+- [web tree & readme](/web/)
+- [packages tree](/packages/)
 
-| service           | video + audio | only audio | only video | metadata | rich file names |
-| :--------         | :-----------: | :--------: | :--------: | :------: | :-------------: |
-| bilibili          | ✅            | ✅         | ✅         | ➖         | ➖              |
-| bluesky           | ✅            | ✅         | ✅         | ➖         | ➖              |
-| dailymotion       | ✅            | ✅         | ✅         | ✅         | ✅              |
-| instagram         | ✅            | ✅         | ✅         | ➖         | ➖              |
-| facebook          | ✅            | ❌         | ✅         | ➖         | ➖              |
-| loom              | ✅            | ❌         | ✅         | ✅         | ➖              |
-| ok.ru             | ✅            | ❌         | ✅         | ✅         | ✅              |
-| pinterest         | ✅            | ✅         | ✅         | ➖         | ➖              |
-| reddit            | ✅            | ✅         | ✅         | ❌         | ❌              |
-| rutube            | ✅            | ✅         | ✅         | ✅         | ✅              |
-| snapchat          | ✅            | ✅         | ✅         | ➖         | ➖              |
-| soundcloud        | ➖            | ✅         | ➖         | ✅         | ✅              |
-| streamable        | ✅            | ✅         | ✅         | ➖         | ➖              |
-| tiktok            | ✅            | ✅         | ✅         | ❌         | ❌              |
-| tumblr            | ✅            | ✅         | ✅         | ➖         | ➖              |
-| twitch clips      | ✅            | ✅         | ✅         | ✅         | ✅              |
-| twitter/x         | ✅            | ✅         | ✅         | ➖         | ➖              |
-| vimeo             | ✅            | ✅         | ✅         | ✅         | ✅              |
-| vine              | ✅            | ✅         | ✅         | ➖         | ➖              |
-| vk videos & clips | ✅            | ❌         | ✅         | ✅         | ✅              |
-| youtube           | ✅            | ✅         | ✅         | ✅         | ✅              |
+it also includes documentation in the [docs tree](/docs/):
+- [cobalt api documentation](/docs/api.md)
+- [how to run a cobalt instance](/docs/run-an-instance.md)
+- [how to protect a cobalt instance](/docs/protect-an-instance.md) (recommended if you host a public instance)
 
-| emoji   | meaning                 |
-| :-----: | :---------------------- |
-| ✅      | supported               |
-| ➖      | impossible/unreasonable |
-| ❌      | not supported           |
+### thank you
+cobalt is sponsored by [royalehosting.net](https://royalehosting.net/?partner=cobalt). a part of our infrastructure is hosted on their network. we really appreciate their kindness and support!
 
-### additional notes or features (per service)
-| service    | notes or features                                                                                                    |
-| :--------  | :-----                                                                                                               |
-| instagram  | supports reels, photos, and videos. lets you pick what to save from multi-media posts.                               |
-| facebook   | supports public accessible videos content only.                                                                      |
-| pinterest  | supports photos, gifs, videos and stories.                                                                           |
-| reddit     | supports gifs and videos.                                                                                            |
-| snapchat   | supports spotlights and stories. lets you pick what to save from stories.                                            |
-| rutube     | supports yappy & private links.                                                                                      |
-| soundcloud | supports private links.                                                                                              |
-| tiktok     | supports videos with or without watermark, images from slideshow without watermark, and full (original) audios.      |
-| twitter/x  | lets you pick what to save from multi-media posts. may not be 100% reliable due to current management.               |
-| vimeo      | audio downloads are only available for dash.                                                                         |
-| youtube    | supports videos, music, and shorts. 8K, 4K, HDR, VR, and high FPS videos. rich metadata & dubs. h264/av1/vp9 codecs. |
+### ethics
+cobalt is a tool that makes downloading public content easier. it takes **zero liability**.
+the end user is responsible for what they download, how they use and distribute that content.
+cobalt never caches any content, it [works like a fancy proxy](/api/src/stream/).
 
-### partners
-cobalt is sponsored by [royalehosting.net](https://royalehosting.net/?partner=cobalt), all main instances are currently hosted on their network :)
+cobalt is in no way a piracy tool and cannot be used as such.
+it can only download free & publicly accessible content.
+same content can be downloaded via dev tools of any modern web browser.
 
-### ethics and disclaimer
-cobalt is a tool for easing content downloads from internet and takes ***zero liability***. you are responsible for what you download, how you use and distribute that content. please be mindful when using content of others and always credit original creators. fair use and credits benefit everyone.
+### contributing
+if you're considering contributing to cobalt, first of all, thank you! check the [contribution guidelines here](/CONTRIBUTING.md) before getting started, they'll help you do your best right away.
 
-cobalt is ***NOT*** a piracy tool and cannot be used as such. it can only download free, publicly accessible content. such content can be easily downloaded through any browser's dev tools. pressing one button is easier, so i made a convenient, ad-less tool for such repeated actions.
-
-### cobalt license
+### licenses
 for relevant licensing information, see the [api](api/README.md) and [web](web/README.md) READMEs.
 unless specified otherwise, the remainder of this repository is licensed under [AGPL-3.0](LICENSE).
-
-## acknowledgements
-### ffmpeg
-cobalt heavily relies on ffmpeg for converting and merging media files. it's an absolutely amazing piece of software offered for anyone for free, yet doesn't receive as much credit as it should.
-
-you can [support ffmpeg here](https://ffmpeg.org/donations.html)!
-
-#### ffmpeg-static
-we use [ffmpeg-static](https://github.com/eugeneware/ffmpeg-static) to get binaries for ffmpeg depending on the platform.
-
-you can support the developer via various methods listed on their github page! (linked above)
-
-### youtube.js
-cobalt relies on [youtube.js](https://github.com/LuanRT/YouTube.js) for interacting with the innertube api, it wouldn't have been possible without it.
-
-you can support the developer via various methods listed on their github page! (linked above)
-
-### many others
-cobalt also depends on:
-
-- [content-disposition-header](https://www.npmjs.com/package/content-disposition-header) to simplify the provision of `content-disposition` headers.
-- [cors](https://www.npmjs.com/package/cors) to manage cross-origin resource sharing within expressjs.
-- [dotenv](https://www.npmjs.com/package/dotenv) to load environment variables from the `.env` file.
-- [esbuild](https://www.npmjs.com/package/esbuild) to minify the frontend files.
-- [express](https://www.npmjs.com/package/express) as the backbone of cobalt servers.
-- [express-rate-limit](https://www.npmjs.com/package/express-rate-limit) to rate limit api endpoints.
-- [hls-parser](https://www.npmjs.com/package/hls-parser) to parse `m3u8` playlists for certain services.
-- [ipaddr.js](https://www.npmjs.com/package/ipaddr.js) to parse ip addresses (for rate limiting).
-- [nanoid](https://www.npmjs.com/package/nanoid) to generate unique (temporary) identifiers for each requested stream.
-- [node-cache](https://www.npmjs.com/package/node-cache) to cache stream info in server ram for a limited amount of time.
-- [psl](https://www.npmjs.com/package/psl) as the domain name parser.
-- [set-cookie-parser](https://www.npmjs.com/package/set-cookie-parser) to parse cookies that cobalt receives from certain services.
-- [undici](https://www.npmjs.com/package/undici) for making http requests.
-- [url-pattern](https://www.npmjs.com/package/url-pattern) to match provided links with supported patterns.
-
-...and many other packages that these packages rely on.

api/README.md

@@ -1,4 +1,64 @@
 # cobalt api
+this directory includes the source code for cobalt api. it's made with [express.js](https://www.npmjs.com/package/express) and love!
+
+## running your own instance
+if you want to run your own instance for whatever purpose, [follow this guide](/docs/run-an-instance.md).
+we recommend to use docker compose unless you intend to run cobalt for developing/debugging purposes.
+
+## accessing the api
+there is currently no publicly available pre-hosted api.
+we recommend [deploying your own instance](/docs/run-an-instance.md) if you wish to use the cobalt api.
+
+you can read [the api documentation here](/docs/api.md).
+
+## supported services
+this list is not final and keeps expanding over time!
+if the desired service isn't supported yet, feel free to create an appropriate issue (or a pull request 👀).
+
+| service           | video + audio | only audio | only video | metadata | rich file names |
+| :--------         | :-----------: | :--------: | :--------: | :------: | :-------------: |
+| bilibili          | ✅            | ✅         | ✅         | ➖         | ➖              |
+| bluesky           | ✅            | ✅         | ✅         | ➖         | ➖              |
+| dailymotion       | ✅            | ✅         | ✅         | ✅         | ✅              |
+| instagram         | ✅            | ✅         | ✅         | ➖         | ➖              |
+| facebook          | ✅            | ❌         | ✅         | ➖         | ➖              |
+| loom              | ✅            | ❌         | ✅         | ✅         | ➖              |
+| ok.ru             | ✅            | ❌         | ✅         | ✅         | ✅              |
+| pinterest         | ✅            | ✅         | ✅         | ➖         | ➖              |
+| reddit            | ✅            | ✅         | ✅         | ❌         | ❌              |
+| rutube            | ✅            | ✅         | ✅         | ✅         | ✅              |
+| snapchat          | ✅            | ✅         | ✅         | ➖         | ➖              |
+| soundcloud        | ➖            | ✅         | ➖         | ✅         | ✅              |
+| streamable        | ✅            | ✅         | ✅         | ➖         | ➖              |
+| tiktok            | ✅            | ✅         | ✅         | ❌         | ❌              |
+| tumblr            | ✅            | ✅         | ✅         | ➖         | ➖              |
+| twitch clips      | ✅            | ✅         | ✅         | ✅         | ✅              |
+| twitter/x         | ✅            | ✅         | ✅         | ➖         | ➖              |
+| vimeo             | ✅            | ✅         | ✅         | ✅         | ✅              |
+| vk videos & clips | ✅            | ❌         | ✅         | ✅         | ✅              |
+| xiaohongshu       | ✅            | ✅         | ✅         | ➖         | ➖              |
+| youtube           | ✅            | ✅         | ✅         | ✅         | ✅              |
+
+| emoji   | meaning                 |
+| :-----: | :---------------------- |
+| ✅      | supported               |
+| ➖      | unreasonable/impossible |
+| ❌      | not supported           |
+
+### additional notes or features (per service)
+| service    | notes or features                                                                                                    |
+| :--------  | :-----                                                                                                               |
+| instagram  | supports reels, photos, and videos. lets you pick what to save from multi-media posts.                               |
+| facebook   | supports public accessible videos content only.                                                                      |
+| pinterest  | supports photos, gifs, videos and stories.                                                                           |
+| reddit     | supports gifs and videos.                                                                                            |
+| snapchat   | supports spotlights and stories. lets you pick what to save from stories.                                            |
+| rutube     | supports yappy & private links.                                                                                      |
+| soundcloud | supports private links.                                                                                              |
+| tiktok     | supports videos with or without watermark, images from slideshow without watermark, and full (original) audios.      |
+| twitter/x  | lets you pick what to save from multi-media posts. may not be 100% reliable due to current management.               |
+| vimeo      | audio downloads are only available for dash.                                                                         |
+| youtube    | supports videos, music, and shorts. 8K, 4K, HDR, VR, and high FPS videos. rich metadata & dubs. h264/av1/vp9 codecs. |
 
 ## license
 cobalt api code is licensed under [AGPL-3.0](LICENSE).
@@ -9,14 +69,35 @@ as long as you:
 - provide a link to the license and indicate if changes to the code were made, and
 - release the code under the **same license**
 
-## running your own instance
-if you want to run your own instance for whatever purpose, [follow this guide](/docs/run-an-instance.md).
-it's *highly* recommended to use a docker compose method unless you run for developing/debugging purposes.
+## open source acknowledgements
+### ffmpeg
+cobalt relies on ffmpeg for muxing and encoding media files. ffmpeg is absolutely spectacular and we're privileged to have an ability to use it for free, just like anyone else. we believe it should be way more recognized.
 
-## accessing the api
-currently, there is no publicly accessible main api. we plan on providing a public api for
-cobalt 10 in some form in the future. we recommend deploying your own instance if you wish
-to use the latest api. you can access [the documentation](/docs/api.md) for it here.
+you can [support ffmpeg here](https://ffmpeg.org/donations.html)!
 
-if you are looking for the documentation for the old (7.x) api, you can find
-it [here](https://github.com/imputnet/cobalt/blob/7/docs/api.md)
+### youtube.js
+cobalt relies on **[youtube.js](https://github.com/LuanRT/YouTube.js)** for interacting with youtube's innertube api, it wouldn't have been possible without this package.
+
+you can support the developer via various methods listed on their github page!
+(linked above)
+
+### many others
+cobalt-api also depends on:
+
+- **[content-disposition-header](https://www.npmjs.com/package/content-disposition-header)** to simplify the provision of `content-disposition` headers.
+- **[cors](https://www.npmjs.com/package/cors)** to manage cross-origin resource sharing within expressjs.
+- **[dotenv](https://www.npmjs.com/package/dotenv)** to load environment variables from the `.env` file.
+- **[express](https://www.npmjs.com/package/express)** as the backbone of cobalt servers.
+- **[express-rate-limit](https://www.npmjs.com/package/express-rate-limit)** to rate limit api endpoints.
+- **[ffmpeg-static](https://www.npmjs.com/package/ffmpeg-static)** to get binaries for ffmpeg depending on the platform.
+- **[hls-parser](https://www.npmjs.com/package/hls-parser)** to parse HLS playlists according to spec (very impressive stuff).
+- **[ipaddr.js](https://www.npmjs.com/package/ipaddr.js)** to parse ip addresses (used for rate limiting).
+- **[nanoid](https://www.npmjs.com/package/nanoid)** to generate unique identifiers for each requested tunnel.
+- **[set-cookie-parser](https://www.npmjs.com/package/set-cookie-parser)** to parse cookies that cobalt receives from certain services.
+- **[undici](https://www.npmjs.com/package/undici)** for making http requests.
+- **[url-pattern](https://www.npmjs.com/package/url-pattern)** to match provided links with supported patterns.
+- **[zod](https://www.npmjs.com/package/zod)** to lock down the api request schema.
+- **[@datastructures-js/priority-queue](https://www.npmjs.com/package/@datastructures-js/priority-queue)** for sorting stream caches for future clean up (without redis).
+- **[@imput/psl](https://www.npmjs.com/package/@imput/psl)** as the domain name parser, our fork of [psl](https://www.npmjs.com/package/psl).
+
+...and many other packages that these packages rely on.

api/package.json

@@ -1,7 +1,7 @@
 {
     "name": "@imput/cobalt-api",
     "description": "save what you love",
-    "version": "10.0.0",
+    "version": "10.8.2",
     "author": "imput",
     "exports": "./src/cobalt.js",
     "type": "module",
@@ -10,9 +10,8 @@
     },
     "scripts": {
         "start": "node src/cobalt",
-        "setup": "node src/util/setup",
         "test": "node src/util/test",
-        "token:youtube": "node src/util/generate-youtube-tokens"
+        "token:jwt": "node src/util/generate-jwt-secret"
     },
     "repository": {
         "type": "git",
@@ -24,26 +23,27 @@
     },
     "homepage": "https://github.com/imputnet/cobalt#readme",
     "dependencies": {
+        "@datastructures-js/priority-queue": "^6.3.1",
+        "@imput/psl": "^2.0.4",
         "@imput/version-info": "workspace:^",
         "content-disposition-header": "0.6.0",
         "cors": "^2.8.5",
         "dotenv": "^16.0.1",
-        "esbuild": "^0.14.51",
-        "express": "^4.18.1",
-        "express-rate-limit": "^6.3.0",
+        "express": "^4.21.2",
+        "express-rate-limit": "^7.4.1",
         "ffmpeg-static": "^5.1.0",
         "hls-parser": "^0.10.7",
-        "ipaddr.js": "2.1.0",
-        "nanoid": "^4.0.2",
-        "node-cache": "^5.1.2",
-        "psl": "1.9.0",
+        "ipaddr.js": "2.2.0",
+        "nanoid": "^5.0.9",
         "set-cookie-parser": "2.6.0",
         "undici": "^5.19.1",
         "url-pattern": "1.0.3",
-        "youtubei.js": "^10.3.0",
+        "youtubei.js": "^13.2.0",
         "zod": "^3.23.8"
     },
     "optionalDependencies": {
-        "freebind": "^0.2.2"
+        "freebind": "^0.2.2",
+        "rate-limit-redis": "^4.2.0",
+        "redis": "^4.7.0"
     }
 }

api/src/cobalt.js

@@ -1,27 +1,32 @@
 import "dotenv/config";
 
 import express from "express";
+import cluster from "node:cluster";
 
-import path from 'path';
-import { fileURLToPath } from 'url';
+import path from "path";
+import { fileURLToPath } from "url";
 
-import { env } from "./config.js"
-import { Bright, Green, Red } from "./misc/console-text.js";
+import { env, isCluster } from "./config.js"
+import { Red } from "./misc/console-text.js";
+import { initCluster } from "./misc/cluster.js";
 
 const app = express();
 
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename).slice(0, -4);
 
-app.disable('x-powered-by');
+app.disable("x-powered-by");
 
 if (env.apiURL) {
-    const { runAPI } = await import('./core/api.js');
-    runAPI(express, app, __dirname)
+    const { runAPI } = await import("./core/api.js");
+
+    if (isCluster) {
+       await initCluster();
+    }
+
+    runAPI(express, app, __dirname, cluster.isPrimary);
 } else {
     console.log(
-        Red(`cobalt wasn't configured yet or configuration is invalid.\n`)
-        + Bright(`please run the setup script to fix this: `)
-        + Green(`npm run setup`)
+        Red("API_URL env variable is missing, cobalt api can't start.")
     )
 }

api/src/config.js

@@ -1,5 +1,7 @@
+import { Constants } from "youtubei.js";
 import { getVersion } from "@imput/version-info";
 import { services } from "./processing/service-config.js";
+import { supportsReusePort } from "./misc/cluster.js";
 
 const version = await getVersion();
 
@@ -13,6 +15,7 @@ const enabledServices = new Set(Object.keys(services).filter(e => {
 const env = {
     apiURL: process.env.API_URL || '',
     apiPort: process.env.API_PORT || 9000,
+    tunnelPort: process.env.API_PORT || 9000,
 
     listenAddress: process.env.API_LISTEN_ADDRESS,
     freebindCIDR: process.platform === 'linux' && process.env.FREEBIND_CIDR,
@@ -26,7 +29,7 @@ const env = {
     rateLimitMax: (process.env.RATELIMIT_MAX && parseInt(process.env.RATELIMIT_MAX)) || 20,
 
     durationLimit: (process.env.DURATION_LIMIT && parseInt(process.env.DURATION_LIMIT)) || 10800,
-    streamLifespan: 90,
+    streamLifespan: (process.env.TUNNEL_LIFESPAN && parseInt(process.env.TUNNEL_LIFESPAN)) || 90,
 
     processingPriority: process.platform !== 'win32'
         && process.env.PROCESSING_PRIORITY
@@ -34,16 +37,55 @@ const env = {
 
     externalProxy: process.env.API_EXTERNAL_PROXY,
 
+    turnstileSitekey: process.env.TURNSTILE_SITEKEY,
     turnstileSecret: process.env.TURNSTILE_SECRET,
     jwtSecret: process.env.JWT_SECRET,
     jwtLifetime: process.env.JWT_EXPIRY || 120,
 
+    sessionEnabled: process.env.TURNSTILE_SITEKEY
+                        && process.env.TURNSTILE_SECRET
+                        && process.env.JWT_SECRET,
+
+    apiKeyURL: process.env.API_KEY_URL && new URL(process.env.API_KEY_URL),
+    authRequired: process.env.API_AUTH_REQUIRED === '1',
+    redisURL: process.env.API_REDIS_URL,
+    instanceCount: (process.env.API_INSTANCE_COUNT && parseInt(process.env.API_INSTANCE_COUNT)) || 1,
+    keyReloadInterval: 900,
+
     enabledServices,
+
+    customInnertubeClient: process.env.CUSTOM_INNERTUBE_CLIENT,
+    ytSessionServer: process.env.YOUTUBE_SESSION_SERVER,
+    ytSessionReloadInterval: 300,
+    ytSessionInnertubeClient: process.env.YOUTUBE_SESSION_INNERTUBE_CLIENT,
 }
 
 const genericUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36";
 const cobaltUserAgent = `cobalt/${version} (+https://github.com/imputnet/cobalt)`;
 
+export const setTunnelPort = (port) => env.tunnelPort = port;
+export const isCluster = env.instanceCount > 1;
+
+if (env.sessionEnabled && env.jwtSecret.length < 16) {
+    throw new Error("JWT_SECRET env is too short (must be at least 16 characters long)");
+}
+
+if (env.instanceCount > 1 && !env.redisURL) {
+    throw new Error("API_REDIS_URL is required when API_INSTANCE_COUNT is >= 2");
+} else if (env.instanceCount > 1 && !await supportsReusePort()) {
+    console.error('API_INSTANCE_COUNT is not supported in your environment. to use this env, your node.js');
+    console.error('version must be >= 23.1.0, and you must be running a recent enough version of linux');
+    console.error('(or other OS that supports it). for more info, see `reusePort` option on');
+    console.error('https://nodejs.org/api/net.html#serverlistenoptions-callback');
+    throw new Error('SO_REUSEPORT is not supported');
+}
+
+if (env.customInnertubeClient && !Constants.SUPPORTED_CLIENTS.includes(env.customInnertubeClient)) {
+    console.error("CUSTOM_INNERTUBE_CLIENT is invalid. Provided client is not supported.");
+    console.error(`Supported clients are: ${Constants.SUPPORTED_CLIENTS.join(', ')}\n`);
+    throw new Error("Invalid CUSTOM_INNERTUBE_CLIENT");
+}
+
 export {
     env,
     genericUserAgent,

api/src/core/api.js

@@ -1,4 +1,5 @@
 import cors from "cors";
+import http from "node:http";
 import rateLimit from "express-rate-limit";
 import { setGlobalDispatcher, ProxyAgent } from "undici";
 import { getCommit, getBranch, getRemote, getVersion } from "@imput/version-info";
@@ -7,17 +8,21 @@ import jwt from "../security/jwt.js";
 import stream from "../stream/stream.js";
 import match from "../processing/match.js";
 
-import { env } from "../config.js";
+import { env, isCluster, setTunnelPort } from "../config.js";
 import { extract } from "../processing/url.js";
-import { languageCode } from "../misc/utils.js";
-import { Bright, Cyan } from "../misc/console-text.js";
-import { generateHmac, generateSalt } from "../misc/crypto.js";
+import { Green, Bright, Cyan } from "../misc/console-text.js";
+import { hashHmac } from "../security/secrets.js";
+import { createStore } from "../store/redis-ratelimit.js";
 import { randomizeCiphers } from "../misc/randomize-ciphers.js";
 import { verifyTurnstileToken } from "../security/turnstile.js";
 import { friendlyServiceName } from "../processing/service-alias.js";
 import { verifyStream, getInternalStream } from "../stream/manage.js";
 import { createResponse, normalizeRequest, getIP } from "../processing/request.js";
 
+import * as APIKeys from "../security/api-keys.js";
+import * as Cookies from "../processing/cookie/manager.js";
+import * as YouTubeSession from "../processing/helpers/youtube-session.js";
+
 const git = {
     branch: await getBranch(),
     commit: await getCommit(),
@@ -28,7 +33,6 @@ const version = await getVersion();
 
 const acceptRegex = /^application\/json(; charset=utf-8)?$/;
 
-const ipSalt = generateSalt();
 const corsConfig = env.corsWildcard ? {} : {
     origin: env.corsURL,
     optionsSuccessStatus: 200
@@ -39,7 +43,7 @@ const fail = (res, code, context) => {
     res.status(status).json(body);
 }
 
-export const runAPI = (express, app, __dirname) => {
+export const runAPI = async (express, app, __dirname, isPrimary = true) => {
     const startTime = new Date();
     const startTimestamp = startTime.getTime();
 
@@ -49,6 +53,7 @@ export const runAPI = (express, app, __dirname) => {
             url: env.apiURL,
             startTime: `${startTimestamp}`,
             durationLimit: env.durationLimit,
+            turnstileSitekey: env.sessionEnabled ? env.turnstileSitekey : undefined,
             services: [...env.enabledServices].map(e => {
                 return friendlyServiceName(e);
             }),
@@ -56,35 +61,46 @@ export const runAPI = (express, app, __dirname) => {
         git,
     })
 
+    const handleRateExceeded = (_, res) => {
+        const { status, body } = createResponse("error", {
+            code: "error.api.rate_exceeded",
+            context: {
+                limit: env.rateLimitWindow
+            }
+        });
+        return res.status(status).json(body);
+    };
+
+    const keyGenerator = (req) => hashHmac(getIP(req), 'rate').toString('base64url');
+
+    const sessionLimiter = rateLimit({
+        windowMs: 60000,
+        limit: 10,
+        standardHeaders: 'draft-6',
+        legacyHeaders: false,
+        keyGenerator,
+        store: await createStore('session'),
+        handler: handleRateExceeded
+    });
+
     const apiLimiter = rateLimit({
         windowMs: env.rateLimitWindow * 1000,
-        max: env.rateLimitMax,
-        standardHeaders: true,
+        limit: (req) => req.rateLimitMax || env.rateLimitMax,
+        standardHeaders: 'draft-6',
         legacyHeaders: false,
-        keyGenerator: req => {
-            if (req.authorized) {
-                return generateHmac(req.header("Authorization"), ipSalt);
-            }
-            return generateHmac(getIP(req), ipSalt);
-        },
-        handler: (req, res) => {
-            const { status, body } = createResponse("error", {
-                code: "error.api.rate_exceeded",
-                context: {
-                    limit: env.rateLimitWindow
-                }
-            });
-            return res.status(status).json(body);
-        }
+        keyGenerator: req => req.rateLimitKey || keyGenerator(req),
+        store: await createStore('api'),
+        handler: handleRateExceeded
     })
 
-    const apiLimiterStream = rateLimit({
+    const apiTunnelLimiter = rateLimit({
         windowMs: env.rateLimitWindow * 1000,
-        max: env.rateLimitMax,
-        standardHeaders: true,
+        limit: (req) => req.rateLimitMax || env.rateLimitMax,
+        standardHeaders: 'draft-6',
         legacyHeaders: false,
-        keyGenerator: req => generateHmac(getIP(req), ipSalt),
-        handler: (req, res) => {
+        keyGenerator: req => req.rateLimitKey || keyGenerator(req),
+        store: await createStore('tunnel'),
+        handler: (_, res) => {
             return res.sendStatus(429)
         }
     })
@@ -102,11 +118,45 @@ export const runAPI = (express, app, __dirname) => {
         ...corsConfig,
     }));
 
-    app.post('/', apiLimiter);
-    app.use('/tunnel', apiLimiterStream);
+    app.post('/', (req, res, next) => {
+        if (!acceptRegex.test(req.header('Accept'))) {
+            return fail(res, "error.api.header.accept");
+        }
+        if (!acceptRegex.test(req.header('Content-Type'))) {
+            return fail(res, "error.api.header.content_type");
+        }
+        next();
+    });
 
     app.post('/', (req, res, next) => {
-        if (!env.turnstileSecret || !env.jwtSecret) {
+        if (!env.apiKeyURL) {
+            return next();
+        }
+
+        const { success, error } = APIKeys.validateAuthorization(req);
+        if (!success) {
+            // We call next() here if either if:
+            // a) we have user sessions enabled, meaning the request
+            //    will still need a Bearer token to not be rejected, or
+            // b) we do not require the user to be authenticated, and
+            //    so they can just make the request with the regular
+            //    rate limit configuration;
+            // otherwise, we reject the request.
+            if (
+                (env.sessionEnabled || !env.authRequired)
+                && ['missing', 'not_api_key'].includes(error)
+            ) {
+                return next();
+            }
+
+            return fail(res, `error.api.auth.key.${error}`);
+        }
+
+        return next();
+    });
+
+    app.post('/', (req, res, next) => {
+        if (!env.sessionEnabled || req.rateLimitKey) {
             return next();
         }
 
@@ -116,34 +166,29 @@ export const runAPI = (express, app, __dirname) => {
                 return fail(res, "error.api.auth.jwt.missing");
             }
 
-            if (!authorization.startsWith("Bearer ") || authorization.length > 256) {
+            if (authorization.length >= 256) {
                 return fail(res, "error.api.auth.jwt.invalid");
             }
 
-            const verifyJwt = jwt.verify(
-                authorization.split("Bearer ", 2)[1]
-            );
-
-            if (!verifyJwt) {
+            const [ type, token, ...rest ] = authorization.split(" ");
+            if (!token || type.toLowerCase() !== 'bearer' || rest.length) {
                 return fail(res, "error.api.auth.jwt.invalid");
             }
 
-            if (!acceptRegex.test(req.header('Accept'))) {
-                return fail(res, "error.api.header.accept");
+            if (!jwt.verify(token)) {
+                return fail(res, "error.api.auth.jwt.invalid");
             }
 
-            if (!acceptRegex.test(req.header('Content-Type'))) {
-                return fail(res, "error.api.header.content_type");
-            }
-
-            req.authorized = true;
+            req.rateLimitKey = hashHmac(token, 'rate');
         } catch {
             return fail(res, "error.api.generic");
         }
         next();
     });
 
+    app.post('/', apiLimiter);
     app.use('/', express.json({ limit: 1024 }));
+
     app.use('/', (err, _, res, next) => {
         if (err) {
             const { status, body } = createResponse("error", {
@@ -155,8 +200,8 @@ export const runAPI = (express, app, __dirname) => {
         next();
     });
 
-    app.post("/session", async (req, res) => {
-        if (!env.turnstileSecret || !env.jwtSecret) {
+    app.post("/session", sessionLimiter, async (req, res) => {
+        if (!env.sessionEnabled) {
             return fail(res, "error.api.auth.not_configured")
         }
 
@@ -184,16 +229,11 @@ export const runAPI = (express, app, __dirname) => {
 
     app.post('/', async (req, res) => {
         const request = req.body;
-        const lang = languageCode(req);
 
         if (!request.url) {
             return fail(res, "error.api.link.missing");
         }
 
-        if (request.youtubeDubBrowserLang) {
-            request.youtubeDubLang = lang;
-        }
-
         const { success, data: normalizedRequest } = await normalizeRequest(request);
         if (!success) {
             return fail(res, "error.api.invalid_body");
@@ -225,7 +265,7 @@ export const runAPI = (express, app, __dirname) => {
         }
     })
 
-    app.get('/tunnel', (req, res) => {
+    app.get('/tunnel', apiTunnelLimiter, async (req, res) => {
         const id = String(req.query.id);
         const exp = String(req.query.exp);
         const sig = String(req.query.sig);
@@ -244,7 +284,7 @@ export const runAPI = (express, app, __dirname) => {
             return res.status(200).end();
         }
 
-        const streamInfo = verifyStream(id, sig, exp, sec, iv);
+        const streamInfo = await verifyStream(id, sig, exp, sec, iv);
         if (!streamInfo?.service) {
             return res.status(streamInfo.status).end();
         }
@@ -256,7 +296,7 @@ export const runAPI = (express, app, __dirname) => {
         return stream(res, streamInfo);
     })
 
-    app.get('/itunnel', (req, res) => {
+    const itunnelHandler = (req, res) => {
         if (!req.ip.endsWith('127.0.0.1')) {
             return res.sendStatus(403);
         }
@@ -275,8 +315,10 @@ export const runAPI = (express, app, __dirname) => {
             ...Object.entries(req.headers)
         ]);
 
-        return stream(res, { type: 'internal', ...streamInfo });
-    })
+        return stream(res, { type: 'internal', data: streamInfo });
+    };
+
+    app.get('/itunnel', itunnelHandler);
 
     app.get('/', (_, res) => {
         res.type('json');
@@ -307,20 +349,52 @@ export const runAPI = (express, app, __dirname) => {
         setGlobalDispatcher(new ProxyAgent(env.externalProxy))
     }
 
-    app.listen(env.apiPort, env.listenAddress, () => {
-        console.log(`\n` +
-            Bright(Cyan("cobalt ")) + Bright("API ^ω⁠^") + "\n" +
+    http.createServer(app).listen({
+        port: env.apiPort,
+        host: env.listenAddress,
+        reusePort: env.instanceCount > 1 || undefined
+    }, () => {
+        if (isPrimary) {
+            console.log(`\n` +
+                Bright(Cyan("cobalt ")) + Bright("API ^ω^") + "\n" +
 
-            "~~~~~~\n" +
-            Bright("version: ") + version + "\n" +
-            Bright("commit: ") + git.commit + "\n" +
-            Bright("branch: ") + git.branch + "\n" +
-            Bright("remote: ") + git.remote + "\n" +
-            Bright("start time: ") + startTime.toUTCString() + "\n" +
-            "~~~~~~\n" +
+                "~~~~~~\n" +
+                Bright("version: ") + version + "\n" +
+                Bright("commit: ") + git.commit + "\n" +
+                Bright("branch: ") + git.branch + "\n" +
+                Bright("remote: ") + git.remote + "\n" +
+                Bright("start time: ") + startTime.toUTCString() + "\n" +
+                "~~~~~~\n" +
 
-            Bright("url: ") + Bright(Cyan(env.apiURL)) + "\n" +
-            Bright("port: ") + env.apiPort + "\n"
-        )
-    })
+                Bright("url: ") + Bright(Cyan(env.apiURL)) + "\n" +
+                Bright("port: ") + env.apiPort + "\n"
+            );
+        }
+
+        if (env.apiKeyURL) {
+            APIKeys.setup(env.apiKeyURL);
+        }
+
+        if (env.cookiePath) {
+            Cookies.setup(env.cookiePath);
+        }
+
+        if (env.ytSessionServer) {
+            YouTubeSession.setup();
+        }
+    });
+
+    if (isCluster) {
+        const istreamer = express();
+        istreamer.get('/itunnel', itunnelHandler);
+        const server = istreamer.listen({
+            port: 0,
+            host: '127.0.0.1',
+            exclusive: true
+        }, () => {
+            const { port } = server.address();
+            console.log(`${Green('[✓]')} cobalt sub-instance running on 127.0.0.1:${port}`);
+            setTunnelPort(port);
+        });
+    }
 }

api/src/misc/cluster.js

@@ -0,0 +1,71 @@
+import cluster from "node:cluster";
+import net from "node:net";
+import { syncSecrets } from "../security/secrets.js";
+import { env, isCluster } from "../config.js";
+
+export { isPrimary, isWorker } from "node:cluster";
+
+export const supportsReusePort = async () => {
+    try {
+        await new Promise((resolve, reject) => {
+            const server = net.createServer().listen({ port: 0, reusePort: true });
+            server.on('listening', () => server.close(resolve));
+            server.on('error', (err) => (server.close(), reject(err)));
+        });
+
+        return true;
+    } catch {
+        return false;
+    }
+}
+
+export const initCluster = async () => {
+    if (cluster.isPrimary) {
+        for (let i = 1; i < env.instanceCount; ++i) {
+            cluster.fork();
+        }
+    }
+
+    await syncSecrets();
+}
+
+export const broadcast = (message) => {
+    if (!isCluster || !cluster.isPrimary || !cluster.workers) {
+        return;
+    }
+
+    for (const worker of Object.values(cluster.workers)) {
+        worker.send(message);
+    }
+}
+
+export const send = (message) => {
+    if (!isCluster) {
+        return;
+    }
+
+    if (cluster.isPrimary) {
+        return broadcast(message);
+    } else {
+        return process.send(message);
+    }
+}
+
+export const waitFor = (key) => {
+    return new Promise(resolve => {
+        const listener = (message) => {
+            if (key in message) {
+                process.off('message', listener);
+                return resolve(message);
+            }
+        }
+
+        process.on('message', listener);
+    });
+}
+
+export const mainOnMessage = (cb) => {
+    for (const worker of Object.values(cluster.workers)) {
+        worker.on('message', cb);
+    }
+}

api/src/misc/console-text.js

@@ -1,16 +1,36 @@
-function t(color, tt) {
-    return color + tt + "\x1b[0m"
+const ANSI = {
+    RESET: "\x1b[0m",
+    BRIGHT: "\x1b[1m",
+    RED: "\x1b[31m",
+    GREEN: "\x1b[32m",
+    CYAN: "\x1b[36m",
+    YELLOW: "\x1b[93m"
 }
 
-export function Bright(tt) {
-    return t("\x1b[1m", tt)
+function wrap(color, text) {
+    if (!ANSI[color.toUpperCase()]) {
+        throw "invalid color";
+    }
+
+    return ANSI[color.toUpperCase()] + text + ANSI.RESET;
 }
-export function Red(tt) {
-    return t("\x1b[31m", tt)
+
+export function Bright(text) {
+    return wrap('bright', text);
 }
-export function Green(tt) {
-    return t("\x1b[32m", tt)
+
+export function Red(text) {
+    return wrap('red', text);
 }
-export function Cyan(tt) {
-    return t("\x1b[36m", tt)
+
+export function Green(text) {
+    return wrap('green', text);
+}
+
+export function Cyan(text) {
+    return wrap('cyan', text);
+}
+
+export function Yellow(text) {
+    return wrap('yellow', text);
 }

api/src/misc/crypto.js

@@ -1,15 +1,7 @@
-import { createHmac, createCipheriv, createDecipheriv, randomBytes } from "crypto";
+import { createCipheriv, createDecipheriv } from "crypto";
 
 const algorithm = "aes256";
 
-export function generateSalt() {
-    return randomBytes(64).toString('hex');
-}
-
-export function generateHmac(str, salt) {
-    return createHmac("sha256", salt).update(str).digest("base64url");
-}
-
 export function encryptStream(plaintext, iv, secret) {
     const buff = Buffer.from(JSON.stringify(plaintext));
     const key = Buffer.from(secret, "base64url");

api/src/misc/run-test.js

@@ -23,6 +23,15 @@ export async function runTest(url, params, expect) {
     if (expect.status !== result.body.status) {
         const detail = `${expect.status} (expected) != ${result.body.status} (actual)`;
         error.push(`status mismatch: ${detail}`);
+
+        if (result.body.status === 'error') {
+            error.push(`error code: ${result.body?.error?.code}`);
+        }
+    }
+
+    if (expect.errorCode && expect.errorCode !== result.body?.error?.code) {
+        const detail = `${expect.errorCode} (expected) != ${result.body.error.code} (actual)`
+        error.push(`error mismatch: ${detail}`);
     }
 
     if (expect.code !== result.status) {
@@ -41,4 +50,4 @@ export async function runTest(url, params, expect) {
     if (result.body.status === 'tunnel') {
         // TODO: stream testing
     }
-}
+}

api/src/misc/utils.js

@@ -1,55 +1,18 @@
-const forbiddenCharsString = ['}', '{', '%', '>', '<', '^', ';', '`', '$', '"', "@", '='];
+import { request } from 'undici';
+const redirectStatuses = new Set([301, 302, 303, 307, 308]);
 
-export function metadataManager(obj) {
-    const keys = Object.keys(obj);
-    const tags = [
-        "album",
-        "copyright",
-        "title",
-        "artist",
-        "track",
-        "date"
-    ]
-    let commands = []
-
-    for (const i in keys) {
-        if (tags.includes(keys[i]))
-            commands.push('-metadata', `${keys[i]}=${obj[keys[i]]}`)
+export async function getRedirectingURL(url, dispatcher, userAgent) {
+    const location = await request(url, {
+        dispatcher,
+        method: 'HEAD',
+        headers: { 'user-agent': userAgent }
+    }).then(r => {
+        if (redirectStatuses.has(r.statusCode) && r.headers['location']) {
+            return r.headers['location'];
         }
-    return commands;
-}
-
-export function cleanString(string) {
-    for (const i in forbiddenCharsString) {
-        string = string.replaceAll("/", "_")
-                       .replaceAll(forbiddenCharsString[i], '')
-    }
-    return string;
-}
-export function verifyLanguageCode(code) {
-    const langCode = String(code.slice(0, 2).toLowerCase());
-    if (RegExp(/[a-z]{2}/).test(code)) {
-        return langCode
-    }
-    return "en"
-}
-export function languageCode(req) {
-    if (req.header('Accept-Language')) {
-        return verifyLanguageCode(req.header('Accept-Language'))
-    }
-    return "en"
-}
-export function cleanHTML(html) {
-    let clean = html.replace(/ {4}/g, '');
-    clean = clean.replace(/\n/g, '');
-    return clean
-}
-
-export function getRedirectingURL(url) {
-    return fetch(url, { redirect: 'manual' }).then((r) => {
-        if ([301, 302, 303].includes(r.status) && r.headers.has('location'))
-            return r.headers.get('location');
     }).catch(() => null);
+
+    return location;
 }
 
 export function merge(a, b) {
@@ -65,3 +28,18 @@ export function merge(a, b) {
 
     return a;
 }
+
+export function splitFilenameExtension(filename) {
+    const parts = filename.split('.');
+    const ext = parts.pop();
+
+    if (!parts.length) {
+        return [ ext, "" ]
+    } else {
+        return [ parts.join('.'), ext ]
+    }
+}
+
+export function zip(a, b) {
+    return a.map((value, i) => [ value, b[i] ]);
+}

api/src/processing/cookie/cookie.js

@@ -4,16 +4,24 @@ export default class Cookie {
     constructor(input) {
         assert(typeof input === 'object');
         this._values = {};
-        this.set(input)
+
+        for (const [ k, v ] of Object.entries(input))
+            this.set(k, v);
     }
-    set(values) {
-        Object.entries(values).forEach(
-            ([ key, value ]) => this._values[key] = value
-        )
+
+    set(key, value) {
+        const old = this._values[key];
+        if (old === value)
+            return false;
+
+        this._values[key] = value;
+        return true;
     }
+
     unset(keys) {
         for (const key of keys) delete this._values[key]
     }
+
     static fromString(str) {
         const obj = {};
 
@@ -25,12 +33,15 @@ export default class Cookie {
 
         return new Cookie(obj)
     }
+
     toString() {
         return Object.entries(this._values).map(([ name, value ]) => `${name}=${value}`).join('; ')
     }
+
     toJSON() {
         return this.toString()
     }
+
     values() {
         return Object.freeze({ ...this._values })
     }

api/src/processing/cookie/manager.js

@@ -1,50 +1,144 @@
 import Cookie from './cookie.js';
+
 import { readFile, writeFile } from 'fs/promises';
+import { Red, Green, Yellow } from '../../misc/console-text.js';
 import { parse as parseSetCookie, splitCookiesString } from 'set-cookie-parser';
-import { env } from '../../config.js';
+import * as cluster from '../../misc/cluster.js';
+import { isCluster } from '../../config.js';
 
-const WRITE_INTERVAL = 60000,
-      cookiePath = env.cookiePath,
-      COUNTER = Symbol('counter');
+const WRITE_INTERVAL = 60000;
+const VALID_SERVICES = new Set([
+    'instagram',
+    'instagram_bearer',
+    'reddit',
+    'twitter',
+    'youtube',
+]);
 
+const invalidCookies = {};
 let cookies = {}, dirty = false, intervalId;
 
-const setup = async () => {
-    try {
-        if (!cookiePath) return;
-
-        cookies = await readFile(cookiePath, 'utf8');
-        cookies = JSON.parse(cookies);
-        intervalId = setInterval(writeChanges, WRITE_INTERVAL)
-    } catch { /* no cookies for you */ }
-}
-
-setup();
-
-function writeChanges() {
+function writeChanges(cookiePath) {
     if (!dirty) return;
     dirty = false;
 
-    writeFile(cookiePath, JSON.stringify(cookies, null, 4)).catch(() => {
-        clearInterval(intervalId)
+    const cookieData = JSON.stringify({ ...cookies, ...invalidCookies }, null, 4);
+    writeFile(cookiePath, cookieData).catch((e) => {
+        console.warn(`${Yellow('[!]')} failed writing updated cookies to storage`);
+        console.warn(e);
+        clearInterval(intervalId);
+        intervalId = null;
     })
 }
 
-export function getCookie(service) {
-    if (!cookies[service] || !cookies[service].length) return;
+const setupMain = async (cookiePath) => {
+    try {
+        cookies = await readFile(cookiePath, 'utf8');
+        cookies = JSON.parse(cookies);
+        for (const serviceName in cookies) {
+            if (!VALID_SERVICES.has(serviceName)) {
+                console.warn(`${Yellow('[!]')} ignoring unknown service in cookie file: ${serviceName}`);
+            } else if (!Array.isArray(cookies[serviceName])) {
+                console.warn(`${Yellow('[!]')} ${serviceName} in cookies file is not an array, ignoring it`);
+            } else if (cookies[serviceName].some(c => typeof c !== 'string')) {
+                console.warn(`${Yellow('[!]')} some cookie for ${serviceName} contains non-string value in cookies file`);
+            } else continue;
 
-    let n;
-    if (cookies[service][COUNTER] === undefined) {
-        n = cookies[service][COUNTER] = 0
-    } else {
-        ++cookies[service][COUNTER]
-        n = (cookies[service][COUNTER] %= cookies[service].length)
+            invalidCookies[serviceName] = cookies[serviceName];
+            delete cookies[serviceName];
+        }
+
+        if (!intervalId) {
+            intervalId = setInterval(() => writeChanges(cookiePath), WRITE_INTERVAL);
+        }
+
+        cluster.broadcast({ cookies });
+
+        console.log(`${Green('[✓]')} cookies loaded successfully!`);
+    } catch (e) {
+        console.error(`${Yellow('[!]')} failed to load cookies.`);
+        console.error('error:', e);
+    }
+}
+
+const setupWorker = async () => {
+    cookies = (await cluster.waitFor('cookies')).cookies;
+}
+
+export const loadFromFile = async (path) => {
+    if (cluster.isPrimary) {
+        await setupMain(path);
+    } else if (cluster.isWorker) {
+        await setupWorker();
     }
 
-    const cookie = cookies[service][n];
-    if (typeof cookie === 'string') cookies[service][n] = Cookie.fromString(cookie);
+    dirty = false;
+}
 
-    return cookies[service][n]
+export const setup = async (path) => {
+    await loadFromFile(path);
+
+    if (isCluster) {
+        const messageHandler = (message) => {
+            if ('cookieUpdate' in message) {
+                const { cookieUpdate } = message;
+
+                if (cluster.isPrimary) {
+                    dirty = true;
+                    cluster.broadcast({ cookieUpdate });
+                }
+
+                const { service, idx, cookie } = cookieUpdate;
+                cookies[service][idx] = cookie;
+            }
+        }
+
+        if (cluster.isPrimary) {
+            cluster.mainOnMessage(messageHandler);
+        } else {
+            process.on('message', messageHandler);
+        }
+    }
+}
+
+export function getCookie(service) {
+    if (!VALID_SERVICES.has(service)) {
+        console.error(
+            `${Red('[!]')} ${service} not in allowed services list for cookies.`
+            + ' if adding a new cookie type, include it there.'
+        );
+        return;
+    }
+
+    if (!cookies[service] || !cookies[service].length) return;
+
+    const idx = Math.floor(Math.random() * cookies[service].length);
+
+    const cookie = cookies[service][idx];
+    if (typeof cookie === 'string') {
+        cookies[service][idx] = Cookie.fromString(cookie);
+    }
+
+    cookies[service][idx].meta = { service, idx };
+    return cookies[service][idx];
+}
+
+export function updateCookieValues(cookie, values) {
+    let changed = false;
+
+    for (const [ key, value ] of Object.entries(values)) {
+        changed = cookie.set(key, value) || changed;
+    }
+
+    if (changed && cookie.meta) {
+        dirty = true;
+        if (isCluster) {
+            const message = { cookieUpdate: { ...cookie.meta, cookie } };
+            cluster.send(message);
+        }
+    }
+
+    return changed;
 }
 
 export function updateCookie(cookie, headers) {
@@ -57,10 +151,6 @@ export function updateCookie(cookie, headers) {
 
     cookie.unset(parsed.filter(c => c.expires < new Date()).map(c => c.name));
     parsed.filter(c => !c.expires || c.expires > new Date()).forEach(c => values[c.name] = c.value);
+
     updateCookieValues(cookie, values);
 }
-
-export function updateCookieValues(cookie, values) {
-    cookie.set(values);
-    if (Object.keys(values).length) dirty = true
-}

api/src/processing/create-filename.js

@@ -1,3 +1,13 @@
+const illegalCharacters = ['}', '{', '%', '>', '<', '^', ';', ':', '`', '$', '"', "@", '=', '?', '|', '*'];
+
+const sanitizeString = (string) => {
+    for (const i in illegalCharacters) {
+        string = string.replaceAll("/", "_").replaceAll("\\", "_")
+                       .replaceAll(illegalCharacters[i], '')
+    }
+    return string;
+}
+
 export default (f, style, isAudioOnly, isAudioMuted) => {
     let filename = '';
 
@@ -5,7 +15,11 @@ export default (f, style, isAudioOnly, isAudioMuted) => {
     let classicTags = [...infoBase];
     let basicTags = [];
 
-    const title = `${f.title} - ${f.author}`;
+    let title = sanitizeString(f.title);
+
+    if (f.author) {
+        title += ` - ${sanitizeString(f.author)}`;
+    }
 
     if (f.resolution) {
         classicTags.push(f.resolution);

api/src/processing/helpers/youtube-session.js

@@ -0,0 +1,74 @@
+import * as cluster from "../../misc/cluster.js";
+
+import { env } from "../../config.js";
+import { Green, Yellow } from "../../misc/console-text.js";
+
+let session;
+
+const validateSession = (sessionResponse) => {
+    if (!sessionResponse.potoken) {
+        throw "no poToken in session response";
+    }
+
+    if (!sessionResponse.visitor_data) {
+        throw "no visitor_data in session response";
+    }
+
+    if (!sessionResponse.updated) {
+        throw "no last update timestamp in session response";
+    }
+
+    // https://github.com/iv-org/youtube-trusted-session-generator/blob/c2dfe3f/potoken_generator/main.py#L25
+    if (sessionResponse.potoken.length < 160) {
+        console.error(`${Yellow('[!]')} poToken is too short and might not work (${new Date().toISOString()})`);
+    }
+}
+
+const updateSession = (newSession) => {
+    session = newSession;
+}
+
+const loadSession = async () => {
+    const sessionServerUrl = new URL(env.ytSessionServer);
+    sessionServerUrl.pathname = "/token";
+
+    const newSession = await fetch(sessionServerUrl).then(a => a.json());
+    validateSession(newSession);
+
+    if (!session || session.updated < newSession?.updated) {
+        cluster.broadcast({ youtube_session: newSession });
+        updateSession(newSession);
+    }
+}
+
+const wrapLoad = (initial = false) => {
+    loadSession()
+    .then(() => {
+        if (initial) {
+            console.log(`${Green('[✓]')} poToken & visitor_data loaded successfully!`);
+        }
+    })
+    .catch((e) => {
+        console.error(`${Yellow('[!]')} Failed loading poToken & visitor_data at ${new Date().toISOString()}.`);
+        console.error('Error:', e);
+    })
+}
+
+export const getYouTubeSession = () => {
+    return session;
+}
+
+export const setup = () => {
+    if (cluster.isPrimary) {
+        wrapLoad(true);
+        if (env.ytSessionReloadInterval > 0) {
+            setInterval(wrapLoad, env.ytSessionReloadInterval * 1000);
+        }
+    } else if (cluster.isWorker) {
+        process.on('message', (message) => {
+            if ('youtube_session' in message) {
+                updateSession(message.youtube_session);
+            }
+        });
+    }
+}

api/src/processing/match-action.js

@@ -3,18 +3,20 @@ import createFilename from "./create-filename.js";
 import { createResponse } from "./request.js";
 import { audioIgnore } from "./service-config.js";
 import { createStream } from "../stream/manage.js";
+import { splitFilenameExtension } from "../misc/utils.js";
 
 export default function({ r, host, audioFormat, isAudioOnly, isAudioMuted, disableMetadata, filenameStyle, twitterGif, requestIP, audioBitrate, alwaysProxy }) {
     let action,
         responseType = "tunnel",
         defaultParams = {
-            u: r.urls,
+            url: r.urls,
             headers: r.headers,
             service: host,
             filename: r.filenameAttributes ?
                     createFilename(r.filenameAttributes, filenameStyle, isAudioOnly, isAudioMuted) : r.filename,
             fileMetadata: !disableMetadata ? r.fileMetadata : false,
-            requestIP
+            requestIP,
+            originalRequest: r.originalRequest
         },
         params = {};
 
@@ -23,7 +25,7 @@ export default function({ r, host, audioFormat, isAudioOnly, isAudioMuted, disab
     else if (r.isGif && twitterGif) action = "gif";
     else if (isAudioOnly) action = "audio";
     else if (isAudioMuted) action = "muteVideo";
-    else if (r.isM3U8) action = "m3u8";
+    else if (r.isHLS) action = "hls";
     else action = "video";
 
     if (action === "picker" || action === "audio") {
@@ -32,10 +34,11 @@ export default function({ r, host, audioFormat, isAudioOnly, isAudioMuted, disab
     }
 
     if (action === "muteVideo" && isAudioMuted && !r.filenameAttributes) {
-        const parts = r.filename.split(".");
-        const ext = parts.pop();
-
-        defaultParams.filename = `${parts.join(".")}_mute.${ext}`;
+        const [ name, ext ] = splitFilenameExtension(r.filename);
+        defaultParams.filename = `${name}_mute.${ext}`;
+    } else if (action === "gif") {
+        const [ name ] = splitFilenameExtension(r.filename);
+        defaultParams.filename = `${name}.gif`;
     }
 
     switch (action) {
@@ -45,27 +48,29 @@ export default function({ r, host, audioFormat, isAudioOnly, isAudioMuted, disab
             });
 
         case "photo":
-            responseType = "redirect";
+            params = { type: "proxy" };
             break;
 
         case "gif":
             params = { type: "gif" };
             break;
 
-        case "m3u8":
+        case "hls":
             params = {
-                type: Array.isArray(r.urls) ? "merge" : "remux"
+                type: Array.isArray(r.urls) ? "merge" : "remux",
+                isHLS: true,
             }
             break;
 
         case "muteVideo":
             let muteType = "mute";
-            if (Array.isArray(r.urls) && !r.isM3U8) {
+            if (Array.isArray(r.urls) && !r.isHLS) {
                 muteType = "proxy";
             }
             params = {
                 type: muteType,
-                u: Array.isArray(r.urls) ? r.urls[0] : r.urls
+                url: Array.isArray(r.urls) ? r.urls[0] : r.urls,
+                isHLS: r.isHLS
             }
             if (host === "reddit" && r.typeId === "redirect") {
                 responseType = "redirect";
@@ -79,6 +84,7 @@ export default function({ r, host, audioFormat, isAudioOnly, isAudioMuted, disab
                 case "twitter":
                 case "snapchat":
                 case "bsky":
+                case "xiaohongshu":
                     params = { picker: r.picker };
                     break;
 
@@ -90,14 +96,15 @@ export default function({ r, host, audioFormat, isAudioOnly, isAudioMuted, disab
                     }
                     params = {
                         picker: r.picker,
-                        u: createStream({
+                        url: createStream({
                             service: "tiktok",
                             type: audioStreamType,
-                            u: r.urls,
+                            url: r.urls,
                             headers: r.headers,
-                            filename: r.audioFilename,
+                            filename: `${r.audioFilename}.${audioFormat}`,
                             isAudioOnly: true,
                             audioFormat,
+                            audioBitrate
                         })
                     }
                     break;
@@ -135,13 +142,14 @@ export default function({ r, host, audioFormat, isAudioOnly, isAudioMuted, disab
                     }
                     break;
 
+                case "ok":
                 case "vk":
                 case "tiktok":
+                case "xiaohongshu":
                     params = { type: "proxy" };
                     break;
 
                 case "facebook":
-                case "vine":
                 case "instagram":
                 case "tumblr":
                 case "pinterest":
@@ -157,7 +165,7 @@ export default function({ r, host, audioFormat, isAudioOnly, isAudioMuted, disab
         case "audio":
             if (audioIgnore.includes(host) || (host === "reddit" && r.typeId === "redirect")) {
                 return createResponse("error", {
-                    code: "error.api.fetch.empty"
+                    code: "error.api.service.audio_not_supported"
                 })
             }
 
@@ -181,18 +189,20 @@ export default function({ r, host, audioFormat, isAudioOnly, isAudioMuted, disab
                 }
             }
 
-            if (r.isM3U8 || host === "vimeo") {
+            if (r.isHLS || host === "vimeo") {
                 copy = false;
                 processType = "audio";
             }
 
             params = {
                 type: processType,
-                u: Array.isArray(r.urls) ? r.urls[1] : r.urls,
+                url: Array.isArray(r.urls) ? r.urls[1] : r.urls,
 
                 audioBitrate,
                 audioCopy: copy,
                 audioFormat,
+
+                isHLS: r.isHLS,
             }
             break;
     }

api/src/processing/match.js

@@ -19,7 +19,6 @@ import tumblr from "./services/tumblr.js";
 import vimeo from "./services/vimeo.js";
 import soundcloud from "./services/soundcloud.js";
 import instagram from "./services/instagram.js";
-import vine from "./services/vine.js";
 import pinterest from "./services/pinterest.js";
 import streamable from "./services/streamable.js";
 import twitch from "./services/twitch.js";
@@ -29,6 +28,7 @@ import snapchat from "./services/snapchat.js";
 import loom from "./services/loom.js";
 import facebook from "./services/facebook.js";
 import bluesky from "./services/bluesky.js";
+import xiaohongshu from "./services/xiaohongshu.js";
 
 let freebind;
 
@@ -78,8 +78,9 @@ export default async function({ host, patternMatch, params }) {
 
             case "vk":
                 r = await vk({
-                    userId: patternMatch.userId,
+                    ownerId: patternMatch.ownerId,
                     videoId: patternMatch.videoId,
+                    accessKey: patternMatch.accessKey,
                     quality: params.videoQuality
                 });
                 break;
@@ -97,17 +98,18 @@ export default async function({ host, patternMatch, params }) {
 
             case "youtube":
                 let fetchInfo = {
+                    dispatcher,
                     id: patternMatch.id.slice(0, 11),
                     quality: params.videoQuality,
                     format: params.youtubeVideoCodec,
                     isAudioOnly,
                     isAudioMuted,
                     dubLang: params.youtubeDubLang,
-                    dispatcher
+                    youtubeHLS: params.youtubeHLS,
                 }
 
                 if (url.hostname === "music.youtube.com" || isAudioOnly) {
-                    fetchInfo.quality = "max";
+                    fetchInfo.quality = "1080";
                     fetchInfo.format = "vp9";
                     fetchInfo.isAudioOnly = true;
                     fetchInfo.isAudioMuted = false;
@@ -118,16 +120,15 @@ export default async function({ host, patternMatch, params }) {
 
             case "reddit":
                 r = await reddit({
-                    sub: patternMatch.sub,
-                    id: patternMatch.id,
-                    user: patternMatch.user
+                    ...patternMatch,
+                    dispatcher,
                 });
                 break;
 
             case "tiktok":
                 r = await tiktok({
                     postId: patternMatch.postId,
-                    id: patternMatch.id,
+                    shortLink: patternMatch.shortLink,
                     fullAudio: params.tiktokFullAudio,
                     isAudioOnly,
                     h265: params.tiktokH265,
@@ -174,12 +175,6 @@ export default async function({ host, patternMatch, params }) {
                 })
                 break;
 
-            case "vine":
-                r = await vine({
-                    id: patternMatch.id
-                });
-                break;
-
             case "pinterest":
                 r = await pinterest({
                     id: patternMatch.id,
@@ -232,14 +227,25 @@ export default async function({ host, patternMatch, params }) {
 
             case "facebook":
                 r = await facebook({
-                    ...patternMatch
+                    ...patternMatch,
+                    dispatcher
                 });
                 break;
 
             case "bsky":
                 r = await bluesky({
                     ...patternMatch,
-                    alwaysProxy: params.alwaysProxy
+                    alwaysProxy: params.alwaysProxy,
+                    dispatcher
+                });
+                break;
+
+            case "xiaohongshu":
+                r = await xiaohongshu({
+                    ...patternMatch,
+                    h265: params.tiktokH265,
+                    isAudioOnly,
+                    dispatcher,
                 });
                 break;
 

api/src/processing/request.js

@@ -37,7 +37,7 @@ export function createResponse(responseType, responseData) {
 
             case "redirect":
                 response = {
-                    url: responseData?.u,
+                    url: responseData?.url,
                     filename: responseData?.filename
                 }
                 break;
@@ -52,7 +52,7 @@ export function createResponse(responseType, responseData) {
             case "picker":
                 response = {
                     picker: responseData?.picker,
-                    audio: responseData?.u,
+                    audio: responseData?.url,
                     audioFilename: responseData?.filename
                 }
                 break;

api/src/processing/schema.js

@@ -1,7 +1,5 @@
 import { z } from "zod";
-
 import { normalizeURL } from "./url.js";
-import { verifyLanguageCode } from "../misc/utils.js";
 
 export const apiSchema = z.object({
     url: z.string()
@@ -33,15 +31,21 @@ export const apiSchema = z.object({
     ).default("1080"),
 
     youtubeDubLang: z.string()
-                     .length(2)
-                     .transform(verifyLanguageCode)
+                     .min(2)
+                     .max(8)
+                     .regex(/^[0-9a-zA-Z\-]+$/)
                      .optional(),
 
+    // TODO: remove this variable as it's no longer used
+    // and is kept for schema compatibility reasons
+    youtubeDubBrowserLang: z.boolean().default(false),
+
     alwaysProxy: z.boolean().default(false),
     disableMetadata: z.boolean().default(false),
     tiktokFullAudio: z.boolean().default(false),
     tiktokH265: z.boolean().default(false),
     twitterGif: z.boolean().default(true),
-    youtubeDubBrowserLang: z.boolean().default(false),
+
+    youtubeHLS: z.boolean().default(false),
 })
 .strict();

api/src/processing/service-config.js

@@ -1,7 +1,7 @@
 import UrlPattern from "url-pattern";
 
 export const audioIgnore = ["vk", "ok", "loom"];
-export const hlsExceptions = ["dailymotion", "vimeo", "rutube", "bsky"];
+export const hlsExceptions = ["dailymotion", "vimeo", "rutube", "bsky", "youtube"];
 
 export const services = {
     bilibili: {
@@ -30,23 +30,35 @@ export const services = {
             "reel/:id",
             "share/:shareType/:id"
         ],
-        subdomains: ["web"],
+        subdomains: ["web", "m"],
         altDomains: ["fb.watch"],
     },
     instagram: {
         patterns: [
-            "reels/:postId",
-            ":username/reel/:postId",
-            "reel/:postId",
             "p/:postId",
-            ":username/p/:postId",
             "tv/:postId",
-            "stories/:username/:storyId"
+            "reel/:postId",
+            "reels/:postId",
+            "stories/:username/:storyId",
+
+            /*
+                share & username links use the same url pattern,
+                so we test the share pattern first, cuz id type is different.
+                however, if someone has the "share" username and the user
+                somehow gets a link of this ancient style, it's joever.
+            */
+
+            "share/:shareId",
+            "share/p/:shareId",
+            "share/reel/:shareId",
+
+            ":username/p/:postId",
+            ":username/reel/:postId",
         ],
         altDomains: ["ddinstagram.com"],
     },
     loom: {
-        patterns: ["share/:id"],
+        patterns: ["share/:id", "embed/:id"],
     },
     ok: {
         patterns: [
@@ -64,8 +76,21 @@ export const services = {
     },
     reddit: {
         patterns: [
+            "comments/:id",
+
+            "r/:sub/comments/:id",
             "r/:sub/comments/:id/:title",
-            "user/:user/comments/:id/:title"
+            "r/:sub/comments/:id/comment/:commentId",
+
+            "user/:user/comments/:id",
+            "user/:user/comments/:id/:title",
+            "user/:user/comments/:id/comment/:commentId",
+
+            "r/u_:user/comments/:id",
+            "r/u_:user/comments/:id/:title",
+            "r/u_:user/comments/:id/comment/:commentId",
+
+            "r/:sub/s/:shareId"
         ],
         subdomains: "*",
     },
@@ -111,10 +136,10 @@ export const services = {
     tiktok: {
         patterns: [
             ":user/video/:postId",
-            ":id",
-            "t/:id",
+            ":shortLink",
+            "t/:shortLink",
             ":user/photo/:postId",
-            "v/:id.html"
+            "v/:postId.html"
         ],
         subdomains: ["vt", "vm", "m"],
     },
@@ -137,15 +162,12 @@ export const services = {
             ":user/status/:id/video/:index",
             ":user/status/:id/photo/:index",
             ":user/status/:id/mediaviewer",
-            ":user/status/:id/mediaViewer"
+            ":user/status/:id/mediaViewer",
+            "i/bookmarks?post_id=:id"
         ],
         subdomains: ["mobile"],
         altDomains: ["x.com", "vxtwitter.com", "fixvx.com"],
     },
-    vine: {
-        patterns: ["v/:id"],
-        tld: "co",
-    },
     vimeo: {
         patterns: [
             ":id",
@@ -157,11 +179,25 @@ export const services = {
     },
     vk: {
         patterns: [
-            "video:userId_:videoId",
-            "clip:userId_:videoId",
-            "clips:duplicate?z=clip:userId_:videoId"
+            "video:ownerId_:videoId",
+            "clip:ownerId_:videoId",
+            "clips:duplicate?z=clip:ownerId_:videoId",
+            "videos:duplicate?z=video:ownerId_:videoId",
+            "video:ownerId_:videoId_:accessKey",
+            "clip:ownerId_:videoId_:accessKey",
+            "clips:duplicate?z=clip:ownerId_:videoId_:accessKey",
+            "videos:duplicate?z=video:ownerId_:videoId_:accessKey"
         ],
         subdomains: ["m"],
+        altDomains: ["vkvideo.ru", "vk.ru"],
+    },
+    xiaohongshu: {
+        patterns: [
+            "explore/:id?xsec_token=:token",
+            "discovery/item/:id?xsec_token=:token",
+            "a/:shareId"
+        ],
+        altDomains: ["xhslink.com"],
     },
     youtube: {
         patterns: [
@@ -176,7 +212,7 @@ export const services = {
 Object.values(services).forEach(service => {
     service.patterns = service.patterns.map(
         pattern => new UrlPattern(pattern, {
-            segmentValueCharset: UrlPattern.defaultOptions.segmentValueCharset + '@\\.'
+            segmentValueCharset: UrlPattern.defaultOptions.segmentValueCharset + '@\\.:'
         })
     )
 })

api/src/processing/service-patterns.js

@@ -6,7 +6,8 @@ export const testers = {
     "dailymotion": pattern => pattern.id?.length <= 32,
 
     "instagram": pattern =>
-        pattern.postId?.length <= 12
+        pattern.postId?.length <= 48
+        || pattern.shareId?.length <= 16
         || (pattern.username?.length <= 30 && pattern.storyId?.length <= 24),
 
     "loom": pattern =>
@@ -19,8 +20,10 @@ export const testers = {
         pattern.id?.length <= 128 || pattern.shortLink?.length <= 32,
 
     "reddit": pattern =>
-        (pattern.sub?.length <= 22 && pattern.id?.length <= 10)
-        || (pattern.user?.length <= 22 && pattern.id?.length <= 10),
+        pattern.id?.length <= 16 && !pattern.sub && !pattern.user
+        || (pattern.sub?.length <= 22 && pattern.id?.length <= 16)
+        || (pattern.user?.length <= 22 && pattern.id?.length <= 16)
+        || (pattern.sub?.length <= 22 && pattern.shareId?.length <= 16),
 
     "rutube": pattern =>
         (pattern.id?.length === 32 && pattern.key?.length <= 32) ||
@@ -36,10 +39,10 @@ export const testers = {
         || pattern.shortLink?.length <= 16,
 
     "streamable": pattern =>
-        pattern.id?.length === 6,
+        pattern.id?.length <= 6,
 
     "tiktok": pattern =>
-        pattern.postId?.length <= 21 || pattern.id?.length <= 13,
+        pattern.postId?.length <= 21 || pattern.shortLink?.length <= 13,
 
     "tumblr": pattern =>
         pattern.id?.length < 21
@@ -55,11 +58,9 @@ export const testers = {
         pattern.id?.length <= 11
         && (!pattern.password || pattern.password.length < 16),
 
-    "vine": pattern =>
-        pattern.id?.length <= 12,
-
     "vk": pattern =>
-        pattern.userId?.length <= 10 && pattern.videoId?.length <= 10,
+        (pattern.ownerId?.length <= 10 && pattern.videoId?.length <= 10) ||
+        (pattern.ownerId?.length <= 10 && pattern.videoId?.length <= 10 && pattern.videoId?.accessKey <= 18),
 
     "youtube": pattern =>
         pattern.id?.length <= 11,
@@ -73,4 +74,8 @@ export const testers = {
 
     "bsky": pattern =>
         pattern.user?.length <= 128 && pattern.post?.length <= 128,
+
+    "xiaohongshu": pattern =>
+        pattern.id?.length <= 24 && pattern.token?.length <= 64
+        || pattern.shareId?.length <= 12,
 }

api/src/processing/services/bilibili.js

@@ -1,19 +1,8 @@
 import { genericUserAgent, env } from "../../config.js";
+import { resolveRedirectingURL } from "../url.js";
 
 // TO-DO: higher quality downloads (currently requires an account)
 
-function com_resolveShortlink(shortId) {
-    return fetch(`https://b23.tv/${shortId}`, { redirect: 'manual' })
-            .then(r => r.status > 300 && r.status < 400 && r.headers.get('location'))
-            .then(url => {
-                if (!url) return;
-                const path = new URL(url).pathname;
-                if (path.startsWith('/video/'))
-                    return path.split('/')[2];
-            })
-            .catch(() => {})
-}
-
 function getBest(content) {
     return content?.filter(v => v.baseUrl || v.url)
                 .map(v => (v.baseUrl = v.baseUrl || v.url, v))
@@ -99,7 +88,8 @@ async function tv_download(id) {
 
 export default async function({ comId, tvId, comShortLink }) {
     if (comShortLink) {
-        comId = await com_resolveShortlink(comShortLink);
+        const patternMatch = await resolveRedirectingURL(`https://b23.tv/${comShortLink}`);
+        comId = patternMatch?.comId;
     }
 
     if (comId) {

api/src/processing/services/bluesky.js

@@ -2,12 +2,19 @@ import HLS from "hls-parser";
 import { cobaltUserAgent } from "../../config.js";
 import { createStream } from "../../stream/manage.js";
 
-const extractVideo = async ({ getPost, filename }) => {
-    const urlMasterHLS = getPost?.thread?.post?.embed?.playlist;
-    if (!urlMasterHLS) return { error: "fetch.empty" };
-    if (!urlMasterHLS.startsWith("https://video.bsky.app/")) return { error: "fetch.empty" };
+const extractVideo = async ({ media, filename, dispatcher }) => {
+    let urlMasterHLS = media?.playlist;
 
-    const masterHLS = await fetch(urlMasterHLS)
+    if (!urlMasterHLS || !urlMasterHLS.startsWith("https://video.bsky.app/")) {
+        return { error: "fetch.empty" };
+    }
+
+    urlMasterHLS = urlMasterHLS.replace(
+        "video.bsky.app/watch/",
+        "video.cdn.bsky.app/hls/"
+    );
+
+    const masterHLS = await fetch(urlMasterHLS, { dispatcher })
         .then(r => {
             if (r.status !== 200) return;
             return r.text();
@@ -26,7 +33,7 @@ const extractVideo = async ({ getPost, filename }) => {
         urls: videoURL,
         filename: `${filename}.mp4`,
         audioFilename: `${filename}_audio`,
-        isM3U8: true,
+        isHLS: true,
     }
 }
 
@@ -48,7 +55,7 @@ const extractImages = ({ getPost, filename, alwaysProxy }) => {
         let proxiedImage = createStream({
             service: "bluesky",
             type: "proxy",
-            u: url,
+            url,
             filename: `${filename}_${i + 1}.jpg`,
         });
 
@@ -64,7 +71,25 @@ const extractImages = ({ getPost, filename, alwaysProxy }) => {
     return { picker };
 }
 
-export default async function ({ user, post, alwaysProxy }) {
+const extractGif = ({ url, filename }) => {
+    const gifUrl = new URL(url);
+
+    if (!gifUrl || gifUrl.hostname !== "media.tenor.com") {
+        return { error: "fetch.empty" };
+    }
+
+    // remove downscaling params from gif url
+    // such as "?hh=498&ww=498"
+    gifUrl.search = "";
+
+    return {
+        urls: gifUrl,
+        isPhoto: true,
+        filename: `${filename}.gif`,
+    }
+}
+
+export default async function ({ user, post, alwaysProxy, dispatcher }) {
     const apiEndpoint = new URL("https://public.api.bsky.app/xrpc/app.bsky.feed.getPostThread?depth=0&parentHeight=0");
     apiEndpoint.searchParams.set(
         "uri",
@@ -73,20 +98,59 @@ export default async function ({ user, post, alwaysProxy }) {
 
     const getPost = await fetch(apiEndpoint, {
         headers: {
-            "user-agent": cobaltUserAgent
-        }
+            "user-agent": cobaltUserAgent,
+        },
+        dispatcher
     }).then(r => r.json()).catch(() => {});
 
-    if (!getPost || getPost?.error) return { error: "fetch.empty" };
+    if (!getPost) return { error: "fetch.empty" };
+
+    if (getPost.error) {
+        switch (getPost.error) {
+            case "NotFound":
+            case "InternalServerError":
+                return { error: "content.post.unavailable" };
+            case "InvalidRequest":
+                return { error: "link.unsupported" };
+            default:
+                return { error: "content.post.unavailable" };
+        }
+    }
 
     const embedType = getPost?.thread?.post?.embed?.$type;
     const filename = `bluesky_${user}_${post}`;
 
-    if (embedType === "app.bsky.embed.video#view") {
-        return extractVideo({ getPost, filename });
-    }
-    if (embedType === "app.bsky.embed.images#view") {
-        return extractImages({ getPost, filename, alwaysProxy });
+    switch (embedType) {
+        case "app.bsky.embed.video#view":
+            return extractVideo({
+                media: getPost.thread?.post?.embed,
+                filename,
+            });
+
+        case "app.bsky.embed.images#view":
+            return extractImages({
+                getPost,
+                filename,
+                alwaysProxy
+            });
+
+        case "app.bsky.embed.external#view":
+            return extractGif({
+                url: getPost?.thread?.post?.embed?.external?.uri,
+                filename,
+            });
+
+        case "app.bsky.embed.recordWithMedia#view":
+            if (getPost?.thread?.post?.embed?.media?.$type === "app.bsky.embed.external#view") {
+                return extractGif({
+                    url: getPost?.thread?.post?.embed?.media?.external?.uri,
+                    filename,
+                });
+            }
+            return extractVideo({
+                media: getPost.thread?.post?.embed?.media,
+                filename,
+            });
     }
 
     return { error: "fetch.empty" };

api/src/processing/services/dailymotion.js

@@ -92,7 +92,7 @@ export default async function({ id }) {
 
     return {
         urls: bestQuality.uri,
-        isM3U8: true,
+        isHLS: true,
         filenameAttributes: {
             service: 'dailymotion',
             id: media.xid,

api/src/processing/services/facebook.js

@@ -8,8 +8,8 @@ const headers = {
     'Sec-Fetch-Site': 'none',
 }
 
-const resolveUrl = (url) => {
-    return fetch(url, { headers })
+const resolveUrl = (url, dispatcher) => {
+    return fetch(url, { headers, dispatcher })
         .then(r => {
             if (r.headers.get('location')) {
                 return decodeURIComponent(r.headers.get('location'));
@@ -23,13 +23,13 @@ const resolveUrl = (url) => {
         .catch(() => false);
 }
 
-export default async function({ id, shareType, shortLink }) {
+export default async function({ id, shareType, shortLink, dispatcher }) {
     let url = `https://web.facebook.com/i/videos/${id}`;
 
     if (shareType) url = `https://web.facebook.com/share/${shareType}/${id}`;
-    if (shortLink) url = await resolveUrl(`https://fb.watch/${shortLink}`);
+    if (shortLink) url = await resolveUrl(`https://fb.watch/${shortLink}`, dispatcher);
 
-    const html = await fetch(url, { headers })
+    const html = await fetch(url, { headers, dispatcher })
         .then(r => r.text())
         .catch(() => false);
 

api/src/processing/services/instagram.js

@@ -1,3 +1,5 @@
+import { randomBytes } from "node:crypto";
+import { resolveRedirectingURL } from "../url.js";
 import { genericUserAgent } from "../../config.js";
 import { createStream } from "../../stream/manage.js";
 import { getCookie, updateCookie } from "../cookie/manager.js";
@@ -8,6 +10,7 @@ const commonHeaders = {
     "sec-fetch-site": "same-origin",
     "x-ig-app-id": "936619743392459"
 }
+
 const mobileHeaders = {
     "x-ig-app-locale": "en_US",
     "x-ig-device-locale": "en_US",
@@ -19,6 +22,7 @@ const mobileHeaders = {
     "x-fb-server-cluster": "True",
     "content-length": "0",
 }
+
 const embedHeaders = {
     "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
     "Accept-Language": "en-GB,en;q=0.9",
@@ -33,7 +37,7 @@ const embedHeaders = {
     "Sec-Fetch-Site": "none",
     "Sec-Fetch-User": "?1",
     "Upgrade-Insecure-Requests": "1",
-    "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+    "User-Agent": genericUserAgent,
 }
 
 const cachedDtsg = {
@@ -41,7 +45,17 @@ const cachedDtsg = {
     expiry: 0
 }
 
-export default function(obj) {
+const getNumberFromQuery = (name, data) => {
+    const s = data?.match(new RegExp(name + '=(\\d+)'))?.[1];
+    if (+s) return +s;
+}
+
+const getObjectFromEntries = (name, data) => {
+    const obj = data?.match(new RegExp('\\["' + name + '",.*?,({.*?}),\\d+\\]'))?.[1];
+    return obj && JSON.parse(obj);
+}
+
+export default function instagram(obj) {
     const dispatcher = obj.dispatcher;
 
     async function findDtsgId(cookie) {
@@ -91,6 +105,7 @@ export default function(obj) {
         updateCookie(cookie, data.headers);
         return data.json();
     }
+
     async function getMediaId(id, { cookie, token } = {}) {
         const oembedURL = new URL('https://i.instagram.com/api/v1/oembed/');
         oembedURL.searchParams.set('url', `https://www.instagram.com/p/${id}/`);
@@ -119,6 +134,7 @@ export default function(obj) {
 
         return mediaInfo?.items?.[0];
     }
+
     async function requestHTML(id, cookie) {
         const data = await fetch(`https://www.instagram.com/p/${id}/embed/captioned/`, {
             headers: {
@@ -136,40 +152,167 @@ export default function(obj) {
 
         return embedData;
     }
-    async function requestGQL(id, cookie) {
-        let dtsgId;
 
-        if (cookie) {
-            dtsgId = await findDtsgId(cookie);
-        }
-        const url = new URL('https://www.instagram.com/api/graphql/');
+    async function getGQLParams(id, cookie) {
+        const req = await fetch(`https://www.instagram.com/p/${id}/`, {
+            headers: {
+                ...embedHeaders,
+                cookie
+            },
+            dispatcher
+        });
 
-        const requestData = {
-            jazoest: '26406',
-            variables: JSON.stringify({
-            shortcode: id,
-            __relay_internal__pv__PolarisShareMenurelayprovider: false
-            }),
-            doc_id: '7153618348081770'
+        const html = await req.text();
+        const siteData = getObjectFromEntries('SiteData', html);
+        const polarisSiteData = getObjectFromEntries('PolarisSiteData', html);
+        const webConfig = getObjectFromEntries('DGWWebConfig', html);
+        const pushInfo = getObjectFromEntries('InstagramWebPushInfo', html);
+        const lsd = getObjectFromEntries('LSD', html)?.token || randomBytes(8).toString('base64url');
+        const csrf = getObjectFromEntries('InstagramSecurityConfig', html)?.csrf_token;
+
+        const anon_cookie = [
+            csrf && "csrftoken=" + csrf,
+            polarisSiteData?.device_id && "ig_did=" + polarisSiteData?.device_id,
+            "wd=1280x720",
+            "dpr=2",
+            polarisSiteData?.machine_id && "mid=" + polarisSiteData.machine_id,
+            "ig_nrcb=1"
+        ].filter(a => a).join('; ');
+
+        return {
+            headers: {
+                'x-ig-app-id': webConfig?.appId || '936619743392459',
+                'X-FB-LSD': lsd,
+                'X-CSRFToken': csrf,
+                'X-Bloks-Version-Id': getObjectFromEntries('WebBloksVersioningID', html)?.versioningID,
+                'x-asbd-id': 129477,
+                cookie: anon_cookie
+            },
+            body: {
+                __d: 'www',
+                __a: '1',
+                __s: '::' + Math.random().toString(36).substring(2).replace(/\d/g, '').slice(0, 6),
+                __hs: siteData?.haste_session || '20126.HYP:instagram_web_pkg.2.1...0',
+                __req: 'b',
+                __ccg: 'EXCELLENT',
+                __rev: pushInfo?.rollout_hash || '1019933358',
+                __hsi: siteData?.hsi || '7436540909012459023',
+                __dyn: randomBytes(154).toString('base64url'),
+                __csr: randomBytes(154).toString('base64url'),
+                __user: '0',
+                __comet_req: getNumberFromQuery('__comet_req', html) || '7',
+                av: '0',
+                dpr: '2',
+                lsd,
+                jazoest: getNumberFromQuery('jazoest', html) || Math.floor(Math.random() * 10000),
+                __spin_r: siteData?.__spin_r || '1019933358',
+                __spin_b: siteData?.__spin_b || 'trunk',
+                __spin_t: siteData?.__spin_t || Math.floor(new Date().getTime() / 1000),
+            }
         };
-        if (dtsgId) {
-            requestData.fb_dtsg = dtsgId;
+    }
+
+    async function requestGQL(id, cookie) {
+        const { headers, body } = await getGQLParams(id, cookie);
+
+        const req = await fetch('https://www.instagram.com/graphql/query', {
+            method: 'POST',
+            dispatcher,
+            headers: {
+                ...embedHeaders,
+                ...headers,
+                cookie,
+                'content-type': 'application/x-www-form-urlencoded',
+                'X-FB-Friendly-Name': 'PolarisPostActionLoadPostQueryQuery',
+            },
+            body: new URLSearchParams({
+                ...body,
+                fb_api_caller_class: 'RelayModern',
+                fb_api_req_friendly_name: 'PolarisPostActionLoadPostQueryQuery',
+                variables: JSON.stringify({
+                    shortcode: id,
+                    fetch_tagged_user_count: null,
+                    hoisted_comment_id: null,
+                    hoisted_reply_id: null
+                }),
+                server_timestamps: true,
+                doc_id: '8845758582119845'
+            }).toString()
+        });
+
+        return {
+            gql_data: await req.json()
+                        .then(r => r.data)
+                        .catch(() => null)
+        };
+    }
+
+    async function getErrorContext(id) {
+        try {
+            const { headers, body } = await getGQLParams(id);
+
+            const req = await fetch('https://www.instagram.com/ajax/bulk-route-definitions/', {
+                method: 'POST',
+                dispatcher,
+                headers: {
+                    ...embedHeaders,
+                    ...headers,
+                    'content-type': 'application/x-www-form-urlencoded',
+                    'X-Ig-D': 'www',
+                },
+                body: new URLSearchParams({
+                    'route_urls[0]': `/p/${id}/`,
+                    routing_namespace: 'igx_www',
+                    ...body
+                }).toString()
+            });
+
+            const response = await req.text();
+            if (response.includes('"tracePolicy":"polaris.privatePostPage"'))
+                return { error: 'content.post.private' };
+
+            const [, mediaId, mediaOwnerId] = response.match(
+                /"media_id":\s*?"(\d+)","media_owner_id":\s*?"(\d+)"/
+            ) || [];
+
+            if (mediaId && mediaOwnerId) {
+                const rulingURL = new URL('https://www.instagram.com/api/v1/web/get_ruling_for_media_content_logged_out');
+                rulingURL.searchParams.set('media_id', mediaId);
+                rulingURL.searchParams.set('owner_id', mediaOwnerId);
+
+                const rulingResponse = await fetch(rulingURL, {
+                    headers: {
+                        ...headers,
+                        ...commonHeaders
+                    },
+                    dispatcher,
+                }).then(a => a.json()).catch(() => ({}));
+
+                if (rulingResponse?.title?.includes('Restricted'))
+                    return { error: "content.post.age" };
+            }
+        } catch {
+            return { error: "fetch.fail" };
         }
 
-        return (await request(url, cookie, 'POST', requestData))
-                .data
-                ?.xdt_api__v1__media__shortcode__web_info
-                ?.items
-                ?.[0];
+        return { error: "fetch.empty" };
     }
 
     function extractOldPost(data, id, alwaysProxy) {
-        const sidecar = data?.gql_data?.shortcode_media?.edge_sidecar_to_children;
+        const shortcodeMedia = data?.gql_data?.shortcode_media || data?.gql_data?.xdt_shortcode_media;
+        const sidecar = shortcodeMedia?.edge_sidecar_to_children;
+
         if (sidecar) {
             const picker = sidecar.edges.filter(e => e.node?.display_url)
                 .map((e, i) => {
-                    const type = e.node?.is_video ? "video" : "photo";
-                    const url = type === "video" ? e.node?.video_url : e.node?.display_url;
+                    const type = e.node?.is_video && e.node?.video_url ? "video" : "photo";
+
+                    let url;
+                    if (type === "video") {
+                        url = e.node?.video_url;
+                    } else if (type === "photo") {
+                        url = e.node?.display_url;
+                    }
 
                     let itemExt = type === "video" ? "mp4" : "jpg";
 
@@ -177,7 +320,7 @@ export default function(obj) {
                     if (alwaysProxy) proxyFile = createStream({
                         service: "instagram",
                         type: "proxy",
-                        u: url,
+                        url,
                         filename: `instagram_${id}_${i + 1}.${itemExt}`
                     });
 
@@ -189,23 +332,28 @@ export default function(obj) {
                         thumb: createStream({
                             service: "instagram",
                             type: "proxy",
-                            u: e.node?.display_url,
+                            url: e.node?.display_url,
                             filename: `instagram_${id}_${i + 1}.jpg`
                         })
                     }
                 });
 
             if (picker.length) return { picker }
-        } else if (data?.gql_data?.shortcode_media?.video_url) {
+        }
+
+        if (shortcodeMedia?.video_url) {
             return {
-                urls: data.gql_data.shortcode_media.video_url,
+                urls: shortcodeMedia.video_url,
                 filename: `instagram_${id}.mp4`,
                 audioFilename: `instagram_${id}_audio`
             }
-        } else if (data?.gql_data?.shortcode_media?.display_url) {
+        }
+
+        if (shortcodeMedia?.display_url) {
             return {
-                urls: data.gql_data?.shortcode_media.display_url,
-                isPhoto: true
+                urls: shortcodeMedia.display_url,
+                isPhoto: true,
+                filename: `instagram_${id}.jpg`,
             }
         }
     }
@@ -230,7 +378,7 @@ export default function(obj) {
                     if (alwaysProxy) proxyFile = createStream({
                         service: "instagram",
                         type: "proxy",
-                        u: url,
+                        url,
                         filename: `instagram_${id}_${i + 1}.${itemExt}`
                     });
 
@@ -242,7 +390,7 @@ export default function(obj) {
                         thumb: createStream({
                             service: "instagram",
                             type: "proxy",
-                            u: imageUrl,
+                            url: imageUrl,
                             filename: `instagram_${id}_${i + 1}.jpg`
                         })
                     }
@@ -266,6 +414,9 @@ export default function(obj) {
     }
 
     async function getPost(id, alwaysProxy) {
+        const hasData = (data) => data
+                                    && data.gql_data !== null
+                                    && data?.gql_data?.xdt_shortcode_media !== null;
         let data, result;
         try {
             const cookie = getCookie('instagram');
@@ -282,19 +433,21 @@ export default function(obj) {
             if (media_id && token) data = await requestMobileApi(media_id, { token });
 
             // mobile api (no cookie, cookie)
-            if (media_id && !data) data = await requestMobileApi(media_id);
-            if (media_id && cookie && !data) data = await requestMobileApi(media_id, { cookie });
+            if (media_id && !hasData(data)) data = await requestMobileApi(media_id);
+            if (media_id && cookie && !hasData(data)) data = await requestMobileApi(media_id, { cookie });
 
             // html embed (no cookie, cookie)
-            if (!data) data = await requestHTML(id);
-            if (!data && cookie) data = await requestHTML(id, cookie);
+            if (!hasData(data)) data = await requestHTML(id);
+            if (!hasData(data) && cookie) data = await requestHTML(id, cookie);
 
             // web app graphql api (no cookie, cookie)
-            if (!data) data = await requestGQL(id);
-            if (!data && cookie) data = await requestGQL(id, cookie);
+            if (!hasData(data)) data = await requestGQL(id);
+            if (!hasData(data) && cookie) data = await requestGQL(id, cookie);
         } catch {}
 
-        if (!data) return { error: "fetch.fail" };
+        if (!hasData(data)) {
+            return getErrorContext(id);
+        }
 
         if (data?.gql_data) {
             result = extractOldPost(data, id, alwaysProxy)
@@ -357,14 +510,30 @@ export default function(obj) {
         if (item.image_versions2?.candidates) {
             return {
                 urls: item.image_versions2.candidates[0].url,
-                isPhoto: true
+                isPhoto: true,
+                filename: `instagram_${id}.jpg`,
             }
         }
 
         return { error: "link.unsupported" };
     }
 
-    const { postId, storyId, username, alwaysProxy } = obj;
+    const { postId, shareId, storyId, username, alwaysProxy } = obj;
+
+    if (shareId) {
+        return resolveRedirectingURL(
+            `https://www.instagram.com/share/${shareId}/`,
+            dispatcher,
+            // for some reason instagram decides to return HTML
+            // instead of a redirect when requesting with a normal
+            // browser user-agent
+            'curl/7.88.1'
+        ).then(match => instagram({
+            ...obj, ...match,
+            shareId: undefined
+        }));
+    }
+
     if (postId) return getPost(postId, alwaysProxy);
     if (username && storyId) return getStory(username, storyId);
 

api/src/processing/services/ok.js

@@ -1,5 +1,4 @@
 import { genericUserAgent, env } from "../../config.js";
-import { cleanString } from "../../misc/utils.js";
 
 const resolutions = {
     "ultra": "2160",
@@ -44,8 +43,8 @@ export default async function(o) {
     let bestVideo = videos.find(v => resolutions[v.name] === quality) || videos[videos.length - 1];
 
     let fileMetadata = {
-        title: cleanString(videoData.movie.title.trim()),
-        author: cleanString((videoData.author?.name || videoData.compilationTitle).trim()),
+        title: videoData.movie.title.trim(),
+        author: (videoData.author?.name || videoData.compilationTitle)?.trim(),
     }
 
     if (bestVideo) return {

api/src/processing/services/pinterest.js

@@ -1,4 +1,5 @@
 import { genericUserAgent } from "../../config.js";
+import { resolveRedirectingURL } from "../url.js";
 
 const videoRegex = /"url":"(https:\/\/v1\.pinimg\.com\/videos\/.*?)"/g;
 const imageRegex = /src="(https:\/\/i\.pinimg\.com\/.*\.(jpg|gif))"/g;
@@ -7,10 +8,10 @@ export default async function(o) {
     let id = o.id;
 
     if (!o.id && o.shortLink) {
-        id = await fetch(`https://api.pinterest.com/url_shortener/${o.shortLink}/redirect/`, { redirect: "manual" })
-                   .then(r => r.headers.get("location").split('pin/')[1].split('/')[0])
-                   .catch(() => {});
+        const patternMatch = await resolveRedirectingURL(`https://api.pinterest.com/url_shortener/${o.shortLink}/redirect/`);
+        id = patternMatch?.id;
     }
+
     if (id.includes("--")) id = id.split("--")[1];
     if (!id) return { error: "fetch.fail" };
 
@@ -22,12 +23,12 @@ export default async function(o) {
 
     const videoLink = [...html.matchAll(videoRegex)]
                     .map(([, link]) => link)
-                    .find(a => a.endsWith('.mp4') && a.includes('720p'));
+                    .find(a => a.endsWith('.mp4'));
 
     if (videoLink) return {
         urls: videoLink,
-        filename: `pinterest_${o.id}.mp4`,
-        audioFilename: `pinterest_${o.id}_audio`
+        filename: `pinterest_${id}.mp4`,
+        audioFilename: `pinterest_${id}_audio`
     }
 
     const imageLink = [...html.matchAll(imageRegex)]
@@ -39,7 +40,7 @@ export default async function(o) {
     if (imageLink) return {
         urls: imageLink,
         isPhoto: true,
-        filename: `pinterest_${o.id}.${imageType}`
+        filename: `pinterest_${id}.${imageType}`
     }
 
     return { error: "fetch.empty" };

api/src/processing/services/reddit.js

@@ -1,3 +1,4 @@
+import { resolveRedirectingURL } from "../url.js";
 import { genericUserAgent, env } from "../../config.js";
 import { getCookie, updateCookieValues } from "../cookie/manager.js";
 
@@ -48,12 +49,20 @@ async function getAccessToken() {
 }
 
 export default async function(obj) {
-    let url = new URL(`https://www.reddit.com/r/${obj.sub}/comments/${obj.id}.json`);
+    let params = obj;
 
-    if (obj.user) {
-        url.pathname = `/user/${obj.user}/comments/${obj.id}.json`;
+    if (!params.id && params.shareId) {
+        params = await resolveRedirectingURL(
+            `https://www.reddit.com/r/${params.sub}/s/${params.shareId}`,
+            obj.dispatcher,
+            genericUserAgent
+        );
     }
 
+    if (!params?.id) return { error: "fetch.short_link" };
+
+    const url = new URL(`https://www.reddit.com/comments/${params.id}.json`);
+
     const accessToken = await getAccessToken();
     if (accessToken) url.hostname = 'oauth.reddit.com';
 
@@ -73,12 +82,17 @@ export default async function(obj) {
 
     data = data[0]?.data?.children[0]?.data;
 
-    const id = `${String(obj.sub).toLowerCase()}_${obj.id}`;
+    let sourceId;
+    if (params.sub || params.user) {
+        sourceId = `${String(params.sub || params.user).toLowerCase()}_${params.id}`;
+    } else {
+        sourceId = params.id;
+    }
 
     if (data?.url?.endsWith('.gif')) return {
         typeId: "redirect",
         urls: data.url,
-        filename: `reddit_${id}.gif`,
+        filename: `reddit_${sourceId}.gif`,
     }
 
     if (!data.secure_media?.reddit_video)
@@ -87,8 +101,9 @@ export default async function(obj) {
     if (data.secure_media?.reddit_video?.duration > env.durationLimit)
         return { error: "content.too_long" };
 
+    const video = data.secure_media?.reddit_video?.fallback_url?.split('?')[0];
+
     let audio = false,
-        video = data.secure_media?.reddit_video?.fallback_url?.split('?')[0],
         audioFileLink = `${data.secure_media?.reddit_video?.fallback_url?.split('DASH')[0]}audio`;
 
     if (video.match('.mp4')) {
@@ -121,7 +136,7 @@ export default async function(obj) {
         typeId: "tunnel",
         type: "merge",
         urls: [video, audioFileLink],
-        audioFilename: `reddit_${id}_audio`,
-        filename: `reddit_${id}.mp4`
+        audioFilename: `reddit_${sourceId}_audio`,
+        filename: `reddit_${sourceId}.mp4`
     }
 }

api/src/processing/services/rutube.js

@@ -1,7 +1,5 @@
 import HLS from "hls-parser";
-
 import { env } from "../../config.js";
-import { cleanString } from "../../misc/utils.js";
 
 async function requestJSON(url) {
     try {
@@ -35,6 +33,10 @@ export default async function(obj) {
     const play = await requestJSON(requestURL);
     if (!play) return { error: "fetch.fail" };
 
+    if (play.detail?.type === "blocking_rule") {
+        return { error: "content.video.region" };
+    }
+
     if (play.detail || !play.video_balancer) return { error: "fetch.empty" };
     if (play.live_streams?.hls) return { error: "content.video.live" };
 
@@ -59,13 +61,13 @@ export default async function(obj) {
     });
 
     const fileMetadata = {
-        title: cleanString(play.title.trim()),
-        artist: cleanString(play.author.name.trim()),
+        title: play.title.trim(),
+        artist: play.author.name.trim(),
     }
 
     return {
         urls: matchingQuality.uri,
-        isM3U8: true,
+        isHLS: true,
         filenameAttributes: {
             service: "rutube",
             id: obj.id,

api/src/processing/services/snapchat.js

@@ -1,7 +1,6 @@
-import { extract, normalizeURL } from "../url.js";
+import { resolveRedirectingURL } from "../url.js";
 import { genericUserAgent } from "../../config.js";
 import { createStream } from "../../stream/manage.js";
-import { getRedirectingURL } from "../../misc/utils.js";
 
 const SPOTLIGHT_VIDEO_REGEX = /<link data-react-helmet="true" rel="preload" href="([^"]+)" as="video"\/>/;
 const NEXT_DATA_REGEX = /<script id="__NEXT_DATA__" type="application\/json">({.+})<\/script><\/body><\/html>$/;
@@ -41,9 +40,9 @@ async function getStory(username, storyId, alwaysProxy) {
     const nextDataString = html.match(NEXT_DATA_REGEX)?.[1];
     if (nextDataString) {
         const data = JSON.parse(nextDataString);
-        const storyIdParam = data.query.profileParams[1];
+        const storyIdParam = data?.query?.profileParams?.[1];
 
-        if (storyIdParam && data.props.pageProps.story) {
+        if (storyIdParam && data?.props?.pageProps?.story) {
             const story = data.props.pageProps.story.snapList.find((snap) => snap.snapId.value === storyIdParam);
             if (story) {
                 if (story.snapMediaType === 0) {
@@ -62,7 +61,7 @@ async function getStory(username, storyId, alwaysProxy) {
             }
         }
 
-        const defaultStory = data.props.pageProps.curatedHighlights[0];
+        const defaultStory = data?.props?.pageProps?.curatedHighlights?.[0];
         if (defaultStory) {
             return {
                 picker: defaultStory.snapList.map(snap => {
@@ -73,7 +72,7 @@ async function getStory(username, storyId, alwaysProxy) {
                     const proxy = createStream({
                         service: "snapchat",
                         type: "proxy",
-                        u: snapUrl,
+                        url: snapUrl,
                         filename: `snapchat_${username}_${snap.timestampInSec.value}.${snapExt}`,
                     });
 
@@ -81,7 +80,7 @@ async function getStory(username, storyId, alwaysProxy) {
                     if (snapType === "video") thumbProxy = createStream({
                         service: "snapchat",
                         type: "proxy",
-                        u: snap.snapUrls.mediaPreviewUrl.value,
+                        url: snap.snapUrls.mediaPreviewUrl.value,
                     });
 
                     if (alwaysProxy) snapUrl = proxy;
@@ -100,18 +99,7 @@ async function getStory(username, storyId, alwaysProxy) {
 export default async function (obj) {
     let params = obj;
     if (obj.shortLink) {
-        const link = await getRedirectingURL(`https://t.snapchat.com/${obj.shortLink}`);
-
-        if (!link?.startsWith('https://www.snapchat.com/')) {
-            return { error: "fetch.short_link" };
-        }
-
-        const extractResult = extract(normalizeURL(link));
-        if (extractResult?.host !== 'snapchat') {
-            return { error: "fetch.short_link" };
-        }
-
-        params = extractResult.patternMatch;
+        params = await resolveRedirectingURL(`https://t.snapchat.com/${obj.shortLink}`);
     }
 
     if (params.spotlightId) {

api/src/processing/services/soundcloud.js

@@ -1,5 +1,4 @@
 import { env } from "../../config.js";
-import { cleanString } from "../../misc/utils.js";
 
 const cachedID = {
     version: '',
@@ -63,7 +62,17 @@ export default async function(obj) {
 
     if (!json) return { error: "fetch.fail" };
 
-    if (!json.media.transcodings) return { error: "fetch.empty" };
+    if (json?.policy === "BLOCK") {
+        return { error: "content.region" };
+    }
+
+    if (json?.policy === "SNIP") {
+        return { error: "content.paid" };
+    }
+
+    if (!json?.media?.transcodings || !json?.media?.transcodings.length === 0) {
+        return { error: "fetch.empty" };
+    }
 
     let bestAudio = "opus",
         selectedStream = json.media.transcodings.find(v => v.preset === "opus_0_0"),
@@ -75,6 +84,10 @@ export default async function(obj) {
         bestAudio = "mp3"
     }
 
+    if (!selectedStream) {
+        return { error: "fetch.empty" };
+    }
+
     let fileUrlBase = selectedStream.url;
     let fileUrl = `${fileUrlBase}${fileUrlBase.includes("?") ? "&" : "?"}client_id=${clientId}&track_authorization=${json.track_authorization}`;
 
@@ -91,8 +104,8 @@ export default async function(obj) {
     if (!file) return { error: "fetch.empty" };
 
     let fileMetadata = {
-        title: cleanString(json.title.trim()),
-        artist: cleanString(json.user.username.trim()),
+        title: json.title.trim(),
+        artist: json.user.username.trim(),
     }
 
     return {

api/src/processing/services/tiktok.js

@@ -12,7 +12,7 @@ export default async function(obj) {
     let postId = obj.postId;
 
     if (!postId) {
-        let html = await fetch(`${shortDomain}${obj.id}`, {
+        let html = await fetch(`${shortDomain}${obj.shortLink}`, {
             redirect: "manual",
             headers: {
                 "user-agent": genericUserAgent.split(' Chrome/1')[0]
@@ -24,13 +24,13 @@ export default async function(obj) {
         if (html.startsWith('<a href="https://')) {
             const extractedURL = html.split('<a href="')[1].split('?')[0];
             const { patternMatch } = extract(extractedURL);
-            postId = patternMatch.postId
+            postId = patternMatch.postId;
         }
     }
     if (!postId) return { error: "fetch.short_link" };
 
     // should always be /video/, even for photos
-    const res = await fetch(`https://tiktok.com/@i/video/${postId}`, {
+    const res = await fetch(`https://www.tiktok.com/@i/video/${postId}`, {
         headers: {
             "user-agent": genericUserAgent,
             cookie,
@@ -44,20 +44,39 @@ export default async function(obj) {
     try {
         const json = html
             .split('<script id="__UNIVERSAL_DATA_FOR_REHYDRATION__" type="application/json">')[1]
-            .split('</script>')[0]
-        const data = JSON.parse(json)
-        detail = data["__DEFAULT_SCOPE__"]["webapp.video-detail"]["itemInfo"]["itemStruct"]
+            .split('</script>')[0];
+
+        const data = JSON.parse(json);
+        const videoDetail = data["__DEFAULT_SCOPE__"]["webapp.video-detail"];
+
+        if (!videoDetail) throw "no video detail found";
+
+        // status_deleted or etc
+        if (videoDetail.statusMsg) {
+            return { error: "content.post.unavailable"};
+        }
+
+        detail = videoDetail?.itemInfo?.itemStruct;
     } catch {
         return { error: "fetch.fail" };
     }
 
+    if (detail.isContentClassified) {
+        return { error: "content.post.age" };
+    }
+
+    if (!detail.author) {
+        return { error: "fetch.empty" };
+    }
+
     let video, videoFilename, audioFilename, audio, images,
-        filenameBase = `tiktok_${detail.author.uniqueId}_${postId}`,
+        filenameBase = `tiktok_${detail.author?.uniqueId}_${postId}`,
         bestAudio; // will get defaulted to m4a later on in match-action
 
     images = detail.imagePost?.images;
 
-    let playAddr = detail.video.playAddr;
+    let playAddr = detail.video?.playAddr;
+
     if (obj.h265) {
         const h265PlayAddr = detail?.video?.bitrateInfo?.find(b => b.CodecType.includes("h265"))?.PlayAddr.UrlList[0]
         playAddr = h265PlayAddr || playAddr
@@ -102,7 +121,7 @@ export default async function(obj) {
                 if (obj.alwaysProxy) url = createStream({
                     service: "tiktok",
                     type: "proxy",
-                    u: url,
+                    url,
                     filename: `${filenameBase}_photo_${i + 1}.jpg`
                 })
 

api/src/processing/services/tumblr.js

@@ -1,4 +1,4 @@
-import psl from "psl";
+import psl from "@imput/psl";
 
 const API_KEY = 'jrsCWX1XDuVxAFO4GkK147syAoN8BJZ5voz8tS80bPcj26Vc5Z';
 const API_BASE = 'https://api-http2.tumblr.com';

api/src/processing/services/twitch.js

@@ -1,5 +1,4 @@
 import { env } from "../../config.js";
-import { cleanString } from '../../misc/utils.js';
 
 const gqlURL = "https://gql.twitch.tv/gql";
 const clientIdHead = { "client-id": "kimne78kx3ncx6brgo4mv6wki5h1ko" };
@@ -73,13 +72,13 @@ export default async function (obj) {
             token: req_token[0].data.clip.playbackAccessToken.value
         })}`,
         fileMetadata: {
-            title: cleanString(clipMetadata.title.trim()),
+            title: clipMetadata.title.trim(),
             artist: `Twitch Clip by @${clipMetadata.broadcaster.login}, clipped by @${clipMetadata.curator.login}`,
         },
         filenameAttributes: {
             service: "twitch",
             id: clipMetadata.id,
-            title: cleanString(clipMetadata.title.trim()),
+            title: clipMetadata.title.trim(),
             author: `${clipMetadata.broadcaster.login}, clipped by ${clipMetadata.curator.login}`,
             qualityLabel: `${format.quality}p`,
             extension: 'mp4'

api/src/processing/services/twitter.js

@@ -24,6 +24,11 @@ const badContainerEnd = new Date(1702605600000);
 
 function needsFixing(media) {
     const representativeId = media.source_status_id_str ?? media.id_str;
+
+    // syndication api doesn't have media ids in its response,
+    // so we just assume it's all good
+    if (!representativeId) return false;
+
     const mediaTimestamp = new Date(
         Number((BigInt(representativeId) >> 22n) + TWITTER_EPOCH)
     );
@@ -53,6 +58,25 @@ const getGuestToken = async (dispatcher, forceReload = false) => {
     }
 }
 
+const requestSyndication = async(dispatcher, tweetId) => {
+    // thank you
+    // https://github.com/yt-dlp/yt-dlp/blob/05c8023a27dd37c49163c0498bf98e3e3c1cb4b9/yt_dlp/extractor/twitter.py#L1334
+    const token = (id) => ((Number(id) / 1e15) * Math.PI).toString(36).replace(/(0+|\.)/g, '');
+    const syndicationUrl = new URL("https://cdn.syndication.twimg.com/tweet-result");
+
+    syndicationUrl.searchParams.set("id", tweetId);
+    syndicationUrl.searchParams.set("token", token(tweetId));
+
+    const result = await fetch(syndicationUrl, {
+        headers: {
+            "user-agent": genericUserAgent
+        },
+        dispatcher
+    });
+
+    return result;
+}
+
 const requestTweet = async(dispatcher, tweetId, token, cookie) => {
     const graphqlTweetURL = new URL(graphqlURL);
 
@@ -87,36 +111,24 @@ const requestTweet = async(dispatcher, tweetId, token, cookie) => {
     let result = await fetch(graphqlTweetURL, { headers, dispatcher });
     updateCookie(cookie, result.headers);
 
-    // we might have been missing the `ct0` cookie, retry
+    // we might have been missing the ct0 cookie, retry
     if (result.status === 403 && result.headers.get('set-cookie')) {
-        result = await fetch(graphqlTweetURL, {
-            headers: {
-                ...headers,
-                'x-csrf-token': cookie.values().ct0
-            },
-            dispatcher
-        });
+        const cookieValues = cookie?.values();
+        if (cookieValues?.ct0) {
+            result = await fetch(graphqlTweetURL, {
+                headers: {
+                    ...headers,
+                    'x-csrf-token': cookieValues.ct0
+                },
+                dispatcher
+            });
+        }
     }
 
     return result
 }
 
-export default async function({ id, index, toGif, dispatcher, alwaysProxy }) {
-    const cookie = await getCookie('twitter');
-
-    let guestToken = await getGuestToken(dispatcher);
-    if (!guestToken) return { error: "fetch.fail" };
-
-    let tweet = await requestTweet(dispatcher, id, guestToken);
-
-    // get new token & retry if old one expired
-    if ([403, 429].includes(tweet.status)) {
-        guestToken = await getGuestToken(dispatcher, true);
-        tweet = await requestTweet(dispatcher, id, guestToken)
-    }
-
-    tweet = await tweet.json();
-
+const extractGraphqlMedia = async (tweet, dispatcher, id, guestToken, cookie) => {
     let tweetTypename = tweet?.data?.tweetResult?.result?.__typename;
 
     if (!tweetTypename) {
@@ -127,13 +139,13 @@ export default async function({ id, index, toGif, dispatcher, alwaysProxy }) {
         const reason = tweet?.data?.tweetResult?.result?.reason;
         switch(reason) {
             case "Protected":
-                return { error: "content.post.private" }
+                return { error: "content.post.private" };
             case "NsfwLoggedOut":
                 if (cookie) {
                     tweet = await requestTweet(dispatcher, id, guestToken, cookie);
                     tweet = await tweet.json();
                     tweetTypename = tweet?.data?.tweetResult?.result?.__typename;
-                } else return { error: "content.post.age" }
+                } else return { error: "content.post.age" };
         }
     }
 
@@ -150,7 +162,69 @@ export default async function({ id, index, toGif, dispatcher, alwaysProxy }) {
         repostedTweet = baseTweet?.retweeted_status_result?.result.tweet.legacy.extended_entities;
     }
 
-    let media = (repostedTweet?.media || baseTweet?.extended_entities?.media);
+    return (repostedTweet?.media || baseTweet?.extended_entities?.media);
+}
+
+const testResponse = (result) => {
+    const contentLength = result.headers.get("content-length");
+
+    if (!contentLength || contentLength === '0') {
+        return false;
+    }
+
+    if (!result.headers.get("content-type").startsWith("application/json")) {
+        return false;
+    }
+
+    return true;
+}
+
+export default async function({ id, index, toGif, dispatcher, alwaysProxy }) {
+    const cookie = await getCookie('twitter');
+
+    let syndication = false;
+
+    let guestToken = await getGuestToken(dispatcher);
+    if (!guestToken) return { error: "fetch.fail" };
+
+    // for now we assume that graphql api will come back after some time,
+    // so we try it first
+
+    let tweet = await requestTweet(dispatcher, id, guestToken);
+
+    // get new token & retry if old one expired
+    if ([403, 429].includes(tweet.status)) {
+        guestToken = await getGuestToken(dispatcher, true);
+        if (cookie) {
+            tweet = await requestTweet(dispatcher, id, guestToken, cookie);
+        } else {
+            tweet = await requestTweet(dispatcher, id, guestToken);
+        }
+    }
+
+    const testGraphql = testResponse(tweet);
+
+    // if graphql requests fail, then resort to tweet embed api
+    if (!testGraphql) {
+        syndication = true;
+        tweet = await requestSyndication(dispatcher, id);
+
+        const testSyndication = testResponse(tweet);
+
+        // if even syndication request failed, then cry out loud
+        if (!testSyndication) {
+            return { error: "fetch.fail" };
+        }
+    }
+
+    tweet = await tweet.json();
+
+    let media =
+        syndication
+            ? tweet.mediaDetails
+            : await extractGraphqlMedia(tweet, dispatcher, id, guestToken, cookie);
+
+    if (!media) return { error: "fetch.empty" };
 
     // check if there's a video at given index (/video/<index>)
     if (index >= 0 && index < media?.length) {
@@ -159,11 +233,11 @@ export default async function({ id, index, toGif, dispatcher, alwaysProxy }) {
 
     const getFileExt = (url) => new URL(url).pathname.split(".", 2)[1];
 
-    const proxyMedia = (u, filename) => createStream({
+    const proxyMedia = (url, filename) => createStream({
         service: "twitter",
         type: "proxy",
-        u, filename,
-    })
+        url, filename,
+    });
 
     switch (media?.length) {
         case undefined:
@@ -208,7 +282,7 @@ export default async function({ id, index, toGif, dispatcher, alwaysProxy }) {
 
                 let url = bestQuality(content.video_info.variants);
                 const shouldRenderGif = content.type === "animated_gif" && toGif;
-                const videoFilename = `twitter_${id}_${i + 1}.mp4`;
+                const videoFilename = `twitter_${id}_${i + 1}.${shouldRenderGif ? "gif" : "mp4"}`;
 
                 let type = "video";
                 if (shouldRenderGif) type = "gif";
@@ -217,7 +291,7 @@ export default async function({ id, index, toGif, dispatcher, alwaysProxy }) {
                     url = createStream({
                         service: "twitter",
                         type: shouldRenderGif ? "gif" : "remux",
-                        u: url,
+                        url,
                         filename: videoFilename,
                     })
                 } else if (alwaysProxy) {

api/src/processing/services/vimeo.js

@@ -1,7 +1,6 @@
 import HLS from "hls-parser";
-
 import { env } from "../../config.js";
-import { cleanString, merge } from '../../misc/utils.js';
+import { merge } from '../../misc/utils.js';
 
 const resolutionMatch = {
     "3840": 2160,
@@ -122,7 +121,7 @@ const getHLS = async (configURL, obj) => {
 
     return {
         urls,
-        isM3U8: true,
+        isHLS: true,
         filenameAttributes: {
             resolution: `${bestQuality.resolution.width}x${bestQuality.resolution.height}`,
             qualityLabel: `${resolutionMatch[bestQuality.resolution.width]}p`,
@@ -152,8 +151,8 @@ export default async function(obj) {
     }
 
     const fileMetadata = {
-        title: cleanString(info.name),
-        artist: cleanString(info.user.name),
+        title: info.name,
+        artist: info.user.name,
     };
 
     return merge(

api/src/processing/services/vine.js

@@ -1,15 +0,0 @@
-export default async function(obj) {
-    let post = await fetch(`https://archive.vine.co/posts/${obj.id}.json`)
-                    .then(r => r.json())
-                    .catch(() => {});
-
-    if (!post) return { error: "fetch.empty" };
-
-    if (post.videoUrl) return {
-        urls: post.videoUrl.replace("http://", "https://"),
-        filename: `vine_${obj.id}.mp4`,
-        audioFilename: `vine_${obj.id}_audio`
-    }
-
-    return { error: "fetch.empty" }
-}

api/src/processing/services/vk.js

@@ -1,63 +1,140 @@
-import { cleanString } from "../../misc/utils.js";
-import { genericUserAgent, env } from "../../config.js";
+import { env } from "../../config.js";
 
-const resolutions = ["2160", "1440", "1080", "720", "480", "360", "240"];
+const resolutions = ["2160", "1440", "1080", "720", "480", "360", "240", "144"];
 
-export default async function(o) {
-    let html, url, quality = o.quality === "max" ? 2160 : o.quality;
+const oauthUrl = "https://oauth.vk.com/oauth/get_anonym_token";
+const apiUrl = "https://api.vk.com/method";
 
-    html = await fetch(`https://vk.com/video${o.userId}_${o.videoId}`, {
-        headers: {
-            "user-agent": genericUserAgent
-        }
-    })
-    .then(r => r.arrayBuffer())
-    .catch(() => {});
+const clientId = "51552953";
+const clientSecret = "qgr0yWwXCrsxA1jnRtRX";
 
-    if (!html) return { error: "fetch.fail" };
+// used in stream/shared.js for accessing media files
+export const vkClientAgent = "com.vk.vkvideo.prod/822 (iPhone, iOS 16.7.7, iPhone10,4, Scale/2.0) SAK/1.119";
 
-    // decode cyrillic from windows-1251 because vk still uses apis from prehistoric times
-    let decoder = new TextDecoder('windows-1251');
-    html = decoder.decode(html);
+const cachedToken = {
+    token: "",
+    expiry: 0,
+    device_id: "",
+};
 
-    if (!html.includes(`{"lang":`)) return { error: "fetch.empty" };
-
-    let js = JSON.parse('{"lang":' + html.split(`{"lang":`)[1].split(']);')[0]);
-
-    if (Number(js.mvData.is_active_live) !== 0) {
-        return { error: "content.video.live" };
+const getToken = async () => {
+    if (cachedToken.expiry - 10 > Math.floor(new Date().getTime() / 1000)) {
+        return cachedToken.token;
     }
 
-    if (js.mvData.duration > env.durationLimit) {
+    const randomDeviceId = crypto.randomUUID().toUpperCase();
+
+    const anonymOauth = new URL(oauthUrl);
+    anonymOauth.searchParams.set("client_id", clientId);
+    anonymOauth.searchParams.set("client_secret", clientSecret);
+    anonymOauth.searchParams.set("device_id", randomDeviceId);
+
+    const oauthResponse = await fetch(anonymOauth.toString(), {
+        headers: {
+            "user-agent": vkClientAgent,
+        }
+    }).then(r => {
+        if (r.status === 200) {
+            return r.json();
+        }
+    });
+
+    if (!oauthResponse) return;
+
+    if (oauthResponse?.token && oauthResponse?.expired_at && typeof oauthResponse?.expired_at === "number") {
+        cachedToken.token = oauthResponse.token;
+        cachedToken.expiry = oauthResponse.expired_at;
+        cachedToken.device_id = randomDeviceId;
+    }
+
+    if (!cachedToken.token) return;
+
+    return cachedToken.token;
+}
+
+const getVideo = async (ownerId, videoId, accessKey) => {
+    const video = await fetch(`${apiUrl}/video.get`, {
+        method: "POST",
+        headers: {
+            "content-type": "application/x-www-form-urlencoded; charset=utf-8",
+            "user-agent": vkClientAgent,
+        },
+        body: new URLSearchParams({
+            anonymous_token: cachedToken.token,
+            device_id: cachedToken.device_id,
+            lang: "en",
+            v: "5.244",
+            videos: `${ownerId}_${videoId}${accessKey ? `_${accessKey}` : ''}`
+        }).toString()
+    })
+    .then(r => {
+        if (r.status === 200) {
+            return r.json();
+        }
+    });
+
+    return video;
+}
+
+export default async function ({ ownerId, videoId, accessKey, quality }) {
+    const token = await getToken();
+    if (!token) return { error: "fetch.fail" };
+
+    const videoGet = await getVideo(ownerId, videoId, accessKey);
+
+    if (!videoGet || !videoGet.response || videoGet.response.items.length !== 1) {
+        return { error: "fetch.empty" };
+    }
+
+    const video = videoGet.response.items[0];
+
+    if (video.restriction) {
+        const title = video.restriction.title;
+        if (title.endsWith("country") || title.endsWith("region.")) {
+            return { error: "content.video.region" };
+        }
+        if (title === "Processing video") {
+            return { error: "fetch.empty" };
+        }
+        return { error: "content.video.unavailable" };
+    }
+
+    if (!video.files || !video.duration) {
+        return { error: "fetch.fail" };
+    }
+
+    if (video.duration > env.durationLimit) {
         return { error: "content.too_long" };
     }
 
-    for (let i in resolutions) {
-        if (js.player.params[0][`url${resolutions[i]}`]) {
-            quality = resolutions[i];
+    const userQuality = quality === "max" ? resolutions[0] : quality;
+    let pickedQuality;
+
+    for (const resolution of resolutions) {
+        if (video.files[`mp4_${resolution}`] && +resolution <= +userQuality) {
+            pickedQuality = resolution;
             break
         }
     }
-    if (Number(quality) > Number(o.quality)) quality = o.quality;
 
-    url = js.player.params[0][`url${quality}`];
+    const url = video.files[`mp4_${pickedQuality}`];
 
-    let fileMetadata = {
-        title: cleanString(js.player.params[0].md_title.trim()),
-        author: cleanString(js.player.params[0].md_author.trim()),
+    if (!url) return { error: "fetch.fail" };
+
+    const fileMetadata = {
+        title: video.title.trim(),
     }
 
-    if (url) return {
+    return {
         urls: url,
+        fileMetadata,
         filenameAttributes: {
             service: "vk",
-            id: `${o.userId}_${o.videoId}`,
+            id: `${ownerId}_${videoId}${accessKey ? `_${accessKey}` : ''}`,
             title: fileMetadata.title,
-            author: fileMetadata.author,
-            resolution: `${quality}p`,
-            qualityLabel: `${quality}p`,
+            resolution: `${pickedQuality}p`,
+            qualityLabel: `${pickedQuality}p`,
             extension: "mp4"
         }
     }
-    return { error: "fetch.empty" }
 }

api/src/processing/services/xiaohongshu.js

@@ -0,0 +1,109 @@
+import { resolveRedirectingURL } from "../url.js";
+import { genericUserAgent } from "../../config.js";
+import { createStream } from "../../stream/manage.js";
+
+const https = (url) => {
+    return url.replace(/^http:/i, 'https:');
+}
+
+export default async function ({ id, token, shareId, h265, isAudioOnly, dispatcher }) {
+    let noteId = id;
+    let xsecToken = token;
+
+    if (!noteId) {
+        const patternMatch = await resolveRedirectingURL(
+            `https://xhslink.com/a/${shareId}`,
+            dispatcher
+        );
+
+        noteId = patternMatch?.id;
+        xsecToken = patternMatch?.token;
+    }
+
+    if (!noteId || !xsecToken) return { error: "fetch.short_link" };
+
+    const res = await fetch(`https://www.xiaohongshu.com/explore/${noteId}?xsec_token=${xsecToken}`, {
+        headers: {
+            "user-agent": genericUserAgent,
+        },
+        dispatcher,
+    });
+
+    const html = await res.text();
+
+    let note;
+    try {
+        const initialState = html
+            .split('<script>window.__INITIAL_STATE__=')[1]
+            .split('</script>')[0]
+            .replace(/:\s*undefined/g, ":null");
+
+        const data = JSON.parse(initialState);
+
+        const noteInfo = data?.note?.noteDetailMap;
+        if (!noteInfo) throw "no note detail map";
+
+        const currentNote = noteInfo[noteId];
+        if (!currentNote) throw "no current note in detail map";
+
+        note = currentNote.note;
+    } catch {}
+
+    if (!note) return { error: "fetch.empty" };
+
+    const video = note.video;
+    const images = note.imageList;
+
+    const filenameBase = `xiaohongshu_${noteId}`;
+
+    if (video) {
+        const videoFilename = `${filenameBase}.mp4`;
+        const audioFilename = `${filenameBase}_audio`;
+
+        let videoURL;
+
+        if (h265 && !isAudioOnly && video.consumer?.originVideoKey) {
+            videoURL = `https://sns-video-bd.xhscdn.com/${video.consumer.originVideoKey}`;
+        } else {
+            const h264Streams = video.media?.stream?.h264;
+
+            if (h264Streams?.length) {
+                videoURL = h264Streams.reduce((a, b) => Number(a?.videoBitrate) > Number(b?.videoBitrate) ? a : b).masterUrl;
+            }
+        }
+
+        if (!videoURL) return { error: "fetch.empty" };
+
+        return {
+            urls: https(videoURL),
+            filename: videoFilename,
+            audioFilename: audioFilename,
+        }
+    }
+
+    if (!images || images.length === 0) {
+        return { error: "fetch.empty" };
+    }
+
+    if (images.length === 1) {
+        return {
+            isPhoto: true,
+            urls: https(images[0].urlDefault),
+            filename: `${filenameBase}.jpg`,
+        }
+    }
+
+    const picker = images.map((image, i) => {
+        return {
+            type: "photo",
+            url: createStream({
+                service: "xiaohongshu",
+                type: "proxy",
+                url: https(image.urlDefault),
+                filename: `${filenameBase}_${i + 1}.jpg`,
+            })
+        }
+    });
+
+    return { picker };
+}

api/src/processing/services/youtube.js

@@ -1,16 +1,17 @@
-import { fetch } from "undici";
+import HLS from "hls-parser";
 
+import { fetch } from "undici";
 import { Innertube, Session } from "youtubei.js";
 
 import { env } from "../../config.js";
-import { cleanString } from "../../misc/utils.js";
-import { getCookie, updateCookieValues } from "../cookie/manager.js";
+import { getCookie } from "../cookie/manager.js";
+import { getYouTubeSession } from "../helpers/youtube-session.js";
 
 const PLAYER_REFRESH_PERIOD = 1000 * 60 * 15; // ms
 
 let innertube, lastRefreshedAt;
 
-const codecMatch = {
+const codecList = {
     h264: {
         videoCodec: "avc1",
         audioCodec: "mp4a",
@@ -18,8 +19,8 @@ const codecMatch = {
     },
     av1: {
         videoCodec: "av01",
-        audioCodec: "mp4a",
-        container: "mp4"
+        audioCodec: "opus",
+        container: "webm"
     },
     vp9: {
         videoCodec: "vp9",
@@ -28,32 +29,43 @@ const codecMatch = {
     }
 }
 
-const transformSessionData = (cookie) => {
-    if (!cookie)
-        return;
-
-    const values = { ...cookie.values() };
-    const REQUIRED_VALUES = [ 'access_token', 'refresh_token' ];
-
-    if (REQUIRED_VALUES.some(x => typeof values[x] !== 'string')) {
-        return;
+const hlsCodecList = {
+    h264: {
+        videoCodec: "avc1",
+        audioCodec: "mp4a",
+        container: "mp4"
+    },
+    vp9: {
+        videoCodec: "vp09",
+        audioCodec: "mp4a",
+        container: "webm"
     }
-
-    if (values.expires) {
-        values.expiry_date = values.expires;
-        delete values.expires;
-    } else if (!values.expiry_date) {
-        return;
-    }
-
-    return values;
 }
 
-const cloneInnertube = async (customFetch) => {
+const clientsWithNoCipher = ['IOS', 'ANDROID', 'YTSTUDIO_ANDROID', 'YTMUSIC_ANDROID'];
+
+const videoQualities = [144, 240, 360, 480, 720, 1080, 1440, 2160, 4320];
+
+const cloneInnertube = async (customFetch, useSession) => {
     const shouldRefreshPlayer = lastRefreshedAt + PLAYER_REFRESH_PERIOD < new Date();
+
+    const rawCookie = getCookie('youtube');
+    const cookie = rawCookie?.toString();
+
+    const sessionTokens = getYouTubeSession();
+    const retrieve_player = Boolean(sessionTokens || cookie);
+
+    if (useSession && env.ytSessionServer && !sessionTokens?.potoken) {
+        throw "no_session_tokens";
+    }
+
     if (!innertube || shouldRefreshPlayer) {
         innertube = await Innertube.create({
-            fetch: customFetch
+            fetch: customFetch,
+            retrieve_player,
+            cookie,
+            po_token: useSession ? sessionTokens?.potoken : undefined,
+            visitor_data: useSession ? sessionTokens?.visitor_data : undefined,
         });
         lastRefreshedAt = +new Date();
     }
@@ -64,81 +76,88 @@ const cloneInnertube = async (customFetch) => {
         innertube.session.api_version,
         innertube.session.account_index,
         innertube.session.player,
-        undefined,
+        cookie,
         customFetch ?? innertube.session.http.fetch,
         innertube.session.cache
     );
 
-    const cookie = getCookie('youtube_oauth');
-    const oauthData = transformSessionData(cookie);
-
-    if (!session.logged_in && oauthData) {
-        await session.oauth.init(oauthData);
-        session.logged_in = true;
-    }
-
-    if (session.logged_in) {
-        if (session.oauth.shouldRefreshToken()) {
-            await session.oauth.refreshAccessToken();
-        }
-
-        const cookieValues = cookie.values();
-        const oldExpiry = new Date(cookieValues.expiry_date);
-        const newExpiry = new Date(session.oauth.oauth2_tokens.expiry_date);
-
-        if (oldExpiry.getTime() !== newExpiry.getTime()) {
-            updateCookieValues(cookie, {
-                ...session.oauth.client_id,
-                ...session.oauth.oauth2_tokens,
-                expiry_date: newExpiry.toISOString()
-            });
-        }
-    }
-
     const yt = new Innertube(session);
     return yt;
 }
 
-export default async function(o) {
+export default async function (o) {
+    const quality = o.quality === "max" ? 9000 : Number(o.quality);
+
+    let useHLS = o.youtubeHLS;
+    let innertubeClient = o.innertubeClient || env.customInnertubeClient || "IOS";
+
+    // HLS playlists from the iOS client don't contain the av1 video format.
+    if (useHLS && o.format === "av1") {
+        useHLS = false;
+    }
+
+    if (useHLS) {
+        innertubeClient = "IOS";
+    }
+
+    // iOS client doesn't have adaptive formats of resolution >1080p,
+    // so we use the WEB_EMBEDDED client instead for those cases
+    const useSession =
+        env.ytSessionServer && (
+            (
+                !useHLS
+                && innertubeClient === "IOS"
+                && (
+                    (quality > 1080 && o.format !== "h264")
+                    || (quality > 1080 && o.format !== "vp9")
+                )
+            )
+        );
+
+    if (useSession) {
+        innertubeClient = env.ytSessionInnertubeClient || "WEB_EMBEDDED";
+    }
+
     let yt;
     try {
         yt = await cloneInnertube(
             (input, init) => fetch(input, {
                 ...init,
                 dispatcher: o.dispatcher
-            })
+            }),
+            useSession
         );
-    } catch(e) {
-        if (e.message?.endsWith("decipher algorithm")) {
+    } catch (e) {
+        if (e === "no_session_tokens") {
+            return { error: "youtube.no_session_tokens" };
+        } else if (e.message?.endsWith("decipher algorithm")) {
             return { error: "youtube.decipher" }
         } else if (e.message?.includes("refresh access token")) {
             return { error: "youtube.token_expired" }
         } else throw e;
     }
 
-    const quality = o.quality === "max" ? "9000" : o.quality;
-
-    let info, isDubbed,
-        format = o.format || "h264";
-
-    function qual(i) {
-        if (!i.quality_label) {
-            return;
-        }
-
-        return i.quality_label.split('p')[0].split('s')[0]
-    }
-
+    let info;
     try {
-        info = await yt.getBasicInfo(o.id, yt.session.logged_in ? 'ANDROID' : 'IOS');
-    } catch(e) {
-        if (e?.info?.reason === "This video is private") {
-            return { error: "content.video.private" };
-        } else if (e?.message === "This video is unavailable") {
-            return { error: "content.video.unavailable" };
-        } else {
-            return { error: "fetch.fail" };
+        info = await yt.getBasicInfo(o.id, innertubeClient);
+    } catch (e) {
+        if (e?.info) {
+            let errorInfo;
+            try { errorInfo = JSON.parse(e?.info); } catch {}
+
+            if (errorInfo?.reason === "This video is private") {
+                return { error: "content.video.private" };
+            }
+            if (["INVALID_ARGUMENT", "UNAUTHENTICATED"].includes(errorInfo?.error?.status)) {
+                return { error: "youtube.api_error" };
+            }
         }
+
+        if (e?.message === "This video is unavailable") {
+            return { error: "content.video.unavailable" };
+        }
+
+        return { error: "fetch.fail" };
     }
 
     if (!info) return { error: "fetch.fail" };
@@ -146,37 +165,47 @@ export default async function(o) {
     const playability = info.playability_status;
     const basicInfo = info.basic_info;
 
-    if (playability.status === "LOGIN_REQUIRED") {
-        if (playability.reason.endsWith("bot")) {
-            return { error: "youtube.login" }
-        }
-        if (playability.reason.endsWith("age")) {
-            return { error: "content.video.age" }
-        }
-        if (playability?.error_screen?.reason?.text === "Private video") {
-            return { error: "content.video.private" }
-        }
-    }
+    switch (playability.status) {
+        case "LOGIN_REQUIRED":
+            if (playability.reason.endsWith("bot")) {
+                return { error: "youtube.login" }
+            }
+            if (playability.reason.endsWith("age") || playability.reason.endsWith("inappropriate for some users.")) {
+                return { error: "content.video.age" }
+            }
+            if (playability?.error_screen?.reason?.text === "Private video") {
+                return { error: "content.video.private" }
+            }
+            break;
 
-    if (playability.status === "UNPLAYABLE") {
-        if (playability?.reason?.endsWith("request limit.")) {
-            return { error: "fetch.rate" }
-        }
-        if (playability?.error_screen?.subreason?.text?.endsWith("in your country")) {
-            return { error: "content.video.region" }
-        }
-        if (playability?.error_screen?.reason?.text === "Private video") {
-            return { error: "content.video.private" }
-        }
+        case "UNPLAYABLE":
+            if (playability?.reason?.endsWith("request limit.")) {
+                return { error: "fetch.rate" }
+            }
+            if (playability?.error_screen?.subreason?.text?.endsWith("in your country")) {
+                return { error: "content.video.region" }
+            }
+            if (playability?.error_screen?.reason?.text === "Private video") {
+                return { error: "content.video.private" }
+            }
+            break;
+
+        case "AGE_VERIFICATION_REQUIRED":
+            return { error: "content.video.age" };
     }
 
     if (playability.status !== "OK") {
         return { error: "content.video.unavailable" };
     }
+
     if (basicInfo.is_live) {
         return { error: "content.video.live" };
     }
 
+    if (basicInfo.duration > env.durationLimit) {
+        return { error: "content.too_long" };
+    }
+
     // return a critical error if returned video is "Video Not Available"
     // or a similar stub by youtube
     if (basicInfo.id !== o.id) {
@@ -186,126 +215,303 @@ export default async function(o) {
         }
     }
 
-    const filterByCodec = (formats) =>
-        formats
-        .filter(e =>
-            e.mime_type.includes(codecMatch[format].videoCodec)
-            || e.mime_type.includes(codecMatch[format].audioCodec)
-        )
-        .sort((a, b) => Number(b.bitrate) - Number(a.bitrate));
-
-    let adaptive_formats = filterByCodec(info.streaming_data.adaptive_formats);
-
-    if (adaptive_formats.length === 0 && format === "vp9") {
-        format = "h264"
-        adaptive_formats = filterByCodec(info.streaming_data.adaptive_formats)
+    const normalizeQuality = res => {
+        const shortestSide = Math.min(res.height, res.width);
+        return videoQualities.find(qual => qual >= shortestSide);
     }
 
-    let bestQuality;
+    let video, audio, dubbedLanguage,
+        codec = o.format || "h264", itag = o.itag;
 
-    const bestVideo = adaptive_formats.find(i => i.has_video && i.content_length);
-    const hasAudio = adaptive_formats.find(i => i.has_audio && i.content_length);
+    if (useHLS) {
+        const hlsManifest = info.streaming_data.hls_manifest_url;
 
-    if (bestVideo) bestQuality = qual(bestVideo);
+        if (!hlsManifest) {
+            return { error: "youtube.no_hls_streams" };
+        }
 
-    if ((!bestQuality && !o.isAudioOnly) || !hasAudio)
-        return { error: "youtube.codec" };
+        const fetchedHlsManifest = await fetch(hlsManifest, {
+            dispatcher: o.dispatcher,
+        }).then(r => {
+            if (r.status === 200) {
+                return r.text();
+            } else {
+                throw new Error("couldn't fetch the HLS playlist");
+            }
+        }).catch(() => { });
 
-    if (basicInfo.duration > env.durationLimit)
-        return { error: "content.too_long" };
+        if (!fetchedHlsManifest) {
+            return { error: "youtube.no_hls_streams" };
+        }
 
-    const checkBestAudio = (i) => (i.has_audio && !i.has_video);
+        const variants = HLS.parse(fetchedHlsManifest).variants.sort(
+            (a, b) => Number(b.bandwidth) - Number(a.bandwidth)
+        );
 
-    let audio = adaptive_formats.find(i =>
-        checkBestAudio(i) && i.is_original
-    );
+        if (!variants || variants.length === 0) {
+            return { error: "youtube.no_hls_streams" };
+        }
 
-    if (o.dubLang) {
-        let dubbedAudio = adaptive_formats.find(i =>
-            checkBestAudio(i)
-            && i.language === o.dubLang
-            && i.audio_track
-        )
+        const matchHlsCodec = codecs => (
+            codecs.includes(hlsCodecList[codec].videoCodec)
+        );
 
-        if (dubbedAudio && !dubbedAudio?.audio_track?.audio_is_default) {
-            audio = dubbedAudio;
-            isDubbed = true;
+        const best = variants.find(i => matchHlsCodec(i.codecs));
+
+        const preferred = variants.find(i =>
+            matchHlsCodec(i.codecs) && normalizeQuality(i.resolution) === quality
+        );
+
+        let selected = preferred || best;
+
+        if (!selected) {
+            codec = "h264";
+            selected = variants.find(i => matchHlsCodec(i.codecs));
+        }
+
+        if (!selected) {
+            return { error: "youtube.no_matching_format" };
+        }
+
+        audio = selected.audio.find(i => i.isDefault);
+
+        // some videos (mainly those with AI dubs) don't have any tracks marked as default
+        // why? god knows, but we assume that a default track is marked as such in the title
+        if (!audio) {
+            audio = selected.audio.find(i => i.name.endsWith("- original"));
+        }
+
+        if (o.dubLang) {
+            const dubbedAudio = selected.audio.find(i =>
+                i.language?.startsWith(o.dubLang)
+            );
+
+            if (dubbedAudio && !dubbedAudio.isDefault) {
+                dubbedLanguage = dubbedAudio.language;
+                audio = dubbedAudio;
+            }
+        }
+
+        selected.audio = [];
+        selected.subtitles = [];
+        video = selected;
+    } else {
+        // i miss typescript so bad
+        const sorted_formats = {
+            h264: {
+                video: [],
+                audio: [],
+                bestVideo: undefined,
+                bestAudio: undefined,
+            },
+            vp9: {
+                video: [],
+                audio: [],
+                bestVideo: undefined,
+                bestAudio: undefined,
+            },
+            av1: {
+                video: [],
+                audio: [],
+                bestVideo: undefined,
+                bestAudio: undefined,
+            },
+        }
+
+        const checkFormat = (format, pCodec) => format.content_length &&
+            (format.mime_type.includes(codecList[pCodec].videoCodec)
+                || format.mime_type.includes(codecList[pCodec].audioCodec));
+
+        // sort formats & weed out bad ones
+        info.streaming_data.adaptive_formats.sort((a, b) =>
+            Number(b.bitrate) - Number(a.bitrate)
+        ).forEach(format => {
+            Object.keys(codecList).forEach(yCodec => {
+                const matchingItag = slot => !itag?.[slot] || itag[slot] === format.itag;
+                const sorted = sorted_formats[yCodec];
+                const goodFormat = checkFormat(format, yCodec);
+                if (!goodFormat) return;
+
+                if (format.has_video && matchingItag('video')) {
+                    sorted.video.push(format);
+                    if (!sorted.bestVideo)
+                        sorted.bestVideo = format;
+                }
+
+                if (format.has_audio && matchingItag('audio')) {
+                    sorted.audio.push(format);
+                    if (!sorted.bestAudio)
+                        sorted.bestAudio = format;
+                }
+            })
+        });
+
+        const noBestMedia = () => {
+            const vid = sorted_formats[codec]?.bestVideo;
+            const aud = sorted_formats[codec]?.bestAudio;
+            return (!vid && !o.isAudioOnly) || (!aud && o.isAudioOnly)
+        };
+
+        if (noBestMedia()) {
+            if (codec === "av1") codec = "vp9";
+            else if (codec === "vp9") codec = "av1";
+
+            // if there's no higher quality fallback, then use h264
+            if (noBestMedia()) codec = "h264";
+        }
+
+        // if there's no proper combo of av1, vp9, or h264, then give up
+        if (noBestMedia()) {
+            return { error: "youtube.no_matching_format" };
+        }
+
+        audio = sorted_formats[codec].bestAudio;
+
+        if (audio?.audio_track && !audio?.audio_track?.audio_is_default) {
+            audio = sorted_formats[codec].audio.find(i =>
+                i?.audio_track?.audio_is_default
+            );
+        }
+
+        if (o.dubLang) {
+            const dubbedAudio = sorted_formats[codec].audio.find(i =>
+                i.language?.startsWith(o.dubLang) && i.audio_track
+            );
+
+            if (dubbedAudio && !dubbedAudio?.audio_track?.audio_is_default) {
+                audio = dubbedAudio;
+                dubbedLanguage = dubbedAudio.language;
+            }
+        }
+
+        if (!o.isAudioOnly) {
+            const qual = (i) => {
+                return normalizeQuality({
+                    width: i.width,
+                    height: i.height,
+                })
+            }
+
+            const bestQuality = qual(sorted_formats[codec].bestVideo);
+            const useBestQuality = quality >= bestQuality;
+
+            video = useBestQuality
+                ? sorted_formats[codec].bestVideo
+                : sorted_formats[codec].video.find(i => qual(i) === quality);
+
+            if (!video) video = sorted_formats[codec].bestVideo;
         }
     }
 
-    if (!audio) {
-        audio = adaptive_formats.find(i => checkBestAudio(i));
+    if (video?.drm_families || audio?.drm_families) {
+        return { error: "youtube.drm" };
     }
 
-    let fileMetadata = {
-        title: cleanString(basicInfo.title.trim()),
-        artist: cleanString(basicInfo.author.replace("- Topic", "").trim()),
+    const fileMetadata = {
+        title: basicInfo.title.trim(),
+        artist: basicInfo.author.replace("- Topic", "").trim()
     }
 
     if (basicInfo?.short_description?.startsWith("Provided to YouTube by")) {
-        let descItems = basicInfo.short_description.split("\n\n");
-        fileMetadata.album = descItems[2];
-        fileMetadata.copyright = descItems[3];
-        if (descItems[4].startsWith("Released on:")) {
-            fileMetadata.date = descItems[4].replace("Released on: ", '').trim()
+        const descItems = basicInfo.short_description.split("\n\n", 5);
+
+        if (descItems.length === 5) {
+            fileMetadata.album = descItems[2];
+            fileMetadata.copyright = descItems[3];
+            if (descItems[4].startsWith("Released on:")) {
+                fileMetadata.date = descItems[4].replace("Released on: ", '').trim();
+            }
         }
     }
 
-    let filenameAttributes = {
+    const filenameAttributes = {
         service: "youtube",
         id: o.id,
         title: fileMetadata.title,
         author: fileMetadata.artist,
-        youtubeDubName: isDubbed ? o.dubLang : false
+        youtubeDubName: dubbedLanguage || false,
     }
 
-    if (audio && o.isAudioOnly) return {
-        type: "audio",
-        isAudioOnly: true,
-        urls: audio.decipher(yt.session.player),
-        filenameAttributes: filenameAttributes,
-        fileMetadata: fileMetadata,
-        bestAudio: format === "h264" ? "m4a" : "opus"
-    }
+    itag = {
+        video: video?.itag,
+        audio: audio?.itag
+    };
 
-    const matchingQuality = Number(quality) > Number(bestQuality) ? bestQuality : quality,
-        checkSingle = i =>
-            qual(i) === matchingQuality && i.mime_type.includes(codecMatch[format].videoCodec),
-        checkRender = i =>
-            qual(i) === matchingQuality && i.has_video && !i.has_audio;
+    const originalRequest = {
+        ...o,
+        dispatcher: undefined,
+        itag,
+        innertubeClient
+    };
 
-    let match, type, urls;
+    if (audio && o.isAudioOnly) {
+        let bestAudio = codec === "h264" ? "m4a" : "opus";
+        let urls = audio.url;
 
-    // prefer good premuxed videos if available
-    if (!o.isAudioOnly && !o.isAudioMuted && format === "h264" && bestVideo.fps <= 30) {
-        match = info.streaming_data.formats.find(checkSingle);
-        type = "proxy";
-        urls = match?.decipher(yt.session.player);
-    }
+        if (useHLS) {
+            bestAudio = "mp3";
+            urls = audio.uri;
+        }
 
-    const video = adaptive_formats.find(checkRender);
+        if (!clientsWithNoCipher.includes(innertubeClient) && innertube) {
+            urls = audio.decipher(innertube.session.player);
+        }
 
-    if (!match && video && audio) {
-        match = video;
-        type = "merge";
-        urls = [
-            video.decipher(yt.session.player),
-            audio.decipher(yt.session.player)
-        ]
-    }
-
-    if (match) {
-        filenameAttributes.qualityLabel = match.quality_label;
-        filenameAttributes.resolution = `${match.width}x${match.height}`;
-        filenameAttributes.extension = codecMatch[format].container;
-        filenameAttributes.youtubeFormat = format;
         return {
-            type,
+            type: "audio",
+            isAudioOnly: true,
             urls,
             filenameAttributes,
-            fileMetadata
+            fileMetadata,
+            bestAudio,
+            isHLS: useHLS,
+            originalRequest
         }
     }
 
-    return { error: "fetch.fail" }
+    if (video && audio) {
+        let resolution;
+
+        if (useHLS) {
+            resolution = normalizeQuality(video.resolution);
+            filenameAttributes.resolution = `${video.resolution.width}x${video.resolution.height}`;
+            filenameAttributes.extension = hlsCodecList[codec].container;
+
+            video = video.uri;
+            audio = audio.uri;
+        } else {
+            resolution = normalizeQuality({
+                width: video.width,
+                height: video.height,
+            });
+
+            filenameAttributes.resolution = `${video.width}x${video.height}`;
+            filenameAttributes.extension = codecList[codec].container;
+
+            if (!clientsWithNoCipher.includes(innertubeClient) && innertube) {
+                video = video.decipher(innertube.session.player);
+                audio = audio.decipher(innertube.session.player);
+            } else {
+                video = video.url;
+                audio = audio.url;
+            }
+        }
+
+        filenameAttributes.qualityLabel = `${resolution}p`;
+        filenameAttributes.youtubeFormat = codec;
+
+        return {
+            type: "merge",
+            urls: [
+                video,
+                audio,
+            ],
+            filenameAttributes,
+            fileMetadata,
+            isHLS: useHLS,
+            originalRequest
+        }
+    }
+
+    return { error: "youtube.no_matching_format" };
 }

api/src/processing/url.js

@@ -1,8 +1,9 @@
-import psl from "psl";
+import psl from "@imput/psl";
 import { strict as assert } from "node:assert";
 
 import { env } from "../config.js";
 import { services } from "./service-config.js";
+import { getRedirectingURL } from "../misc/utils.js";
 import { friendlyServiceName } from "./service-alias.js";
 
 function aliasURL(url) {
@@ -42,7 +43,7 @@ function aliasURL(url) {
         case "fixvx":
         case "x":
             if (services.twitter.altDomains.includes(url.hostname)) {
-                url.hostname = 'twitter.com'
+                url.hostname = 'twitter.com';
             }
             break;
 
@@ -85,9 +86,29 @@ function aliasURL(url) {
                 url.hostname = 'instagram.com';
             }
             break;
+
+        case "vk":
+        case "vkvideo":
+            if (services.vk.altDomains.includes(url.hostname)) {
+                url.hostname = 'vk.com';
+            }
+            break;
+
+        case "xhslink":
+            if (url.hostname === 'xhslink.com' && parts.length === 3) {
+                url = new URL(`https://www.xiaohongshu.com/a/${parts[2]}`);
+            }
+            break;
+
+        case "loom":
+            const idPart = parts[parts.length - 1];
+            if (idPart.length > 32) {
+                url.pathname = `/share/${idPart.slice(-32)}`;
+            }
+            break;
     }
 
-    return url
+    return url;
 }
 
 function cleanURL(url) {
@@ -107,31 +128,41 @@ function cleanURL(url) {
             break;
         case "vk":
             if (url.pathname.includes('/clip') && url.searchParams.get('z')) {
-                limitQuery('z')
+                limitQuery('z');
             }
             break;
         case "youtube":
             if (url.searchParams.get('v')) {
-                limitQuery('v')
+                limitQuery('v');
             }
             break;
         case "rutube":
             if (url.searchParams.get('p')) {
-                limitQuery('p')
+                limitQuery('p');
+            }
+            break;
+        case "twitter":
+            if (url.searchParams.get('post_id')) {
+                limitQuery('post_id');
+            }
+            break;
+        case "xiaohongshu":
+            if (url.searchParams.get('xsec_token')) {
+                limitQuery('xsec_token');
             }
             break;
     }
 
     if (stripQuery) {
-        url.search = ''
+        url.search = '';
     }
 
-    url.username = url.password = url.port = url.hash = ''
+    url.username = url.password = url.port = url.hash = '';
 
     if (url.pathname.endsWith('/'))
         url.pathname = url.pathname.slice(0, -1);
 
-    return url
+    return url;
 }
 
 function getHostIfValid(url) {
@@ -169,6 +200,11 @@ export function extract(url) {
     }
 
     if (!env.enabledServices.has(host)) {
+        // show a different message when youtube is disabled on official instances
+        // as it only happens when shit hits the fan
+        if (new URL(env.apiURL).hostname.endsWith(".imput.net") && host === "youtube") {
+            return { error: "youtube.temporary_disabled" };
+        }
         return { error: "service.disabled" };
     }
 
@@ -194,3 +230,17 @@ export function extract(url) {
 
     return { host, patternMatch };
 }
+
+export async function resolveRedirectingURL(url, dispatcher, userAgent) {
+    const originalService = getHostIfValid(normalizeURL(url));
+    if (!originalService) return;
+
+    const canonicalURL = await getRedirectingURL(url, dispatcher, userAgent);
+    if (!canonicalURL) return;
+
+    const { host, patternMatch } = extract(normalizeURL(canonicalURL));
+
+    if (host === originalService) {
+        return patternMatch;
+    }
+}

api/src/security/api-keys.js

@@ -0,0 +1,227 @@
+import { env } from "../config.js";
+import { readFile } from "node:fs/promises";
+import { Green, Yellow } from "../misc/console-text.js";
+import ip from "ipaddr.js";
+import * as cluster from "../misc/cluster.js";
+
+// this function is a modified variation of code
+// from https://stackoverflow.com/a/32402438/14855621
+const generateWildcardRegex = rule => {
+    var escapeRegex = (str) => str.replace(/([.*+?^=!:${}()|\[\]\/\\])/g, "\\$1");
+    return new RegExp("^" + rule.split("*").map(escapeRegex).join(".*") + "$");
+}
+
+const UUID_REGEX = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
+
+let keys = {};
+
+const ALLOWED_KEYS = new Set(['name', 'ips', 'userAgents', 'limit']);
+
+/* Expected format pseudotype:
+** type KeyFileContents = Record<
+**    UUIDv4String,
+**    {
+**        name?: string,
+**        limit?: number | "unlimited",
+**        ips?: CIDRString[],
+**        userAgents?: string[]
+**    }
+** >;
+*/
+
+const validateKeys = (input) => {
+    if (typeof input !== 'object' || input === null) {
+        throw "input is not an object";
+    }
+
+    if (Object.keys(input).some(x => !UUID_REGEX.test(x))) {
+        throw "key file contains invalid key(s)";
+    }
+
+    Object.values(input).forEach(details => {
+        if (typeof details !== 'object' || details === null) {
+            throw "some key(s) are incorrectly configured";
+        }
+
+        const unexpected_key = Object.keys(details).find(k => !ALLOWED_KEYS.has(k));
+        if (unexpected_key) {
+            throw "detail object contains unexpected key: " + unexpected_key;
+        }
+
+        if (details.limit && details.limit !== 'unlimited') {
+            if (typeof details.limit !== 'number')
+                throw "detail object contains invalid limit (not a number)";
+            else if (details.limit < 1)
+                throw "detail object contains invalid limit (not a positive number)";
+        }
+
+        if (details.ips) {
+            if (!Array.isArray(details.ips))
+                throw "details object contains value for `ips` which is not an array";
+
+            const invalid_ip = details.ips.find(
+                addr => typeof addr !== 'string' || (!ip.isValidCIDR(addr) && !ip.isValid(addr))
+            );
+
+            if (invalid_ip) {
+                throw "`ips` in details contains an invalid IP or CIDR range: " + invalid_ip;
+            }
+        }
+
+        if (details.userAgents) {
+            if (!Array.isArray(details.userAgents))
+                throw "details object contains value for `userAgents` which is not an array";
+
+            const invalid_ua = details.userAgents.find(ua => typeof ua !== 'string');
+            if (invalid_ua) {
+                throw "`userAgents` in details contains an invalid user agent: " + invalid_ua;
+            }
+        }
+    });
+}
+
+const formatKeys = (keyData) => {
+    const formatted = {};
+
+    for (let key in keyData) {
+        const data = keyData[key];
+        key = key.toLowerCase();
+
+        formatted[key] = {};
+
+        if (data.limit) {
+            if (data.limit === "unlimited") {
+                data.limit = Infinity;
+            }
+
+            formatted[key].limit = data.limit;
+        }
+
+        if (data.ips) {
+            formatted[key].ips = data.ips.map(addr => {
+                if (ip.isValid(addr)) {
+                    const parsed = ip.parse(addr);
+                    const range = parsed.kind() === 'ipv6' ? 128 : 32;
+                    return [ parsed, range ];
+                }
+
+                return ip.parseCIDR(addr);
+            });
+        }
+
+        if (data.userAgents) {
+            formatted[key].userAgents = data.userAgents.map(generateWildcardRegex);
+        }
+    }
+
+    return formatted;
+}
+
+const updateKeys = (newKeys) => {
+    keys = formatKeys(newKeys);
+}
+
+const loadKeys = async (source) => {
+    let updated;
+    if (source.protocol === 'file:') {
+        const pathname = source.pathname === '/' ? '' : source.pathname;
+        updated = JSON.parse(
+            await readFile(
+                decodeURIComponent(source.host + pathname),
+                'utf8'
+            )
+        );
+    } else {
+        updated = await fetch(source).then(a => a.json());
+    }
+
+    validateKeys(updated);
+
+    cluster.broadcast({ api_keys: updated });
+
+    updateKeys(updated);
+}
+
+const wrapLoad = (url, initial = false) => {
+    loadKeys(url)
+    .then(() => {
+        if (initial) {
+            console.log(`${Green('[✓]')} api keys loaded successfully!`)
+        }
+    })
+    .catch((e) => {
+        console.error(`${Yellow('[!]')} Failed loading API keys at ${new Date().toISOString()}.`);
+        console.error('Error:', e);
+    })
+}
+
+const err = (reason) => ({ success: false, error: reason });
+
+export const validateAuthorization = (req) => {
+    const authHeader = req.get('Authorization');
+
+    if (typeof authHeader !== 'string') {
+        return err("missing");
+    }
+
+    const [ authType, keyString ] = authHeader.split(' ', 2);
+    if (authType.toLowerCase() !== 'api-key') {
+        return err("not_api_key");
+    }
+
+    if (!UUID_REGEX.test(keyString) || `${authType} ${keyString}` !== authHeader) {
+        return err("invalid");
+    }
+
+    const matchingKey = keys[keyString.toLowerCase()];
+    if (!matchingKey) {
+        return err("not_found");
+    }
+
+    if (matchingKey.ips) {
+        let addr;
+        try {
+            addr = ip.parse(req.ip);
+        } catch {
+            return err("invalid_ip");
+        }
+
+        const ip_allowed = matchingKey.ips.some(
+            ([ allowed, size ]) => {
+                return addr.kind() === allowed.kind()
+                        && addr.match(allowed, size);
+            }
+        );
+
+        if (!ip_allowed) {
+            return err("ip_not_allowed");
+        }
+    }
+
+    if (matchingKey.userAgents) {
+        const userAgent = req.get('User-Agent');
+        if (!matchingKey.userAgents.some(regex => regex.test(userAgent))) {
+            return err("ua_not_allowed");
+        }
+    }
+
+    req.rateLimitKey = keyString.toLowerCase();
+    req.rateLimitMax = matchingKey.limit;
+
+    return { success: true };
+}
+
+export const setup = (url) => {
+    if (cluster.isPrimary) {
+        wrapLoad(url, true);
+        if (env.keyReloadInterval > 0) {
+            setInterval(() => wrapLoad(url), env.keyReloadInterval * 1000);
+        }
+    } else if (cluster.isWorker) {
+        process.on('message', (message) => {
+            if ('api_keys' in message) {
+                updateKeys(message.api_keys);
+            }
+        });
+    }
+}

api/src/security/secrets.js

@@ -0,0 +1,62 @@
+import cluster from "node:cluster";
+import { createHmac, randomBytes } from "node:crypto";
+
+const generateSalt = () => {
+    if (cluster.isPrimary)
+        return randomBytes(64);
+
+    return null;
+}
+
+let rateSalt = generateSalt();
+let streamSalt = generateSalt();
+
+export const syncSecrets = () => {
+    return new Promise((resolve, reject) => {
+        if (cluster.isPrimary) {
+            let remaining = Object.values(cluster.workers).length;
+            const handleReady = (worker, m) => {
+                if (m.ready)
+                    worker.send({ rateSalt, streamSalt });
+
+                if (!--remaining)
+                    resolve();
+            }
+
+            for (const worker of Object.values(cluster.workers)) {
+                worker.once(
+                    'message',
+                    (m) => handleReady(worker, m)
+                );
+            }
+        } else if (cluster.isWorker) {
+            if (rateSalt || streamSalt)
+                return reject();
+
+            process.send({ ready: true });
+            process.once('message', (message) => {
+                if (rateSalt || streamSalt)
+                    return reject();
+
+                if (message.rateSalt && message.streamSalt) {
+                    streamSalt = Buffer.from(message.streamSalt);
+                    rateSalt = Buffer.from(message.rateSalt);
+                    resolve();
+                }
+            });
+        } else reject();
+    });
+}
+
+
+export const hashHmac = (value, type) => {
+    let salt;
+    if (type === 'rate')
+        salt = rateSalt;
+    else if (type === 'stream')
+        salt = streamSalt;
+    else
+        throw "unknown salt";
+
+    return createHmac("sha256", salt).update(value).digest();
+}

api/src/store/base-store.js

@@ -0,0 +1,48 @@
+const _stores = new Set();
+
+export class Store {
+    id;
+
+    constructor(name) {
+        name = name.toUpperCase();
+
+        if (_stores.has(name))
+            throw `${name} store already exists`;
+        _stores.add(name);
+
+        this.id = name;
+    }
+
+    async _has(_key) { await Promise.reject("needs implementation"); }
+    has(key) {
+        if (typeof key !== 'string') {
+            key = key.toString();
+        }
+
+        return this._has(key);
+    }
+
+    async _get(_key) { await Promise.reject("needs implementation"); }
+    async get(key) {
+        if (typeof key !== 'string') {
+            key = key.toString();
+        }
+
+        const val = await this._get(key);
+        if (val === null)
+            return null;
+
+        return val;
+    }
+
+    async _set(_key, _val, _exp_sec = -1) { await Promise.reject("needs implementation") }
+    set(key, val, exp_sec = -1) {
+        if (typeof key !== 'string') {
+            key = key.toString();
+        }
+
+        exp_sec = Math.round(exp_sec);
+
+        return this._set(key, val, exp_sec);
+    }
+};

api/src/store/memory-store.js

@@ -0,0 +1,77 @@
+import { MinPriorityQueue } from '@datastructures-js/priority-queue';
+import { Store } from './base-store.js';
+
+// minimum delay between sweeps to avoid repeatedly
+// sweeping entries close in proximity one by one.
+const MIN_THRESHOLD_MS = 2500;
+
+export default class MemoryStore extends Store {
+    #store = new Map();
+    #timeouts = new MinPriorityQueue/*<{ t: number, k: unknown }>*/((obj) => obj.t);
+    #nextSweep = { id: null, t: null };
+
+    constructor(name) {
+        super(name);
+    }
+
+    _has(key) {
+        return this.#store.has(key);
+    }
+
+    _get(key) {
+        const val = this.#store.get(key);
+
+        return val === undefined ? null : val;
+    }
+
+    _set(key, val, exp_sec = -1) {
+        if (this.#store.has(key)) {
+            this.#timeouts.remove(o => o.k === key);
+        }
+
+        if (exp_sec > 0) {
+            const exp = 1000 * exp_sec;
+            const timeout_at = +new Date() + exp;
+
+            this.#timeouts.enqueue({ k: key, t: timeout_at });
+        }
+
+        this.#store.set(key, val);
+        this.#reschedule();
+    }
+
+    #reschedule() {
+        const current_time = new Date().getTime();
+        const time = this.#timeouts.front()?.t;
+        if (!time) {
+            return;
+        } else if (time < current_time) {
+            return this.#sweepNow();
+        }
+
+        const sweep = this.#nextSweep;
+        if (sweep.id === null || sweep.t > time) {
+            if (sweep.id) {
+                clearTimeout(sweep.id);
+            }
+
+            sweep.t = time;
+            sweep.id = setTimeout(
+                () => this.#sweepNow(),
+                Math.max(MIN_THRESHOLD_MS, time - current_time)
+            );
+            sweep.id.unref();
+        }
+    }
+
+    #sweepNow() {
+        while (this.#timeouts.front()?.t < new Date().getTime()) {
+            const item = this.#timeouts.dequeue();
+            this.#store.delete(item.k);
+        }
+
+        this.#nextSweep.id = null;
+        this.#nextSweep.t = null;
+        this.#reschedule();
+    }
+}

api/src/store/redis-ratelimit.js

@@ -0,0 +1,19 @@
+import { env } from "../config.js";
+
+let client, redis, redisLimiter;
+
+export const createStore = async (name) => {
+    if (!env.redisURL) return;
+
+    if (!client) {
+        redis = await import('redis');
+        redisLimiter = await import('rate-limit-redis');
+        client = redis.createClient({ url: env.redisURL });
+        await client.connect();
+    }
+
+    return new redisLimiter.default({
+        prefix: `RL${name}_`,
+        sendCommand: (...args) => client.sendCommand(args),
+    });
+}

api/src/store/redis-store.js

@@ -0,0 +1,64 @@
+import { commandOptions, createClient } from "redis";
+import { env } from "../config.js";
+import { Store } from "./base-store.js";
+
+export default class RedisStore extends Store {
+    #client = createClient({
+        url: env.redisURL,
+    });
+    #connected;
+
+    constructor(name) {
+        super(name);
+        this.#connected = this.#client.connect();
+    }
+
+    #keyOf(key) {
+        return this.id + '_' + key;
+    }
+
+    async _has(key) {
+        await this.#connected;
+
+        return this.#client.hExists(key);
+    }
+
+    async _get(key) {
+        await this.#connected;
+
+        const valueType = await this.#client.get(this.#keyOf(key) + '_t');
+        const value = await this.#client.get(
+            commandOptions({ returnBuffers: true }),
+            this.#keyOf(key)
+        );
+
+        if (!value) {
+            return null;
+        }
+
+        if (valueType === 'b')
+            return value;
+        else
+            return JSON.parse(value);
+    }
+
+    async _set(key, val, exp_sec = -1) {
+        await this.#connected;
+
+        const options = exp_sec > 0 ? { EX: exp_sec } : undefined;
+
+        if (val instanceof Buffer) {
+            await this.#client.set(
+                this.#keyOf(key) + '_t',
+                'b',
+                options
+            );
+        }
+
+        await this.#client.set(
+            this.#keyOf(key),
+            val,
+            options
+        );
+    }
+}

api/src/store/store.js

@@ -0,0 +1,10 @@
+import { env } from '../config.js';
+
+let _export;
+if (env.redisURL) {
+    _export = await import('./redis-store.js');
+} else {
+    _export = await import('./memory-store.js');
+}
+
+export default _export.default;

api/src/stream/internal-hls.js

@@ -16,15 +16,17 @@ function transformObject(streamInfo, hlsObject) {
 
     let fullUrl;
     if (getURL(hlsObject.uri)) {
-        fullUrl = hlsObject.uri;
+        fullUrl = new URL(hlsObject.uri);
     } else {
         fullUrl = new URL(hlsObject.uri, streamInfo.url);
     }
 
-    hlsObject.uri = createInternalStream(fullUrl.toString(), streamInfo);
+    if (fullUrl.hostname !== '127.0.0.1') {
+        hlsObject.uri = createInternalStream(fullUrl.toString(), streamInfo);
 
-    if (hlsObject.map) {
-        hlsObject.map = transformObject(streamInfo, hlsObject.map);
+        if (hlsObject.map) {
+            hlsObject.map = transformObject(streamInfo, hlsObject.map);
+        }
     }
 
     return hlsObject;
@@ -53,7 +55,7 @@ function transformMediaPlaylist(streamInfo, hlsPlaylist) {
 
 const HLS_MIME_TYPES = ["application/vnd.apple.mpegurl", "audio/mpegurl", "application/x-mpegURL"];
 
-export function isHlsRequest (req) {
+export function isHlsResponse (req) {
     return HLS_MIME_TYPES.includes(req.headers['content-type']);
 }
 

api/src/stream/internal.js

@@ -1,13 +1,13 @@
 import { request } from "undici";
 import { Readable } from "node:stream";
 import { closeRequest, getHeaders, pipe } from "./shared.js";
-import { handleHlsPlaylist, isHlsRequest } from "./internal-hls.js";
+import { handleHlsPlaylist, isHlsResponse } from "./internal-hls.js";
 
 const CHUNK_SIZE = BigInt(8e6); // 8 MB
 const min = (a, b) => a < b ? a : b;
 
 async function* readChunks(streamInfo, size) {
-    let read = 0n;
+    let read = 0n, chunksSinceTransplant = 0;
     while (read < size) {
         if (streamInfo.controller.signal.aborted) {
             throw new Error("controller aborted");
@@ -19,9 +19,20 @@ async function* readChunks(streamInfo, size) {
                 Range: `bytes=${read}-${read + CHUNK_SIZE}`
             },
             dispatcher: streamInfo.dispatcher,
-            signal: streamInfo.controller.signal
+            signal: streamInfo.controller.signal,
+            maxRedirections: 4
         });
 
+        if (chunk.statusCode === 403 && chunksSinceTransplant >= 3 && streamInfo.transplant) {
+            chunksSinceTransplant = 0;
+            try {
+                await streamInfo.transplant(streamInfo.dispatcher);
+                continue;
+            } catch {}
+        }
+
+        chunksSinceTransplant++;
+
         const expected = min(CHUNK_SIZE, size - read);
         const received = BigInt(chunk.headers['content-length']);
 
@@ -42,14 +53,25 @@ async function handleYoutubeStream(streamInfo, res) {
     const cleanup = () => (res.end(), closeRequest(streamInfo.controller));
 
     try {
-        const req = await fetch(streamInfo.url, {
-            headers: getHeaders('youtube'),
-            method: 'HEAD',
-            dispatcher: streamInfo.dispatcher,
-            signal
-        });
+        let req, attempts = 3;
+        while (attempts--) {
+            req = await fetch(streamInfo.url, {
+                headers: getHeaders('youtube'),
+                method: 'HEAD',
+                dispatcher: streamInfo.dispatcher,
+                signal
+            });
+
+            streamInfo.url = req.url;
+            if (req.status === 403 && streamInfo.transplant) {
+                try {
+                    await streamInfo.transplant(streamInfo.dispatcher);
+                } catch {
+                    break;
+                }
+            } else break;
+        }
 
-        streamInfo.url = req.url;
         const size = BigInt(req.headers.get('content-length'));
 
         if (req.status !== 200 || !size) {
@@ -83,7 +105,7 @@ async function handleGenericStream(streamInfo, res) {
     const cleanup = () => res.end();
 
     try {
-        const req = await request(streamInfo.url, {
+        const fileResponse = await request(streamInfo.url, {
             headers: {
                 ...Object.fromEntries(streamInfo.headers),
                 host: undefined
@@ -93,19 +115,28 @@ async function handleGenericStream(streamInfo, res) {
             maxRedirections: 16
         });
 
-        res.status(req.statusCode);
-        req.body.on('error', () => {});
+        res.status(fileResponse.statusCode);
+        fileResponse.body.on('error', () => {});
 
-        for (const [ name, value ] of Object.entries(req.headers))
-            res.setHeader(name, value)
+        // bluesky's cdn responds with wrong content-type for the hls playlist,
+        // so we enforce it here until they fix it
+        const isHls = isHlsResponse(fileResponse)
+                      || (streamInfo.service === "bsky" && streamInfo.url.endsWith('.m3u8'));
 
-        if (req.statusCode < 200 || req.statusCode > 299)
+        for (const [ name, value ] of Object.entries(fileResponse.headers)) {
+            if (!isHls || name.toLowerCase() !== 'content-length') {
+                res.setHeader(name, value);
+            }
+        }
+
+        if (fileResponse.statusCode < 200 || fileResponse.statusCode > 299) {
             return cleanup();
+        }
 
-        if (isHlsRequest(req)) {
-            await handleHlsPlaylist(streamInfo, req, res);
+        if (isHls) {
+            await handleHlsPlaylist(streamInfo, fileResponse, res);
         } else {
-            pipe(req.body, res, cleanup);
+            pipe(fileResponse.body, res, cleanup);
         }
     } catch {
         closeRequest(streamInfo.controller);
@@ -114,7 +145,11 @@ async function handleGenericStream(streamInfo, res) {
 }
 
 export function internalStream(streamInfo, res) {
-    if (streamInfo.service === 'youtube') {
+    if (streamInfo.headers) {
+        streamInfo.headers.delete('icy-metadata');
+    }
+
+    if (streamInfo.service === 'youtube' && !streamInfo.isHLS) {
         return handleYoutubeStream(streamInfo, res);
     }
 

api/src/stream/manage.js

@@ -1,4 +1,4 @@
-import NodeCache from "node-cache";
+import Store from "../store/store.js";
 
 import { nanoid } from "nanoid";
 import { randomBytes } from "crypto";
@@ -7,34 +7,27 @@ import { setMaxListeners } from "node:events";
 
 import { env } from "../config.js";
 import { closeRequest } from "./shared.js";
-import { decryptStream, encryptStream, generateHmac } from "../misc/crypto.js";
+import { decryptStream, encryptStream } from "../misc/crypto.js";
+import { hashHmac } from "../security/secrets.js";
+import { zip } from "../misc/utils.js";
 
 // optional dependency
 const freebind = env.freebindCIDR && await import('freebind').catch(() => {});
 
-const streamCache = new NodeCache({
-    stdTTL: env.streamLifespan,
-    checkperiod: 10,
-    deleteOnExpire: true
-})
-
-streamCache.on("expired", (key) => {
-    streamCache.del(key);
-})
+const streamCache = new Store('streams');
 
 const internalStreamCache = new Map();
-const hmacSalt = randomBytes(64).toString('hex');
 
 export function createStream(obj) {
     const streamID = nanoid(),
         iv = randomBytes(16).toString('base64url'),
         secret = randomBytes(32).toString('base64url'),
         exp = new Date().getTime() + env.streamLifespan * 1000,
-        hmac = generateHmac(`${streamID},${exp},${iv},${secret}`, hmacSalt),
+        hmac = hashHmac(`${streamID},${exp},${iv},${secret}`, 'stream').toString('base64url'),
         streamData = {
             exp: exp,
             type: obj.type,
-            urls: obj.u,
+            urls: obj.url,
             service: obj.service,
             filename: obj.filename,
 
@@ -46,12 +39,19 @@ export function createStream(obj) {
             audioBitrate: obj.audioBitrate,
             audioCopy: !!obj.audioCopy,
             audioFormat: obj.audioFormat,
+
+            isHLS: obj.isHLS || false,
+            originalRequest: obj.originalRequest
         };
 
+    // FIXME: this is now a Promise, but it is not awaited
+    //        here. it may happen that the stream is not
+    //        stored in the Store before it is requested.
     streamCache.set(
         streamID,
-        encryptStream(streamData, iv, secret)
-    )
+        encryptStream(streamData, iv, secret),
+        env.streamLifespan
+    );
 
     let streamLink = new URL('/tunnel', env.apiURL);
 
@@ -77,7 +77,7 @@ export function getInternalStream(id) {
 export function createInternalStream(url, obj = {}) {
     assert(typeof url === 'string');
 
-    let dispatcher;
+    let dispatcher = obj.dispatcher;
     if (obj.requestIP) {
         dispatcher = freebind?.dispatcherFromIP(obj.requestIP, { strict: false })
     }
@@ -100,10 +100,12 @@ export function createInternalStream(url, obj = {}) {
         service: obj.service,
         headers,
         controller,
-        dispatcher
+        dispatcher,
+        isHLS: obj.isHLS,
+        transplant: obj.transplant
     });
 
-    let streamLink = new URL('/itunnel', `http://127.0.0.1:${env.apiPort}`);
+    let streamLink = new URL('/itunnel', `http://127.0.0.1:${env.tunnelPort}`);
     streamLink.searchParams.set('id', streamID);
 
     const cleanup = () => {
@@ -116,13 +118,17 @@ export function createInternalStream(url, obj = {}) {
     return streamLink.toString();
 }
 
-export function destroyInternalStream(url) {
+function getInternalTunnelId(url) {
     url = new URL(url);
     if (url.hostname !== '127.0.0.1') {
         return;
     }
 
-    const id = url.searchParams.get('id');
+    return url.searchParams.get('id');
+}
+
+export function destroyInternalStream(url) {
+    const id = getInternalTunnelId(url);
 
     if (internalStreamCache.has(id)) {
         closeRequest(getInternalStream(id)?.controller);
@@ -130,9 +136,68 @@ export function destroyInternalStream(url) {
     }
 }
 
+const transplantInternalTunnels = function(tunnelUrls, transplantUrls) {
+    if (tunnelUrls.length !== transplantUrls.length) {
+        return;
+    }
+
+    for (const [ tun, url ] of zip(tunnelUrls, transplantUrls)) {
+        const id = getInternalTunnelId(tun);
+        const itunnel = getInternalStream(id);
+
+        if (!itunnel) continue;
+        itunnel.url = url;
+    }
+}
+
+const transplantTunnel = async function (dispatcher) {
+    if (this.pendingTransplant) {
+        await this.pendingTransplant;
+        return;
+    }
+
+    let finished;
+    this.pendingTransplant = new Promise(r => finished = r);
+
+    try {
+        const handler = await import(`../processing/services/${this.service}.js`);
+        const response = await handler.default({
+            ...this.originalRequest,
+            dispatcher
+        });
+
+        if (!response.urls) {
+            return;
+        }
+
+        response.urls = [response.urls].flat();
+        if (this.originalRequest.isAudioOnly && response.urls.length > 1) {
+            response.urls = [response.urls[1]];
+        } else if (this.originalRequest.isAudioMuted) {
+            response.urls = [response.urls[0]];
+        }
+
+        const tunnels = [this.urls].flat();
+        if (tunnels.length !== response.urls.length) {
+            return;
+        }
+
+        transplantInternalTunnels(tunnels, response.urls);
+    }
+    catch {}
+    finally {
+        finished();
+        delete this.pendingTransplant;
+    }
+}
+
 function wrapStream(streamInfo) {
     const url = streamInfo.urls;
 
+    if (streamInfo.originalRequest) {
+        streamInfo.transplant = transplantTunnel.bind(streamInfo);
+    }
+
     if (typeof url === 'string') {
         streamInfo.urls = createInternalStream(url, streamInfo);
     } else if (Array.isArray(url)) {
@@ -146,10 +211,10 @@ function wrapStream(streamInfo) {
     return streamInfo;
 }
 
-export function verifyStream(id, hmac, exp, secret, iv) {
+export async function verifyStream(id, hmac, exp, secret, iv) {
     try {
-        const ghmac = generateHmac(`${id},${exp},${iv},${secret}`, hmacSalt);
-        const cache = streamCache.get(id.toString());
+        const ghmac = hashHmac(`${id},${exp},${iv},${secret}`, 'stream').toString('base64url');
+        const cache = await streamCache.get(id.toString());
 
         if (ghmac !== String(hmac)) return { status: 401 };
         if (!cache) return { status: 404 };

api/src/stream/shared.js

@@ -1,4 +1,5 @@
 import { genericUserAgent } from "../config.js";
+import { vkClientAgent } from "../processing/services/vk.js";
 
 const defaultHeaders = {
     'user-agent': genericUserAgent
@@ -13,6 +14,9 @@ const serviceHeaders = {
         origin: 'https://www.youtube.com',
         referer: 'https://www.youtube.com',
         DNT: '?1'
+    },
+    vk: {
+        'user-agent': vkClientAgent
     }
 }
 

api/src/stream/stream.js

@@ -10,7 +10,7 @@ export default async function(res, streamInfo) {
                 return await stream.proxy(streamInfo, res);
 
             case "internal":
-                return internalStream(streamInfo, res);
+                return internalStream(streamInfo.data, res);
 
             case "merge":
                 return stream.merge(streamInfo, res);

api/src/stream/types.js

@@ -1,10 +1,9 @@
-import { request } from "undici";
+import { Agent, request } from "undici";
 import ffmpeg from "ffmpeg-static";
 import { spawn } from "child_process";
 import { create as contentDisposition } from "content-disposition-header";
 
 import { env } from "../config.js";
-import { metadataManager } from "../misc/utils.js";
 import { destroyInternalStream } from "./manage.js";
 import { hlsExceptions } from "../processing/service-config.js";
 import { getHeaders, closeRequest, closeResponse, pipe } from "./shared.js";
@@ -16,6 +15,29 @@ const ffmpegArgs = {
     gif: ["-vf", "scale=-1:-1:flags=lanczos,split[s0][s1];[s0]palettegen[p];[s1][p]paletteuse", "-loop", "0"]
 }
 
+const metadataTags = [
+    "album",
+    "copyright",
+    "title",
+    "artist",
+    "track",
+    "date",
+];
+
+const convertMetadataToFFmpeg = (metadata) => {
+    let args = [];
+
+    for (const [ name, value ] of Object.entries(metadata)) {
+        if (metadataTags.includes(name)) {
+            args.push('-metadata', `${name}=${value.replace(/[\u0000-\u0009]/g, "")}`);
+        } else {
+            throw `${name} metadata tag is not supported.`;
+        }
+    }
+
+    return args;
+}
+
 const toRawHeaders = (headers) => {
     return Object.entries(headers)
                  .map(([key, value]) => `${key}: ${value}\r\n`)
@@ -38,6 +60,8 @@ const getCommand = (args) => {
     return [ffmpeg, args]
 }
 
+const defaultAgent = new Agent();
+
 const proxy = async (streamInfo, res) => {
     const abortController = new AbortController();
     const shutdown = () => (
@@ -56,7 +80,8 @@ const proxy = async (streamInfo, res) => {
                 Range: streamInfo.range
             },
             signal: abortController.signal,
-            maxRedirections: 16
+            maxRedirections: 16,
+            dispatcher: defaultAgent,
         });
 
         res.status(statusCode);
@@ -101,12 +126,16 @@ const merge = (streamInfo, res) => {
 
         args = args.concat(ffmpegArgs[format]);
 
-        if (hlsExceptions.includes(streamInfo.service)) {
-            args.push('-bsf:a', 'aac_adtstoasc')
+        if (hlsExceptions.includes(streamInfo.service) && streamInfo.isHLS) {
+            if (streamInfo.service === "youtube" && format === "webm") {
+                args.push('-c:a', 'libopus');
+            } else {
+                args.push('-c:a', 'aac', '-bsf:a', 'aac_adtstoasc');
+            }
         }
 
         if (streamInfo.metadata) {
-            args = args.concat(metadataManager(streamInfo.metadata))
+            args = args.concat(convertMetadataToFFmpeg(streamInfo.metadata))
         }
 
         args.push('-f', format, 'pipe:3');
@@ -238,7 +267,7 @@ const convertAudio = (streamInfo, res) => {
         }
 
         if (streamInfo.metadata) {
-            args = args.concat(metadataManager(streamInfo.metadata))
+            args = args.concat(convertMetadataToFFmpeg(streamInfo.metadata))
         }
 
         args.push('-f', streamInfo.audioFormat === "m4a" ? "ipod" : streamInfo.audioFormat, 'pipe:3');
@@ -291,7 +320,7 @@ const convertGif = (streamInfo, res) => {
         const [,,, muxOutput] = process.stdio;
 
         res.setHeader('Connection', 'keep-alive');
-        res.setHeader('Content-Disposition', contentDisposition(streamInfo.filename.split('.')[0] + ".gif"));
+        res.setHeader('Content-Disposition', contentDisposition(streamInfo.filename));
 
         pipe(muxOutput, res, shutdown);
 

api/src/util/generate-jwt-secret.js

@@ -0,0 +1,22 @@
+// run with `pnpm -r token:jwt`
+
+const makeSecureString = (length = 64) => {
+    const alphabet = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-';
+    const out = [];
+
+    while (out.length < length) {
+        for (const byte of crypto.getRandomValues(new Uint8Array(length))) {
+            if (byte < alphabet.length) {
+                out.push(alphabet[byte]);
+            }
+
+            if (out.length === length) {
+                break;
+            }
+        }
+    }
+
+    return out.join('');
+}
+
+console.log(`JWT_SECRET: ${JSON.stringify(makeSecureString(64))}`)

api/src/util/generate-youtube-tokens.js

@@ -1,38 +0,0 @@
-import { Innertube } from 'youtubei.js';
-import { Red } from '../misc/console-text.js'
-
-const bail = (...msg) => {
-    console.error(...msg);
-    throw new Error(msg);
-};
-
-const tube = await Innertube.create();
-
-tube.session.once(
-    'auth-pending',
-    ({ verification_url, user_code }) => {
-        console.log(`${Red('[!]')} The token generated by this script is sensitive and you should not share it with anyone!`);
-        console.log(`    By using this token, you are risking your Google account getting terminated.`);
-        console.log(`    You should ${Red('NOT')} use your personal account!`);
-        console.log();
-        console.log(`Open ${verification_url} in a browser and enter ${user_code} when asked for the code.`);
-    }
-);
-
-tube.session.once('auth-error', (err) => bail('An error occurred:', err));
-tube.session.once('auth', ({ credentials }) => {
-    if (!credentials.access_token) {
-        bail('something went wrong');
-    }
-
-    console.log(
-        'add this cookie to the youtube_oauth array in your cookies file:',
-        JSON.stringify(
-            Object.entries(credentials)
-                .map(([k, v]) => `${k}=${v instanceof Date ? v.toISOString() : v}`)
-                .join('; ')
-        )
-    );
-});
-
-await tube.session.signIn();

api/src/util/setup.js

@@ -1,105 +0,0 @@
-import { existsSync, unlinkSync, appendFileSync } from "fs";
-import { createInterface } from "readline";
-import { Cyan, Bright } from "./misc/console-text.js";
-import { loadJSON } from "./misc/load-from-fs.js";
-import { execSync } from "child_process";
-
-const { version } = loadJSON("./package.json");
-
-let envPath = './.env';
-let q = `${Cyan('?')} \x1b[1m`;
-let ob = {};
-let rl = createInterface({ input: process.stdin, output: process.stdout });
-
-let final = () => {
-    if (existsSync(envPath)) unlinkSync(envPath);
-
-    for (let i in ob) {
-        appendFileSync(envPath, `${i}=${ob[i]}\n`)
-    }
-    console.log(Bright("\nAwesome! I've created a fresh .env file for you."));
-    console.log(`${Bright("Now I'll run")} ${Cyan("npm install")} ${Bright("to install all dependencies. It shouldn't take long.\n\n")}`);
-    execSync('npm install', { stdio: [0, 1, 2] });
-    console.log(`\n\n${Cyan("All done!\n")}`);
-    console.log(Bright("You can re-run this script at any time to update the configuration."));
-    console.log(Bright("\nYou're now ready to start cobalt. Simply run ") + Cyan("npm start") + Bright('!\nHave fun :)'));
-    rl.close()
-}
-
-console.log(
-    `${Cyan(`Hey, this is cobalt v.${version}!`)}\n${Bright("Let's start by creating a new ")}${Cyan(".env")}${Bright(" file. You can always change it later.")}`
-)
-
-function setup() {
-    console.log(Bright("\nWhat kind of server will this instance be?\nOptions: api, web."));
-
-    rl.question(q, r1 => {
-        switch (r1.toLowerCase()) {
-            case 'api':
-                console.log(Bright("\nCool! What's the domain this API instance will be running on? (localhost)\nExample: api.cobalt.tools"));
-
-                rl.question(q, apiURL => {
-                    ob.API_URL = `http://localhost:9000/`;
-                    ob.API_PORT = 9000;
-                    if (apiURL && apiURL !== "localhost") ob.API_URL = `https://${apiURL.toLowerCase()}/`;
-
-                    console.log(Bright("\nGreat! Now, what port will it be running on? (9000)"));
-
-                    rl.question(q, apiPort => {
-                        if (apiPort) ob.API_PORT = apiPort;
-                        if (apiPort && (apiURL === "localhost" || !apiURL)) ob.API_URL = `http://localhost:${apiPort}/`;
-
-                        console.log(Bright("\nWhat will your instance's name be? Usually it's something like eu-nl aka region-country. (local)"));
-
-                        rl.question(q, apiName => {
-                            ob.API_NAME = apiName.toLowerCase();
-                            if (!apiName || apiName === "local") ob.API_NAME = "local";
-
-                            console.log(Bright("\nOne last thing: would you like to enable CORS? It allows other websites and extensions to use your instance's API.\ny/n (n)"));
-
-                            rl.question(q, apiCors => {
-                                let answCors = apiCors.toLowerCase().trim();
-                                if (answCors !== "y" && answCors !== "yes") ob.CORS_WILDCARD = '0'
-                                final()
-                            })
-                        })
-                    });
-    
-                })
-                break;
-            case 'web':
-                console.log(Bright("\nAwesome! What's the domain this web app instance will be running on? (localhost)\nExample: cobalt.tools"));
-    
-                rl.question(q, webURL => {
-                    ob.WEB_URL = `http://localhost:9001/`;
-                    ob.WEB_PORT = 9001;
-                    if (webURL && webURL !== "localhost") ob.WEB_URL = `https://${webURL.toLowerCase()}/`;
-    
-                    console.log(
-                        Bright("\nGreat! Now, what port will it be running on? (9001)")
-                    )
-                    rl.question(q, webPort => {
-                        if (webPort) ob.WEB_PORT = webPort;
-                        if (webPort && (webURL === "localhost" || !webURL)) ob.WEB_URL = `http://localhost:${webPort}/`;
-
-                        console.log(
-                            Bright("\nOne last thing: what default API domain should be used? (api.cobalt.tools)\nIf it's hosted locally, make sure to include the port:") + Cyan(" localhost:9000")
-                        );
-
-                        rl.question(q, apiURL => {
-                            ob.API_URL = `https://${apiURL.toLowerCase()}/`;
-                            if (apiURL.includes(':')) ob.API_URL = `http://${apiURL.toLowerCase()}/`;
-                            if (!apiURL) ob.API_URL = "https://api.cobalt.tools/";
-                            final()
-                        })
-                    });
-    
-                });
-                break;
-            default:
-                console.log(Bright("\nThis is not an option. Try again."));
-                setup()
-        }
-    })
-}
-setup()

api/src/util/test-ci.js

@@ -1,82 +0,0 @@
-import { env } from "../config.js";
-import { runTest } from "../misc/run-test.js";
-import { loadJSON } from "../misc/load-from-fs.js";
-import { Red, Bright } from "../misc/console-text.js";
-import { randomizeCiphers } from "../misc/randomize-ciphers.js";
-
-import { services } from "../processing/service-config.js";
-
-const tests = loadJSON('./src/util/tests.json');
-
-// services that are known to frequently fail due to external
-// factors (e.g. rate limiting)
-const finnicky = new Set(['bilibili', 'instagram', 'facebook', 'youtube'])
-
-const action = process.argv[2];
-switch (action) {
-    case "get-services":
-        const fromConfig = Object.keys(services);
-
-        const missingTests = fromConfig.filter(
-            service => !tests[service] || tests[service].length === 0
-        );
-
-        if (missingTests.length) {
-            console.error('services have no tests:', missingTests);
-            console.log('[]');
-            process.exitCode = 1;
-            break;
-        }
-
-        console.log(JSON.stringify(fromConfig));
-        break;
-
-    case "run-tests-for":
-        const service = process.argv[3];
-        let failed = false;
-
-        if (!tests[service]) {
-            console.error('no such service:', service);
-        }
-
-        env.streamLifespan = 10000;
-        env.apiURL = 'http://x';
-        randomizeCiphers();
-
-        for (const test of tests[service]) {
-            const { name, url, params, expected } = test;
-            const canFail = test.canFail || finnicky.has(service);
-
-            try {
-                await runTest(url, params, expected);
-                console.log(`${service}/${name}: ok`);
-
-            } catch(e) {
-                failed = !canFail;
-
-                let failText = canFail ? `${Red('FAIL')} (ignored)` : Bright(Red('FAIL'));
-                if (canFail && process.env.GITHUB_ACTION) {
-                    console.log(`::warning title=${service}/${name.replace(/,/g, ';')}::failed and was ignored`);
-                }
-
-                console.error(`${service}/${name}: ${failText}`);
-                const errorString = e.toString().split('\n');
-                let c = '┃';
-                errorString.forEach((line, index) => {
-                    line = line.replace('!=', Red('!='));
-
-                    if (index === errorString.length - 1) {
-                        c = '┗';
-                    }
-
-                    console.error(`   ${c}`, line);
-                });
-            }
-        }
-
-        process.exitCode = Number(failed);
-        break;
-    default:
-        console.error('invalid action:', action);
-        process.exitCode = 1;
-}

api/src/util/test.js

@@ -1,84 +1,135 @@
-import "dotenv/config";
+import path from "node:path";
+
+import { env } from "../config.js";
+import { runTest } from "../misc/run-test.js";
+import { loadJSON } from "../misc/load-from-fs.js";
+import { Red, Bright } from "../misc/console-text.js";
+import { setGlobalDispatcher, ProxyAgent } from "undici";
+import { randomizeCiphers } from "../misc/randomize-ciphers.js";
 
 import { services } from "../processing/service-config.js";
-import { extract } from "../processing/url.js";
-import match from "../processing/match.js";
-import { loadJSON } from "../misc/load-from-fs.js";
-import { normalizeRequest } from "../processing/request.js";
-import { env } from "../config.js";
 
-env.apiURL = 'http://localhost:9000'
-let tests = loadJSON('./src/util/tests.json');
+const getTestPath = service => path.join('./src/util/tests/', `./${service}.json`);
+const getTests = (service) => loadJSON(getTestPath(service));
 
-let noTest = [];
-let failed = [];
-let success = 0;
+// services that are known to frequently fail due to external
+// factors (e.g. rate limiting)
+const finnicky = new Set(
+    process.env.TEST_IGNORE_SERVICES
+    ? process.env.TEST_IGNORE_SERVICES.split(',')
+    : ['bilibili', 'instagram', 'facebook', 'youtube', 'vk', 'twitter', 'reddit']
+);
 
-function addToFail(service, testName, url, status, response) {
-    failed.push({
-        service: service,
-        name: testName,
-        url: url,
-        status: status,
-        response: response
-    })
-}
-for (let i in services) {
-    if (tests[i]) {
-        console.log(`\nRunning tests for ${i}...\n`)
-        for (let k = 0; k < tests[i].length; k++) {
-            let test = tests[i][k];
+const runTestsFor = async (service) => {
+    const tests = getTests(service);
+    let softFails = 0, fails = 0;
 
-            console.log(`Running test ${k+1}: ${test.name}`);
-            console.log('params:');
-            let params = {...{url: test.url}, ...test.params};
-            console.log(params);
-
-            let chck = await normalizeRequest(params);
-            if (chck.success) {
-                chck = chck.data;
-
-                const parsed = extract(chck.url);
-                if (parsed === null) {
-                    throw `Invalid URL: ${chck.url}`
-                }
-
-                let j = await match({
-                    host: parsed.host,
-                    patternMatch: parsed.patternMatch,
-                    params: chck,
-                });
-                console.log('\nReceived:');
-                console.log(j)
-                if (j.status === test.expected.code && j.body.status === test.expected.status) {
-                    console.log("\n✅ Success.\n");
-                    success++
-                } else {
-                    console.log(`\n❌ Fail. Expected: ${test.expected.code} & ${test.expected.status}, received: ${j.status} & ${j.body.status}\n`);
-                    addToFail(i, test.name, test.url, j.body.status, j)
-                }
-            } else {
-                console.log("\n❌ couldn't validate the request JSON.\n");
-                addToFail(i, test.name, test.url, "unknown", {})
-            }
-        }
-        console.log("\n\n")
-    } else {
-        console.warn(`No tests found for ${i}.`);
-        noTest.push(i)
+    if (!tests) {
+        throw "no such service: " + service;
     }
+
+    for (const test of tests) {
+        const { name, url, params, expected } = test;
+        const canFail = test.canFail || finnicky.has(service);
+
+        try {
+            await runTest(url, params, expected);
+            console.log(`${service}/${name}: ok`);
+
+        } catch (e) {
+            softFails += !canFail;
+            fails++;
+
+            let failText = canFail ? `${Red('FAIL')} (ignored)` : Bright(Red('FAIL'));
+            if (canFail && process.env.GITHUB_ACTION) {
+                console.log(`::warning title=${service}/${name.replace(/,/g, ';')}::failed and was ignored`);
+            }
+
+            console.error(`${service}/${name}: ${failText}`);
+            const errorString = e.toString().split('\n');
+            let c = '┃';
+            errorString.forEach((line, index) => {
+                line = line.replace('!=', Red('!='));
+
+                if (index === errorString.length - 1) {
+                    c = '┗';
+                }
+
+                console.error(`   ${c}`, line);
+            });
+        }
+    }
+
+    return { fails, softFails };
 }
 
-console.log(`✅ ${success} tests succeeded.`);
-console.log(`❌ ${failed.length} tests failed.`);
-console.log(`❔ ${noTest.length} services weren't tested.`);
-
-if (failed.length > 0) {
-    console.log(`\nFailed tests:`);
-    console.log(failed)
+const printHeader = (service, padLen) => {
+    const padding = padLen - service.length;
+    service = service.padEnd(1 + service.length + padding, ' ');
+    console.log(service + '='.repeat(50));
 }
 
-if (noTest.length > 0) {
-    console.log(`\nMissing tests:`);
-    console.log(noTest)
+if (env.externalProxy) {
+    setGlobalDispatcher(new ProxyAgent(env.externalProxy));
+}
+
+env.streamLifespan = 10000;
+env.apiURL = 'http://x/';
+randomizeCiphers();
+
+const action = process.argv[2];
+switch (action) {
+    case "get-services":
+        const fromConfig = Object.keys(services);
+
+        const missingTests = fromConfig.filter(
+            service => {
+                const tests = getTests(service);
+                return !tests || tests.length === 0
+            }
+        );
+
+        if (missingTests.length) {
+            console.error('services have no tests:', missingTests);
+            process.exitCode = 1;
+            break;
+        }
+
+        console.log(JSON.stringify(fromConfig));
+        break;
+
+    case "run-tests-for":
+
+        try {
+            const { softFails } = await runTestsFor(process.argv[3]);
+            process.exitCode = Number(!!softFails);
+        } catch (e) {
+            console.error(e);
+            process.exitCode = 1;
+            break;
+        }
+
+        break;
+    default:
+        const maxHeaderLen = Object.keys(services).reduce((n, v) => v.length > n ? v.length : n, 0);
+        const failCounters = {};
+
+        for (const service in services) {
+            printHeader(service, maxHeaderLen);
+            const { fails, softFails } = await runTestsFor(service);
+            failCounters[service] = fails;
+            console.log();
+
+            if (!process.exitCode && softFails)
+                process.exitCode = 1;
+        }
+
+        console.log('='.repeat(50 + maxHeaderLen));
+        console.log(
+            Bright('total fails:'),
+            Object.values(failCounters).reduce((a, b) => a + b)
+        );
+        for (const [ service, fails ] of Object.entries(failCounters)) {
+            if (fails) console.log(`${Bright(service)} fails: ${fails}`);
+        }
 }

api/src/util/tests/bilibili.json

@@ -0,0 +1,60 @@
+[
+    {
+        "name": "1080p video",
+        "url": "https://www.bilibili.com/video/BV18i4y1m7xV/",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "1080p video muted",
+        "url": "https://www.bilibili.com/video/BV18i4y1m7xV/",
+        "params": {
+            "downloadMode": "mute"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "1080p vertical video",
+        "url": "https://www.bilibili.com/video/BV1uu411z7VV/",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "1080p vertical video muted",
+        "url": "https://www.bilibili.com/video/BV1uu411z7VV/",
+        "params": {
+            "downloadMode": "mute"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "b23.tv shortlink",
+        "url": "https://b23.tv/av32430100",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "bilibili.tv link",
+        "url": "https://www.bilibili.tv/en/video/4789599404426256",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    }
+]

api/src/util/tests/bsky.json

@@ -0,0 +1,96 @@
+[
+    {
+        "name": "horizontal video",
+        "url": "https://bsky.app/profile/did:plc:oisofpd7lj26yvgiivf3lxsi/post/3l3giwtwp222m",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "horizontal video, recordWithMedia",
+        "url": "https://bsky.app/profile/did:plc:ywbm3iywnhzep3ckt6efhoh7/post/3l3wonhk23g2i",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "vertical video",
+        "url": "https://bsky.app/profile/did:plc:oisofpd7lj26yvgiivf3lxsi/post/3l3jhpomhjk2m",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "vertical video (muted)",
+        "url": "https://bsky.app/profile/did:plc:oisofpd7lj26yvgiivf3lxsi/post/3l3jhpomhjk2m",
+        "params": {
+            "downloadMode": "mute"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "vertical video (audio)",
+        "url": "https://bsky.app/profile/did:plc:oisofpd7lj26yvgiivf3lxsi/post/3l3jhpomhjk2m",
+        "params": {
+            "downloadMode": "audio"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "single image",
+        "url": "https://bsky.app/profile/did:plc:k4a7d65fcyevbrnntjxh57go/post/3l33flpoygt26",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "gif with a quoted post",
+        "url": "https://bsky.app/profile/imlunahey.com/post/3lgajpn5dtk2t",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "gif alone in a post",
+        "url": "https://bsky.app/profile/imlunahey.com/post/3lgah3ovxnc2q",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "several images",
+        "url": "https://bsky.app/profile/did:plc:rai7s6su2sy22ss7skouedl7/post/3kzxuxbiul626",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "picker"
+        }
+    },
+    {
+        "name": "deleted post/invalid user",
+        "url": "https://bsky.app/profile/notreal.bsky.team/post/3l2udah76ch2c",
+        "params": {},
+        "expected": {
+            "code": 400,
+            "status": "error"
+        }
+    }
+]

api/src/util/tests/dailymotion.json

@@ -0,0 +1,29 @@
+[
+    {
+        "name": "regular video",
+        "url": "https://www.dailymotion.com/video/x8t1eho",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "private video",
+        "url": "https://www.dailymotion.com/video/k41fZWpx2TaAORA2nok",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "dai.ly shortened link",
+        "url": "https://dai.ly/k41fZWpx2TaAORA2nok",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    }
+]

api/src/util/tests/facebook.json

@@ -0,0 +1,65 @@
+[
+    {
+        "name": "direct video with username and id",
+        "url": "https://web.facebook.com/100048111287134/videos/1157798148685638/",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "direct video with id as query param",
+        "url": "https://web.facebook.com/watch/?v=883839773514682&ref=sharing",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "direct video with caption",
+        "url": "https://web.facebook.com/wood57/videos/𝐒𝐞𝐛𝐚𝐬𝐤𝐨𝐦-𝐟𝐮𝐥𝐥/883839773514682",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "shortlink video",
+        "url": "https://fb.watch/r1K6XHMfGT/",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "reel video",
+        "url": "https://web.facebook.com/reel/730293269054758",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "shared video link",
+        "url": "https://www.facebook.com/share/v/6EJK4Z8EAEAHtz8K/",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "shared video link v2",
+        "url": "https://web.facebook.com/share/r/JFZfPVgLkiJQmWrr/",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    }
+]

api/src/util/tests/instagram.json

@@ -0,0 +1,133 @@
+[
+    {
+        "name": "single photo post",
+        "url": "https://www.instagram.com/p/DFx6KVduFWy/",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "various picker (photos + video)",
+        "url": "https://www.instagram.com/p/CvYrSgnsKjv/",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "picker"
+        }
+    },
+    {
+        "name": "reel",
+        "url": "https://www.instagram.com/reel/DFQe23tOWKz/",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "regular video",
+        "url": "https://www.instagram.com/p/CmCVWoIr9OH/",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "reel (isAudioOnly)",
+        "url": "https://www.instagram.com/reel/DFQe23tOWKz/",
+        "params": {
+            "downloadMode": "audio"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "reel (isAudioMuted)",
+        "url": "https://www.instagram.com/reel/DFQe23tOWKz/",
+        "params": {
+            "downloadMode": "mute"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "inexistent reel",
+        "url": "https://www.instagram.com/reel/XXXXXXXXXX/",
+        "params": {},
+        "expected": {
+            "code": 400,
+            "status": "error"
+        }
+    },
+    {
+        "name": "inexistent post",
+        "url": "https://www.instagram.com/p/XXXXXXXXXX/",
+        "params": {},
+        "expected": {
+            "code": 400,
+            "status": "error"
+        }
+    },
+    {
+        "name": "post info in an array (for whatever reason??)",
+        "url": "https://www.instagram.com/reel/CrVB9tatUDv/?igshid=blaBlABALALbLABULLSHIT==",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "prone to get rate limited",
+        "url": "https://www.instagram.com/reel/CrO-T7Qo6rq/?igshid=fuckYouNoTrackingIdForYou==",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "ddinstagram link",
+        "url": "https://ddinstagram.com/p/CmCVWoIr9OH/",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "d.ddinstagram.com link",
+        "url": "https://d.ddinstagram.com/p/CmCVWoIr9OH/",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "g.ddinstagram.com link",
+        "url": "https://g.ddinstagram.com/p/CmCVWoIr9OH/",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "private instagram post",
+        "url": "https://www.instagram.com/p/C5_A1TQNPrYw4c2g9KAUTPUl8RVHqiAdAcOOSY0",
+        "params": {},
+        "expected": {
+            "code": 400,
+            "status": "error",
+            "errorCode": "error.api.content.post.private"
+        }
+    }
+]

api/src/util/tests/loom.json

@@ -0,0 +1,33 @@
+[
+    {
+        "name": "1080p video",
+        "url": "https://www.loom.com/share/d165fd054a294d8a8587807bcc50c885",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "1080p video (muted)",
+        "url": "https://www.loom.com/share/d165fd054a294d8a8587807bcc50c885",
+        "params": {
+            "downloadMode": "mute"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "1080p video (audio only)",
+        "url": "https://www.loom.com/share/d165fd054a294d8a8587807bcc50c885",
+        "params": {
+            "downloadMode": "audio"
+        },
+        "expected": {
+            "code": 400,
+            "status": "error"
+        }
+    }
+]

api/src/util/tests/ok.json

@@ -0,0 +1,11 @@
+[
+    {
+        "name": "regular video",
+        "url": "https://ok.ru/video/7204071410346",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    }
+]

api/src/util/tests/pinterest.json

@@ -0,0 +1,87 @@
+[
+    {
+        "name": "regular video",
+        "url": "https://www.pinterest.com/pin/70437485604616/",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "regular video (isAudioOnly)",
+        "url": "https://www.pinterest.com/pin/70437485604616/",
+        "params": {
+            "downloadMode": "audio"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "regular video (isAudioMuted)",
+        "url": "https://www.pinterest.com/pin/70437485604616/",
+        "params": {
+            "downloadMode": "mute"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "regular video (.ca TLD)",
+        "url": "https://www.pinterest.ca/pin/70437485604616/",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "story",
+        "url": "https://www.pinterest.com/pin/gadget-cool-products-amazon-product-technology-kitchen-gadgets--1084663891475263837/",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "regular picture",
+        "url": "https://www.pinterest.com/pin/412994228343400946/",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "regular picture (.ca TLD)",
+        "url": "https://www.pinterest.ca/pin/412994228343400946/",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "regular gif",
+        "url": "https://www.pinterest.com/pin/643170390530326178/",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "regular gif (.ca TLD)",
+        "url": "https://www.pinterest.ca/pin/643170390530326178/",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    }
+]

api/src/util/tests/reddit.json

@@ -0,0 +1,60 @@
+[
+    {
+        "name": "video with audio",
+        "url": "https://www.reddit.com/r/TikTokCringe/comments/wup1fg/id_be_escaping_at_the_first_chance_i_got/?utm_source=share&utm_medium=web2x&context=3",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "video with audio (isAudioOnly)",
+        "url": "https://www.reddit.com/r/TikTokCringe/comments/wup1fg/id_be_escaping_at_the_first_chance_i_got/?utm_source=share&utm_medium=web2x&context=3",
+        "params": {
+            "downloadMode": "audio"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "video with audio (isAudioMuted)",
+        "url": "https://www.reddit.com/r/TikTokCringe/comments/wup1fg/id_be_escaping_at_the_first_chance_i_got/?utm_source=share&utm_medium=web2x&context=3",
+        "params": {
+            "downloadMode": "mute"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "video without audio",
+        "url": "https://www.reddit.com/r/catvideos/comments/ftoeo7/luna_doesnt_want_to_be_bothered_while_shes_napping/?utm_source=share&utm_medium=web2x&context=3",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "actual gif, not looping video",
+        "url": "https://www.reddit.com/r/whenthe/comments/109wqy1/god_really_did_some_trolling/?utm_source=share&utm_medium=web2x&context=3",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "different audio link, live render",
+        "url": "https://www.reddit.com/r/TikTokCringe/comments/15hce91/asian_daddy_kink/",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    }
+]

api/src/util/tests/rutube.json

@@ -0,0 +1,100 @@
+[
+    {
+        "name": "regular video",
+        "url": "https://rutube.ru/video/b2f6c27649907c2fde0af411b03825eb/",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "vertical video (isAudioMuted)",
+        "url": "https://rutube.ru/video/18a281399b96f9184c647455a86f6724/",
+        "params": {
+            "downloadMode": "mute"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "russian region lock",
+        "url": "https://rutube.ru/video/b521653b4f71ece57b8ff54e57ca9b82/",
+        "params": {},
+        "expected": {
+            "code": 400,
+            "status": "error"
+        }
+    },
+    {
+        "name": "vertical video",
+        "url": "https://rutube.ru/video/18a281399b96f9184c647455a86f6724/",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "yappy",
+        "url": "https://rutube.ru/yappy/c8c32bf7aee04412837656ea26c2b25b/",
+        "canFail": true,
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "shorts",
+        "url": "https://rutube.ru/shorts/935c1afafd0e7d52836d671967d53dac/",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "vertical video (isAudioOnly)",
+        "url": "https://rutube.ru/video/18a281399b96f9184c647455a86f6724/",
+        "params": {
+            "downloadMode": "audio"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "vertical video (isAudioMuted)",
+        "url": "https://rutube.ru/video/18a281399b96f9184c647455a86f6724/",
+        "params": {
+            "downloadMode": "mute"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "private video",
+        "url": "https://rutube.ru/video/private/1161415be0e686214bb2a498165cab3e/?p=_IL1G8RSnKutunnTYwhZ5A",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "region locked video, should fail",
+        "canFail": true,
+        "url": "https://rutube.ru/video/e7ac82708cc22bd068a3bf6a7004d1b1/",
+        "params": {},
+        "expected": {
+            "code": 400,
+            "status": "error"
+        }
+    }
+]

api/src/util/tests/snapchat.json

@@ -0,0 +1,29 @@
+[
+    {
+        "name": "spotlight",
+        "url": "https://www.snapchat.com/spotlight/W7_EDlXWTBiXAEEniNoMPwAAYdWxucG9pZmNqAY46j0a5AY46j0YbAAAAAQ",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "shortlinked spotlight",
+        "url": "https://t.snapchat.com/4ZsiBLDi",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "story",
+        "url": "https://www.snapchat.com/add/bazerkmakane",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "picker"
+        }
+    }
+]

api/src/util/tests/soundcloud.json

@@ -0,0 +1,106 @@
+[
+    {
+        "name": "public song (best)",
+        "url": "https://soundcloud.com/l2share77/loona-butterfly?utm_source=clipboard&utm_medium=text&utm_campaign=social_sharing",
+        "params": {
+            "audioFormat": "best"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "public song (mp3, isAudioMuted)",
+        "url": "https://soundcloud.com/l2share77/loona-butterfly?utm_source=clipboard&utm_medium=text&utm_campaign=social_sharing",
+        "params": {
+            "downloadMode": "mute",
+            "audioFormat": "mp3"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "private song",
+        "url": "https://soundcloud.com/user-798052861/asdasdasdsdsd/s-9TqZ7edLJ90",
+        "params": {
+            "audioFormat": "mp3"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "private song (wav, isAudioMuted)",
+        "url": "https://soundcloud.com/user-798052861/asdasdasdsdsd/s-9TqZ7edLJ90",
+        "params": {
+            "downloadMode": "mute",
+            "audioFormat": "wav"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "private song (ogg, isAudioMuted, isAudioOnly)",
+        "url": "https://soundcloud.com/user-798052861/asdasdasdsdsd/s-9TqZ7edLJ90",
+        "params": {
+            "downloadMode": "audio",
+            "audioFormat": "ogg"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "on.soundcloud link",
+        "url": "https://on.soundcloud.com/wLZre",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "on.soundcloud link, different stream type",
+        "url": "https://on.soundcloud.com/AG4c",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "no opus audio, fallback to mp3",
+        "url": "https://soundcloud.com/frums/credits",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "go+ song, should fail",
+        "url": "https://soundcloud.com/dualipa/illusion-1",
+        "params": {},
+        "expected": {
+            "code": 400,
+            "status": "error"
+        }
+    },
+    {
+        "name": "region locked song, should fail",
+        "canFail": true,
+        "url": "https://soundcloud.com/gotye/somebody-2024-feat-kimbra",
+        "params": {},
+        "expected": {
+            "code": 400,
+            "status": "error"
+        }
+    }
+]

api/src/util/tests/streamable.json

@@ -0,0 +1,51 @@
+[
+    {
+        "name": "regular video",
+        "url": "https://streamable.com/p9cln4",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "embedded link",
+        "url": "https://streamable.com/e/rsmo56",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "regular video (isAudioOnly)",
+        "url": "https://streamable.com/p9cln4",
+        "params": {
+            "downloadMode": "audio"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "regular video (isAudioMuted)",
+        "url": "https://streamable.com/p9cln4",
+        "params": {
+            "downloadMode": "mute"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "inexistent video",
+        "url": "https://streamable.com/XXXXXX",
+        "params": {},
+        "expected": {
+            "code": 400,
+            "status": "error"
+        }
+    }
+]

api/src/util/tests/tiktok.json

@@ -0,0 +1,47 @@
+[
+    {
+        "name": "long link video",
+        "url": "https://www.tiktok.com/@fatfatmillycat/video/7195741644585454894",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "images",
+        "url": "https://www.tiktok.com/@matryoshk4/video/7231234675476532526",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "picker"
+        }
+    },
+    {
+        "name": "long link inexistent",
+        "url": "https://www.tiktok.com/@blablabla/video/7120851458451417478",
+        "params": {},
+        "expected": {
+            "code": 400,
+            "status": "error"
+        }
+    },
+    {
+        "name": "short link inexistent",
+        "url": "https://vt.tiktok.com/2p4ewa7/",
+        "params": {},
+        "expected": {
+            "code": 400,
+            "status": "error"
+        }
+    },
+    {
+        "name": "age restricted video",
+        "url": "https://www.tiktok.com/@.kyle.films/video/7415757181145877793",
+        "params": {},
+        "expected": {
+            "code": 400,
+            "status": "error"
+        }
+    }
+]

api/src/util/tests/tumblr.json

@@ -0,0 +1,49 @@
+[
+    {
+        "name": "at.tumblr link",
+        "url": "https://at.tumblr.com/music/704177038274281472/n7x7pr7x4w2b",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "user subdomain link",
+        "url": "https://garfield-69.tumblr.com/post/696499862852780032",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "web app link",
+        "url": "https://www.tumblr.com/rongzhi/707729381162958848/english-added-by-me?source=share",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "tumblr audio",
+        "url": "https://www.tumblr.com/zedneon/737815079301562368/zedneon-ft-mr-sauceman-tech-n9ne-speed-of?source=share",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "tumblr video converted to audio",
+        "url": "https://garfield-69.tumblr.com/post/696499862852780032",
+        "params": {
+            "downloadMode": "audio"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    }
+]

api/src/util/tests/twitch.json

@@ -0,0 +1,33 @@
+[
+    {
+        "name": "clip",
+        "url": "https://twitch.tv/rtgame/clip/TubularInventiveSardineCorgiDerp-PM47mJQQ2vsL5B5G",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "clip (isAudioOnly)",
+        "url": "https://twitch.tv/rtgame/clip/TubularInventiveSardineCorgiDerp-PM47mJQQ2vsL5B5G",
+        "params": {
+            "downloadMode": "audio"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "clip (isAudioMuted)",
+        "url": "https://twitch.tv/rtgame/clip/TubularInventiveSardineCorgiDerp-PM47mJQQ2vsL5B5G",
+        "params": {
+            "downloadMode": "mute"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    }
+]

api/src/util/tests/twitter.json

@@ -0,0 +1,221 @@
+[
+    {
+        "name": "regular video",
+        "url": "https://twitter.com/X/status/1697304622749086011",
+        "params": {
+            "audioFormat": "mp3"
+        },
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "video with mobile web mediaviewer",
+        "url": "https://twitter.com/X/status/1697304622749086011/mediaViewer?currentTweet=1697304622749086011&currentTweetUser=X&currentTweet=1697304622749086011&currentTweetUser=X",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "embedded twitter video",
+        "url": "https://twitter.com/dustbin_nie/status/1624596567188717568?s=20",
+        "params": {
+            "audioFormat": "mp3"
+        },
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "mixed media (image + gif)",
+        "url": "https://twitter.com/sky_mj26/status/1807756010712428565",
+        "params": {
+            "audioFormat": "mp3"
+        },
+        "expected": {
+            "code": 200,
+            "status": "picker"
+        }
+    },
+    {
+        "name": "picker: mixed media (3 videos)",
+        "url": "https://twitter.com/DankGameAlert/status/1584726006094794774",
+        "params": {
+            "audioFormat": "mp3"
+        },
+        "expected": {
+            "code": 200,
+            "status": "picker"
+        }
+    },
+    {
+        "name": "audio from embedded twitter video (mp3, isAudioOnly)",
+        "url": "https://twitter.com/dustbin_nie/status/1624596567188717568?s=20",
+        "params": {
+            "downloadMode": "audio",
+            "audioFormat": "mp3"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "audio from embedded twitter video (best, isAudioOnly)",
+        "url": "https://twitter.com/dustbin_nie/status/1624596567188717568?s=20",
+        "params": {
+            "downloadMode": "audio",
+            "audioFormat": "best"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "audio from embedded twitter video (ogg, isAudioOnly, isAudioMuted)",
+        "url": "https://twitter.com/dustbin_nie/status/1624596567188717568?s=20",
+        "params": {
+            "downloadMode": "audio",
+            "audioFormat": "best"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "muted embedded twitter video",
+        "url": "https://twitter.com/dustbin_nie/status/1624596567188717568?s=20",
+        "params": {
+            "downloadMode": "mute",
+            "audioFormat": "mp3"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "retweeted video",
+        "url": "https://twitter.com/schlizzawg/status/1869017025055793405",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "age restricted video",
+        "url": "https://x.com/XSpaces/status/1526955853743546372",
+        "params": {},
+        "canFail": true,
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "twitter voice + x.com link",
+        "url": "https://x.com/eggsaladscreams/status/1693089534886506756?s=46",
+        "params": {},
+        "canFail": true,
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "vxtwitter link",
+        "url": "https://vxtwitter.com/dustbin_nie/status/1624596567188717568?s=20",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "post with 1 image",
+        "url": "https://x.com/PopCrave/status/1815960083475423235",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "post with 4 images",
+        "url": "https://x.com/PopCrave/status/1816260887147114696",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "picker"
+        }
+    },
+    {
+        "name": "retweeted video, isAudioOnly",
+        "url": "https://twitter.com/hugekiwinuts/status/1618671150829309953?s=46&t=gItGzgwGQQJJaJrO6qc1Pg",
+        "params": {
+            "downloadMode": "mute",
+            "audioFormat": "mp3"
+        },
+        "canFail": true,
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "gif",
+        "url": "https://x.com/thelastromances/status/1897839691212202479",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "inexistent post",
+        "url": "https://twitter.com/test/status/9487653",
+        "params": {
+            "audioFormat": "best"
+        },
+        "expected": {
+            "code": 400,
+            "status": "error"
+        }
+    },
+    {
+        "name": "post with no media content",
+        "url": "https://twitter.com/elonmusk/status/1604617643973124097?s=20",
+        "params": {
+            "audioFormat": "best"
+        },
+        "expected": {
+            "code": 400,
+            "status": "error"
+        }
+    },
+    {
+        "name": "bookmarked video",
+        "url": "https://twitter.com/i/bookmarks?post_id=1828099210220294314",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "bookmarked photo",
+        "url": "https://twitter.com/i/bookmarks?post_id=1887450602164396149",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    }
+]

api/src/util/tests/vimeo.json

@@ -0,0 +1,64 @@
+[
+    {
+        "name": "4k progressive",
+        "url": "https://vimeo.com/288386543",
+        "params": {
+            "videoQuality": "2160"
+        },
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "720p progressive",
+        "url": "https://vimeo.com/288386543",
+        "params": {
+            "videoQuality": "720"
+        },
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "1080p dash parcel",
+        "url": "https://vimeo.com/967252742",
+        "params": {
+            "videoQuality": "1440"
+        },
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "720p dash parcel",
+        "url": "https://vimeo.com/967252742",
+        "params": {
+            "videoQuality": "360"
+        },
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "private video",
+        "url": "https://vimeo.com/903115595/f14d06da38",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "mature video",
+        "url": "https://vimeo.com/973212054",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    }
+]

api/src/util/tests/vk.json

@@ -0,0 +1,82 @@
+[
+    {
+        "name": "clip, defaults",
+        "url": "https://vk.com/clip-57274055_456239788",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "clip, 360",
+        "url": "https://vk.com/clip-57274055_456239788",
+        "params": {
+            "videoQuality": "360"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "clip different link, max",
+        "url": "https://vk.com/clips-57274055?z=clip-57274055_456239788",
+        "params": {
+            "videoQuality": "max"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "video, defaults",
+        "url": "https://vk.com/video-57274055_456239399",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "big 4k video",
+        "url": "https://vk.com/video-1112285_456248465",
+        "params": {
+            "videoQuality": "max"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "short 4k video, 480p, vkvideo.ru domain",
+        "url": "https://vkvideo.ru/video-26006257_456245538",
+        "params": {
+            "videoQuality": "480"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "ancient video (fallback to 240p)",
+        "url": "https://vk.com/video-1959_28496479",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "inexistent video",
+        "url": "https://vk.com/video-53333333_456233333",
+        "params": {},
+        "expected": {
+            "code": 400,
+            "status": "error"
+        }
+    }
+]

api/src/util/tests/xiaohongshu.json

@@ -0,0 +1,60 @@
+[
+    {
+        "name": "video (might have expired)",
+        "url": "https://www.xiaohongshu.com/explore/67cc17a3000000000e00726a?xsec_token=CBSFRtbF57so920elY1kbIX4fE1nhrwlpGZs9m6pIFpwo=",
+        "canFail": true,
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "picker with multiple live photos (might have expired)",
+        "url": "https://www.xiaohongshu.com/explore/67c691b4000000000d0159cc?xsec_token=CB8p1eyB5DiFkwlUpy1BTeVsI9oOve6ppNjuDzo8V8p5w=",
+        "canFail": true,
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "picker"
+        }
+    },
+    {
+        "name": "one photo (might have expired)",
+        "url": "https://www.xiaohongshu.com/explore/676e132d000000000b016f68?xsec_token=ABRv6LKzizOFeSaf2HnnBkdBqniB5Ak1fI8tMAHzO31jA",
+        "canFail": true,
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "short link (might have expired)",
+        "url": "https://xhslink.com/a/czn4z6c1tic4",
+        "canFail": true,
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "wrong note id",
+        "url": "https://www.xiaohongshu.com/discovery/item/6789065911100000210035fc?source=webshare&xhsshare=pc_web&xsec_token=CBustnz_Twf1BSybpe5-D-BzUb-Bx28DPLb418TN9S9Kk&xsec_source=pc_share",
+        "params": {},
+        "expected": {
+            "code": 400,
+            "status": "error"
+        }
+    },
+    {
+        "name": "short link, wrong id",
+        "url": "https://xhslink.com/a/aaaaaa",
+        "params": {},
+        "expected": {
+            "code": 400,
+            "status": "error"
+        }
+    }
+]

api/src/util/tests/youtube.json

@@ -0,0 +1,244 @@
+[
+    {
+        "name": "4k video (h264, 1440)",
+        "url": "https://www.youtube.com/watch?v=vPwaXytZcgI",
+        "params": {
+            "youtubeVideoCodec": "h264",
+            "videoQuality": "1440"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "4k video (vp9, 720)",
+        "url": "https://www.youtube.com/watch?v=vPwaXytZcgI",
+        "params": {
+            "youtubeVideoCodec": "vp9",
+            "videoQuality": "720"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "4k video (av1, max)",
+        "url": "https://www.youtube.com/watch?v=vPwaXytZcgI",
+        "params": {
+            "youtubeVideoCodec": "av1",
+            "videoQuality": "max"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "4k video (h264, 720)",
+        "url": "https://www.youtube.com/watch?v=vPwaXytZcgI",
+        "params": {
+            "youtubeVideoCodec": "h264",
+            "videoQuality": "720"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "4k video (vp9, max, isAudioMuted)",
+        "url": "https://www.youtube.com/watch?v=vPwaXytZcgI",
+        "params": {
+            "downloadMode": "mute",
+            "youtubeVideoCodec": "vp9",
+            "videoQuality": "max"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "4k video (h264, max, isAudioMuted)",
+        "url": "https://www.youtube.com/watch?v=vPwaXytZcgI",
+        "params": {
+            "downloadMode": "mute",
+            "youtubeVideoCodec": "h264",
+            "videoQuality": "max"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "4k video (av1, max, isAudioMuted, isAudioOnly, mp3)",
+        "url": "https://www.youtube.com/watch?v=vPwaXytZcgI",
+        "params": {
+            "downloadMode": "audio",
+            "audioFormat": "mp3",
+            "youtubeVideoCodec": "av1",
+            "videoQuality": "max"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "4k video (av1, max, isAudioMuted, isAudioOnly, best)",
+        "url": "https://www.youtube.com/watch?v=vPwaXytZcgI",
+        "params": {
+            "downloadMode": "audio",
+            "audioFormat": "best",
+            "youtubeVideoCodec": "av1",
+            "videoQuality": "max"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "music (mp3, isAudioOnly, isAudioMuted)",
+        "url": "https://music.youtube.com/watch?v=5rGTsvZCEdk&feature=share",
+        "params": {
+            "downloadMode": "audio",
+            "audioFormat": "mp3"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "music (mp3)",
+        "url": "https://music.youtube.com/watch?v=5rGTsvZCEdk&feature=share",
+        "params": {
+            "audioFormat": "mp3"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "audio bitrate higher than video, no vp9 video in response (mp3, isAudioOnly)",
+        "url": "https://www.youtube.com/watch?v=t5nC_ucYBrc",
+        "params": {
+            "downloadMode": "audio",
+            "audioFormat": "mp3"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "short, defaults",
+        "url": "https://www.youtube.com/shorts/r5FpeOJItbw",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "vr 360, av1, max",
+        "url": "https://www.youtube.com/watch?v=hEdzv7D4CbQ",
+        "params": {
+            "youtubeVideoCodec": "vp9",
+            "videoQuality": "max"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "live link, defaults",
+        "url": "https://www.youtube.com/live/ENxZS6PUDuI?feature=shared",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "inexistent video",
+        "url": "https://youtube.com/watch?v=gnjuHYWGEW",
+        "params": {},
+        "expected": {
+            "code": 400,
+            "status": "error"
+        }
+    },
+    {
+        "name": "broken audioOnly download",
+        "url": "https://www.youtube.com/watch?v=ink80Al5nbw",
+        "params": {
+            "downloadMode": "audio"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "hls video (h264, 1440p)",
+        "url": "https://www.youtube.com/watch?v=vPwaXytZcgI",
+        "canFail": true,
+        "params": {
+            "youtubeVideoCodec": "h264",
+            "videoQuality": "1440",
+            "youtubeHLS": true
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "hls video (vp9, 360p)",
+        "url": "https://www.youtube.com/watch?v=vPwaXytZcgI",
+        "canFail": true,
+        "params": {
+            "youtubeVideoCodec": "vp9",
+            "videoQuality": "360",
+            "youtubeHLS": true
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "hls video (audio mode)",
+        "url": "https://www.youtube.com/watch?v=vPwaXytZcgI",
+        "canFail": true,
+        "params": {
+            "downloadMode": "audio",
+            "youtubeHLS": true
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "hls video (audio mode, best format)",
+        "url": "https://www.youtube.com/watch?v=vPwaXytZcgI",
+        "canFail": true,
+        "params": {
+            "downloadMode": "audio",
+            "youtubeHLS": true,
+            "audioFormat": "best"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    }
+]

docs/api.md

@@ -1,9 +1,44 @@
 # cobalt api documentation
 this document provides info about methods and acceptable variables for all cobalt api requests.
 
-> if you are looking for the documentation for the old (7.x) api, you can find
-> it [here](https://github.com/imputnet/cobalt/blob/7/docs/api.md)
-<!-- TODO: authorization -->
+> [!IMPORTANT]
+> hosted api instances (such as `api.cobalt.tools`) use bot protection and are **not** intended to be used in other projects without explicit permission. if you want to access the cobalt api reliably, you should [host your own instance](/docs/run-an-instance.md) or ask an instance owner for access.
+
+## authentication
+an api instance may be configured to require you to authenticate yourself.
+if this is the case, you will typically receive an [error response](#error-response)
+with a **`api.auth.<method>.missing`** code, which tells you that a particular method
+of authentication is required.
+
+authentication is done by passing the `Authorization` header, containing
+the authentication scheme and the token:
+```
+Authorization: <scheme> <token>
+```
+
+currently, cobalt supports two ways of authentication. an instance can
+choose to configure both, or neither:
+- [`Api-Key`](#api-key-authentication)
+- [`Bearer`](#bearer-authentication)
+
+### api-key authentication
+the api key authentication is the most straightforward. the instance owner
+will assign you an api key which you can then use to authenticate like so:
+```
+Authorization: Api-Key aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee
+```
+
+if you are an instance owner and wish to configure api key authentication,
+see the [instance](run-an-instance.md#api-key-file-format) documentation!
+
+### bearer authentication
+the cobalt server may be configured to issue JWT bearers, which are short-lived
+tokens intended for use by regular users (e.g. after passing a challenge).
+currently, cobalt can issue tokens for successfully solved [turnstile](run-an-instance.md#list-of-all-environment-variables)
+challenge, if the instance has turnstile configured. the resulting token is passed like so:
+```
+Authorization: Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9...
+```
 
 ## POST: `/`
 cobalt's main processing endpoint.
@@ -11,9 +46,10 @@ cobalt's main processing endpoint.
 request body type: `application/json`
 response body type: `application/json`
 
-```
-⚠️ you must include Accept and Content-Type headers with every `POST /` request.
+> [!IMPORTANT]
+> you must include `Accept` and `Content-Type` headers with every `POST /` request.
 
+```
 Accept: application/json
 Content-Type: application/json
 ```
@@ -28,13 +64,13 @@ Content-Type: application/json
 | `filenameStyle`              | `string`  | `classic / pretty / basic / nerdy` | `classic` | changes the way files are named. previews can be seen in the web app.           |
 | `downloadMode`               | `string`  | `auto / audio / mute`              | `auto`    | `audio` downloads only the audio, `mute` skips the audio track in videos.       |
 | `youtubeVideoCodec`          | `string`  | `h264 / av1 / vp9`                 | `h264`    | `h264` is recommended for phones.                                               |
-| `youtubeDubLang`             | `string`  | `en / ru / cs / ja / ...`          | --        | specifies the language of audio to download, when the youtube video is dubbed   |
-| `youtubeDubBrowserLang`      | `boolean` | `true / false`                     | `false`   | uses value from the Accept-Language header for `youtubeDubLang`.                |
+| `youtubeDubLang`             | `string`  | `en / ru / cs / ja / es-US / ...`  | --        | specifies the language of audio to download when a youtube video is dubbed.     |
 | `alwaysProxy`                | `boolean` | `true / false`                     | `false`   | tunnels all downloads through the processing server, even when not necessary.   |
 | `disableMetadata`            | `boolean` | `true / false`                     | `false`   | disables file metadata when set to `true`.                                      |
 | `tiktokFullAudio`            | `boolean` | `true / false`                     | `false`   | enables download of original sound used in a tiktok video.                      |
-| `tiktokH265`                 | `boolean` | `true / false`                     | `false`   | changes whether 1080p h265 videos are preferred or not.                         |
+| `tiktokH265`                 | `boolean` | `true / false`                     | `false`   | allows h265 videos when enabled. applies to tiktok & xiaohongshu.               |
 | `twitterGif`                 | `boolean` | `true / false`                     | `true`    | changes whether twitter gifs are converted to .gif                              |
+| `youtubeHLS`                 | `boolean` | `true / false`                     | `false`   | specifies whether to use HLS for downloading video or audio from youtube.       |
 
 ### response
 the response will always be a JSON object containing the `status` key, which will be one of:
@@ -108,3 +144,18 @@ response body type: `application/json`
 | `commit`    | `string` | commit hash       |
 | `branch`    | `string` | git branch        |
 | `remote`    | `string` | git remote        |
+
+## POST: `/session`
+
+used for generating JWT tokens, if enabled. currently, cobalt only supports
+generating tokens when a [turnstile](run-an-instance.md#list-of-all-environment-variables) challenge solution
+is submitted by the client.
+
+the turnstile challenge response is submitted via the `cf-turnstile-response` header.
+### response body
+| key             | type       | description                                            |
+|:----------------|:-----------|:-------------------------------------------------------|
+| `token`         | `string`   | a `Bearer` token used for later request authentication |
+| `exp`           | `number`   | number in seconds indicating the token lifetime        |
+
+on failure, an [error response](#error-response) is returned.

docs/examples/docker-compose.example.yml

@@ -1,33 +1,54 @@
 services:
     cobalt-api:
         image: ghcr.io/imputnet/cobalt:10
+
+        init: true
+        read_only: true
         restart: unless-stopped
         container_name: cobalt-api
 
-        init: true
-
         ports:
             - 9000:9000/tcp
-            # if you're using a reverse proxy, uncomment the next line and remove the one above (9000:9000/tcp):
-            #- 127.0.0.1:9000:9000
+            # if you use a reverse proxy (such as nginx),
+            # uncomment the next line and remove the one above (9000:9000/tcp):
+            # - 127.0.0.1:9000:9000
 
         environment:
-            # replace https://api.cobalt.tools/ with your instance's target url in same format
-            API_URL: "https://api.cobalt.tools/"
-            # if you want to use cookies when fetching data from services, uncomment the next line and the lines under volume
+            # replace https://api.url.example/ with your instance's url
+            # or else tunneling functionality won't work properly
+            API_URL: "https://api.url.example/"
+
+            # if you want to use cookies for fetching data from services,
+            # uncomment the next line & volumes section
             # COOKIE_PATH: "/cookies.json"
-            # see docs/run-an-instance.md for more information
+
+            # it's recommended to configure bot protection or api keys if the instance is public,
+            # see /docs/protect-an-instance.md for more info
+
+            # see /docs/run-an-instance.md for more variables that you can use here
+
         labels:
             - com.centurylinklabs.watchtower.scope=cobalt
 
-        # if you want to use cookies when fetching data from services, uncomment volumes and next line
-        #volumes:
-            #- ./cookies.json:/cookies.json
+        # uncomment only if you use the COOKIE_PATH variable
+        # volumes:
+            # - ./cookies.json:/cookies.json
 
-    # update the cobalt image automatically with watchtower
+    # watchtower updates the cobalt image automatically
     watchtower:
         image: ghcr.io/containrrr/watchtower
         restart: unless-stopped
         command: --cleanup --scope cobalt --interval 900 --include-restarting
         volumes:
             - /var/run/docker.sock:/var/run/docker.sock
+
+    # if needed, use this image for automatically generating poToken & visitor_data
+    # yt-session-generator:
+    #     image: ghcr.io/imputnet/yt-session-generator:webserver
+
+    #     init: true
+    #     restart: unless-stopped
+    #     container_name: yt-session-generator
+
+    #     ports:
+    #         - 127.0.0.1:1280:8080