api/core: remove non-printable unicode character in boot message (#1182)

all links expire apparently

api/package.json

@@ -1,7 +1,7 @@
 {
     "name": "@imput/cobalt-api",
     "description": "save what you love",
-    "version": "10.7.10",
+    "version": "10.8.2",
     "author": "imput",
     "exports": "./src/cobalt.js",
     "type": "module",
@@ -11,7 +11,6 @@
     "scripts": {
         "start": "node src/cobalt",
         "test": "node src/util/test",
-        "token:youtube": "node src/util/generate-youtube-tokens",
         "token:jwt": "node src/util/generate-jwt-secret"
     },
     "repository": {
@@ -39,7 +38,7 @@
         "set-cookie-parser": "2.6.0",
         "undici": "^5.19.1",
         "url-pattern": "1.0.3",
-        "youtubei.js": "^13.1.0",
+        "youtubei.js": "^13.2.0",
         "zod": "^3.23.8"
     },
     "optionalDependencies": {

api/src/config.js

@@ -53,7 +53,11 @@ const env = {
     keyReloadInterval: 900,
 
     enabledServices,
+
     customInnertubeClient: process.env.CUSTOM_INNERTUBE_CLIENT,
+    ytSessionServer: process.env.YOUTUBE_SESSION_SERVER,
+    ytSessionReloadInterval: 300,
+    ytSessionInnertubeClient: process.env.YOUTUBE_SESSION_INNERTUBE_CLIENT,
 }
 
 const genericUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36";

api/src/core/api.js

@@ -18,8 +18,10 @@ import { verifyTurnstileToken } from "../security/turnstile.js";
 import { friendlyServiceName } from "../processing/service-alias.js";
 import { verifyStream, getInternalStream } from "../stream/manage.js";
 import { createResponse, normalizeRequest, getIP } from "../processing/request.js";
+
 import * as APIKeys from "../security/api-keys.js";
 import * as Cookies from "../processing/cookie/manager.js";
+import * as YouTubeSession from "../processing/helpers/youtube-session.js";
 
 const git = {
     branch: await getBranch(),
@@ -354,7 +356,7 @@ export const runAPI = async (express, app, __dirname, isPrimary = true) => {
     }, () => {
         if (isPrimary) {
             console.log(`\n` +
-                Bright(Cyan("cobalt ")) + Bright("API ^ω⁠^") + "\n" +
+                Bright(Cyan("cobalt ")) + Bright("API ^ω^") + "\n" +
 
                 "~~~~~~\n" +
                 Bright("version: ") + version + "\n" +
@@ -376,6 +378,10 @@ export const runAPI = async (express, app, __dirname, isPrimary = true) => {
         if (env.cookiePath) {
             Cookies.setup(env.cookiePath);
         }
+
+        if (env.ytSessionServer) {
+            YouTubeSession.setup();
+        }
     });
 
     if (isCluster) {

api/src/processing/cookie/manager.js

@@ -13,7 +13,6 @@ const VALID_SERVICES = new Set([
     'reddit',
     'twitter',
     'youtube',
-    'youtube_oauth'
 ]);
 
 const invalidCookies = {};

api/src/processing/helpers/youtube-session.js

@@ -0,0 +1,74 @@
+import * as cluster from "../../misc/cluster.js";
+
+import { env } from "../../config.js";
+import { Green, Yellow } from "../../misc/console-text.js";
+
+let session;
+
+const validateSession = (sessionResponse) => {
+    if (!sessionResponse.potoken) {
+        throw "no poToken in session response";
+    }
+
+    if (!sessionResponse.visitor_data) {
+        throw "no visitor_data in session response";
+    }
+
+    if (!sessionResponse.updated) {
+        throw "no last update timestamp in session response";
+    }
+
+    // https://github.com/iv-org/youtube-trusted-session-generator/blob/c2dfe3f/potoken_generator/main.py#L25
+    if (sessionResponse.potoken.length < 160) {
+        console.error(`${Yellow('[!]')} poToken is too short and might not work (${new Date().toISOString()})`);
+    }
+}
+
+const updateSession = (newSession) => {
+    session = newSession;
+}
+
+const loadSession = async () => {
+    const sessionServerUrl = new URL(env.ytSessionServer);
+    sessionServerUrl.pathname = "/token";
+
+    const newSession = await fetch(sessionServerUrl).then(a => a.json());
+    validateSession(newSession);
+
+    if (!session || session.updated < newSession?.updated) {
+        cluster.broadcast({ youtube_session: newSession });
+        updateSession(newSession);
+    }
+}
+
+const wrapLoad = (initial = false) => {
+    loadSession()
+    .then(() => {
+        if (initial) {
+            console.log(`${Green('[✓]')} poToken & visitor_data loaded successfully!`);
+        }
+    })
+    .catch((e) => {
+        console.error(`${Yellow('[!]')} Failed loading poToken & visitor_data at ${new Date().toISOString()}.`);
+        console.error('Error:', e);
+    })
+}
+
+export const getYouTubeSession = () => {
+    return session;
+}
+
+export const setup = () => {
+    if (cluster.isPrimary) {
+        wrapLoad(true);
+        if (env.ytSessionReloadInterval > 0) {
+            setInterval(wrapLoad, env.ytSessionReloadInterval * 1000);
+        }
+    } else if (cluster.isWorker) {
+        process.on('message', (message) => {
+            if ('youtube_session' in message) {
+                updateSession(message.youtube_session);
+            }
+        });
+    }
+}

api/src/processing/match.js

@@ -109,7 +109,7 @@ export default async function({ host, patternMatch, params }) {
                 }
 
                 if (url.hostname === "music.youtube.com" || isAudioOnly) {
-                    fetchInfo.quality = "max";
+                    fetchInfo.quality = "1080";
                     fetchInfo.format = "vp9";
                     fetchInfo.isAudioOnly = true;
                     fetchInfo.isAudioMuted = false;

api/src/processing/services/youtube.js

@@ -4,7 +4,8 @@ import { fetch } from "undici";
 import { Innertube, Session } from "youtubei.js";
 
 import { env } from "../../config.js";
-import { getCookie, updateCookieValues } from "../cookie/manager.js";
+import { getCookie } from "../cookie/manager.js";
+import { getYouTubeSession } from "../helpers/youtube-session.js";
 
 const PLAYER_REFRESH_PERIOD = 1000 * 60 * 15; // ms
 
@@ -45,41 +46,26 @@ const clientsWithNoCipher = ['IOS', 'ANDROID', 'YTSTUDIO_ANDROID', 'YTMUSIC_ANDR
 
 const videoQualities = [144, 240, 360, 480, 720, 1080, 1440, 2160, 4320];
 
-const transformSessionData = (cookie) => {
-    if (!cookie)
-        return;
-
-    const values = { ...cookie.values() };
-    const REQUIRED_VALUES = ['access_token', 'refresh_token'];
-
-    if (REQUIRED_VALUES.some(x => typeof values[x] !== 'string')) {
-        return;
-    }
-
-    if (values.expires) {
-        values.expiry_date = values.expires;
-        delete values.expires;
-    } else if (!values.expiry_date) {
-        return;
-    }
-
-    return values;
-}
-
-const cloneInnertube = async (customFetch) => {
+const cloneInnertube = async (customFetch, useSession) => {
     const shouldRefreshPlayer = lastRefreshedAt + PLAYER_REFRESH_PERIOD < new Date();
 
     const rawCookie = getCookie('youtube');
-    const rawCookieValues = rawCookie?.values();
     const cookie = rawCookie?.toString();
 
+    const sessionTokens = getYouTubeSession();
+    const retrieve_player = Boolean(sessionTokens || cookie);
+
+    if (useSession && env.ytSessionServer && !sessionTokens?.potoken) {
+        throw "no_session_tokens";
+    }
+
     if (!innertube || shouldRefreshPlayer) {
         innertube = await Innertube.create({
             fetch: customFetch,
-            retrieve_player: !!cookie,
+            retrieve_player,
             cookie,
-            po_token: rawCookieValues?.po_token,
-            visitor_data: rawCookieValues?.visitor_data,
+            po_token: useSession ? sessionTokens?.potoken : undefined,
+            visitor_data: useSession ? sessionTokens?.visitor_data : undefined,
         });
         lastRefreshedAt = +new Date();
     }
@@ -95,73 +81,62 @@ const cloneInnertube = async (customFetch) => {
         innertube.session.cache
     );
 
-    const oauthCookie = getCookie('youtube_oauth');
-    const oauthData = transformSessionData(oauthCookie);
-
-    if (!session.logged_in && oauthData) {
-        await session.oauth.init(oauthData);
-        session.logged_in = true;
-    }
-
-    if (session.logged_in && oauthData) {
-        if (session.oauth.shouldRefreshToken()) {
-            await session.oauth.refreshAccessToken();
-        }
-
-        const cookieValues = oauthCookie.values();
-        const oldExpiry = new Date(cookieValues.expiry_date);
-        const newExpiry = new Date(session.oauth.oauth2_tokens.expiry_date);
-
-        if (oldExpiry.getTime() !== newExpiry.getTime()) {
-            updateCookieValues(oauthCookie, {
-                ...session.oauth.client_id,
-                ...session.oauth.oauth2_tokens,
-                expiry_date: newExpiry.toISOString()
-            });
-        }
-    }
-
     const yt = new Innertube(session);
     return yt;
 }
 
 export default async function (o) {
+    const quality = o.quality === "max" ? 9000 : Number(o.quality);
+
+    let useHLS = o.youtubeHLS;
+    let innertubeClient = o.innertubeClient || env.customInnertubeClient || "IOS";
+
+    // HLS playlists from the iOS client don't contain the av1 video format.
+    if (useHLS && o.format === "av1") {
+        useHLS = false;
+    }
+
+    if (useHLS) {
+        innertubeClient = "IOS";
+    }
+
+    // iOS client doesn't have adaptive formats of resolution >1080p,
+    // so we use the WEB_EMBEDDED client instead for those cases
+    const useSession =
+        env.ytSessionServer && (
+            (
+                !useHLS
+                && innertubeClient === "IOS"
+                && (
+                    (quality > 1080 && o.format !== "h264")
+                    || (quality > 1080 && o.format !== "vp9")
+                )
+            )
+        );
+
+    if (useSession) {
+        innertubeClient = env.ytSessionInnertubeClient || "WEB_EMBEDDED";
+    }
+
     let yt;
     try {
         yt = await cloneInnertube(
             (input, init) => fetch(input, {
                 ...init,
                 dispatcher: o.dispatcher
-            })
+            }),
+            useSession
         );
     } catch (e) {
-        if (e.message?.endsWith("decipher algorithm")) {
+        if (e === "no_session_tokens") {
+            return { error: "youtube.no_session_tokens" };
+        } else if (e.message?.endsWith("decipher algorithm")) {
             return { error: "youtube.decipher" }
         } else if (e.message?.includes("refresh access token")) {
             return { error: "youtube.token_expired" }
         } else throw e;
     }
 
-    const cookie = getCookie('youtube')?.toString();
-
-    let useHLS = o.youtubeHLS;
-
-    // HLS playlists don't contain the av1 video format, at least with the iOS client
-    if (useHLS && o.format === "av1") {
-        useHLS = false;
-    }
-
-    let innertubeClient = o.innertubeClient || env.customInnertubeClient || "ANDROID";
-
-    if (cookie) {
-        useHLS = false;
-        innertubeClient = "WEB";
-    }
-
-    if (useHLS) {
-        innertubeClient = "IOS";
-    }
-
     let info;
     try {
         info = await yt.getBasicInfo(o.id, innertubeClient);
@@ -240,8 +215,6 @@ export default async function (o) {
         }
     }
 
-    const quality = o.quality === "max" ? 9000 : Number(o.quality);
-
     const normalizeQuality = res => {
         const shortestSide = Math.min(res.height, res.width);
         return videoQualities.find(qual => qual >= shortestSide);

api/src/util/generate-youtube-tokens.js

@@ -1,38 +0,0 @@
-import { Innertube } from 'youtubei.js';
-import { Red } from '../misc/console-text.js'
-
-const bail = (...msg) => {
-    console.error(...msg);
-    throw new Error(msg);
-};
-
-const tube = await Innertube.create();
-
-tube.session.once(
-    'auth-pending',
-    ({ verification_url, user_code }) => {
-        console.log(`${Red('[!]')} The token generated by this script is sensitive and you should not share it with anyone!`);
-        console.log(`    By using this token, you are risking your Google account getting terminated.`);
-        console.log(`    You should ${Red('NOT')} use your personal account!`);
-        console.log();
-        console.log(`Open ${verification_url} in a browser and enter ${user_code} when asked for the code.`);
-    }
-);
-
-tube.session.once('auth-error', (err) => bail('An error occurred:', err));
-tube.session.once('auth', ({ credentials }) => {
-    if (!credentials.access_token) {
-        bail('something went wrong');
-    }
-
-    console.log(
-        'add this cookie to the youtube_oauth array in your cookies file:',
-        JSON.stringify(
-            Object.entries(credentials)
-                .map(([k, v]) => `${k}=${v instanceof Date ? v.toISOString() : v}`)
-                .join('; ')
-        )
-    );
-});
-
-await tube.session.signIn();

api/src/util/tests/xiaohongshu.json

@@ -1,7 +1,8 @@
 [
     {
-        "name": "long link video",
-        "url": "https://www.xiaohongshu.com/discovery/item/6789065900000000210035fc?source=webshare&xhsshare=pc_web&xsec_token=CBustnz_Twf1BSybpe5-D-BzUb-Bx28DPLb418TN9S9Kk&xsec_source=pc_share",
+        "name": "video (might have expired)",
+        "url": "https://www.xiaohongshu.com/explore/67cc17a3000000000e00726a?xsec_token=CBSFRtbF57so920elY1kbIX4fE1nhrwlpGZs9m6pIFpwo=",
+        "canFail": true,
         "params": {},
         "expected": {
             "code": 200,
@@ -9,8 +10,9 @@
         }
     },
     {
-        "name": "picker with multiple live photos",
-        "url": "https://www.xiaohongshu.com/explore/67847fa1000000000203e6ed?xsec_token=CBzyP7Y44PPpsM20lgxqrIIJMHqOLemusDsRcmsX0cTpk",
+        "name": "picker with multiple live photos (might have expired)",
+        "url": "https://www.xiaohongshu.com/explore/67c691b4000000000d0159cc?xsec_token=CB8p1eyB5DiFkwlUpy1BTeVsI9oOve6ppNjuDzo8V8p5w=",
+        "canFail": true,
         "params": {},
         "expected": {
             "code": 200,
@@ -18,8 +20,9 @@
         }
     },
     {
-        "name": "one photo",
+        "name": "one photo (might have expired)",
         "url": "https://www.xiaohongshu.com/explore/676e132d000000000b016f68?xsec_token=ABRv6LKzizOFeSaf2HnnBkdBqniB5Ak1fI8tMAHzO31jA",
+        "canFail": true,
         "params": {},
         "expected": {
             "code": 200,
@@ -27,7 +30,7 @@
         }
     },
     {
-        "name": "short link, might expire eventually",
+        "name": "short link (might have expired)",
         "url": "https://xhslink.com/a/czn4z6c1tic4",
         "canFail": true,
         "params": {},

api/src/util/tests/youtube.json

@@ -189,6 +189,7 @@
     {
         "name": "hls video (h264, 1440p)",
         "url": "https://www.youtube.com/watch?v=vPwaXytZcgI",
+        "canFail": true,
         "params": {
             "youtubeVideoCodec": "h264",
             "videoQuality": "1440",
@@ -202,6 +203,7 @@
     {
         "name": "hls video (vp9, 360p)",
         "url": "https://www.youtube.com/watch?v=vPwaXytZcgI",
+        "canFail": true,
         "params": {
             "youtubeVideoCodec": "vp9",
             "videoQuality": "360",
@@ -215,6 +217,7 @@
     {
         "name": "hls video (audio mode)",
         "url": "https://www.youtube.com/watch?v=vPwaXytZcgI",
+        "canFail": true,
         "params": {
             "downloadMode": "audio",
             "youtubeHLS": true
@@ -227,6 +230,7 @@
     {
         "name": "hls video (audio mode, best format)",
         "url": "https://www.youtube.com/watch?v=vPwaXytZcgI",
+        "canFail": true,
         "params": {
             "downloadMode": "audio",
             "youtubeHLS": true,

docs/configure-for-youtube.md

@@ -1,33 +0,0 @@
-# how to configure a cobalt instance for youtube
-if you get various errors when attempting to download videos that are:
-publicly available, not region locked, and not age-restricted;
-then your instance's ip address may have bad reputation.
-
-in this case you have to use disposable google accounts.
-there's no other known workaround as of time of writing this document.
-
-> [!CAUTION]
-> **NEVER** use your personal google account for downloading videos via any means.
-> you can use any google accounts that you're willing to sacrifice,
-> but be prepared to have them **permanently suspended**.
->
-> we recommend that you use accounts that don't link back to your personal google account or identity, just in case.
->
-> use incognito mode when signing in.
-> we also recommend using vpn/proxy services (such as [mullvad](https://mullvad.net/)).
-
-1. if you haven't done it already, clone the cobalt repo, go to the cloned directory, and run `pnpm install`
-
-2. run `pnpm -C api token:youtube`
-
-3. follow instructions, use incognito mode in your browser when signing in.
-i cannot stress this enough, but again, **DO NOT USE YOUR PERSONAL GOOGLE ACCOUNT**.
-
-4. once you have the oauth token, add it to `youtube_oauth` in your cookies file.
-you can see an [example here](/docs/examples/cookies.example.json).
-you can have several account tokens in this file, if you like.
-
-5. all done! enjoy freedom.
-
-### liability
-you're responsible for any damage done to any of your google accounts or any other damages. you do this by yourself and at your own risk.

docs/examples/cookies.example.json

@@ -10,8 +10,5 @@
     ],
     "twitter": [
         "auth_token=<replace_this>; ct0=<replace_this>"
-    ],
-    "youtube_oauth": [
-        "<output from running `pnpm run token:youtube` in `api` folder goes here>"
     ]
 }

docs/examples/docker-compose.example.yml

@@ -41,3 +41,14 @@ services:
         command: --cleanup --scope cobalt --interval 900 --include-restarting
         volumes:
             - /var/run/docker.sock:/var/run/docker.sock
+
+    # if needed, use this image for automatically generating poToken & visitor_data
+    # yt-session-generator:
+    #     image: ghcr.io/imputnet/yt-session-generator:webserver
+
+    #     init: true
+    #     restart: unless-stopped
+    #     container_name: yt-session-generator
+
+    #     ports:
+    #         - 127.0.0.1:1280:8080

docs/run-an-instance.md

@@ -81,6 +81,7 @@ sudo service nscd start
 | `API_INSTANCE_COUNT`  | ➖        | `2`                     | supported only on Linux and node.js `>=23.1.0`. when configured, cobalt will spawn multiple sub-instances amongst which requests will be balanced. |
 | `DISABLED_SERVICES`   | ➖        | `bilibili,youtube`       | comma-separated list which disables certain services from being used. |
 | `CUSTOM_INNERTUBE_CLIENT` | ➖    | `IOS`                    | innertube client that will be used instead of the default one. |
+| `YOUTUBE_SESSION_SERVER` | ➖    | `http://localhost:8080/`  | url to an instance of [invidious' youtube-trusted-session-generator](https://github.com/iv-org/youtube-trusted-session-generator) or its fork/counterpart. used for automatically pulling poToken & visitor_data for youtube. can be local or remote. |
 
 \* the higher the nice value, the lower the priority. [read more here](https://en.wikipedia.org/wiki/Nice_(Unix)).
 

pnpm-lock.yaml

@@ -56,8 +56,8 @@ importers:
         specifier: 1.0.3
         version: 1.0.3
       youtubei.js:
-        specifier: ^13.1.0
-        version: 13.1.0
+        specifier: ^13.2.0
+        version: 13.2.0
       zod:
         specifier: ^3.23.8
         version: 3.23.8
@@ -188,8 +188,8 @@ packages:
     resolution: {integrity: sha512-30iZtAPgz+LTIYoeivqYo853f02jBYSd5uGnGpkFV0M3xOt9aN73erkgYAmZU43x4VfqcnLxW9Kpg3R5LC4YYw==}
     engines: {node: '>=6.0.0'}
 
-  '@bufbuild/protobuf@2.1.0':
-    resolution: {integrity: sha512-+2Mx67Y3skJ4NCD/qNSdBJNWtu6x6Qr53jeNg+QcwiL6mt0wK+3jwHH2x1p7xaYH6Ve2JKOVn0OxU35WsmqI9A==}
+  '@bufbuild/protobuf@2.2.5':
+    resolution: {integrity: sha512-/g5EzJifw5GF8aren8wZ/G5oMuPoGeS6MQD3ca8ddcvdXR5UELUfdTZITCGNhNXynY/AYl3Z4plmxdj/tRl/hQ==}
 
   '@datastructures-js/heap@4.3.3':
     resolution: {integrity: sha512-UcUu/DLh/aM4W3C8zZfwxxm6/6FIZUlm3mcAXuNOCa6Aj4iizNvNXQyb8DjZQH2jKSQbMRyNlngP6TPimuGjpQ==}
@@ -2286,8 +2286,8 @@ packages:
     resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==}
     engines: {node: '>=10'}
 
-  youtubei.js@13.1.0:
-    resolution: {integrity: sha512-uL4TyojAYET0c5NGFD7+ScCod/k8Pc/B+D5tLrunFcz1GaBjRMOGRPcNGaRmnhwisegU7ibtw0iUxCN+BZ0ang==}
+  youtubei.js@13.2.0:
+    resolution: {integrity: sha512-esbSvWS12Dz/cVlHhnL/PSE84a/mVpQdzwPDIkRQu/NHJVxv0isBUcm3hJnYB1jg1LYvomV0YeOrYv5qWwJREA==}
 
   zod@3.23.8:
     resolution: {integrity: sha512-XBx9AXhXktjUqnepgTiE5flcKIYWi/rme0Eaj+5Y0lftuGBq+jyRu/md4WnuxqgP1ubdpNCsYEYPxrzVHD8d6g==}
@@ -2299,7 +2299,7 @@ snapshots:
       '@jridgewell/gen-mapping': 0.3.5
       '@jridgewell/trace-mapping': 0.3.25
 
-  '@bufbuild/protobuf@2.1.0': {}
+  '@bufbuild/protobuf@2.2.5': {}
 
   '@datastructures-js/heap@4.3.3': {}
 
@@ -4242,9 +4242,9 @@ snapshots:
 
   yocto-queue@0.1.0: {}
 
-  youtubei.js@13.1.0:
+  youtubei.js@13.2.0:
     dependencies:
-      '@bufbuild/protobuf': 2.1.0
+      '@bufbuild/protobuf': 2.2.5
       jintr: 3.2.1
       tslib: 2.6.3
       undici: 5.28.4

web/i18n/en/error.json

@@ -68,5 +68,6 @@
     "api.youtube.no_hls_streams": "couldn't find any matching HLS streams for this video. try downloading it without HLS!",
     "api.youtube.api_error": "youtube updated something about its api and i couldn't get any info about this video. try again in a few seconds, but if this issue sticks, please report it!",
     "api.youtube.temporary_disabled": "youtube downloading is temporarily disabled due to restrictions from youtube's side. we're already looking for ways to go around them.\n\nwe apologize for the inconvenience and are doing our best to restore this functionality. check cobalt's socials or github for timely updates!",
-    "api.youtube.drm": "this youtube video is protected by widevine DRM, so i can't download it. try a different link!"
+    "api.youtube.drm": "this youtube video is protected by widevine DRM, so i can't download it. try a different link!",
+    "api.youtube.no_session_tokens": "couldn't get required session tokens for youtube. this may be caused by a restriction on youtube's side. try again in a few seconds, but if this issue sticks, please report it!"
 }