diff --git a/.github/workflows/build-assets.yml b/.github/bakup_workflows/build-assets.yml similarity index 100% rename from .github/workflows/build-assets.yml rename to .github/bakup_workflows/build-assets.yml diff --git a/.github/workflows/codeql.yml b/.github/bakup_workflows/codeql.yml similarity index 100% rename from .github/workflows/codeql.yml rename to .github/bakup_workflows/codeql.yml diff --git a/.github/workflows/comment-on-issue.yml b/.github/bakup_workflows/comment-on-issue.yml similarity index 100% rename from .github/workflows/comment-on-issue.yml rename to .github/bakup_workflows/comment-on-issue.yml diff --git a/.github/workflows/dependabot-fork.yml b/.github/bakup_workflows/dependabot-fork.yml similarity index 100% rename from .github/workflows/dependabot-fork.yml rename to .github/bakup_workflows/dependabot-fork.yml diff --git a/.github/workflows/docker-release.yml b/.github/bakup_workflows/docker-release.yml similarity index 100% rename from .github/workflows/docker-release.yml rename to .github/bakup_workflows/docker-release.yml diff --git a/.github/workflows/docker-test-cont.yml b/.github/bakup_workflows/docker-test-cont.yml similarity index 100% rename from .github/workflows/docker-test-cont.yml rename to .github/bakup_workflows/docker-test-cont.yml diff --git a/.github/workflows/docker-test.yml b/.github/bakup_workflows/docker-test.yml similarity index 100% rename from .github/workflows/docker-test.yml rename to .github/bakup_workflows/docker-test.yml diff --git a/.github/workflows/format.yml b/.github/bakup_workflows/format.yml similarity index 100% rename from .github/workflows/format.yml rename to .github/bakup_workflows/format.yml diff --git a/.github/workflows/issue-command.yml b/.github/bakup_workflows/issue-command.yml similarity index 100% rename from .github/workflows/issue-command.yml rename to .github/bakup_workflows/issue-command.yml diff --git a/.github/workflows/lint.yml b/.github/bakup_workflows/lint.yml similarity index 100% rename from .github/workflows/lint.yml rename to .github/bakup_workflows/lint.yml diff --git a/.github/workflows/npm-publish.yml b/.github/bakup_workflows/npm-publish.yml similarity index 100% rename from .github/workflows/npm-publish.yml rename to .github/bakup_workflows/npm-publish.yml diff --git a/.github/bakup_workflows/release.pubfi.yml b/.github/bakup_workflows/release.pubfi.yml new file mode 100644 index 000000000000..6cf783976314 --- /dev/null +++ b/.github/bakup_workflows/release.pubfi.yml @@ -0,0 +1,96 @@ +name: 'Docker Release (Pubfi)' + +on: + push: + branches: + - pubfi + +jobs: + release: + runs-on: ubuntu-latest + timeout-minutes: 60 + permissions: + packages: write + id-token: write + steps: + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - name: Install cosign + if: github.event_name != 'pull_request' + uses: sigstore/cosign-installer@398d4b0eeef1380460a10c8013a76f728fb906ac # v3.9.1 + + - name: Set up QEMU + uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v3.6.0 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 + + - name: Log in to the Container registry + uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract Docker metadata (ordinary version) + id: meta-ordinary + uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804 # v5.7.0 + with: + images: | + ghcr.io/${{ github.repository }} + tags: | + type=raw,value=latest,enable=true + type=raw,value={{date 'YYYY-MM-DD'}},enable=true + type=sha,format=long,prefix=,enable=true + flavor: latest=false + + - name: Build and push Docker image (ordinary version) + id: build-and-push + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + with: + context: . + push: true + tags: ${{ steps.meta-ordinary.outputs.tags }} + labels: ${{ steps.meta-ordinary.outputs.labels }} + platforms: linux/amd64,linux/arm64 + cache-from: type=gha,scope=docker-release + cache-to: type=gha,mode=max,scope=docker-release + + - name: Sign the published Docker image + if: ${{ github.event_name != 'pull_request' }} + env: + COSIGN_EXPERIMENTAL: 'true' + run: echo "${{ steps.meta-ordinary.outputs.tags }}" | xargs -I {} cosign sign --yes {}@${{ steps.build-and-push.outputs.digest }} + + - name: Extract Docker metadata (Chromium-bundled version) + id: meta-chromium-bundled + uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804 # v5.7.0 + with: + images: | + ghcr.io/${{ github.repository }} + tags: | + type=raw,value=chromium-bundled,enable=true + type=raw,value=chromium-bundled-{{date 'YYYY-MM-DD'}},enable=true + type=sha,format=long,prefix=chromium-bundled-,enable=true + flavor: latest=false + + - name: Build and push Docker image (Chromium-bundled version) + id: build-and-push-chromium + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + with: + context: . + build-args: PUPPETEER_SKIP_DOWNLOAD=0 + push: true + tags: ${{ steps.meta-chromium-bundled.outputs.tags }} + labels: ${{ steps.meta-chromium-bundled.outputs.labels }} + platforms: linux/amd64,linux/arm64 + cache-from: | + type=registry,ref=${{ secrets.DOCKER_USERNAME }}/rsshub:chromium-bundled + cache-to: type=inline,ref=${{ secrets.DOCKER_USERNAME }}/rsshub:chromium-bundled # inline cache is enough + + - name: Sign the published Docker image + if: ${{ github.event_name != 'pull_request' }} + env: + COSIGN_EXPERIMENTAL: 'true' + run: echo "${{ steps.meta-chromium-bundled.outputs.tags }}" | xargs -I {} cosign sign --yes {}@${{ steps.build-and-push-chromium.outputs.digest }} diff --git a/.github/workflows/semgrep.yml b/.github/bakup_workflows/semgrep.yml similarity index 100% rename from .github/workflows/semgrep.yml rename to .github/bakup_workflows/semgrep.yml diff --git a/.github/workflows/stale.yml b/.github/bakup_workflows/stale.yml similarity index 100% rename from .github/workflows/stale.yml rename to .github/bakup_workflows/stale.yml diff --git a/.github/workflows/test-full-routes.yml b/.github/bakup_workflows/test-full-routes.yml similarity index 100% rename from .github/workflows/test-full-routes.yml rename to .github/bakup_workflows/test-full-routes.yml diff --git a/.github/workflows/test.yml b/.github/bakup_workflows/test.yml similarity index 100% rename from .github/workflows/test.yml rename to .github/bakup_workflows/test.yml diff --git a/.github/workflows/release.pubfi.yml b/.github/workflows/release.pubfi.yml new file mode 100644 index 000000000000..db203efaffcc --- /dev/null +++ b/.github/workflows/release.pubfi.yml @@ -0,0 +1,104 @@ +name: 'Docker Release (Pubfi)' + +on: + push: + branches: + - pubfi + +jobs: + release: + runs-on: ubuntu-latest + timeout-minutes: 60 + permissions: + packages: write + id-token: write + steps: + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - name: Install cosign + if: github.event_name != 'pull_request' + uses: sigstore/cosign-installer@398d4b0eeef1380460a10c8013a76f728fb906ac # v3.9.1 + + - name: Set up QEMU + uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v3.6.0 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 + + - name: Log in to the Container registry + uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract Docker metadata (ordinary version) + id: meta-ordinary + uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804 # v5.7.0 + with: + images: | + ghcr.io/${{ github.repository }} + tags: | + type=raw,value=latest,enable=true + type=raw,value={{date 'YYYY-MM-DD'}},enable=true + type=sha,format=long,prefix=,enable=true + flavor: latest=false + + - name: Build and push Docker image (ordinary version) + id: build-and-push + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + with: + context: . + push: true + tags: ${{ steps.meta-ordinary.outputs.tags }} + labels: ${{ steps.meta-ordinary.outputs.labels }} + platforms: linux/amd64,linux/arm64 + cache-from: type=gha,scope=docker-release + cache-to: type=gha,mode=max,scope=docker-release + + - name: Sign the published Docker image + if: ${{ github.event_name != 'pull_request' }} + env: + COSIGN_EXPERIMENTAL: 'true' + run: echo "${{ steps.meta-ordinary.outputs.tags }}" | xargs -I {} cosign sign --yes {}@${{ steps.build-and-push.outputs.digest }} + + - name: Extract Docker metadata (Chromium-bundled version) + id: meta-chromium-bundled + uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804 # v5.7.0 + with: + images: | + ghcr.io/${{ github.repository }} + tags: | + type=raw,value=chromium-bundled,enable=true + type=raw,value=chromium-bundled-{{date 'YYYY-MM-DD'}},enable=true + type=sha,format=long,prefix=chromium-bundled-,enable=true + flavor: latest=false + + - name: Build and push Docker image (Chromium-bundled version) + id: build-and-push-chromium + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + with: + context: . + build-args: PUPPETEER_SKIP_DOWNLOAD=0 + push: true + tags: ${{ steps.meta-chromium-bundled.outputs.tags }} + labels: ${{ steps.meta-chromium-bundled.outputs.labels }} + platforms: linux/amd64,linux/arm64 + cache-from: | + type=registry,ref=${{ secrets.DOCKER_USERNAME }}/rsshub:chromium-bundled + cache-to: type=inline,ref=${{ secrets.DOCKER_USERNAME }}/rsshub:chromium-bundled # inline cache is enough + + - name: Sign the published Docker image + if: ${{ github.event_name != 'pull_request' }} + env: + COSIGN_EXPERIMENTAL: 'true' + run: echo "${{ steps.meta-chromium-bundled.outputs.tags }}" | xargs -I {} cosign sign --yes {}@${{ steps.build-and-push-chromium.outputs.digest }} + + - name: Deploy + uses: tarasyarema/dokploy-deploy-action@main + with: + auth_token: ${{ secrets.DOKPLOY_TOKEN }} + application_id: xRDUMgJGh52qvtaJakCAq + dokploy_url: ${{ secrets.DOKPLOY_URL }} + wait_for_completion: true diff --git a/lib/config.ts b/lib/config.ts index 2834a3d8899d..b7fdb761c243 100644 --- a/lib/config.ts +++ b/lib/config.ts @@ -361,6 +361,7 @@ export type Config = { authenticationSecret?: string[]; phoneOrEmail?: string[]; authToken?: string[]; + proxyUris?: string[]; thirdPartyApi?: string; }; uestc: { @@ -823,6 +824,9 @@ const calculateValue = () => { authenticationSecret: envs.TWITTER_AUTHENTICATION_SECRET?.split(','), phoneOrEmail: envs.TWITTER_PHONE_OR_EMAIL?.split(','), authToken: envs.TWITTER_AUTH_TOKEN?.split(','), + proxyUris: envs.TWITTER_PROXY_URIS?.split(',') + .map((uri) => uri.trim()) + .filter(Boolean), thirdPartyApi: envs.TWITTER_THIRD_PARTY_API, }, uestc: { diff --git a/lib/routes/twitter/api/web-api/constants.ts b/lib/routes/twitter/api/web-api/constants.ts index aecefd1ab4ca..087e277308f3 100644 --- a/lib/routes/twitter/api/web-api/constants.ts +++ b/lib/routes/twitter/api/web-api/constants.ts @@ -9,7 +9,7 @@ const graphQLEndpointsPlain = [ '/graphql/dexO_2tohK86JDudXXG3Yw/UserMedia', '/graphql/Qw77dDjp9xCpUY-AXwt-yQ/UserByRestId', '/graphql/UN1i3zUiCWa-6r-Uaho4fw/SearchTimeline', - '/graphql/Pa45JvqZuKcW1plybfgBlQ/ListLatestTweetsTimeline', + '/graphql/nXOf2bJbJBtuzgtKxOqJ-A/ListLatestTweetsTimeline', '/graphql/QuBlQ6SxNAQCt6-kBiCXCQ/TweetDetail', ]; @@ -32,14 +32,22 @@ const gqlFeatureUser = { responsive_web_graphql_timeline_navigation_enabled: true, }; const gqlFeatureFeed = { + rweb_video_screen_enabled: false, + payments_enabled: false, + rweb_xchat_enabled: false, + profile_label_improvements_pcf_label_in_post_enabled: true, rweb_tipjar_consumption_enabled: true, - responsive_web_graphql_exclude_directive_enabled: true, verified_phone_label_enabled: false, creator_subscriptions_tweet_preview_api_enabled: true, responsive_web_graphql_timeline_navigation_enabled: true, responsive_web_graphql_skip_user_profile_image_extensions_enabled: false, + premium_content_api_read_enabled: false, communities_web_enable_tweet_community_results_fetch: true, c9s_tweet_anatomy_moderator_badge_enabled: true, + responsive_web_grok_analyze_button_fetch_trends_enabled: false, + responsive_web_grok_analyze_post_followups_enabled: true, + responsive_web_jetfuel_frame: true, + responsive_web_grok_share_attachment_enabled: true, articles_preview_enabled: true, responsive_web_edit_tweet_api_enabled: true, graphql_is_translatable_rweb_tweet_is_translatable_enabled: true, @@ -47,13 +55,17 @@ const gqlFeatureFeed = { longform_notetweets_consumption_enabled: true, responsive_web_twitter_article_tweet_consumption_enabled: true, tweet_awards_web_tipping_enabled: false, + responsive_web_grok_show_grok_translated_post: false, + responsive_web_grok_analysis_button_from_backend: false, creator_subscriptions_quote_tweet_preview_enabled: false, freedom_of_speech_not_reach_fetch_enabled: true, standardized_nudges_misinfo: true, tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: true, - rweb_video_timestamps_enabled: true, longform_notetweets_rich_text_read_enabled: true, longform_notetweets_inline_media_enabled: true, + responsive_web_grok_image_annotation_enabled: true, + responsive_web_grok_imagine_annotation_enabled: true, + responsive_web_grok_community_note_auto_translation_is_enabled: false, responsive_web_enhance_cards_enabled: false, }; diff --git a/lib/routes/twitter/api/web-api/utils.ts b/lib/routes/twitter/api/web-api/utils.ts index 02653eacff39..14e6ff21efbf 100644 --- a/lib/routes/twitter/api/web-api/utils.ts +++ b/lib/routes/twitter/api/web-api/utils.ts @@ -13,7 +13,7 @@ import login from './login'; let authTokenIndex = 0; -const token2Cookie = async (token) => { +const token2Cookie = async (token, proxyUri?) => { const c = await cache.get(`twitter:cookie:${token}`); if (c) { return c; @@ -21,10 +21,14 @@ const token2Cookie = async (token) => { const jar = new CookieJar(); await jar.setCookie(`auth_token=${token}`, 'https://x.com'); try { - const agent = proxy.proxyUri + const effectiveProxyUri = proxyUri || proxy.proxyUri; + if (effectiveProxyUri) { + logger.info(`twitter token2Cookie via proxy: ${effectiveProxyUri}`); + } + const agent = effectiveProxyUri ? new ProxyAgent({ factory: (origin, opts) => new CookieClient(origin as string, { ...opts, cookies: { jar } }), - uri: proxy.proxyUri, + uri: effectiveProxyUri, }) : new CookieAgent({ cookies: { jar } }); if (token) { @@ -58,13 +62,15 @@ const getAuth = async (retry: number) => { const lock = await cache.get(`${lockPrefix}${token}`, false); if (lock) { logger.debug(`twitter debug: twitter cookie for token ${token} is locked, retry: ${retry}`); - await new Promise((resolve) => setTimeout(resolve, Math.random() * 500 + 500)); - return await getAuth(retry - 1); + // await new Promise((resolve) => setTimeout(resolve, Math.random() * 500 + 500)); + // return await getAuth(retry - 1); + return; } else { logger.debug(`twitter debug: lock twitter cookie for token ${token}`); await cache.set(`${lockPrefix}${token}`, '1', 20); return { token, + proxyUri: config.twitter.proxyUris?.[index], username: config.twitter.username?.[index], password: config.twitter.password?.[index], authenticationSecret: config.twitter.authenticationSecret?.[index], @@ -88,7 +94,12 @@ export const twitterGot = async ( const requestUrl = `${url}?${queryString.stringify(params)}`; - let cookie: string | Record | null | undefined = await token2Cookie(auth?.token); + const effectiveProxyUri = auth?.proxyUri || proxy.proxyUri; + if (effectiveProxyUri) { + logger.info(`twitter request via proxy: ${effectiveProxyUri}`); + } + + let cookie: string | Record | null | undefined = await token2Cookie(auth?.token, auth?.proxyUri); if (!cookie && auth) { cookie = await login({ username: auth.username, @@ -108,14 +119,15 @@ export const twitterGot = async ( cookie = JSON.parse(cookie); } const jar = CookieJar.deserializeSync(cookie as any); - const agent = proxy.proxyUri + const effectiveProxyUri = auth?.proxyUri || proxy.proxyUri; + const agent = effectiveProxyUri ? new ProxyAgent({ factory: (origin, opts) => new CookieClient(origin as string, { ...opts, cookies: { jar } }), - uri: proxy.proxyUri, + uri: effectiveProxyUri, }) : new CookieAgent({ cookies: { jar } }); - if (proxy.proxyUri) { - logger.debug(`twitter debug: Proxying request: ${requestUrl}`); + if (effectiveProxyUri) { + logger.info(`twitter info: Proxying request: ${requestUrl} via ${effectiveProxyUri}`); } dispatchers = { jar, diff --git a/lib/utils/ofetch.ts b/lib/utils/ofetch.ts index bb3443183698..34a901825d06 100644 --- a/lib/utils/ofetch.ts +++ b/lib/utils/ofetch.ts @@ -7,7 +7,7 @@ config.enableRemoteDebugging && process.env.NODE_ENV === 'dev' && register(); const rofetch = createFetch().create({ retryStatusCodes: [400, 408, 409, 425, 429, 500, 502, 503, 504], - retry: config.requestRetry, + retry: 0, retryDelay: 1000, // timeout: config.requestTimeout, onResponseError({ request, response, options }) {