Add media mirroring

alopatindev · alopatindev · commit 94781ca614dd · 2025-05-08T01:52:06.000+08:00
diff --git a/_config.i2p.yml b/_config.i2p.yml
@@ -4,6 +4,7 @@ theme_settings:
   p2p_player:
     media_urls_overwrite:
     - http://codnaft43k7ncna2hfsxrzi2nqoxieu22vbyjkmhkwdrrta2ghlq.b32.i2p
+    - http://codonfttnvztewul3qgathdxjr2fnv34udrqyc6quw3zptqojnea.b32.i2p
   nostr:
     spam_api_overwrite: http://codnaft43k7ncna2hfsxrzi2nqoxieu22vbyjkmhkwdrrta2ghlq.b32.i2p/nostr/spam.nostr.band/spam_api
     profile_relays_overwrite:
diff --git a/_config.tor.yml b/_config.tor.yml
@@ -4,6 +4,7 @@ theme_settings:
   p2p_player:
     media_urls_overwrite:
     - http://codonaftct3jsouvfyrjq4yumyngzv3el2msndf5oddccktgghnw7eyd.onion
+    - http://codonafte3ygy3szn5qwsdje4vp6mwobvzd75jl4ytyrb262cpkaegid.onion
   nostr:
     spam_api_overwrite: http://codonaftct3jsouvfyrjq4yumyngzv3el2msndf5oddccktgghnw7eyd.onion/nostr/spam.nostr.band/spam_api
     profile_relays_overwrite:
diff --git a/_config.yml b/_config.yml
@@ -160,7 +160,7 @@ theme_settings:
   post_navigation: true
 
   p2p_player:
-    swarmPrefix: https://media.codonaft.com
+    swarm_prefix: https://media.codonaft.com
     media_urls:
     - https://media.codonaft.com
     - https://media-cached.codonaft.com
diff --git a/_do.cr b/_do.cr
@@ -60,7 +60,13 @@ CACHE_DIR        = BUILD_DIR.join(".cache")
 BANLISTS         = CACHE_DIR.join("banlists.txt")
 BROKEN_POST_URLS = CACHE_DIR.join("broken_post_urls.txt")
 COMMON_ROOT      = Path["_ohmyvps/alpine/alpine-root"]
-MEDIA_BUILD_DIR  = BUILD_DIR.join(MEDIA_HOST, "/var/www", PUBLIC_MEDIA_HOST)
+
+SERVER_MEDIA_DIR = Path["var/www"].join(PUBLIC_MEDIA_HOST)
+MEDIA_BUILD_DIR  = BUILD_DIR.join(MEDIA_HOST, SERVER_MEDIA_DIR)
+
+MIRROR_FROM_TO = [
+  {MEDIA_HOST, [{MIRROR_HOST, SERVER_MEDIA_DIR}]},
+]
 
 USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36"
 
@@ -317,6 +323,10 @@ def sync
   hosts = all_hosts()
   puts("hosts: #{hosts}")
 
+  mirror = MIRROR_FROM_TO.map { |source_host, destination_and_files|
+    {source_host, destination_and_files.select { |destination_host, _| hosts.includes?(destination_host) }}
+  }.to_h
+
   common_files = git_ls(COMMON_ROOT)
   hosts_files : Hash(String, Set(Path)) = hosts.map { |host|
     host_dir = HOSTS_DIR.join(host)
@@ -327,7 +337,7 @@ def sync
   hosts.map { |host|
     done = Channel(Nil).new
     spawn do
-      sync_host(host, hosts_files, common_files)
+      sync_host(host, hosts_files: hosts_files, common_files: common_files, mirror_to: mirror[host]?)
       done.send(nil)
     end
     done
@@ -336,7 +346,7 @@ def sync
   ok("sync finished\n")
 end
 
-def sync_host(host : String, hosts_files : Hash(String, Set(Path)), common_files : Set(Path))
+def sync_host(host : String, *, hosts_files : Hash(String, Set(Path)), common_files : Set(Path), mirror_to : Array({String, Path}) | Nil)
   host_files : Set(Path) = hosts_files[host]
   all_hosts_files : Set(Path) = hosts_files
     .flat_map { |_, files| files.to_a }
@@ -352,6 +362,15 @@ def sync_host(host : String, hosts_files : Hash(String, Set(Path)), common_files
   host_build_files = children_recursive(host_build_dir).to_set
   filtered_sync(host, host_build_dir, upload: host_build_files) # TODO: with --delete specifically for jekyll ?
 
+  unless mirror_to.nil?
+    mirror_to.each { |destination_host, dir|
+      mirror_host_files = host_files.select { |i| i.to_s.starts_with?(dir.to_s) }
+      mirror_host_build_files = host_build_files.select { |i| i.to_s.starts_with?(dir.to_s) }
+      filtered_sync(destination_host, HOSTS_DIR.join(host), upload: mirror_host_files.to_set)
+      filtered_sync(destination_host, BUILD_DIR.join(host), upload: mirror_host_build_files.to_set)
+    }
+  end
+
   ssh(host, ["sudo etckeeper commit sync 2>>/dev/null"])
 end
 
@@ -1066,23 +1085,25 @@ end
 
 def check_i2p_host(host : String)
   puts("checking i2p configuration at #{host}")
-  private_key = File.read_lines(HOSTS_DIR.join(host).join("etc/i2pd/tunnels.conf"))
-    .select { |i| i.starts_with?("keys =") }
-    .map { |i| i.split(" = ")[1] }[0]
   service_dir = Path["/var/lib/i2pd"]
+  private_keys = File.read_lines(HOSTS_DIR.join(host).join("etc/i2pd/tunnels.conf"))
+    .select { |i| i.starts_with?("keys =") }
+    .map { |i| i.split(" = ")[1] }
   check_manual_upload(host, owner: "i2pd", group: "i2pd", mode: 700, path: service_dir)
-  check_manual_upload(host, owner: "i2pd", group: "i2pd", mode: 440, path: service_dir.join(private_key))
+  private_keys.each { |i| check_manual_upload(host, owner: "i2pd", group: "i2pd", mode: 440, path: service_dir.join(i)) }
 end
 
 def check_tor_host(host : String)
   puts("checking tor configuration at #{host}")
-  service_dir = Path[File.read_lines(HOSTS_DIR.join(host).join("etc/tor/torrc"))
+  service_dirs = File.read_lines(HOSTS_DIR.join(host).join("etc/tor/torrc"))
     .select { |i| i.starts_with?("HiddenServiceDir") }
-    .map { |i| i.split(" ")[1] }[0]]
-  check_manual_upload(host, owner: "tor", group: "tor", mode: 700, path: service_dir)
-  check_manual_upload(host, owner: "tor", group: "tor", mode: 400, path: service_dir.join("hs_ed25519_secret_key"))
-  check_manual_upload(host, owner: "tor", group: "tor", mode: 400, path: service_dir.join("hs_ed25519_public_key"))
-  check_manual_upload(host, owner: "tor", group: "tor", mode: 400, path: service_dir.join("hostname"), data: service_dir.basename)
+    .map { |i| Path[i.split(" ")[1]] }
+  service_dirs.map { |i|
+    check_manual_upload(host, owner: "tor", group: "tor", mode: 700, path: i)
+    check_manual_upload(host, owner: "tor", group: "tor", mode: 400, path: i.join("hs_ed25519_secret_key"))
+    check_manual_upload(host, owner: "tor", group: "tor", mode: 400, path: i.join("hs_ed25519_public_key"))
+    check_manual_upload(host, owner: "tor", group: "tor", mode: 400, path: i.join("hostname"), data: i.basename)
+  }
 end
 
 def check_ssh_hosts(ps : Array(Tuple(Int64, String)))
@@ -1132,12 +1153,15 @@ def check_existing_ssh_connection(host : String, ps : Array(Tuple(Int64, String)
 end
 
 def check_manual_upload(host : String, *, owner : String, group : String, mode : UInt16, path : Path, data : String? = nil)
-  raise "#{path}: unexpected owner" unless ssh(host, ["sudo stat -c %U #{path}"]) == owner
-  raise "#{path}: unexpected group" unless ssh(host, ["sudo stat -c %G #{path}"]) == group
-  raise "#{path}: unexpected mode" unless ssh(host, ["sudo stat -c %a #{path}"]).to_i == mode
-  unless data.nil?
-    raise "#{path}: unexpected data" unless ssh(host, ["sudo cat #{path}"]).strip == data
-  end
+  commands = ["stat -c %U,%G,%a"]
+  commands += ["cat"] unless data.nil?
+  output = ssh(host, commands.map { |i| "sudo #{i} #{path}" }).strip.split('\n')
+  stat = output[0].split(',')
+
+  raise "#{path}: unexpected owner" unless stat[0] == owner
+  raise "#{path}: unexpected group" unless stat[1] == group
+  raise "#{path}: unexpected mode" unless stat[2].to_i == mode
+  raise "#{path}: unexpected data" unless data.nil? || output[1] == data
 end
 
 def sync_nostr(config, *, profiles : Bool, output_relays : Array(String))
diff --git a/_hosts/media.codonaft/etc/i2pd/tunnels.conf b/_hosts/media.codonaft/etc/i2pd/tunnels.conf
@@ -1,4 +1,4 @@
-[anon-website]
+[codonaft-media]
 type = http
 host = 127.0.0.1
 port = 80
diff --git a/_hosts/media.codonaft/etc/nginx/http.d/media.conf b/_hosts/media.codonaft/etc/nginx/http.d/media.conf
@@ -64,10 +64,6 @@ server {
   }
 
   location ~ \.(m3u8|mp4|ts|vtt|webp)$ {
-    if ($http_origin != $origin) {
-      return 404;
-    }
-
     add_header "Access-Control-Allow-Origin" $origin always;
 
     add_header Cache-Control "public, max-age=31536000, immutable";
@@ -136,10 +132,6 @@ server {
   ignore_invalid_headers on;
 
   location ~ \.(m3u8|mp4|ts|vtt|webp)$ {
-    if ($http_origin != $origin) {
-      return 404;
-    }
-
     add_header "Access-Control-Allow-Origin" $origin always;
 
     add_header Cache-Control "public, max-age=31536000, immutable";
diff --git a/_hosts/mirror.codonaft/etc/i2pd/tunnels.conf b/_hosts/mirror.codonaft/etc/i2pd/tunnels.conf
@@ -1,4 +1,4 @@
-[anon-website]
+[codonaft]
 type = http
 host = 127.0.0.1
 port = 80
@@ -8,3 +8,9 @@ keys = codonftbnpdkjwyflssto3iklawhuthbe37l6swigegqkyyfmiqa.b32.i2p.dat
 #inbound.quantity = 3
 #outbound.length = 2
 #outbound.quantity = 3
+
+[codonaft-media-mirror]
+type = http
+host = 127.0.0.1
+port = 80
+keys = codonfttnvztewul3qgathdxjr2fnv34udrqyc6quw3zptqojnea.b32.i2p.dat
diff --git a/_hosts/mirror.codonaft/etc/nginx/http.d/mirror.conf b/_hosts/mirror.codonaft/etc/nginx/http.d/mirror.conf
@@ -25,7 +25,6 @@ server {
 
 server {
   set $i2p_host "codonaft.i2p";
-
   listen 127.0.0.1:80;
   server_name codonftbnpdkjwyflssto3iklawhuthbe37l6swigegqkyyfmiqa.b32.i2p;
   root /var/www/$i2p_host;
@@ -57,3 +56,40 @@ server {
   index index.html;
   error_page 404 =200 /404.html;
 }
+
+server {
+  listen 127.0.0.1:80;
+  server_name codonafte3ygy3szn5qwsdje4vp6mwobvzd75jl4ytyrb262cpkaegid.onion;
+  root /var/www/media.codonaft.com;
+
+  location ~ \.(m3u8|mp4|ts|vtt|webp)$ {
+    set $origin http://codonaftbvv4j5k7nsrdivbdblycqrng5ls2qkng6lm77svepqjyxgid.onion;
+
+    add_header "Access-Control-Allow-Origin" $origin always;
+
+    add_header Cache-Control "public, max-age=31536000, immutable";
+    expires 1y;
+
+    limit_rate_after 5M;
+    limit_rate 5M;
+  }
+}
+
+server {
+  set $i2p_host "codonaft.i2p";
+  listen 127.0.0.1:80;
+  server_name codonfttnvztewul3qgathdxjr2fnv34udrqyc6quw3zptqojnea.b32.i2p;
+  root /var/www/media.codonaft.com;
+
+  location ~ \.(m3u8|mp4|ts|vtt|webp)$ {
+    set $origin http://$i2p_host;
+
+    add_header "Access-Control-Allow-Origin" $origin always;
+
+    add_header Cache-Control "public, max-age=31536000, immutable";
+    expires 1y;
+
+    limit_rate_after 5M;
+    limit_rate 5M;
+  }
+}
diff --git a/_hosts/mirror.codonaft/etc/tor/torrc b/_hosts/mirror.codonaft/etc/tor/torrc
@@ -4,3 +4,6 @@ ExcludeExitNodes {am}, {az}, {by}, {ee}, {ge}, {kg}, {kz}, {lt}, {ly}, {md}, {ru
 
 HiddenServiceDir /var/lib/tor/codonaftbvv4j5k7nsrdivbdblycqrng5ls2qkng6lm77svepqjyxgid.onion/
 HiddenServicePort 80 127.0.0.1:80
+
+HiddenServiceDir /var/lib/tor/codonafte3ygy3szn5qwsdje4vp6mwobvzd75jl4ytyrb262cpkaegid.onion/
+HiddenServicePort 80 127.0.0.1:80
diff --git a/_includes/head.html b/_includes/head.html
@@ -120,7 +120,7 @@
  const getSource = player => new URL(player.getAttribute('src'));
  const getSwarmId = player => {
   const source = getSource(player);
-  return '{{ site.theme_settings.p2p_player.swarmPrefix }}' + source.pathname;
+  return '{{ site.theme_settings.p2p_player.swarm_prefix }}' + source.pathname;
  };
 
  const loadStoryboard = player => {
diff --git a/_includes/p2p_player.html b/_includes/p2p_player.html
@@ -7,7 +7,8 @@
  {% assign video_id = page.permalink | split: '/' | last %}
  {% endif %}
 
- {% assign stream_url_prefix = site.theme_settings.p2p_player.media_urls[0] | append: '/' | append: video_id %}
+ {% assign media_urls = site.theme_settings.p2p_player.media_urls_overwrite | concat: site.theme_settings.p2p_player.media_urls %}
+ {% assign stream_url_prefix = media_urls[0] | append: '/' | append: video_id %}
  {% assign stream_url = stream_url_prefix | append: "/main.m3u8" %}
  {% assign thumbnail_url = stream_url_prefix | append: "/thumbnail.webp" %}
 {% endif %}

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-[anon-website]`
	`1`	`+[codonaft-media]`
`2`	`2`	`type = http`
`3`	`3`	`host = 127.0.0.1`
`4`	`4`	`port = 80`