diff --git a/README.md b/README.md index 04851682e..2ecf9b5e9 100644 --- a/README.md +++ b/README.md @@ -36,11 +36,26 @@ These configuration preferences indicate - what i18n languages you wish to support - what is your default "fallback" language for your content - what root level files/folders are excluded from localization, based on if their paths start with any of the excluded regexp substrings. (this is different from the jekyll `exclude: [ .gitignore ]` ; you should `exclude` files and directories in your repo you dont want in your built site at all, and `exclude_from_localization` files and directories you want to see in your built site, but not in your sublanguage sites.) -- whether to run language processing in parallel or serial. Set to `false` if building on Windows hosts, or if Polyglot collides with other Jekyll plugins. +- whether to run language processing in parallel or serial. Set to `false` if building on Windows hosts, or if Polyglot collides with other Jekyll plugins. If a plugin breaks only when you turn this on, try [`serial_default_lang`](#parallel-safe-plugins-serial_default_lang) before giving up on parallel builds. - your jekyll website production url. Make sure this value is set; Polyglot requires this to relative site urls correctly, and to make functioning language switchers. The optional `lang_from_path: true` option enables getting the page language from a filepath segment seperated by `/` or `.`, e.g `de/first-one.md`, or `_posts/zh_HK/use-second-segment.md` , if the lang frontmatter isn't defined. +#### Parallel-safe plugins (`serial_default_lang`) + +```yaml +parallel_localization: true +serial_default_lang: true +``` + +Default: `false`. Has no effect when `parallel_localization` is `false`. + +With `parallel_localization` on, polyglot forks one process per language and runs them at once. Some plugins aren't safe to run that way — typically ones that do expensive setup once per build and share state (a cache, a manifest, a generated directory) across the whole site. When every fork tries to do that setup at the same time, they race and the build fails. [`jekyll-assets`](https://github.com/envygeeks/jekyll-assets) is the common example. + +`serial_default_lang: true` makes polyglot build the default language first, on its own, before forking the rest. The expensive one-time setup happens once in that first pass, and the language forks inherit the finished state instead of each redoing it. Enable this if a plugin that works fine with `parallel_localization: false` breaks when you turn it on. + +The trade-off is small: the default language no longer runs alongside the others, so you get one fewer concurrent fork. + #### Netlify _redirects localization If you are deploying to Netlify and use a `_redirects` file, you can enable automatic localization of redirects: ```yaml diff --git a/lib/jekyll/polyglot/patches/jekyll/site.rb b/lib/jekyll/polyglot/patches/jekyll/site.rb index 06aee3a41..c242bf4e0 100644 --- a/lib/jekyll/polyglot/patches/jekyll/site.rb +++ b/lib/jekyll/polyglot/patches/jekyll/site.rb @@ -3,13 +3,20 @@ include Process module Jekyll class Site - attr_reader :default_lang, :languages, :exclude_from_localization, :lang_vars, :lang_from_path, :fallback_canonical_to_default_lang + attr_reader :default_lang, :languages, :exclude_from_localization, :lang_vars, :lang_from_path, :fallback_canonical_to_default_lang, :serial_default_lang attr_accessor :file_langs, :active_lang def prepare @file_langs = {} fetch_languages @parallel_localization = config.fetch('parallel_localization', true) + # When true (and parallel_localization is also true), the default + # language is processed synchronously in the parent before any forks + # are spawned for the other languages. This makes parallel builds + # safe for plugins that do expensive one-time setup and share state + # across the site (e.g. jekyll-assets), which otherwise race when + # every fork runs that setup at once. See README for details. + @serial_default_lang = config.fetch('serial_default_lang', false) @lang_from_path = config.fetch('lang_from_path', false) @fallback_canonical_to_default_lang = config.fetch('fallback_canonical_to_default_lang', false) @exclude_from_localization = config.fetch('exclude_from_localization', []).map do |e| @@ -34,14 +41,23 @@ def process prepare all_langs = ([@default_lang] + @languages).uniq if @parallel_localization + if @serial_default_lang + # Run the default language in the parent first to prime shared + # state (e.g. jekyll-assets' Sprockets cache) before forking + # for the remaining languages. + process_language @default_lang + langs_to_fork = @languages - [@default_lang] + else + langs_to_fork = all_langs + end nproc = Etc.nprocessors pids = {} begin - all_langs.each do |lang| + langs_to_fork.each do |lang| pids[lang] = fork do process_language lang end - while pids.length >= (lang == all_langs[-1] ? 1 : nproc) + while pids.length >= (lang == langs_to_fork[-1] ? 1 : nproc) sleep 0.1 pids.map do |pid_lang, pid| next unless waitpid pid, Process::WNOHANG @@ -52,7 +68,7 @@ def process end end rescue Interrupt - all_langs.each do |lang| + langs_to_fork.each do |lang| next unless pids.key? lang puts "Killing #{pids[lang]} : #{lang}" diff --git a/spec/jekyll/polyglot/patches/jekyll/site_spec.rb b/spec/jekyll/polyglot/patches/jekyll/site_spec.rb index fcddd7a7c..134e919df 100644 --- a/spec/jekyll/polyglot/patches/jekyll/site_spec.rb +++ b/spec/jekyll/polyglot/patches/jekyll/site_spec.rb @@ -413,6 +413,39 @@ expect(forks).to eq((@langs + [@default_lang]).uniq.length) end + it 'runs the default language in the parent when serial_default_lang is set' do + site_with_serial = Site.new( + Jekyll.configuration( + 'languages' => @langs, + 'default_lang' => @default_lang, + 'exclude_from_localization' => @exclude_from_localization, + 'serial_default_lang' => true, + 'source' => File.expand_path('fixtures', __dir__), + 'url' => 'https://test.github.io' + ) + ) + + parent_lang_calls = [] + allow(Etc).to receive(:nprocessors).and_return(99) + # Real but immediate forks so polyglot's waitpid loop terminates. + allow(site_with_serial).to receive(:fork) { fork { exit 0 } } + # Track parent-process calls to process_language. Calls inside fork + # blocks happen in child processes (with copied state) and don't + # show up in this list. + allow(site_with_serial).to receive(:process_language) do |lang| + parent_lang_calls << lang + end + + site_with_serial.process + + expect(site_with_serial.serial_default_lang).to be true + # The default language is the only one processed in the parent — + # all others are dispatched to fork blocks. This is the whole point + # of the option: prime shared on-disk state once in the parent, + # then let forks inherit it via fork(2) copy-on-write. + expect(parent_lang_calls).to eq([@default_lang]) + end + describe 'assignPageRedirects' do before do @collection = Jekyll::Collection.new(@site, 'test')