From 3c80925bdd5d3f568b3871910fbd26fdb9539fc5 Mon Sep 17 00:00:00 2001 From: Misha Nasledov Date: Sun, 4 Oct 2026 11:18:56 -0700 Subject: [PATCH 01/17] osc: sample zones, and files that arrive late A file of the form "a.wav@48 b.wav@55" is a multisample: the trigger picks the recording nearest the voice's pitch and plays it from its own note. A file that was missing when first asked for is read again once the host calls thUtil::dataFilesChanged(), which the browser's tw_sample now does, so a pack downloaded while its instrument is loaded is heard without a reload. --- docs/NODES.md | 2 +- libthink/thUtil.cpp | 13 +++++ libthink/thUtil.h | 6 +++ plugins/osc/sample.cpp | 36 ++++++++++--- plugins/osc/sampleslot.h | 97 +++++++++++++++++++++++++++++++++- scripts/statecheck.cpp | 110 +++++++++++++++++++++++++++++++++++++++ wasm/web/thinkweb.cpp | 9 +++- 7 files changed, 262 insertions(+), 11 deletions(-) diff --git a/docs/NODES.md b/docs/NODES.md index 9829a9f3..90378ee9 100644 --- a/docs/NODES.md +++ b/docs/NODES.md @@ -1064,7 +1064,7 @@ Sample Player (a wav at a voice's pitch) | Arg | Dir | Description | Default | Range | Units | Values | |---|---|---|---|---|---|---| -| `file` | in | The wav to play, found under samples/ on THINK_DSP_PATH | | | | | +| `file` | in | The wav to play, found under samples/ on THINK_DSP_PATH; or zones, `a.wav@48 b.wav@55', the one nearest the note played from its own | | | | | | `file2` | in | Middle layer wav; an empty slot uses file | | | | | | `file3` | in | Upper layer wav; an empty slot uses file2 or file | | | | | | `freq` | in | The note to play it at | | | Hz | | diff --git a/libthink/thUtil.cpp b/libthink/thUtil.cpp index b1b48b34..39b44988 100644 --- a/libthink/thUtil.cpp +++ b/libthink/thUtil.cpp @@ -25,6 +25,7 @@ #include /* MoveFileExA -- see replaceFile */ #endif +#include #include #include #include @@ -561,3 +562,15 @@ bool thUtil::findEmbeddedFile (const string &name, const unsigned char *&data, return true; } + +static std::atomic dataGeneration{0}; + +void thUtil::dataFilesChanged () +{ + dataGeneration++; +} + +unsigned thUtil::dataFilesGeneration () +{ + return dataGeneration.load(); +} diff --git a/libthink/thUtil.h b/libthink/thUtil.h index 94a3c4d5..e81561de 100644 --- a/libthink/thUtil.h +++ b/libthink/thUtil.h @@ -136,6 +136,12 @@ class THINK_API thUtil { size_t size); static bool findEmbeddedFile (const string &name, const unsigned char *&data, size_t &size); + + /* A count a host bumps after putting new files where findDataFile + * looks -- a sample pack downloaded while the synth runs. A reader that + * cached "not found" retries when the count has moved on. */ + static void dataFilesChanged (); + static unsigned dataFilesGeneration (); }; #endif /* TH_UTIL_H */ diff --git a/plugins/osc/sample.cpp b/plugins/osc/sample.cpp index 6511d5ea..576455b7 100644 --- a/plugins/osc/sample.cpp +++ b/plugins/osc/sample.cpp @@ -70,6 +70,13 @@ * Omitted layers fall back to the one below them. With `alternate = 1', * a counter shared by this synth's voices chooses 1, 2, 3, 1 instead. * + * A `file' MAY BE A SET OF ZONES: "vln_G3.wav@55 vln_D4.wav@62 ...", each + * recording with the MIDI note it was made at. The trigger edge picks the + * one nearest the voice's pitch and plays it from that note, so `root' is + * not read; every layer may be zoned the same way. That is a multisample + * -- an instrument recorded every few semitones, so no note is pitched + * far enough to sound like a sampler. + * * THE FILE IS READ ONCE PER SYNTH, not once per voice -- sixteen voices * of a kit share one copy of the kick. See osc/sampleslot.h, which also * says why the read happens on the first window that asks for it rather @@ -137,7 +144,8 @@ int module_init (thPlugin *plugin) node per drum rather than one node with the name swept. */ plugin->setArgDesc(args[IN_FILE], "The wav to play, found under samples/ on " - "THINK_DSP_PATH"); + "THINK_DSP_PATH; or zones, `a.wav@48 b.wav@55', the " + "one nearest the note played from its own"); args[IN_FILE2] = plugin->regArg("file2", thPlugin::ARG_IN); plugin->setArgDesc(args[IN_FILE2], "Middle layer wav; an empty slot uses file"); @@ -196,7 +204,8 @@ int module_init (thPlugin *plugin) "can be the sample's own length"); plugin->setArgRange(args[OUT_PLAY], 0, 1); - /* [0] the playhead, [1] the last trigger, [2] the chosen layer. */ + /* [0] the playhead, [1] the last trigger, [2] the chosen layer, [3] the + note a zone was chosen for. */ args[INOUT_STATE] = plugin->regArg("state", thPlugin::ARG_STATE); /* Registered last, so every arg above keeps the index it had. */ @@ -219,7 +228,7 @@ int module_callback (thNode *node, thSynthTree *mod, unsigned int windowlen, thArg *out_arg, *out_play; thArg *inout_state; unsigned int i; - float at, lastTrigger; + float at, lastTrigger, note; unsigned layer; in_file = mod->getArg(node, args[IN_FILE]); @@ -247,7 +256,8 @@ int module_callback (thNode *node, thSynthTree *mod, unsigned int windowlen, at = (*inout_state)[0]; lastTrigger = (*inout_state)[1]; layer = (unsigned)thClampArg((*inout_state)[2], 0, 2); - state = inout_state->allocate(3); + note = (*inout_state)[3]; + state = inout_state->allocate(4); out_arg = mod->getArg(node, args[OUT_ARG]); out = out_arg->allocate(windowlen); @@ -260,6 +270,7 @@ int module_callback (thNode *node, thSynthTree *mod, unsigned int windowlen, const thSampleData *smp = NULL; size_t len = 0; unsigned loadedLayer = 3; + double zoneRoot = 0; for (i = 0; i < windowlen; i++) { @@ -269,6 +280,12 @@ int module_callback (thNode *node, thSynthTree *mod, unsigned int windowlen, if (edge) { + /* The pitch the zone is chosen for, kept for the windows + after this one. */ + note = (float)(69 + 12 * log2(thBoundFreq((*in_freq)[i], samples) / + 440.0)); + loadedLayer = 3; + if ((*in_alternate)[i] > 0) layer = thSampleNextLayer(node->plugin(), in_file->text()); else @@ -285,12 +302,16 @@ int module_callback (thNode *node, thSynthTree *mod, unsigned int windowlen, if (loadedLayer != layer) { - const std::string &name = + const std::string &text = (layer == 2 && !in_file3->text().empty()) ? in_file3->text() : (layer >= 1 && !in_file2->text().empty()) ? in_file2->text() : in_file->text(); + const thSampleZone *zone = thSampleIsZoned(text) + ? thSampleZoneFor(node->plugin(), text, note) : NULL; - smp = thSampleGet(node->plugin(), name, samples); + zoneRoot = zone != NULL ? zone->root : 0; + smp = thSampleGet(node->plugin(), zone != NULL ? zone->file : text, + samples); len = (smp != NULL) ? smp->frames.size() : 0; loadedLayer = layer; } @@ -306,7 +327,7 @@ int module_callback (thNode *node, thSynthTree *mod, unsigned int windowlen, } const float loop = thClampArg((*in_loop)[i], 0, (float)len); - const float root = (*in_root)[i]; + const float root = zoneRoot > 0 ? (float)zoneRoot : (*in_root)[i]; /* Both bounded, so a root of 0 -- which is what an unwired `root' reads as before its default is applied -- is middle C @@ -373,6 +394,7 @@ int module_callback (thNode *node, thSynthTree *mod, unsigned int windowlen, state[0] = at; state[1] = lastTrigger; state[2] = (float)layer; + state[3] = note; return 0; } diff --git a/plugins/osc/sampleslot.h b/plugins/osc/sampleslot.h index 9c885678..ffced275 100644 --- a/plugins/osc/sampleslot.h +++ b/plugins/osc/sampleslot.h @@ -59,7 +59,14 @@ * not there would stat() once per window per voice forever and print a * line each time. The entry is remembered with no frames in it, which is * the same thing the reader returns for an empty file, and both play as - * silence. + * silence -- until the host says new files have arrived + * (thUtil::dataFilesChanged), when it is read once more. That is a + * sample pack downloaded while an instrument naming it is loaded. + * + * ZONES. A `file' of the form "a.wav@48 b.wav@55 c.wav@62" is a set of + * recordings with the MIDI note each was made at; thSampleZoneFor picks + * the one nearest a note, the lower on a tie. The list is parsed once per + * synth and kept beside the wavs. * * MONO, AT THE SYNTH'S RATE. A stereo file is summed -- a voice has one * output and a graph that wants two instantiates two nodes, the way @@ -71,6 +78,7 @@ #include #include +#include #include #include #include @@ -86,6 +94,14 @@ struct thSampleData { std::vector frames; + unsigned generation = 0; /* thUtil::dataFilesGeneration at the read */ +}; + +/* One recording of a zoned `file', and the frequency it was made at. */ +struct thSampleZone +{ + std::string file; + double root; }; struct thSampleSlot @@ -95,6 +111,8 @@ struct thSampleSlot /* The next velocity layer for each primary file. All voices in a synth reach the same slot, so alternate hits do not restart at layer one. */ std::map alternates; + /* Zoned `file' texts, parsed. Owner's thread only. */ + std::map> zones; }; static thSampleSlot thSampleSlots[THINK_SAMPLE_SLOTS]; @@ -519,10 +537,14 @@ thSampleGet (const thPlugin *plugin, const std::string &name, unsigned rate) std::map::const_iterator i = slot->table->find(name); - if (i != slot->table->end()) + const unsigned generation = thUtil::dataFilesGeneration(); + + if (i != slot->table->end() && + (!i->second.frames.empty() || i->second.generation == generation)) return &i->second; thSampleData &entry = (*slot->table)[name]; + entry.generation = generation; if (name.empty()) { @@ -564,4 +586,75 @@ thSampleGet (const thPlugin *plugin, const std::string &name, unsigned rate) return &entry; } +static inline bool thSampleIsZoned (const std::string &text) +{ + return text.find('@') != std::string::npos; +} + +/* The zone of a zoned `file' nearest `note', or NULL for a list with no + zones in it. */ +static inline const thSampleZone * +thSampleZoneFor (const thPlugin *plugin, const std::string &text, + double note) +{ + thSampleSlot *slot = thSampleSlotFor(plugin); + + if (slot == NULL) + return NULL; + + std::map>::iterator i = + slot->zones.find(text); + + if (i == slot->zones.end()) + { + std::vector &list = slot->zones[text]; + size_t at = 0; + + while (at < text.size()) + { + const size_t end = text.find_first_of(" \t\n,", at); + const std::string word = + text.substr(at, end == std::string::npos ? std::string::npos + : end - at); + const size_t mark = word.rfind('@'); + + if (mark != std::string::npos && mark > 0) + { + const double midi = atof(word.c_str() + mark + 1); + list.push_back({word.substr(0, mark), + 440.0 * pow(2.0, (midi - 69) / 12.0)}); + } + else if (!word.empty()) + { + fprintf(stderr, "osc::sample: zone '%s' has no @note\n", + word.c_str()); + } + + if (end == std::string::npos) + break; + at = end + 1; + } + + i = slot->zones.find(text); + } + + const thSampleZone *best = NULL; + double bestDistance = 0; + const double freq = 440.0 * pow(2.0, (note - 69) / 12.0); + + for (const thSampleZone &zone : i->second) + { + const double distance = fabs(log2(freq / zone.root)); + + if (best == NULL || distance < bestDistance - 1e-9 || + (fabs(distance - bestDistance) <= 1e-9 && zone.root < best->root)) + { + best = &zone; + bestDistance = distance; + } + } + + return best; +} + #endif /* THINK_SAMPLESLOT_H */ diff --git a/scripts/statecheck.cpp b/scripts/statecheck.cpp index 257f0398..aa72a8c1 100644 --- a/scripts/statecheck.cpp +++ b/scripts/statecheck.cpp @@ -4652,6 +4652,115 @@ static void checkSample (const string &pluginPath) std::filesystem::remove_all(dir, ec); } +/* Zones, and a file that arrives late. Each wav is one constant level, so + which zone played is the level that came out. */ +static vector zoneGraph (const char *file, float freq) +{ + vector spec = sampleGraph(file, freq, 261.63f, 0, 0); + + spec[0].values.push_back(Value{ "trigger", 1 }); + + return spec; +} + +static float zoneLevel (const string &pluginPath, const char *file, int note, + string &why) +{ + vector out; + const float freq = (float)(440.0 * pow(2.0, (note - 69) / 12.0)); + + if (!render1(pluginPath, zoneGraph(file, freq), "smp", "out", 256, 512, + out, why)) + return -1; + + return out[300]; +} + +static void checkSampleZones (const string &pluginPath) +{ + const string dir = thUtil::tempFile("statecheck-zones-"); + std::error_code ec; + + std::filesystem::remove(dir, ec); + std::filesystem::create_directories(dir + "/samples", ec); + + const vector low(4000, 0.25f), high(4000, 0.5f); + + if (ec || !writeWav(dir + "/samples/low.wav", low, TH_DEFAULT_SAMPLES) || + !writeWav(dir + "/samples/high.wav", high, TH_DEFAULT_SAMPLES)) + { + fail("osc::sample zones: could not write the scratch wavs", ""); + return; + } + +#ifdef _WIN32 + _putenv_s("THINK_DSP_PATH", dir.c_str()); +#else + setenv("THINK_DSP_PATH", dir.c_str(), 1); +#endif + + const char *zones = "low.wav@60 high.wav@72"; + const struct { int note; float level; const char *what; } cases[] = { + { 62, 0.25f, "a note near the lower zone plays it" }, + { 70, 0.5f, "a note near the upper zone plays it" }, + { 66, 0.25f, "a note between two zones plays the lower" }, + { 30, 0.25f, "a note under every zone plays the lowest" }, + }; + + for (const auto &c : cases) + { + string why; + const float got = zoneLevel(pluginPath, zones, c.note, why); + + okOrFail(fabsf(got - c.level) < 1e-3f, + string("osc::sample zones: ") + c.what, + why.empty() ? "note " + num(c.note) + " came out at " + + num(got) : why); + } + + /* One synth throughout: a file missing on its first read is cached as + silence, and read again only once the host says files have changed. */ + { + thSynth synth(pluginPath, 256, TH_DEFAULT_SAMPLES); + thSynthTree tree("statecheck", &synth); + string why; + + if (!buildGraph(synth, tree, zoneGraph("late.wav", 261.63f), why)) + { + fail("osc::sample: a late file renders", why); + } + else + { + thArg *out = tree.findNode("smp")->getArg("out"); + auto window = [&](void) { + tree.setActiveNodes(); + tree.process(256); + return (*out)[100]; + }; + + const float before = window(); + const bool wrote = writeWav(dir + "/samples/late.wav", high, + TH_DEFAULT_SAMPLES); + const float cached = window(); + + thUtil::dataFilesChanged(); + + const float after = window(); + + okOrFail(wrote && before == 0 && cached == 0, + "osc::sample: a missing file stays silent until the " + "host says files changed", + "before " + num(before) + ", after writing " + + num(cached)); + okOrFail(fabsf(after - 0.5f) < 1e-3f, + "osc::sample: once it does, the file plays", + "came out at " + num(after)); + } + } + + std::filesystem::remove_all(dir, ec); +} + /* ---- misc::drift -------------------------------------------------------- */ static vector driftGraph (float rate, float depth, float center, @@ -7256,6 +7365,7 @@ int main (int argc, char **argv) checkFmop(pluginPath); checkSimple(pluginPath); checkSample(pluginPath); + checkSampleZones(pluginPath); checkGrain(pluginPath); checkDrift(pluginPath); checkPad(pluginPath); diff --git a/wasm/web/thinkweb.cpp b/wasm/web/thinkweb.cpp index 99ae8a12..2d4986f8 100644 --- a/wasm/web/thinkweb.cpp +++ b/wasm/web/thinkweb.cpp @@ -87,6 +87,7 @@ #include "thcScheduler.h" #include "thDynLib.h" +#include "thUtil.h" #include "cairo2d.h" #include "cairomm/context.h" @@ -2633,8 +2634,14 @@ EMSCRIPTEN_KEEPALIVE int tw_sample (const char *name, const char *bytes, const bool whole = fwrite(bytes, 1, (size_t)len, f) == (size_t)len; + const bool ok = fclose(f) == 0 && whole; - return (fclose(f) == 0 && whole) ? 1 : 0; + /* A pack downloaded while an instrument naming it is loaded: what + osc::sample cached as missing is read again. */ + if (ok) + thUtil::dataFilesChanged(); + + return ok ? 1 : 0; } /* A .gen, as text. Nonzero if it parsed and built; tw_error_count and From e8e5999e9edb63798db509d78d0667f8641f4381 Mon Sep 17 00:00:00 2001 From: Misha Nasledov Date: Sun, 4 Oct 2026 11:25:44 -0700 Subject: [PATCH 02/17] util: find data files in this user's own data directory too thUtil::userDataDir() is $XDG_DATA_HOME/thinksynth (~/.local/share), ~/Library/Application Support on macOS, %LOCALAPPDATA% on Windows, and findDataFile tries it last, so a downloaded sample pack is found and nothing shipped can be shadowed by one. --- libthink/thUtil.cpp | 32 ++++++++++++++++++++++++++++++++ libthink/thUtil.h | 8 ++++++++ 2 files changed, 40 insertions(+) diff --git a/libthink/thUtil.cpp b/libthink/thUtil.cpp index 39b44988..4d3f77ba 100644 --- a/libthink/thUtil.cpp +++ b/libthink/thUtil.cpp @@ -241,6 +241,31 @@ static string absolutePath (const fs::path &p) return ec ? p.string() : abs.string(); } +string thUtil::userDataDir (void) +{ +#if defined(_WIN32) + const char *base = getenv("LOCALAPPDATA"); + + return (base != NULL && *base != 0) + ? (fs::path(base) / PACKAGE_NAME).string() : ""; +#else + const char *home = getenv("HOME"); +#if defined(__APPLE__) + return (home != NULL && *home != 0) + ? (fs::path(home) / "Library" / "Application Support" / + PACKAGE_NAME).string() : ""; +#else + const char *xdg = getenv("XDG_DATA_HOME"); + + if (xdg != NULL && *xdg == '/') + return (fs::path(xdg) / PACKAGE_NAME).string(); + + return (home != NULL && *home != 0) + ? (fs::path(home) / ".local" / "share" / PACKAGE_NAME).string() : ""; +#endif +#endif +} + string thUtil::findDataFile (const string &name, const string &subdir, const char *envVar, const string &fallback) { @@ -294,6 +319,13 @@ string thUtil::findDataFile (const string &name, const string &subdir, if (!fallback.empty()) tries.push_back(fs::path(fallback) / name); + /* Last: what this user downloaded, so nothing shipped can be shadowed + by it. */ + const string user = userDataDir(); + + if (!user.empty()) + tries.push_back(fs::path(user) / subdir / name); + for (size_t i = 0; i < tries.size(); i++) if (fs::exists(tries[i], ec)) return absolutePath(tries[i]); diff --git a/libthink/thUtil.h b/libthink/thUtil.h index e81561de..279cbd8a 100644 --- a/libthink/thUtil.h +++ b/libthink/thUtil.h @@ -38,6 +38,14 @@ class THINK_API thUtil { determined. Linux, macOS and Windows each need a different call. */ static string exeDir (void); + /* This user's own data directory for thinksynth, whether or not it + * exists: $XDG_DATA_HOME/thinksynth (~/.local/share/thinksynth) on + * Linux, ~/Library/Application Support/thinksynth on macOS, + * %LOCALAPPDATA%\thinksynth on Windows. Where a downloaded sample pack + * goes, because an install directory is usually not writable; empty + * if no home directory can be found. */ + static string userDataDir (void); + /* Find a data file that something referred to by bare name. * * A .patch says `dsp ts1.dsp' and a .dsp says nothing about where it From e17f3c4aca714a83c9575870040f4711e98661f1 Mon Sep 17 00:00:00 2001 From: Misha Nasledov Date: Sun, 4 Oct 2026 11:25:44 -0700 Subject: [PATCH 03/17] scripts: packs.py, sample packs built from their CC0 sources Lists each source at a pinned commit, downloads, trims, converts to mono 16-bit at 44.1 kHz with one gain per pack, and names files by MIDI note. `build' writes a directory the site serves, `install' writes the user's data directory, `zones' prints a pack's file lines. Violins, cellos, horn, trumpet and flute from VSCO 2 CE, an upright piano from the same, a drum kit from Big Rusty Drums. --- scripts/packs.py | 294 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 294 insertions(+) create mode 100755 scripts/packs.py diff --git a/scripts/packs.py b/scripts/packs.py new file mode 100755 index 00000000..a1e6f33c --- /dev/null +++ b/scripts/packs.py @@ -0,0 +1,294 @@ +#!/usr/bin/env python3 +"""Sample packs: recorded instruments, fetched from where they are published. + +The recordings are not in this repository. Every pack is public domain +(CC0) and is built here from its source, pinned to a commit: listed with +the GitHub API, downloaded, trimmed, turned to mono 16-bit at 44.1 kHz and +levelled, and named by MIDI note -- the sources disagree about which +octave "C3" is, and a number cannot. + + packs.py build OUTDIR [--cache DIR] [PACK...] + Writes OUTDIR//*.wav, OUTDIR//pack.json and + OUTDIR/index.json. The deploy runs this into the site's packs/, + which the page downloads from when asked. + + packs.py install [PACK...] + Builds into this user's data directory (thUtil::userDataDir), under + dsp/samples/, where the desktop app looks. + + packs.py zones PACK + Prints the `file' lines the pack's .dsp uses. + +Needs ffmpeg (for FLAC and 24-bit sources). GITHUB_TOKEN, if set, is used +for the directory listings. +""" + +import argparse +import array +import json +import os +import re +import subprocess +import sys +import tempfile +import urllib.parse +import urllib.request +import wave + +VSCO = ("sgossner/VSCO-2-CE", "440300901dfe9275fd84e0b7763af1f8443ae62e", + "VSCO 2 Community Edition, Versilian Studios / Sam Gossner") +RUSTY = ("sfzinstruments/karoryfer.big-rusty-drums", + "f07ce00df34a46b6b08375be56fe116cf15782bc", + "Big Rusty Drums, Karoryfer Samples") + +RATE = 44100 +PEAK = 0.89 + +NOTE = r"(?P[A-G](?:#|b)?-?\d)" + +# A pack: where its files are, how a name gives the note and the velocity +# layer, which layers to keep (as layer 1, 2, 3), how long a file is kept, +# and whether the end is a sustain the .dsp loops. +# +# `octave' is what to add to a name's octave for MIDI: VSCO's orchestral +# folders call middle C "C3", its Upright Nr1 calls it "C4". Each was +# checked against the source's own SFZ `pitch_keycenter'. +PACKS = { + "violins": dict( + title="Violin section", source=VSCO, + dirs=["Strings/Violin Section/susVib"], + match=r"VlnEns_susVib_" + NOTE + r"_v(?P\d)\.wav", + octave=1, layers=["1", "2"], seconds=3.0, sustain=True), + "cellos": dict( + title="Cello section", source=VSCO, + dirs=["Strings/Cello Section/susvib"], + match=r"susvib_" + NOTE + r"_v(?P\d)_1\.wav", + octave=1, layers=["1", "3"], seconds=3.0, sustain=True), + "horn": dict( + title="French horn", source=VSCO, + dirs=["Brass/F Horn/sus"], + match=r"MOHorn_sus_" + NOTE + r"_v(?P\d)_1\.wav", + octave=1, layers=["2", "3", "4"], seconds=2.5, sustain=True), + "trumpet": dict( + title="Trumpet", source=VSCO, + dirs=["Brass/Trumpet/sus"], + match=r"Sum_SHTrumpet_sus_" + NOTE + r"_v(?P\d)_rr1\.wav", + octave=1, layers=["1", "3"], seconds=2.5, sustain=True), + "flute": dict( + title="Flute", source=VSCO, + dirs=["Woodwinds/Flute/susNV"], + match=r"LDFlute_susNV_" + NOTE + r"_v(?P\d)_1\.wav", + octave=1, layers=["1", "3"], seconds=2.5, sustain=True), + "upright": dict( + title="Upright piano", source=VSCO, + dirs=["Keys/Upright Nr1"], + match=r"UR1_" + NOTE + r"_(?Ppp|mf|f)_RR1\.wav", + octave=0, layers=["pp", "mf", "f"], seconds=5.0, sustain=False), +} + +# The kit: each drum's folder, the General MIDI note it answers, how long +# it rings, and the velocity layers to keep, low to high. +DRUMS = [ + ("kick_24/kick/oh", "k", 36, 1.0, ["4", "9", "14"]), + ("snare_14/center/oh", "sn_center", 38, 1.2, ["3", "6", "10"]), + ("tom_22/center/oh", "t22", 41, 1.8, ["2", "4", "7"]), + ("hihat_14/cl/oh", "ht_cl", 42, 0.6, ["2", "4", "6"]), + ("tom_18/center/oh", "t18", 43, 1.6, None), + ("tom_15/center/oh", "t15", 45, 1.5, None), + ("hihat_14/open/oh", "ht_open", 46, 2.5, ["2", "4", "6"]), + ("tom_14/center/oh", "t14", 48, 1.4, ["2", "4", "6"]), + ("crash_17/cr/oh", "cr", 49, 4.0, ["2", "4", "5"]), + ("ride_22/rd/oh", "rd", 51, 3.0, ["3", "6", "10"]), +] + +PACKS["drums"] = dict(title="Drum kit", source=RUSTY, drums=DRUMS, + sustain=False) + + +def midi(name, octave): + m = re.fullmatch(r"([A-G])(#|b)?(-?\d)", name) + pc = {"C": 0, "D": 2, "E": 4, "F": 5, "G": 7, "A": 9, "B": 11}[m.group(1)] + pc += {"#": 1, "b": -1, None: 0}[m.group(2)] + return 12 * (int(m.group(3)) + 1 + octave) + pc + + +def listing(repo, sha, path, cache): + """The file names in a directory at a commit, which never change, so + they are kept beside the downloads: the API allows sixty requests an + hour without a token.""" + local = os.path.join(cache, repo.replace("/", "_"), sha[:12], + path, ".listing.json") + if os.path.exists(local): + with open(local) as f: + return json.load(f) + url = (f"https://api.github.com/repos/{repo}/contents/" + f"{urllib.parse.quote(path)}?ref={sha}") + req = urllib.request.Request(url) + token = os.environ.get("GITHUB_TOKEN") + if token: + req.add_header("Authorization", f"Bearer {token}") + with urllib.request.urlopen(req) as r: + names = [e["name"] for e in json.load(r) if e["type"] == "file"] + os.makedirs(os.path.dirname(local), exist_ok=True) + with open(local, "w") as f: + json.dump(names, f) + return names + + +def fetch(repo, sha, path, cache): + local = os.path.join(cache, repo.replace("/", "_"), sha[:12], path) + if not os.path.exists(local): + os.makedirs(os.path.dirname(local), exist_ok=True) + url = (f"https://raw.githubusercontent.com/{repo}/{sha}/" + f"{urllib.parse.quote(path)}") + with urllib.request.urlopen(url) as r, open(local + ".part", "wb") as f: + f.write(r.read()) + os.replace(local + ".part", local) + return local + + +def decode(src, seconds, sustain): + """The file as mono 16-bit samples at RATE, its leading silence cut.""" + filt = ("silenceremove=start_periods=1:start_threshold=-60dB," + f"atrim=0:{seconds}") + if not sustain: + filt += f",afade=t=out:st={max(seconds - 0.08, 0)}:d=0.08" + out = subprocess.run( + ["ffmpeg", "-v", "error", "-i", src, "-af", filt, "-ac", "1", + "-ar", str(RATE), "-f", "s16le", "-"], + check=True, capture_output=True).stdout + return array.array("h", out) + + +def plan(pack, cache): + """[(source path, output name, MIDI note, layer 1-3)] for a pack.""" + spec = PACKS[pack] + repo, sha, _ = spec["source"] + jobs = [] + if "drums" in spec: + for folder, prefix, note, seconds, layers in spec["drums"]: + names = listing(repo, sha, f"Samples/{folder}", cache) + have = sorted({re.fullmatch(prefix + r"_vl(\d+)_rr1\.flac", n) + .group(1) for n in names + if re.fullmatch(prefix + r"_vl(\d+)_rr1\.flac", n)}, + key=int) + if layers is None: + n = len(have) + layers = [have[max(0, round(n / 3) - 1)], + have[max(0, round(2 * n / 3) - 1)], have[-1]] + for k, vl in enumerate(layers, 1): + if vl not in have: + sys.exit(f"{folder}: no layer vl{vl}; it has {have}") + jobs.append((f"Samples/{folder}/{prefix}_vl{vl}_rr1.flac", + f"{pack}/{prefix}_{note}_{k}.wav", note, k, + seconds)) + return jobs + for d in spec["dirs"]: + for n in sorted(listing(repo, sha, d, cache)): + m = re.fullmatch(spec["match"], n) + if not m or m.group("layer") not in spec["layers"]: + continue + note = midi(m.group("note"), spec["octave"]) + k = spec["layers"].index(m.group("layer")) + 1 + jobs.append((f"{d}/{n}", f"{pack}/{pack}_{note}_{k}.wav", note, + k, spec["seconds"])) + return jobs + + +def zones(jobs, layer): + """A `file' line: each note's file at this layer, or at the nearest + layer the source has for that note -- under it first.""" + have = {} + for _, out, note, k, _ in jobs: + have.setdefault(note, {})[k] = out + picks = [] + for note in sorted(have): + ks = sorted(have[note], key=lambda k: (k > layer, abs(k - layer))) + picks.append(f"{have[note][ks[0]]}@{note}") + return " ".join(picks) + + +def build(pack, outdir, cache): + spec = PACKS[pack] + repo, sha, credit = spec["source"] + jobs = plan(pack, cache) + decoded = [] + for path, out, note, k, seconds in jobs: + decoded.append((out, decode(fetch(repo, sha, path, cache), seconds, + spec["sustain"]))) + # One gain for the whole pack, so the balance across the range and + # between layers is the recording's. + peak = max(max(abs(min(a)), max(a)) for _, a in decoded if len(a)) or 1 + gain = PEAK * 32767 / peak + files = [] + for out, a in decoded: + scaled = array.array("h", (max(-32768, min(32767, round(x * gain))) + for x in a)) + path = os.path.join(outdir, out) + os.makedirs(os.path.dirname(path), exist_ok=True) + with wave.open(path, "wb") as w: + w.setnchannels(1) + w.setsampwidth(2) + w.setframerate(RATE) + w.writeframes(scaled.tobytes()) + files.append({"name": out, "bytes": os.path.getsize(path)}) + manifest = { + "id": pack, "title": spec["title"], "license": "CC0-1.0", + "credit": credit, "source": f"https://github.com/{repo}/tree/{sha}", + "bytes": sum(f["bytes"] for f in files), "files": files, + } + with open(os.path.join(outdir, pack, "pack.json"), "w") as f: + json.dump(manifest, f, indent=1) + return manifest + + +def user_samples(): + if sys.platform == "win32": + base = os.path.join(os.environ["LOCALAPPDATA"], "thinksynth") + elif sys.platform == "darwin": + base = os.path.expanduser("~/Library/Application Support/thinksynth") + else: + xdg = os.environ.get("XDG_DATA_HOME", "") + base = os.path.join(xdg if xdg.startswith("/") + else os.path.expanduser("~/.local/share"), + "thinksynth") + return os.path.join(base, "dsp", "samples") + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + ap.add_argument("mode", choices=["build", "install", "zones"]) + ap.add_argument("args", nargs="*") + ap.add_argument("--cache", default=os.path.join(tempfile.gettempdir(), + "thinksynth-packs")) + opt = ap.parse_args() + + if opt.mode == "zones": + for pack in opt.args: + jobs = plan(pack, opt.cache) + top = max(k for *_, k, _ in jobs) + for layer in range(1, top + 1): + print(f'file{"" if layer == 1 else layer} = "' + f'{zones(jobs, layer)}";') + return + + if opt.mode == "build": + if not opt.args: + ap.error("build needs an output directory") + outdir, names = opt.args[0], opt.args[1:] or list(PACKS) + else: + outdir, names = user_samples(), opt.args or list(PACKS) + + index = [] + for pack in names: + m = build(pack, outdir, opt.cache) + index.append({k: m[k] for k in ("id", "title", "license", "credit", + "bytes")} | {"files": len(m["files"])}) + print(f"{pack}: {len(m['files'])} files, {m['bytes'] / 1e6:.1f} MB") + if opt.mode == "build": + with open(os.path.join(outdir, "index.json"), "w") as f: + json.dump(index, f, indent=1) + + +if __name__ == "__main__": + main() From 5c9c17d8a98f074a16cdfc045f2d5963228ece14 Mon Sep 17 00:00:00 2001 From: Misha Nasledov Date: Sun, 4 Oct 2026 11:25:44 -0700 Subject: [PATCH 04/17] dsp: seven sampled instruments, one osc::sample each Zoned files with two or three velocity layers; the sustained ones loop their last second with a crossfade, and the kit follows the General MIDI drum map. Silent until their pack is downloaded. --- dsp/sampled_cellos.dsp | 69 ++++++++++++++++++++++++++++++++++++++++ dsp/sampled_drums.dsp | 56 +++++++++++++++++++++++++++++++++ dsp/sampled_flute.dsp | 69 ++++++++++++++++++++++++++++++++++++++++ dsp/sampled_horn.dsp | 70 +++++++++++++++++++++++++++++++++++++++++ dsp/sampled_trumpet.dsp | 69 ++++++++++++++++++++++++++++++++++++++++ dsp/sampled_upright.dsp | 65 ++++++++++++++++++++++++++++++++++++++ dsp/sampled_violins.dsp | 69 ++++++++++++++++++++++++++++++++++++++++ 7 files changed, 467 insertions(+) create mode 100644 dsp/sampled_cellos.dsp create mode 100644 dsp/sampled_drums.dsp create mode 100644 dsp/sampled_flute.dsp create mode 100644 dsp/sampled_horn.dsp create mode 100644 dsp/sampled_trumpet.dsp create mode 100644 dsp/sampled_upright.dsp create mode 100644 dsp/sampled_violins.dsp diff --git a/dsp/sampled_cellos.dsp b/dsp/sampled_cellos.dsp new file mode 100644 index 00000000..5f09a18c --- /dev/null +++ b/dsp/sampled_cellos.dsp @@ -0,0 +1,69 @@ +# Cello section (sampled) -- recorded, from the `cellos' sample pack. +# +# VSCO 2 Community Edition, Versilian Studios / Sam Gossner; CC0. The +# recordings are not in this repository: scripts/packs.py fetches them +# from their source, and the page offers the pack for download when +# something in play needs it. Without it, this plays silence. +# +# One osc::sample, its `file' a set of zones -- each recording with the +# note it was made at -- so every note is played from the nearest +# recording. 2 velocity layers, chosen by how hard the note is played. +# +# The recordings are trimmed to a few seconds, and the last second +# loops with a 300 ms crossfade, so a held note holds. + +name "Cello section (sampled)"; +author "Misha Nasledov"; +description "Cello section, recorded (VSCO 2 Community Edition, Versilian Studios / Sam Gossner)."; +category "Strings and pads"; + + @a = 120 ms; + @a.widget = 1; + @a.min = 1 ms; + @a.max = 2000 ms; + @a.label = "Attack"; + + @r = 450 ms; + @r.widget = 1; + @r.min = 70 ms; + @r.max = 4000 ms; + @r.label = "Release"; + +node ionode { + channels = 2; + out0 = vca->out; + out1 = vca->out; + play = env->play; +}; + +node freq misc::midi2freq { + note = ionode->note; +}; + +node smp osc::sample { + file = "cellos/cellos_36_1.wav@36 cellos/cellos_40_1.wav@40 cellos/cellos_43_1.wav@43 cellos/cellos_47_1.wav@47 cellos/cellos_50_1.wav@50 cellos/cellos_53_1.wav@53 cellos/cellos_57_1.wav@57 cellos/cellos_60_1.wav@60 cellos/cellos_64_1.wav@64 cellos/cellos_67_1.wav@67 cellos/cellos_71_1.wav@71 cellos/cellos_74_1.wav@74 cellos/cellos_77_1.wav@77"; + file2 = "cellos/cellos_36_2.wav@36 cellos/cellos_40_2.wav@40 cellos/cellos_43_2.wav@43 cellos/cellos_47_2.wav@47 cellos/cellos_50_2.wav@50 cellos/cellos_53_2.wav@53 cellos/cellos_57_2.wav@57 cellos/cellos_60_2.wav@60 cellos/cellos_64_2.wav@64 cellos/cellos_67_2.wav@67 cellos/cellos_71_2.wav@71 cellos/cellos_74_2.wav@74 cellos/cellos_77_2.wav@77"; + freq = freq->out; + trigger = ionode->trigger; + select = ionode->velocity; + split1 = 0.6; + split2 = 1; + loop = 1000 ms; + xfade = 300 ms; +}; + +node env env::adsr { + a = @a; + d = 0; + s = 1; + r = @r; + p = 0.5 + 0.5 * ionode->velocity; + trigger = ionode->trigger; +}; + +node vca mixer::mul { + in0 = smp->out; + in1 = env->out * 0.75; +}; + +io ionode; diff --git a/dsp/sampled_drums.dsp b/dsp/sampled_drums.dsp new file mode 100644 index 00000000..45712692 --- /dev/null +++ b/dsp/sampled_drums.dsp @@ -0,0 +1,56 @@ +# Drum kit (sampled) -- recorded, from the `drums' sample pack. +# +# Big Rusty Drums, Karoryfer Samples; CC0. The recordings are not in +# this repository: scripts/packs.py fetches them from their source, and +# the page offers the pack for download when something in play needs it. +# Without it, this plays silence. +# +# One osc::sample, its `file' a set of zones -- each recording with the +# note it was made at -- so every note is played from the nearest +# recording. 3 velocity layers, chosen by how hard the note is played. +# +# General MIDI drum notes: 36 kick, 38 snare, 41 43 45 48 toms, +# 42 closed and 46 open hi-hat, 49 crash, 51 ride. Every other +# note plays its nearest drum, retuned. + +name "Drum kit (sampled)"; +author "Misha Nasledov"; +description "Drum kit, recorded (Big Rusty Drums, Karoryfer Samples)."; +category "Drums"; + +node ionode { + channels = 2; + poly = 16; + out0 = vca->out; + out1 = vca->out; + play = max(smp->play, least->play); +}; + +# At least a tenth of a second, so a voice whose file is not +# there yet still lives long enough to be one. +node least env::ad { + a = 0; + d = 100 ms; +}; + +node freq misc::midi2freq { + note = ionode->note; +}; + +node smp osc::sample { + file = "drums/k_36_1.wav@36 drums/sn_center_38_1.wav@38 drums/t22_41_1.wav@41 drums/ht_cl_42_1.wav@42 drums/t18_43_1.wav@43 drums/t15_45_1.wav@45 drums/ht_open_46_1.wav@46 drums/t14_48_1.wav@48 drums/cr_49_1.wav@49 drums/rd_51_1.wav@51"; + file2 = "drums/k_36_2.wav@36 drums/sn_center_38_2.wav@38 drums/t22_41_2.wav@41 drums/ht_cl_42_2.wav@42 drums/t18_43_2.wav@43 drums/t15_45_2.wav@45 drums/ht_open_46_2.wav@46 drums/t14_48_2.wav@48 drums/cr_49_2.wav@49 drums/rd_51_2.wav@51"; + file3 = "drums/k_36_3.wav@36 drums/sn_center_38_3.wav@38 drums/t22_41_3.wav@41 drums/ht_cl_42_3.wav@42 drums/t18_43_3.wav@43 drums/t15_45_3.wav@45 drums/ht_open_46_3.wav@46 drums/t14_48_3.wav@48 drums/cr_49_3.wav@49 drums/rd_51_3.wav@51"; + freq = freq->out; + trigger = 1; + select = ionode->velocity; + split1 = 0.4; + split2 = 0.75; +}; + +node vca mixer::mul { + in0 = smp->out; + in1 = (0.4 + 0.6 * ionode->velocity) * 0.85; +}; + +io ionode; diff --git a/dsp/sampled_flute.dsp b/dsp/sampled_flute.dsp new file mode 100644 index 00000000..161d10f4 --- /dev/null +++ b/dsp/sampled_flute.dsp @@ -0,0 +1,69 @@ +# Flute (sampled) -- recorded, from the `flute' sample pack. +# +# VSCO 2 Community Edition, Versilian Studios / Sam Gossner; CC0. The +# recordings are not in this repository: scripts/packs.py fetches them +# from their source, and the page offers the pack for download when +# something in play needs it. Without it, this plays silence. +# +# One osc::sample, its `file' a set of zones -- each recording with the +# note it was made at -- so every note is played from the nearest +# recording. 2 velocity layers, chosen by how hard the note is played. +# +# The recordings are trimmed to a few seconds, and the last second +# loops with a 300 ms crossfade, so a held note holds. + +name "Flute (sampled)"; +author "Misha Nasledov"; +description "Flute, recorded (VSCO 2 Community Edition, Versilian Studios / Sam Gossner)."; +category "Leads and stabs"; + + @a = 30 ms; + @a.widget = 1; + @a.min = 1 ms; + @a.max = 2000 ms; + @a.label = "Attack"; + + @r = 200 ms; + @r.widget = 1; + @r.min = 70 ms; + @r.max = 4000 ms; + @r.label = "Release"; + +node ionode { + channels = 2; + out0 = vca->out; + out1 = vca->out; + play = env->play; +}; + +node freq misc::midi2freq { + note = ionode->note; +}; + +node smp osc::sample { + file = "flute/flute_60_1.wav@60 flute/flute_64_1.wav@64 flute/flute_69_1.wav@69 flute/flute_72_1.wav@72 flute/flute_76_1.wav@76 flute/flute_81_1.wav@81 flute/flute_84_1.wav@84 flute/flute_88_1.wav@88 flute/flute_93_1.wav@93 flute/flute_96_1.wav@96"; + file2 = "flute/flute_60_2.wav@60 flute/flute_64_2.wav@64 flute/flute_69_2.wav@69 flute/flute_72_2.wav@72 flute/flute_76_2.wav@76 flute/flute_81_2.wav@81 flute/flute_84_1.wav@84 flute/flute_88_1.wav@88 flute/flute_93_2.wav@93 flute/flute_96_1.wav@96"; + freq = freq->out; + trigger = ionode->trigger; + select = ionode->velocity; + split1 = 0.6; + split2 = 1; + loop = 1000 ms; + xfade = 300 ms; +}; + +node env env::adsr { + a = @a; + d = 0; + s = 1; + r = @r; + p = 0.5 + 0.5 * ionode->velocity; + trigger = ionode->trigger; +}; + +node vca mixer::mul { + in0 = smp->out; + in1 = env->out * 3.2; +}; + +io ionode; diff --git a/dsp/sampled_horn.dsp b/dsp/sampled_horn.dsp new file mode 100644 index 00000000..881800bd --- /dev/null +++ b/dsp/sampled_horn.dsp @@ -0,0 +1,70 @@ +# French horn (sampled) -- recorded, from the `horn' sample pack. +# +# VSCO 2 Community Edition, Versilian Studios / Sam Gossner; CC0. The +# recordings are not in this repository: scripts/packs.py fetches them +# from their source, and the page offers the pack for download when +# something in play needs it. Without it, this plays silence. +# +# One osc::sample, its `file' a set of zones -- each recording with the +# note it was made at -- so every note is played from the nearest +# recording. 3 velocity layers, chosen by how hard the note is played. +# +# The recordings are trimmed to a few seconds, and the last second +# loops with a 300 ms crossfade, so a held note holds. + +name "French horn (sampled)"; +author "Misha Nasledov"; +description "French horn, recorded (VSCO 2 Community Edition, Versilian Studios / Sam Gossner)."; +category "Leads and stabs"; + + @a = 40 ms; + @a.widget = 1; + @a.min = 1 ms; + @a.max = 2000 ms; + @a.label = "Attack"; + + @r = 250 ms; + @r.widget = 1; + @r.min = 70 ms; + @r.max = 4000 ms; + @r.label = "Release"; + +node ionode { + channels = 2; + out0 = vca->out; + out1 = vca->out; + play = env->play; +}; + +node freq misc::midi2freq { + note = ionode->note; +}; + +node smp osc::sample { + file = "horn/horn_36_1.wav@36 horn/horn_39_1.wav@39 horn/horn_43_1.wav@43 horn/horn_46_1.wav@46 horn/horn_50_1.wav@50 horn/horn_53_1.wav@53 horn/horn_57_1.wav@57 horn/horn_60_1.wav@60"; + file2 = "horn/horn_36_2.wav@36 horn/horn_39_2.wav@39 horn/horn_43_2.wav@43 horn/horn_46_2.wav@46 horn/horn_50_2.wav@50 horn/horn_53_2.wav@53 horn/horn_57_2.wav@57 horn/horn_60_2.wav@60"; + file3 = "horn/horn_36_2.wav@36 horn/horn_39_2.wav@39 horn/horn_43_2.wav@43 horn/horn_46_2.wav@46 horn/horn_50_3.wav@50 horn/horn_53_2.wav@53 horn/horn_57_2.wav@57 horn/horn_60_3.wav@60"; + freq = freq->out; + trigger = ionode->trigger; + select = ionode->velocity; + split1 = 0.4; + split2 = 0.75; + loop = 1000 ms; + xfade = 300 ms; +}; + +node env env::adsr { + a = @a; + d = 0; + s = 1; + r = @r; + p = 0.5 + 0.5 * ionode->velocity; + trigger = ionode->trigger; +}; + +node vca mixer::mul { + in0 = smp->out; + in1 = env->out * 0.5; +}; + +io ionode; diff --git a/dsp/sampled_trumpet.dsp b/dsp/sampled_trumpet.dsp new file mode 100644 index 00000000..17030037 --- /dev/null +++ b/dsp/sampled_trumpet.dsp @@ -0,0 +1,69 @@ +# Trumpet (sampled) -- recorded, from the `trumpet' sample pack. +# +# VSCO 2 Community Edition, Versilian Studios / Sam Gossner; CC0. The +# recordings are not in this repository: scripts/packs.py fetches them +# from their source, and the page offers the pack for download when +# something in play needs it. Without it, this plays silence. +# +# One osc::sample, its `file' a set of zones -- each recording with the +# note it was made at -- so every note is played from the nearest +# recording. 2 velocity layers, chosen by how hard the note is played. +# +# The recordings are trimmed to a few seconds, and the last second +# loops with a 300 ms crossfade, so a held note holds. + +name "Trumpet (sampled)"; +author "Misha Nasledov"; +description "Trumpet, recorded (VSCO 2 Community Edition, Versilian Studios / Sam Gossner)."; +category "Leads and stabs"; + + @a = 20 ms; + @a.widget = 1; + @a.min = 1 ms; + @a.max = 2000 ms; + @a.label = "Attack"; + + @r = 200 ms; + @r.widget = 1; + @r.min = 70 ms; + @r.max = 4000 ms; + @r.label = "Release"; + +node ionode { + channels = 2; + out0 = vca->out; + out1 = vca->out; + play = env->play; +}; + +node freq misc::midi2freq { + note = ionode->note; +}; + +node smp osc::sample { + file = "trumpet/trumpet_53_1.wav@53 trumpet/trumpet_57_1.wav@57 trumpet/trumpet_60_1.wav@60 trumpet/trumpet_63_1.wav@63 trumpet/trumpet_67_1.wav@67 trumpet/trumpet_70_1.wav@70 trumpet/trumpet_74_1.wav@74 trumpet/trumpet_77_1.wav@77 trumpet/trumpet_81_1.wav@81 trumpet/trumpet_84_1.wav@84"; + file2 = "trumpet/trumpet_53_2.wav@53 trumpet/trumpet_57_2.wav@57 trumpet/trumpet_60_2.wav@60 trumpet/trumpet_63_2.wav@63 trumpet/trumpet_67_2.wav@67 trumpet/trumpet_70_2.wav@70 trumpet/trumpet_74_2.wav@74 trumpet/trumpet_77_2.wav@77 trumpet/trumpet_81_2.wav@81 trumpet/trumpet_84_2.wav@84"; + freq = freq->out; + trigger = ionode->trigger; + select = ionode->velocity; + split1 = 0.6; + split2 = 1; + loop = 1000 ms; + xfade = 300 ms; +}; + +node env env::adsr { + a = @a; + d = 0; + s = 1; + r = @r; + p = 0.5 + 0.5 * ionode->velocity; + trigger = ionode->trigger; +}; + +node vca mixer::mul { + in0 = smp->out; + in1 = env->out * 0.6; +}; + +io ionode; diff --git a/dsp/sampled_upright.dsp b/dsp/sampled_upright.dsp new file mode 100644 index 00000000..21035af2 --- /dev/null +++ b/dsp/sampled_upright.dsp @@ -0,0 +1,65 @@ +# Upright piano (sampled) -- recorded, from the `upright' sample pack. +# +# VSCO 2 Community Edition, Versilian Studios / Sam Gossner; CC0. The +# recordings are not in this repository: scripts/packs.py fetches them +# from their source, and the page offers the pack for download when +# something in play needs it. Without it, this plays silence. +# +# One osc::sample, its `file' a set of zones -- each recording with the +# note it was made at -- so every note is played from the nearest +# recording. 3 velocity layers, chosen by how hard the note is played. + +name "Upright piano (sampled)"; +author "Misha Nasledov"; +description "Upright piano, recorded (VSCO 2 Community Edition, Versilian Studios / Sam Gossner)."; +category "Keys"; + + @a = 1 ms; + @a.widget = 1; + @a.min = 1 ms; + @a.max = 2000 ms; + @a.label = "Attack"; + + @r = 400 ms; + @r.widget = 1; + @r.min = 70 ms; + @r.max = 4000 ms; + @r.label = "Release"; + +node ionode { + channels = 2; + out0 = vca->out; + out1 = vca->out; + play = env->play; +}; + +node freq misc::midi2freq { + note = ionode->note; +}; + +node smp osc::sample { + file = "upright/upright_24_1.wav@24 upright/upright_31_1.wav@31 upright/upright_36_1.wav@36 upright/upright_43_1.wav@43 upright/upright_48_1.wav@48 upright/upright_55_1.wav@55 upright/upright_60_1.wav@60 upright/upright_67_1.wav@67 upright/upright_72_1.wav@72 upright/upright_79_1.wav@79 upright/upright_84_1.wav@84 upright/upright_91_1.wav@91 upright/upright_96_1.wav@96 upright/upright_103_1.wav@103"; + file2 = "upright/upright_24_1.wav@24 upright/upright_31_2.wav@31 upright/upright_36_2.wav@36 upright/upright_43_2.wav@43 upright/upright_48_2.wav@48 upright/upright_55_2.wav@55 upright/upright_60_2.wav@60 upright/upright_67_2.wav@67 upright/upright_72_2.wav@72 upright/upright_79_2.wav@79 upright/upright_84_2.wav@84 upright/upright_91_2.wav@91 upright/upright_96_2.wav@96 upright/upright_103_2.wav@103"; + file3 = "upright/upright_24_3.wav@24 upright/upright_31_2.wav@31 upright/upright_36_3.wav@36 upright/upright_43_3.wav@43 upright/upright_48_3.wav@48 upright/upright_55_3.wav@55 upright/upright_60_3.wav@60 upright/upright_67_3.wav@67 upright/upright_72_3.wav@72 upright/upright_79_3.wav@79 upright/upright_84_3.wav@84 upright/upright_91_3.wav@91 upright/upright_96_3.wav@96 upright/upright_103_3.wav@103"; + freq = freq->out; + trigger = ionode->trigger; + select = ionode->velocity; + split1 = 0.4; + split2 = 0.75; +}; + +node env env::adsr { + a = @a; + d = 0; + s = 1; + r = @r; + p = 0.5 + 0.5 * ionode->velocity; + trigger = ionode->trigger; +}; + +node vca mixer::mul { + in0 = smp->out; + in1 = env->out * 0.85; +}; + +io ionode; diff --git a/dsp/sampled_violins.dsp b/dsp/sampled_violins.dsp new file mode 100644 index 00000000..0def3ba7 --- /dev/null +++ b/dsp/sampled_violins.dsp @@ -0,0 +1,69 @@ +# Violin section (sampled) -- recorded, from the `violins' sample pack. +# +# VSCO 2 Community Edition, Versilian Studios / Sam Gossner; CC0. The +# recordings are not in this repository: scripts/packs.py fetches them +# from their source, and the page offers the pack for download when +# something in play needs it. Without it, this plays silence. +# +# One osc::sample, its `file' a set of zones -- each recording with the +# note it was made at -- so every note is played from the nearest +# recording. 2 velocity layers, chosen by how hard the note is played. +# +# The recordings are trimmed to a few seconds, and the last second +# loops with a 300 ms crossfade, so a held note holds. + +name "Violin section (sampled)"; +author "Misha Nasledov"; +description "Violin section, recorded (VSCO 2 Community Edition, Versilian Studios / Sam Gossner)."; +category "Strings and pads"; + + @a = 120 ms; + @a.widget = 1; + @a.min = 1 ms; + @a.max = 2000 ms; + @a.label = "Attack"; + + @r = 450 ms; + @r.widget = 1; + @r.min = 70 ms; + @r.max = 4000 ms; + @r.label = "Release"; + +node ionode { + channels = 2; + out0 = vca->out; + out1 = vca->out; + play = env->play; +}; + +node freq misc::midi2freq { + note = ionode->note; +}; + +node smp osc::sample { + file = "violins/violins_55_1.wav@55 violins/violins_57_1.wav@57 violins/violins_59_1.wav@59 violins/violins_62_1.wav@62 violins/violins_66_1.wav@66 violins/violins_69_1.wav@69 violins/violins_72_1.wav@72 violins/violins_76_1.wav@76 violins/violins_79_1.wav@79 violins/violins_83_1.wav@83 violins/violins_86_1.wav@86"; + file2 = "violins/violins_55_2.wav@55 violins/violins_57_2.wav@57 violins/violins_59_2.wav@59 violins/violins_62_2.wav@62 violins/violins_66_2.wav@66 violins/violins_69_2.wav@69 violins/violins_72_2.wav@72 violins/violins_76_2.wav@76 violins/violins_79_2.wav@79 violins/violins_83_2.wav@83 violins/violins_86_2.wav@86"; + freq = freq->out; + trigger = ionode->trigger; + select = ionode->velocity; + split1 = 0.6; + split2 = 1; + loop = 1000 ms; + xfade = 300 ms; +}; + +node env env::adsr { + a = @a; + d = 0; + s = 1; + r = @r; + p = 0.5 + 0.5 * ionode->velocity; + trigger = ionode->trigger; +}; + +node vca mixer::mul { + in0 = smp->out; + in1 = env->out * 0.9; +}; + +io ionode; From 7103aeefd4e867f3e3f67edad64571a9d65e3360 Mon Sep 17 00:00:00 2001 From: Misha Nasledov Date: Sun, 4 Oct 2026 11:53:23 -0700 Subject: [PATCH 05/17] scripts: serve packs as FLAC Built packs are FLAC, listed under the .wav name the instrument plays and the .flac they are fetched from: 17 MB for the seven, against 45 MB as WAV. install still writes WAV, which the desktop reads. --- scripts/packs.py | 36 +++++++++++++++++++++++++----------- 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/scripts/packs.py b/scripts/packs.py index a1e6f33c..ce98156c 100755 --- a/scripts/packs.py +++ b/scripts/packs.py @@ -8,13 +8,15 @@ octave "C3" is, and a number cannot. packs.py build OUTDIR [--cache DIR] [PACK...] - Writes OUTDIR//*.wav, OUTDIR//pack.json and + Writes OUTDIR//*.flac, OUTDIR//pack.json and OUTDIR/index.json. The deploy runs this into the site's packs/, - which the page downloads from when asked. + which the page downloads from when asked. Each file is listed under + the .wav name its instrument plays and the .flac URL it is served + at; the page decodes it. packs.py install [PACK...] Builds into this user's data directory (thUtil::userDataDir), under - dsp/samples/, where the desktop app looks. + dsp/samples/, where the desktop app looks, as .wav. packs.py zones PACK Prints the `file' lines the pack's .dsp uses. @@ -208,7 +210,7 @@ def zones(jobs, layer): return " ".join(picks) -def build(pack, outdir, cache): +def build(pack, outdir, cache, flac): spec = PACKS[pack] repo, sha, credit = spec["source"] jobs = plan(pack, cache) @@ -226,12 +228,24 @@ def build(pack, outdir, cache): for x in a)) path = os.path.join(outdir, out) os.makedirs(os.path.dirname(path), exist_ok=True) - with wave.open(path, "wb") as w: - w.setnchannels(1) - w.setsampwidth(2) - w.setframerate(RATE) - w.writeframes(scaled.tobytes()) - files.append({"name": out, "bytes": os.path.getsize(path)}) + if flac: + # The name stays the .wav the instrument plays; what is served + # is FLAC, about half the size, which the page decodes. + served = out[:-4] + ".flac" + subprocess.run( + ["ffmpeg", "-v", "error", "-y", "-f", "s16le", "-ar", + str(RATE), "-ac", "1", "-i", "-", "-compression_level", "8", + os.path.join(outdir, served)], + input=scaled.tobytes(), check=True) + files.append({"name": out, "url": served, "bytes": + os.path.getsize(os.path.join(outdir, served))}) + else: + with wave.open(path, "wb") as w: + w.setnchannels(1) + w.setsampwidth(2) + w.setframerate(RATE) + w.writeframes(scaled.tobytes()) + files.append({"name": out, "bytes": os.path.getsize(path)}) manifest = { "id": pack, "title": spec["title"], "license": "CC0-1.0", "credit": credit, "source": f"https://github.com/{repo}/tree/{sha}", @@ -281,7 +295,7 @@ def main(): index = [] for pack in names: - m = build(pack, outdir, opt.cache) + m = build(pack, outdir, opt.cache, opt.mode == "build") index.append({k: m[k] for k in ("id", "title", "license", "credit", "bytes")} | {"files": len(m["files"])}) print(f"{pack}: {len(m['files'])} files, {m['bytes'] / 1e6:.1f} MB") From 0d48d29c52a22e85b4deee664f08af6c7f570afe Mon Sep 17 00:00:00 2001 From: Misha Nasledov Date: Sun, 4 Oct 2026 11:53:24 -0700 Subject: [PATCH 06/17] web: sample packs, downloaded when asked and kept packs.js keeps downloaded packs in a cache of their own that the service worker leaves alone across updates, decodes their FLAC to WAV for the synth, and loads installed ones at Start. Whatever is in play is checked for packs it names, and a missing one is offered beside the mode menu; a Sample packs pane lists them all. The build ships an empty packs/index.json for the deploy to replace, not precached, and the worker answers it with an empty list offline. --- wasm/web/CMakeLists.txt | 11 +- wasm/web/distcheck.mjs | 6 +- wasm/web/index.html | 17 ++++ wasm/web/main.js | 130 +++++++++++++++++++++++- wasm/web/packs.js | 205 ++++++++++++++++++++++++++++++++++++++ wasm/web/pagetest.mjs | 92 +++++++++++++++++ wasm/web/pwatest.mjs | 2 +- wasm/web/serviceworker.js | 17 +++- 8 files changed, 470 insertions(+), 10 deletions(-) create mode 100644 wasm/web/packs.js diff --git a/wasm/web/CMakeLists.txt b/wasm/web/CMakeLists.txt index ee9e9f83..91f321f8 100644 --- a/wasm/web/CMakeLists.txt +++ b/wasm/web/CMakeLists.txt @@ -244,7 +244,7 @@ set(SITE_SOURCES index.html jam.html style.css main.js keyboard.js mirror.js canvasview.js composerview.js seqview.js nodeview.js rollview.js tapediff.js offline.js - layouts.js manifest.json) + layouts.js packs.js manifest.json) set(SITE_OUTPUTS "") foreach(f IN LISTS SITE_SOURCES) @@ -634,6 +634,15 @@ foreach(dir IN LISTS DIST_PATCH_DIRS) DESTINATION "patches/${dir}" PERMISSIONS ${SITE_FILE_MODE}) endforeach() +# The sample packs' index, empty: the deploy builds the packs into +# dist/packs/ with scripts/packs.py, index and all, after this. Here so a +# build without them answers the page's question rather than 404ing it, and +# not precached, because the service worker would go on answering with the +# copy it had after a deploy added packs. +file(WRITE "${PROJECT_BINARY_DIR}/packs/index.json" "[]\n") +install(FILES "${PROJECT_BINARY_DIR}/packs/index.json" + DESTINATION packs PERMISSIONS ${SITE_FILE_MODE}) + set(DIST_DIR "${PROJECT_BINARY_DIR}/dist") add_custom_target(dist diff --git a/wasm/web/distcheck.mjs b/wasm/web/distcheck.mjs index aff8d027..69d082c3 100644 --- a/wasm/web/distcheck.mjs +++ b/wasm/web/distcheck.mjs @@ -273,14 +273,16 @@ else * is one the page can load online and not offline: a patch picked on a * train is a 404 nobody saw on a desk. So the list is held against the * whole dist, both ways. What is left off on purpose is config.json, which - * no build depends on, the bundles' source maps, and the worker itself. + * no build depends on, the bundles' source maps, the worker itself, and + * the sample packs' index: the deploy writes packs/ after this build, and + * the page keeps a downloaded pack in a cache of its own (packs.js). * * And the manifest's icons, which no page names, so the walk above does * not reach them. */ { const NETWORK = new Set(['jam.js.map', 'sourcebox.js.map', 'config.json', - 'sw.js']); + 'sw.js', 'packs/index.json']); let files = null; try diff --git a/wasm/web/index.html b/wasm/web/index.html index 997bae9f..ee0a5e07 100644 --- a/wasm/web/index.html +++ b/wasm/web/index.html @@ -67,6 +67,13 @@

thinksynth

+ + + @@ -374,6 +381,16 @@

Closed panes

+ +
+

Recorded instruments, public domain (CC0). Each is + downloaded once and kept in this browser, offline too.

+
    + +
    +
    What the browser reports diff --git a/wasm/web/main.js b/wasm/web/main.js index 87a1c13c..e11fb878 100644 --- a/wasm/web/main.js +++ b/wasm/web/main.js @@ -76,6 +76,7 @@ import { createPanes } from './panes.js'; import { numberIn, showPanel } from './panel.js'; import { keepOffline, offerInstall } from './offline.js'; import { moveLayouts } from './layouts.js'; +import * as packs from './packs.js'; import * as patch from './patch.js'; import { createRollView, showClock } from './rollview.js'; import './sourcebox.js'; @@ -132,7 +133,7 @@ const VELOCITY = 100; */ const PANES = ['roll', 'seqview', 'composerview', 'knobs', 'channelbox', 'paramview', 'nodeview', 'keyboard', 'patchsource', - 'piecesource', 'detail']; + 'piecesource', 'detail', 'packs']; /* Which of them belong to which mode. Everything not named here is in all three -- the keys, the parameters, the graph, the numbers. @@ -819,10 +820,12 @@ async function toggleMic () */ const LIVE = /\bionode->live\d/; -async function listens () +/* The text of every graph in play: the patch box's in patch mode, and in + the other two the ones the page aimed and the ones the piece names. */ +async function textsInPlay () { if (mode() === 'patch') - return LIVE.test($('dsp').value); + return [$('dsp').value]; const names = new Set(); @@ -837,10 +840,13 @@ async function listens () /\b(?:dsp|effect)\s+"([^"]+)"/g)) names.add(m[1]); - const texts = await Promise.all([...names].map( + return Promise.all([...names].map( (n) => dspTexts[n] ?? patch.graphText(n).catch(() => ''))); +} - return texts.some((t) => LIVE.test(t)); +async function listens () +{ + return (await textsInPlay()).some((t) => LIVE.test(t)); } /* Asked after anything that changes what is in play; only the latest @@ -857,6 +863,108 @@ async function showLiveIn () $('livein').hidden = !on; $('windowlabel').hidden = synth !== null; + + showPackNeed(); +} + +/* ---- sample packs ---- + * + * Recorded instruments the site offers and the page keeps once asked + * (packs.js). A sampled instrument with its pack missing plays silence, + * so whatever is in play is checked whenever it changes, and a missing + * pack is offered right there rather than in a pane somebody has to know + * to open. The pane lists them all, to download ahead of time or remove. + */ +let packsOffered = []; +let packsHave = new Set(); +let packsAsked = 0; + +const packTitle = (id) => packsOffered.find((p) => p.id === id)?.title ?? id; +const megabytes = (bytes) => `${(bytes / 1e6).toFixed(1)} MB`; + +async function showPackNeed () +{ + const asked = ++packsAsked; + const texts = packsOffered.length > 0 + ? await textsInPlay().catch(() => []) : []; + const missing = [...packs.needed(texts, packsOffered)] + .filter((id) => !packsHave.has(id)); + + if (asked !== packsAsked) + return; + + $('packneed').hidden = missing.length === 0; + $('packneed').dataset.packs = missing.join(' '); + $('packneedtext').textContent = missing.length === 0 ? '' : + 'Needs ' + missing.map((id) => { + const p = packsOffered.find((q) => q.id === id); + + return `${p.title} (${megabytes(p.bytes)})`; + }).join(', '); +} + +async function getPacks (ids) +{ + for (const id of ids) + { + const title = packTitle(id); + + try + { + await packs.download(id, synth, (done, total) => + status(`Downloading ${title}: ` + + `${Math.round(100 * done / total)}%`)); + packsHave.add(id); + status(`${title} downloaded.`); + } + catch (e) + { + status(`Could not download ${title}: ${e.message}`, true); + break; + } + } + + drawPacks(); + showPackNeed(); +} + +function drawPacks () +{ + const list = $('packlist'); + + list.replaceChildren(); + $('packsnone').hidden = packsOffered.length > 0; + + for (const p of packsOffered) + { + const row = document.createElement('li'); + const have = packsHave.has(p.id); + const button = document.createElement('button'); + + row.append(`${p.title}, ${megabytes(p.bytes)} · ${p.credit} `); + button.textContent = have ? 'Remove' : 'Download'; + button.dataset.pack = p.id; + button.addEventListener('click', async () => + { + button.disabled = true; + + if (have) + { + await packs.remove(p.id); + packsHave.delete(p.id); + status(`${p.title} removed.`); + drawPacks(); + showPackNeed(); + } + else + { + await getPacks([p.id]); + } + }); + + row.append(button); + list.append(row); + } } /* ---- what the browser admits to ---- */ @@ -2246,6 +2354,11 @@ async function start () kit.map((name) => served(name).then((r) => r.arrayBuffer()))); kit.forEach((name, i) => synth.sample(name, new Uint8Array(wavs[i]))); + + /* And any sample pack somebody downloaded before. A pack that + will not read is a missing instrument, not a page that will not + start. */ + await packs.loadInstalled(synth).catch((e) => log(e.message)); } catch (e) { @@ -3212,6 +3325,13 @@ async function init () otherwise -- see holdText. */ textMode = 'piece'; + [packsOffered, packsHave] = + await Promise.all([packs.available(), packs.installed() + .catch(() => new Set())]); + drawPacks(); + $('packget').addEventListener('click', () => + getPacks($('packneed').dataset.packs.split(' '))); + /* Sixteen is all there are, counted the way the file counts them and the way the channels row does -- the engine's number is one lower and is what the option carries, since that is what a chain's `input diff --git a/wasm/web/packs.js b/wasm/web/packs.js new file mode 100644 index 00000000..622edad3 --- /dev/null +++ b/wasm/web/packs.js @@ -0,0 +1,205 @@ +/* Sample packs: recordings the page downloads when somebody asks, and keeps. + * + * The site serves them under packs/ -- packs/index.json lists them, and + * packs//pack.json lists a pack's wavs -- but nothing on the page + * fetches them until a Download is pressed: they are tens of megabytes of + * recorded instruments, and most visits play none of them. See + * scripts/packs.py for what they are and where they come from. + * + * KEPT IN A CACHE OF THEIR OWN, `thinksynth-packs', and not the service + * worker's: that one is replaced whole with every new build, and a pack is + * the same pack whatever build plays it. The worker leaves this one alone + * when it clears out old versions (serviceworker.js), the page asks the + * browser to keep the site's storage rather than evict it under pressure, + * and a pack downloaded once plays offline from then on. + * + * A PACK IS INSTALLED WHEN ITS pack.json IS IN THE CACHE, and that goes + * in last: a download that stopped half way leaves wavs and no manifest, + * which reads as not installed and is cleared by the next attempt. + * + * SERVED AS FLAC, about half the size of the WAV it decodes to, and kept + * that way. Each file is listed under the .wav name its instrument plays and + * the .flac it is fetched from; the browser's own decoder turns it into + * samples when it is handed to the synth. + * + * Into the synth the way the shipped kit goes: as WAV bytes, through + * synth.sample, under samples//. osc::sample reads a file it once + * found missing again once new ones arrive (thUtil::dataFilesChanged), so + * an instrument that was loaded before its pack is heard as soon as the + * pack is in, without loading anything again. + */ + +const CACHE = 'thinksynth-packs'; + +const manifestUrl = (id) => `packs/${id}/pack.json`; +const fileUrl = (f) => `packs/${f.url ?? f.name}`; + +/* A file as the WAV osc::sample reads: as it came if it is one, decoded + if it is FLAC, at the rate the packs are built at so nothing is + resampled twice. */ +async function wavBytes (f, bytes) +{ + if (!/\.flac$/.test(f.url ?? '')) + return new Uint8Array(bytes); + + const audio = await new OfflineAudioContext(1, 1, 44100) + .decodeAudioData(bytes); + const pcm = audio.getChannelData(0); + const out = new DataView(new ArrayBuffer(44 + 2 * pcm.length)); + const text = (at, s) => + [...s].forEach((c, i) => out.setUint8(at + i, c.charCodeAt(0))); + + text(0, 'RIFF'); + out.setUint32(4, 36 + 2 * pcm.length, true); + text(8, 'WAVEfmt '); + out.setUint32(16, 16, true); + out.setUint16(20, 1, true); + out.setUint16(22, 1, true); + out.setUint32(24, audio.sampleRate, true); + out.setUint32(28, 2 * audio.sampleRate, true); + out.setUint16(32, 2, true); + out.setUint16(34, 16, true); + text(36, 'data'); + out.setUint32(40, 2 * pcm.length, true); + + for (let i = 0; i < pcm.length; i++) + out.setInt16(44 + 2 * i, + Math.max(-32768, Math.min(32767, + Math.round(pcm[i] * 32767))), + true); + + return new Uint8Array(out.buffer); +} + +/* What the site offers, or nothing -- offline, or a build served without + packs. */ +export async function available () +{ + try + { + const r = await fetch('packs/index.json', { cache: 'no-cache' }); + + return r.ok ? await r.json() : []; + } + catch (e) + { + return []; + } +} + +async function manifests () +{ + /* `has' before `open', which would make the cache: a page that never + downloads a pack leaves no trace of the feature in its storage. */ + if (!('caches' in self) || !await caches.has(CACHE)) + return []; + + const cache = await caches.open(CACHE); + const out = []; + + for (const req of await cache.keys()) + if (/\/packs\/[^/]+\/pack\.json$/.test(new URL(req.url).pathname)) + out.push(await (await cache.match(req)).json()); + + return out; +} + +/* The ids downloaded so far. */ +export async function installed () +{ + return new Set((await manifests()).map((m) => m.id)); +} + +/* Downloads a pack into the cache, `progress(done, total)' in bytes along + the way, and hands it to `synth' if there is one. */ +export async function download (id, synth, progress = () => {}) +{ + const r = await fetch(manifestUrl(id), { cache: 'no-cache' }); + + if (!r.ok) + throw new Error(`${manifestUrl(id)}: ${r.status} ${r.statusText}`); + + const m = await r.json(); + const cache = await caches.open(CACHE); + let done = 0; + + await remove(id); + + /* Four at a time: enough to fill a connection, few enough that a + progress bar moves. */ + const queue = [...m.files]; + + const worker = async () => + { + for (let f; (f = queue.shift()) !== undefined;) + { + const url = fileUrl(f); + const got = await fetch(url, { cache: 'no-cache' }); + + if (!got.ok) + throw new Error(`${url}: ${got.status} ${got.statusText}`); + + const bytes = await got.arrayBuffer(); + + /* A copy into the cache: decoding detaches what it is given. */ + await cache.put(url, new Response(bytes.slice(0))); + + if (synth) + synth.sample(`samples/${f.name}`, await wavBytes(f, bytes)); + done += f.bytes; + progress(done, m.bytes); + } + }; + + await Promise.all([worker(), worker(), worker(), worker()]); + await cache.put(manifestUrl(id), new Response(JSON.stringify(m))); + + navigator.storage?.persist?.().catch(() => {}); + + return m; +} + +export async function remove (id) +{ + if (!await caches.has(CACHE)) + return; + + const cache = await caches.open(CACHE); + + for (const req of await cache.keys()) + if (new URL(req.url).pathname.includes(`/packs/${id}/`)) + await cache.delete(req); +} + +/* Every installed pack into `synth', at Start. */ +export async function loadInstalled (synth) +{ + if (!('caches' in self) || !await caches.has(CACHE)) + return; + + const cache = await caches.open(CACHE); + + for (const m of await manifests()) + for (const f of m.files) + { + const hit = await cache.match(fileUrl(f)); + + if (hit) + synth.sample(`samples/${f.name}`, + await wavBytes(f, await hit.arrayBuffer())); + } +} + +/* The packs these graph texts play from: a sampled instrument names its + files `/...'. */ +export function needed (texts, offered) +{ + const out = new Set(); + + for (const p of offered) + if (texts.some((t) => t.includes(`"${p.id}/`) || + t.includes(` ${p.id}/`))) + out.add(p.id); + + return out; +} diff --git a/wasm/web/pagetest.mjs b/wasm/web/pagetest.mjs index 5926467b..37a5f164 100644 --- a/wasm/web/pagetest.mjs +++ b/wasm/web/pagetest.mjs @@ -2667,6 +2667,98 @@ try await fresh.close(); } + /* Sample packs, served from a route rather than the build, which has + * none: a pack of one tiny FLAC, enough for the page to offer it when + * the sampled kit is in play, download it into its cache, remember it + * on the next page, and take it back out. */ + { + const ctx = await browser.newContext(); + /* Five milliseconds of a sine, as FLAC: what the site serves, and + what the page has to decode before the synth can play it. */ + const flac = Buffer.from( + 'ZkxhQ4AAACIA3QDdAAEJAAEJCsRA8AAAAN0O71C8ehPnWH8ejTPvPlJ+//hp' + + 'CADcB04AAAEAAf8C/QP4BO4F4AbM57dmX5aUUMFAoI94u5hxcADHI4HI4G41' + + 'GYxFwrFIlER8dGpiWE5E9bsVihCj07unn9R1FdZWt1fSOSymWS+YzKaTabze' + + 'bzebzaazOYy+WyqTySQ3txaV9XTUHvtw15sGVWwauiAlKi4zODwQCMTisXDE' + + 'ZjUbDccjkcjkcjgbjUaDIYC0UiYRH52bGRYUEb5y0WqEaLRt5ef1FT1lhaXN' + + '9IZLKZXLphMppNZtN5vN5tN5rNJlMJdKpPJZDe3VrYVlNRfe/LZmxY1C5o6f' + + 'khQWmRsdiASCcVC0YDIaDYbjgcDkcjkcjcbjQZi8WioTiMClPw==', 'base64'); + + const manifest = { + id: 'drums', title: 'Drum kit', license: 'CC0-1.0', + credit: 'a test', bytes: flac.length, + files: [{ name: 'drums/k_36_1.wav', url: 'drums/k_36_1.flac', + bytes: flac.length }], + }; + + await ctx.route('**/packs/index.json', (r) => r.fulfill({ + json: [{ id: 'drums', title: 'Drum kit', license: 'CC0-1.0', + credit: 'a test', bytes: flac.length, files: 1 }] })); + await ctx.route('**/packs/drums/pack.json', + (r) => r.fulfill({ json: manifest })); + await ctx.route('**/packs/drums/k_36_1.flac', + (r) => r.fulfill({ body: flac })); + + const open = async () => + { + const pg = await ctx.newPage(); + + pg.on('pageerror', (e) => errors.push(e.message)); + await pg.goto(url); + await pg.click('#start'); + await pg.waitForFunction(() => window.solo?.settled, null, + { timeout: 20000 }); + await pg.selectOption('#mode', 'patch'); + await pg.evaluate(() => window.solo.settled()); + await pg.selectOption('#patch', 'sampled_drums.dsp'); + await pg.evaluate(() => window.solo.settled()); + + return pg; + }; + /* Polled on a timer rather than on animation frames, which a + browser stops for a page in the background -- and with two pages + open, one of them is. */ + const offered = (pg) => pg.waitForFunction( + () => !document.getElementById('packneed').hidden, null, + { timeout: 5000, polling: 100 }).then(() => true, () => false); + const quiet = (pg) => pg.waitForFunction( + () => document.getElementById('packneed').hidden, null, + { timeout: 5000, polling: 100 }).then(() => true, () => false); + + const pg = await open(); + + check(await offered(pg) && + (await pg.$eval('#packneedtext', (e) => e.textContent)) + .startsWith('Needs Drum kit'), + 'a sampled instrument whose pack is missing offers it: ' + + await pg.$eval('#packneedtext', (e) => e.textContent)); + + await pg.click('#packget'); + + check(await quiet(pg), + 'Download takes the offer away once the pack is in'); + check(await pg.$eval('#packlist button', (b) => b.textContent) === + 'Remove', 'and the pane says it is there'); + + const again = await open(); + + check(await quiet(again), + 'a page opened later finds the pack kept and offers nothing'); + + const button = (pg, text) => pg.waitForFunction( + (t) => document.querySelector('#packlist button') + ?.textContent === t, text, { timeout: 10000, polling: 100 }) + .then(() => true, () => false); + + await button(again, 'Remove'); + await again.$eval('#packlist button', (b) => b.click()); + + check(await button(again, 'Download') && await offered(again), + 'Remove takes it out and the offer comes back'); + + await ctx.close(); + } + for (const e of errors) check(false, `page error: ${e}`); } diff --git a/wasm/web/pwatest.mjs b/wasm/web/pwatest.mjs index e40a53eb..e3567dc0 100644 --- a/wasm/web/pwatest.mjs +++ b/wasm/web/pwatest.mjs @@ -95,7 +95,7 @@ const site = path.join(build, 'pwatest-site'); fs.rmSync(site, { recursive: true, force: true }); -for (const rel of [...FILES, 'sw.js', 'config.json']) +for (const rel of [...FILES, 'sw.js', 'config.json', 'packs/index.json']) { fs.mkdirSync(path.dirname(path.join(site, rel)), { recursive: true }); fs.copyFileSync(path.join(build, rel), path.join(site, rel)); diff --git a/wasm/web/serviceworker.js b/wasm/web/serviceworker.js index 2b878c33..a6a5782e 100644 --- a/wasm/web/serviceworker.js +++ b/wasm/web/serviceworker.js @@ -54,12 +54,17 @@ const PREFIX = 'thinksynth-'; const CACHE = PREFIX + VERSION; +/* The sample packs somebody downloaded (packs.js), which belong to no + build and outlive every one. */ +const PACKS = PREFIX + 'packs'; + /* Every precached file by absolute URL, without a query. */ const PRECACHED = new Set(FILES.map((f) => new URL(f, self.location).href)); /* The site's root, which a navigation reaches as the directory. */ const ROOT = new URL('./', self.location).href; const INDEX = new URL('index.html', self.location).href; +const PACK_INDEX = new URL('packs/index.json', self.location).href; self.addEventListener('install', (e) => { @@ -76,7 +81,7 @@ self.addEventListener('activate', (e) => { for (const name of await caches.keys()) { - if (name.startsWith(PREFIX) && name !== CACHE) + if (name.startsWith(PREFIX) && name !== CACHE && name !== PACKS) await caches.delete(name); } @@ -128,6 +133,16 @@ self.addEventListener('fetch', (e) => with the query a test or a link adds. */ const key = url.href === ROOT ? INDEX : url.href; + /* What sample packs the site offers: always asked of the network, + since a deploy changes it, and offline an empty list rather than a + failed load -- no pack can be downloaded then anyway. */ + if (key === PACK_INDEX) + { + e.respondWith(fetch(e.request).catch(() => new Response('[]', + { headers: { 'Content-Type': 'application/json' } }))); + return; + } + if (!PRECACHED.has(key)) return; From 7fddbd085caa3cdfebfa453388fd1c81f93e19e7 Mon Sep 17 00:00:00 2001 From: Misha Nasledov Date: Sun, 4 Oct 2026 11:53:24 -0700 Subject: [PATCH 07/17] ci: build the sample packs into the demo From master only, with the downloads cached by packs.py's hash. --- .github/workflows/ci.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e9636e11..292006cf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -482,6 +482,26 @@ jobs: run: | jq -n --arg relay "$JAM_RELAY" '{ relay: $relay }' > build-web/dist/config.json + # The sample packs (scripts/packs.py), built from their CC0 sources + # into the site's packs/, which the page downloads from when asked. + # Only for the demo: the downloads are a couple of hundred megabytes, + # cached by the script's own hash, since every source is pinned there. + - name: The sample packs' sources + if: github.event_name != 'pull_request' && github.ref == 'refs/heads/master' + uses: actions/cache@v4 + with: + path: build-packs-cache + key: packs-${{ hashFiles('scripts/packs.py') }} + + - name: The sample packs + if: github.event_name != 'pull_request' && github.ref == 'refs/heads/master' + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + sudo apt-get update -q + sudo apt-get install -y --no-install-recommends ffmpeg + python3 scripts/packs.py build build-web/dist/packs --cache build-packs-cache + - name: The packaged site, for GitHub Pages if: github.event_name != 'pull_request' && github.ref == 'refs/heads/master' uses: actions/upload-pages-artifact@v3 From 76c67f8bd5ccc1dda06a95b72f357f94d020e020 Mon Sep 17 00:00:00 2001 From: Misha Nasledov Date: Sun, 4 Oct 2026 11:53:58 -0700 Subject: [PATCH 08/17] readme: the sampled instruments and packs.py install --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 4a14b2a1..4bf6f131 100644 --- a/README.md +++ b/README.md @@ -182,6 +182,12 @@ browser, the page's MIDI out button does the same through Web MIDI (Chromium and Firefox). Either can also send MIDI clock, so a drum machine or a DAW follows the piece's tempo and transport. +The sampled instruments (`sampled_*.dsp`: strings, horn, trumpet, flute, +upright piano, a drum kit) play public-domain recordings that are not in this +repository. `scripts/packs.py install` (it needs ffmpeg) downloads them from +their sources into `~/.local/share/thinksynth`, where thinksynth looks; the +page offers each one for download when something in play needs it. + If you want JACK on Linux, start `jackd` before thinksynth — RtAudio will use the running server. From 726cb9356bd2871afb1951f53eaa9a878f671e4e Mon Sep 17 00:00:00 2001 From: Misha Nasledov Date: Sun, 4 Oct 2026 12:13:48 -0700 Subject: [PATCH 09/17] packs: a release of their own, built by its own workflow The sample packs workflow (packs.yml, run by hand) builds the packs from their sources and publishes them as the release packs.py names in RELEASE: a tar per pack and the index, marked not the latest so the program's releases stay /releases/latest. It refuses a tag that exists. The deploy and packs.py install now download that release instead of the sources, with no ffmpeg and no API in the deploy; before it is published the deploy warns and the site keeps an empty index. --- .github/workflows/ci.yml | 22 ++---- .github/workflows/packs.yml | 61 ++++++++++++++++ README.md | 7 +- dsp/sampled_cellos.dsp | 7 +- dsp/sampled_drums.dsp | 6 +- dsp/sampled_flute.dsp | 7 +- dsp/sampled_horn.dsp | 7 +- dsp/sampled_trumpet.dsp | 7 +- dsp/sampled_upright.dsp | 7 +- dsp/sampled_violins.dsp | 7 +- scripts/packs.py | 141 +++++++++++++++++++++++++++++------- 11 files changed, 210 insertions(+), 69 deletions(-) create mode 100644 .github/workflows/packs.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 292006cf..3d3824c2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -482,25 +482,13 @@ jobs: run: | jq -n --arg relay "$JAM_RELAY" '{ relay: $relay }' > build-web/dist/config.json - # The sample packs (scripts/packs.py), built from their CC0 sources - # into the site's packs/, which the page downloads from when asked. - # Only for the demo: the downloads are a couple of hundred megabytes, - # cached by the script's own hash, since every source is pinned there. - - name: The sample packs' sources - if: github.event_name != 'pull_request' && github.ref == 'refs/heads/master' - uses: actions/cache@v4 - with: - path: build-packs-cache - key: packs-${{ hashFiles('scripts/packs.py') }} - + # The sample packs, from the release scripts/packs.py names (made by + # the `sample packs' workflow, packs.yml), into the site's packs/, + # which the page downloads from when asked. Before that release is + # published the site keeps the empty index the build wrote. - name: The sample packs if: github.event_name != 'pull_request' && github.ref == 'refs/heads/master' - env: - GITHUB_TOKEN: ${{ github.token }} - run: | - sudo apt-get update -q - sudo apt-get install -y --no-install-recommends ffmpeg - python3 scripts/packs.py build build-web/dist/packs --cache build-packs-cache + run: python3 scripts/packs.py fetch build-web/dist/packs --missing-ok - name: The packaged site, for GitHub Pages if: github.event_name != 'pull_request' && github.ref == 'refs/heads/master' diff --git a/.github/workflows/packs.yml b/.github/workflows/packs.yml new file mode 100644 index 00000000..a9da5ab8 --- /dev/null +++ b/.github/workflows/packs.yml @@ -0,0 +1,61 @@ +# The sample packs, as a release of their own. +# +# scripts/packs.py builds them from their CC0 sources -- a couple of hundred +# megabytes of downloads and an ffmpeg pass -- and this publishes the result +# as the release scripts/packs.py names in RELEASE (packs-1, packs-2, ...): +# a tar per pack and the index, about 17 MB. The site's deploy and +# `packs.py install' download that and never touch the sources. +# +# By hand, and apart from a thinksynth release: the packs change when a pack +# is added or rebuilt, not when the program does. To publish, bump RELEASE, +# merge, and run this. It refuses a tag that already exists -- a release the +# deploy points at is not replaced under it -- and marks its release as not +# the latest, so the program's own releases stay what /releases/latest is. + +name: sample packs + +on: + workflow_dispatch: + +permissions: + contents: write + +jobs: + publish: + name: build and publish the sample packs + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: ffmpeg + run: | + sudo apt-get update -q + sudo apt-get install -y --no-install-recommends ffmpeg + + - name: The release this would make, not made already + env: + GH_TOKEN: ${{ github.token }} + run: | + tag=$(python3 scripts/packs.py tag) + if gh release view "$tag" > /dev/null 2>&1; then + echo "::error::$tag is published already; bump RELEASE in scripts/packs.py" + exit 1 + fi + echo "TAG=$tag" >> "$GITHUB_ENV" + + - name: The packs, from their sources + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + python3 scripts/packs.py build out + python3 scripts/packs.py archive out + + - name: The release + env: + GH_TOKEN: ${{ github.token }} + run: | + gh release create "$TAG" out/*.tar out/index.json \ + --target "$GITHUB_SHA" --latest=false \ + --title "Sample packs ($TAG)" \ + --notes "Recorded instruments for the sampled_*.dsp graphs, built by scripts/packs.py at $GITHUB_SHA. Public domain (CC0): VSCO 2 Community Edition (Versilian Studios / Sam Gossner) and Big Rusty Drums (Karoryfer Samples)." diff --git a/README.md b/README.md index 4bf6f131..a3b8dcb5 100644 --- a/README.md +++ b/README.md @@ -184,9 +184,10 @@ a DAW follows the piece's tempo and transport. The sampled instruments (`sampled_*.dsp`: strings, horn, trumpet, flute, upright piano, a drum kit) play public-domain recordings that are not in this -repository. `scripts/packs.py install` (it needs ffmpeg) downloads them from -their sources into `~/.local/share/thinksynth`, where thinksynth looks; the -page offers each one for download when something in play needs it. +repository: they are published as a release of their own, built by the +`sample packs` workflow. `scripts/packs.py install` (it needs ffmpeg) +downloads it into `~/.local/share/thinksynth`, where thinksynth looks; the +page offers each pack for download when something in play needs it. If you want JACK on Linux, start `jackd` before thinksynth — RtAudio will use the running server. diff --git a/dsp/sampled_cellos.dsp b/dsp/sampled_cellos.dsp index 5f09a18c..6796a1fc 100644 --- a/dsp/sampled_cellos.dsp +++ b/dsp/sampled_cellos.dsp @@ -1,9 +1,10 @@ # Cello section (sampled) -- recorded, from the `cellos' sample pack. # # VSCO 2 Community Edition, Versilian Studios / Sam Gossner; CC0. The -# recordings are not in this repository: scripts/packs.py fetches them -# from their source, and the page offers the pack for download when -# something in play needs it. Without it, this plays silence. +# recordings are not in this repository: scripts/packs.py builds them +# from their source into a release of their own, and the page offers the +# pack for download when something in play needs it. Without it, this +# plays silence. # # One osc::sample, its `file' a set of zones -- each recording with the # note it was made at -- so every note is played from the nearest diff --git a/dsp/sampled_drums.dsp b/dsp/sampled_drums.dsp index 45712692..089713ef 100644 --- a/dsp/sampled_drums.dsp +++ b/dsp/sampled_drums.dsp @@ -1,9 +1,9 @@ # Drum kit (sampled) -- recorded, from the `drums' sample pack. # # Big Rusty Drums, Karoryfer Samples; CC0. The recordings are not in -# this repository: scripts/packs.py fetches them from their source, and -# the page offers the pack for download when something in play needs it. -# Without it, this plays silence. +# this repository: scripts/packs.py builds them from their source into a +# release of their own, and the page offers the pack for download when +# something in play needs it. Without it, this plays silence. # # One osc::sample, its `file' a set of zones -- each recording with the # note it was made at -- so every note is played from the nearest diff --git a/dsp/sampled_flute.dsp b/dsp/sampled_flute.dsp index 161d10f4..15ebf95e 100644 --- a/dsp/sampled_flute.dsp +++ b/dsp/sampled_flute.dsp @@ -1,9 +1,10 @@ # Flute (sampled) -- recorded, from the `flute' sample pack. # # VSCO 2 Community Edition, Versilian Studios / Sam Gossner; CC0. The -# recordings are not in this repository: scripts/packs.py fetches them -# from their source, and the page offers the pack for download when -# something in play needs it. Without it, this plays silence. +# recordings are not in this repository: scripts/packs.py builds them +# from their source into a release of their own, and the page offers the +# pack for download when something in play needs it. Without it, this +# plays silence. # # One osc::sample, its `file' a set of zones -- each recording with the # note it was made at -- so every note is played from the nearest diff --git a/dsp/sampled_horn.dsp b/dsp/sampled_horn.dsp index 881800bd..62705286 100644 --- a/dsp/sampled_horn.dsp +++ b/dsp/sampled_horn.dsp @@ -1,9 +1,10 @@ # French horn (sampled) -- recorded, from the `horn' sample pack. # # VSCO 2 Community Edition, Versilian Studios / Sam Gossner; CC0. The -# recordings are not in this repository: scripts/packs.py fetches them -# from their source, and the page offers the pack for download when -# something in play needs it. Without it, this plays silence. +# recordings are not in this repository: scripts/packs.py builds them +# from their source into a release of their own, and the page offers the +# pack for download when something in play needs it. Without it, this +# plays silence. # # One osc::sample, its `file' a set of zones -- each recording with the # note it was made at -- so every note is played from the nearest diff --git a/dsp/sampled_trumpet.dsp b/dsp/sampled_trumpet.dsp index 17030037..89e69b9c 100644 --- a/dsp/sampled_trumpet.dsp +++ b/dsp/sampled_trumpet.dsp @@ -1,9 +1,10 @@ # Trumpet (sampled) -- recorded, from the `trumpet' sample pack. # # VSCO 2 Community Edition, Versilian Studios / Sam Gossner; CC0. The -# recordings are not in this repository: scripts/packs.py fetches them -# from their source, and the page offers the pack for download when -# something in play needs it. Without it, this plays silence. +# recordings are not in this repository: scripts/packs.py builds them +# from their source into a release of their own, and the page offers the +# pack for download when something in play needs it. Without it, this +# plays silence. # # One osc::sample, its `file' a set of zones -- each recording with the # note it was made at -- so every note is played from the nearest diff --git a/dsp/sampled_upright.dsp b/dsp/sampled_upright.dsp index 21035af2..c6b77558 100644 --- a/dsp/sampled_upright.dsp +++ b/dsp/sampled_upright.dsp @@ -1,9 +1,10 @@ # Upright piano (sampled) -- recorded, from the `upright' sample pack. # # VSCO 2 Community Edition, Versilian Studios / Sam Gossner; CC0. The -# recordings are not in this repository: scripts/packs.py fetches them -# from their source, and the page offers the pack for download when -# something in play needs it. Without it, this plays silence. +# recordings are not in this repository: scripts/packs.py builds them +# from their source into a release of their own, and the page offers the +# pack for download when something in play needs it. Without it, this +# plays silence. # # One osc::sample, its `file' a set of zones -- each recording with the # note it was made at -- so every note is played from the nearest diff --git a/dsp/sampled_violins.dsp b/dsp/sampled_violins.dsp index 0def3ba7..ee587c17 100644 --- a/dsp/sampled_violins.dsp +++ b/dsp/sampled_violins.dsp @@ -1,9 +1,10 @@ # Violin section (sampled) -- recorded, from the `violins' sample pack. # # VSCO 2 Community Edition, Versilian Studios / Sam Gossner; CC0. The -# recordings are not in this repository: scripts/packs.py fetches them -# from their source, and the page offers the pack for download when -# something in play needs it. Without it, this plays silence. +# recordings are not in this repository: scripts/packs.py builds them +# from their source into a release of their own, and the page offers the +# pack for download when something in play needs it. Without it, this +# plays silence. # # One osc::sample, its `file' a set of zones -- each recording with the # note it was made at -- so every note is played from the nearest diff --git a/scripts/packs.py b/scripts/packs.py index ce98156c..11091e11 100755 --- a/scripts/packs.py +++ b/scripts/packs.py @@ -1,28 +1,38 @@ #!/usr/bin/env python3 -"""Sample packs: recorded instruments, fetched from where they are published. +"""Sample packs: recorded instruments, built once and published as a release. The recordings are not in this repository. Every pack is public domain -(CC0) and is built here from its source, pinned to a commit: listed with -the GitHub API, downloaded, trimmed, turned to mono 16-bit at 44.1 kHz and -levelled, and named by MIDI note -- the sources disagree about which -octave "C3" is, and a number cannot. +(CC0) and is built from its source, pinned to a commit: listed with the +GitHub API, downloaded, trimmed, turned to mono 16-bit at 44.1 kHz and +levelled, and named by MIDI note -- the sources disagree about which octave +"C3" is, and a number cannot. That happens once, in the `sample packs' +workflow (.github/workflows/packs.yml), which publishes the result as the +release RELEASE names below; everything else downloads that. packs.py build OUTDIR [--cache DIR] [PACK...] - Writes OUTDIR//*.flac, OUTDIR//pack.json and - OUTDIR/index.json. The deploy runs this into the site's packs/, - which the page downloads from when asked. Each file is listed under - the .wav name its instrument plays and the .flac URL it is served - at; the page decodes it. + From the sources: OUTDIR//*.flac, OUTDIR//pack.json and + OUTDIR/index.json. Each file is listed under the .wav name its + instrument plays and the .flac it is served as; the page decodes it. + + packs.py archive OUTDIR + One OUTDIR/.tar per built pack, the release's assets. + + packs.py fetch OUTDIR [--missing-ok] + RELEASE, unpacked into OUTDIR -- what the deploy puts in the site's + packs/. With --missing-ok a release not published yet is a warning. packs.py install [PACK...] - Builds into this user's data directory (thUtil::userDataDir), under - dsp/samples/, where the desktop app looks, as .wav. + RELEASE into this user's data directory (thUtil::userDataDir), under + dsp/samples/, as .wav, where the desktop app looks. packs.py zones PACK Prints the `file' lines the pack's .dsp uses. -Needs ffmpeg (for FLAC and 24-bit sources). GITHUB_TOKEN, if set, is used -for the directory listings. + packs.py tag + Prints RELEASE. + +build and install need ffmpeg. GITHUB_TOKEN, if set, is used for build's +directory listings. """ import argparse @@ -32,7 +42,9 @@ import re import subprocess import sys +import tarfile import tempfile +import urllib.error import urllib.parse import urllib.request import wave @@ -43,6 +55,11 @@ "f07ce00df34a46b6b08375be56fe116cf15782bc", "Big Rusty Drums, Karoryfer Samples") +# The published packs everything downloads. A change to what a pack holds +# is a new number: a release, once made, is not replaced. +REPO = "mishan/thinksynth" +RELEASE = "packs-1" + RATE = 44100 PEAK = 0.89 @@ -137,7 +154,7 @@ def listing(repo, sha, path, cache): return names -def fetch(repo, sha, path, cache): +def download(repo, sha, path, cache): local = os.path.join(cache, repo.replace("/", "_"), sha[:12], path) if not os.path.exists(local): os.makedirs(os.path.dirname(local), exist_ok=True) @@ -216,7 +233,7 @@ def build(pack, outdir, cache, flac): jobs = plan(pack, cache) decoded = [] for path, out, note, k, seconds in jobs: - decoded.append((out, decode(fetch(repo, sha, path, cache), seconds, + decoded.append((out, decode(download(repo, sha, path, cache), seconds, spec["sustain"]))) # One gain for the whole pack, so the balance across the range and # between layers is the recording's. @@ -269,13 +286,85 @@ def user_samples(): return os.path.join(base, "dsp", "samples") +def asset(name): + return f"https://github.com/{REPO}/releases/download/{RELEASE}/{name}" + + +def archive(outdir): + with open(os.path.join(outdir, "index.json")) as f: + index = json.load(f) + for p in index: + with tarfile.open(os.path.join(outdir, p["id"] + ".tar"), "w") as t: + t.add(os.path.join(outdir, p["id"]), arcname=p["id"]) + + +def fetch(outdir, missing_ok=False): + """RELEASE's packs into outdir; the index, or None if there is none.""" + try: + with urllib.request.urlopen(asset("index.json")) as r: + index = json.load(r) + except urllib.error.HTTPError as e: + if e.code == 404 and missing_ok: + print(f"::warning::{RELEASE} is not published; no sample packs") + return None + raise + os.makedirs(outdir, exist_ok=True) + for p in index: + with urllib.request.urlopen(asset(p["id"] + ".tar")) as r, \ + tempfile.TemporaryFile() as tmp: + tmp.write(r.read()) + tmp.seek(0) + with tarfile.open(fileobj=tmp) as t: + t.extractall(outdir, filter="data") + print(f"{p['id']}: {p['bytes'] / 1e6:.1f} MB") + with open(os.path.join(outdir, "index.json"), "w") as f: + json.dump(index, f, indent=1) + return index + + +def install(names): + """RELEASE, decoded to .wav, into the user's samples directory.""" + dest = user_samples() + with tempfile.TemporaryDirectory() as tmp: + index = fetch(tmp) + for p in index: + if names and p["id"] not in names: + continue + with open(os.path.join(tmp, p["id"], "pack.json")) as f: + m = json.load(f) + for f in m["files"]: + out = os.path.join(dest, f["name"]) + os.makedirs(os.path.dirname(out), exist_ok=True) + subprocess.run(["ffmpeg", "-v", "error", "-y", "-i", + os.path.join(tmp, f.get("url", f["name"])), + out], check=True) + print(f"{p['id']}: installed in {dest}") + + def main(): ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) - ap.add_argument("mode", choices=["build", "install", "zones"]) + ap.add_argument("mode", choices=["build", "archive", "fetch", "install", + "zones", "tag"]) ap.add_argument("args", nargs="*") ap.add_argument("--cache", default=os.path.join(tempfile.gettempdir(), "thinksynth-packs")) - opt = ap.parse_args() + ap.add_argument("--missing-ok", action="store_true") + opt = ap.parse_intermixed_args() + + if opt.mode == "tag": + print(RELEASE) + return + if opt.mode in ("archive", "fetch"): + if not opt.args: + ap.error(f"{opt.mode} needs an output directory") + if opt.mode == "archive": + archive(opt.args[0]) + else: + fetch(opt.args[0], opt.missing_ok) + return + if opt.mode == "install": + install(opt.args) + return if opt.mode == "zones": for pack in opt.args: @@ -286,22 +375,18 @@ def main(): f'{zones(jobs, layer)}";') return - if opt.mode == "build": - if not opt.args: - ap.error("build needs an output directory") - outdir, names = opt.args[0], opt.args[1:] or list(PACKS) - else: - outdir, names = user_samples(), opt.args or list(PACKS) + if not opt.args: + ap.error("build needs an output directory") + outdir, names = opt.args[0], opt.args[1:] or list(PACKS) index = [] for pack in names: - m = build(pack, outdir, opt.cache, opt.mode == "build") + m = build(pack, outdir, opt.cache, True) index.append({k: m[k] for k in ("id", "title", "license", "credit", "bytes")} | {"files": len(m["files"])}) print(f"{pack}: {len(m['files'])} files, {m['bytes'] / 1e6:.1f} MB") - if opt.mode == "build": - with open(os.path.join(outdir, "index.json"), "w") as f: - json.dump(index, f, indent=1) + with open(os.path.join(outdir, "index.json"), "w") as f: + json.dump(index, f, indent=1) if __name__ == "__main__": From ce81cb1c702897f89c30bc15655193a19cd0254b Mon Sep 17 00:00:00 2001 From: Misha Nasledov Date: Sun, 4 Oct 2026 12:26:32 -0700 Subject: [PATCH 10/17] web: check.mjs plays sampled instruments on stand-in tones A sampled_*.dsp plays a pack the site downloads when asked, so the gate gives every file a graph names and the kit lacks a half-second sine, and checks the graph -- zones, layers, envelope -- rather than calling it silent. --- wasm/web/check.mjs | 40 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/wasm/web/check.mjs b/wasm/web/check.mjs index 8f09937a..60b36d9d 100644 --- a/wasm/web/check.mjs +++ b/wasm/web/check.mjs @@ -74,6 +74,32 @@ const kit = Object.fromEntries( .map((n) => [n, new Uint8Array(fs.readFileSync( path.join(dspDir, n)))])); +/* Half a second of a 16-bit mono sine at 440 Hz, as a RIFF. */ +function standIn () +{ + const frames = RATE / 2; + const out = Buffer.alloc(44 + 2 * frames); + + out.write('RIFF', 0); + out.writeUInt32LE(36 + 2 * frames, 4); + out.write('WAVEfmt ', 8); + out.writeUInt32LE(16, 16); + out.writeUInt16LE(1, 20); + out.writeUInt16LE(1, 22); + out.writeUInt32LE(RATE, 24); + out.writeUInt32LE(2 * RATE, 28); + out.writeUInt16LE(2, 32); + out.writeUInt16LE(16, 34); + out.write('data', 36); + out.writeUInt32LE(2 * frames, 40); + + for (let i = 0; i < frames; i++) + out.writeInt16LE(Math.round(16000 * Math.sin(2 * Math.PI * 440 * i / + RATE)), 44 + 2 * i); + + return new Uint8Array(out); +} + for (const name of names) { /* An effect graph has no note to play and is not an instrument; the @@ -95,13 +121,23 @@ for (const name of names) } const text = fs.readFileSync(path.join(dspDir, name), 'utf8'); + + /* A sampled instrument plays a pack the site downloads when asked + (packs.js), not one it ships, so here every file a graph names and + the kit does not have is a stand-in tone: what is under test is the + graph -- its zones, its layers, its envelope -- not the recording. */ + const samples = { ...kit }; + + for (const m of text.matchAll(/([\w./-]+\.wav)(?:@\d+)?/g)) + samples[`samples/${m[1]}`] ??= standIn(); + const events = [ { on: true, frame: 0, ...NOTE }, { on: false, frame: RATE / 2, note: NOTE.note }, ]; const a = await renderDirect(createThinkWeb, - { rate: RATE, text, events, samples: kit, + { rate: RATE, text, events, samples, frames: FRAMES }); if (!a.ok) @@ -121,7 +157,7 @@ for (const name of names) } const b = await renderDirect(createThinkWeb, - { rate: RATE, text, events, samples: kit, + { rate: RATE, text, events, samples, frames: FRAMES }); const same = Buffer.from(a.out.buffer).equals(Buffer.from(b.out.buffer)); From d42d66002e99746613aa02344a785dfbdfd5fc08 Mon Sep 17 00:00:00 2001 From: Misha Nasledov Date: Sun, 4 Oct 2026 12:48:25 -0700 Subject: [PATCH 11/17] ci: the gate trees carry build-web/packs The page asks for packs/index.json at load; without it in the trees the browser jobs see a 404 that the build itself never has. --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3d3824c2..69171392 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -538,6 +538,7 @@ jobs: build-web/gen build-web/patches build-web/icons + build-web/packs build-web/cairo2d/cairo2dtest.js build-web/cairo2d/cairo2dtest.wasm From c3d717b3bd7ca4df5891dd607765fcf9c6798016 Mon Sep 17 00:00:00 2001 From: Misha Nasledov Date: Sun, 4 Oct 2026 14:09:55 -0700 Subject: [PATCH 12/17] osc: sample slots forget their zones with their wavs; sampled sustain follows velocity thSampleClaim and thSampleRelease clear the parsed zones beside the table and the alternates, so a reused slot does not carry a past synth's. The sampled instruments' sustain is 0.5 + 0.5 velocity, as their peak is, so a soft note does not jump to full level after the attack. --- dsp/sampled_cellos.dsp | 2 +- dsp/sampled_flute.dsp | 2 +- dsp/sampled_horn.dsp | 2 +- dsp/sampled_trumpet.dsp | 2 +- dsp/sampled_upright.dsp | 2 +- dsp/sampled_violins.dsp | 2 +- plugins/osc/sampleslot.h | 2 ++ 7 files changed, 8 insertions(+), 6 deletions(-) diff --git a/dsp/sampled_cellos.dsp b/dsp/sampled_cellos.dsp index 6796a1fc..6604ad86 100644 --- a/dsp/sampled_cellos.dsp +++ b/dsp/sampled_cellos.dsp @@ -56,7 +56,7 @@ node smp osc::sample { node env env::adsr { a = @a; d = 0; - s = 1; + s = 0.5 + 0.5 * ionode->velocity; r = @r; p = 0.5 + 0.5 * ionode->velocity; trigger = ionode->trigger; diff --git a/dsp/sampled_flute.dsp b/dsp/sampled_flute.dsp index 15ebf95e..0dd7fe22 100644 --- a/dsp/sampled_flute.dsp +++ b/dsp/sampled_flute.dsp @@ -56,7 +56,7 @@ node smp osc::sample { node env env::adsr { a = @a; d = 0; - s = 1; + s = 0.5 + 0.5 * ionode->velocity; r = @r; p = 0.5 + 0.5 * ionode->velocity; trigger = ionode->trigger; diff --git a/dsp/sampled_horn.dsp b/dsp/sampled_horn.dsp index 62705286..7e43217d 100644 --- a/dsp/sampled_horn.dsp +++ b/dsp/sampled_horn.dsp @@ -57,7 +57,7 @@ node smp osc::sample { node env env::adsr { a = @a; d = 0; - s = 1; + s = 0.5 + 0.5 * ionode->velocity; r = @r; p = 0.5 + 0.5 * ionode->velocity; trigger = ionode->trigger; diff --git a/dsp/sampled_trumpet.dsp b/dsp/sampled_trumpet.dsp index 89e69b9c..bf1b03c1 100644 --- a/dsp/sampled_trumpet.dsp +++ b/dsp/sampled_trumpet.dsp @@ -56,7 +56,7 @@ node smp osc::sample { node env env::adsr { a = @a; d = 0; - s = 1; + s = 0.5 + 0.5 * ionode->velocity; r = @r; p = 0.5 + 0.5 * ionode->velocity; trigger = ionode->trigger; diff --git a/dsp/sampled_upright.dsp b/dsp/sampled_upright.dsp index c6b77558..a7164a2e 100644 --- a/dsp/sampled_upright.dsp +++ b/dsp/sampled_upright.dsp @@ -52,7 +52,7 @@ node smp osc::sample { node env env::adsr { a = @a; d = 0; - s = 1; + s = 0.5 + 0.5 * ionode->velocity; r = @r; p = 0.5 + 0.5 * ionode->velocity; trigger = ionode->trigger; diff --git a/dsp/sampled_violins.dsp b/dsp/sampled_violins.dsp index ee587c17..dc2c6500 100644 --- a/dsp/sampled_violins.dsp +++ b/dsp/sampled_violins.dsp @@ -56,7 +56,7 @@ node smp osc::sample { node env env::adsr { a = @a; d = 0; - s = 1; + s = 0.5 + 0.5 * ionode->velocity; r = @r; p = 0.5 + 0.5 * ionode->velocity; trigger = ionode->trigger; diff --git a/plugins/osc/sampleslot.h b/plugins/osc/sampleslot.h index ffced275..613404d5 100644 --- a/plugins/osc/sampleslot.h +++ b/plugins/osc/sampleslot.h @@ -138,6 +138,7 @@ static inline void thSampleClaim (const thPlugin *plugin) delete thSampleSlots[i].table; thSampleSlots[i].table = new std::map(); thSampleSlots[i].alternates.clear(); + thSampleSlots[i].zones.clear(); return; } } @@ -151,6 +152,7 @@ static inline void thSampleRelease (const thPlugin *plugin) delete thSampleSlots[i].table; thSampleSlots[i].table = NULL; thSampleSlots[i].alternates.clear(); + thSampleSlots[i].zones.clear(); thSampleSlots[i].owner.store(NULL, std::memory_order_release); return; } From 407365b7f538e5c63119ef49efc0d486dfcff269 Mon Sep 17 00:00:00 2001 From: Misha Nasledov Date: Sun, 4 Oct 2026 14:09:55 -0700 Subject: [PATCH 13/17] web: sample packs fixes from review A pack carries a version, a hash of its files, in its manifest and the index; a kept pack whose version is not the one offered is offered again, and its button says Update. A download waits for every worker to settle before reporting a failure, so a retry is not raced. Each pack button's accessible name includes the pack. Picking a patch checks for packs before Start, and the check runs again once the pack list arrives, so a choice made before it is not missed. The page test's pack context blocks the service worker, whose requests the routes do not see, and waits for the page to start before choosing an instrument. --- scripts/packs.py | 10 +++++++++- wasm/web/main.js | 26 +++++++++++++++++++------- wasm/web/packs.js | 20 ++++++++++++++++---- wasm/web/pagetest.mjs | 40 ++++++++++++++++++++++++++++++++-------- 4 files changed, 76 insertions(+), 20 deletions(-) diff --git a/scripts/packs.py b/scripts/packs.py index 11091e11..bc0f5c6d 100755 --- a/scripts/packs.py +++ b/scripts/packs.py @@ -37,6 +37,7 @@ import argparse import array +import hashlib import json import os import re @@ -263,8 +264,14 @@ def build(pack, outdir, cache, flac): w.setframerate(RATE) w.writeframes(scaled.tobytes()) files.append({"name": out, "bytes": os.path.getsize(path)}) + # What the page compares to tell a rebuilt pack from the one it kept. + digest = hashlib.sha256() + for f in files: + with open(os.path.join(outdir, f.get("url", f["name"])), "rb") as h: + digest.update(f["name"].encode() + b"\0" + h.read()) manifest = { "id": pack, "title": spec["title"], "license": "CC0-1.0", + "version": digest.hexdigest()[:16], "credit": credit, "source": f"https://github.com/{repo}/tree/{sha}", "bytes": sum(f["bytes"] for f in files), "files": files, } @@ -383,7 +390,8 @@ def main(): for pack in names: m = build(pack, outdir, opt.cache, True) index.append({k: m[k] for k in ("id", "title", "license", "credit", - "bytes")} | {"files": len(m["files"])}) + "version", "bytes")} + | {"files": len(m["files"])}) print(f"{pack}: {len(m['files'])} files, {m['bytes'] / 1e6:.1f} MB") with open(os.path.join(outdir, "index.json"), "w") as f: json.dump(index, f, indent=1) diff --git a/wasm/web/main.js b/wasm/web/main.js index e11fb878..177ef985 100644 --- a/wasm/web/main.js +++ b/wasm/web/main.js @@ -876,10 +876,14 @@ async function showLiveIn () * to open. The pane lists them all, to download ahead of time or remove. */ let packsOffered = []; -let packsHave = new Set(); +let packsHave = new Map(); let packsAsked = 0; const packTitle = (id) => packsOffered.find((p) => p.id === id)?.title ?? id; + +/* Downloaded, and the version the site offers now. */ +const packCurrent = (p) => packsHave.has(p.id) && + packsHave.get(p.id) === p.version; const megabytes = (bytes) => `${(bytes / 1e6).toFixed(1)} MB`; async function showPackNeed () @@ -888,7 +892,7 @@ async function showPackNeed () const texts = packsOffered.length > 0 ? await textsInPlay().catch(() => []) : []; const missing = [...packs.needed(texts, packsOffered)] - .filter((id) => !packsHave.has(id)); + .filter((id) => !packCurrent(packsOffered.find((p) => p.id === id))); if (asked !== packsAsked) return; @@ -911,10 +915,10 @@ async function getPacks (ids) try { - await packs.download(id, synth, (done, total) => + const m = await packs.download(id, synth, (done, total) => status(`Downloading ${title}: ` + `${Math.round(100 * done / total)}%`)); - packsHave.add(id); + packsHave.set(id, m.version); status(`${title} downloaded.`); } catch (e) @@ -938,11 +942,13 @@ function drawPacks () for (const p of packsOffered) { const row = document.createElement('li'); - const have = packsHave.has(p.id); + const have = packCurrent(p); const button = document.createElement('button'); row.append(`${p.title}, ${megabytes(p.bytes)} · ${p.credit} `); - button.textContent = have ? 'Remove' : 'Download'; + button.textContent = have ? 'Remove' + : packsHave.has(p.id) ? 'Update' : 'Download'; + button.setAttribute('aria-label', `${button.textContent} ${p.title}`); button.dataset.pack = p.id; button.addEventListener('click', async () => { @@ -1035,6 +1041,9 @@ async function pickPatch () showNodes(); + /* Before Start too: a pack can be downloaded ahead of playing. */ + showPackNeed(); + await loadPatch(); })(); @@ -3327,8 +3336,11 @@ async function init () [packsOffered, packsHave] = await Promise.all([packs.available(), packs.installed() - .catch(() => new Set())]); + .catch(() => new Map())]); drawPacks(); + + /* What is in play may have been chosen before the list came. */ + showPackNeed(); $('packget').addEventListener('click', () => getPacks($('packneed').dataset.packs.split(' '))); diff --git a/wasm/web/packs.js b/wasm/web/packs.js index 622edad3..a992c72e 100644 --- a/wasm/web/packs.js +++ b/wasm/web/packs.js @@ -15,7 +15,10 @@ * * A PACK IS INSTALLED WHEN ITS pack.json IS IN THE CACHE, and that goes * in last: a download that stopped half way leaves wavs and no manifest, - * which reads as not installed and is cleared by the next attempt. + * which reads as not installed and is cleared by the next attempt. The + * manifest carries the pack's `version', a hash of its files, and a kept + * pack whose version is not the one offered is one to download again -- + * an instrument may name recordings the old one does not have. * * SERVED AS FLAC, about half the size of the WAV it decodes to, and kept * that way. Each file is listed under the .wav name its instrument plays and @@ -104,10 +107,10 @@ async function manifests () return out; } -/* The ids downloaded so far. */ +/* The packs downloaded so far, id to version. */ export async function installed () { - return new Set((await manifests()).map((m) => m.id)); + return new Map((await manifests()).map((m) => [m.id, m.version])); } /* Downloads a pack into the cache, `progress(done, total)' in bytes along @@ -151,7 +154,16 @@ export async function download (id, synth, progress = () => {}) } }; - await Promise.all([worker(), worker(), worker(), worker()]); + /* Every worker settled before any failure is reported: one that is + still fetching would otherwise go on writing the cache and the + progress under a retry that has already started. */ + const ends = await Promise.allSettled([worker(), worker(), worker(), + worker()]); + const failed = ends.find((e) => e.status === 'rejected'); + + if (failed) + throw failed.reason; + await cache.put(manifestUrl(id), new Response(JSON.stringify(m))); navigator.storage?.persist?.().catch(() => {}); diff --git a/wasm/web/pagetest.mjs b/wasm/web/pagetest.mjs index 37a5f164..f49dfbfb 100644 --- a/wasm/web/pagetest.mjs +++ b/wasm/web/pagetest.mjs @@ -2672,7 +2672,10 @@ try * the sampled kit is in play, download it into its cache, remember it * on the next page, and take it back out. */ { - const ctx = await browser.newContext(); + /* No service worker: a request it makes is not one a route sees, + and once it controls a page the index would come from the + build's empty one instead of the routes below. */ + const ctx = await browser.newContext({ serviceWorkers: 'block' }); /* Five milliseconds of a sine, as FLAC: what the site serves, and what the page has to decode before the synth can play it. */ const flac = Buffer.from( @@ -2684,18 +2687,21 @@ try '9IZLKZXLphMppNZtN5vN5tN5rNJlMJdKpPJZDe3VrYVlNRfe/LZmxY1C5o6f' + 'khQWmRsdiASCcVC0YDIaDYbjgcDkcjkcjcbjQZi8WioTiMClPw==', 'base64'); - const manifest = { + /* What the site offers, which a later deploy can change. */ + let version = '1'; + const manifest = () => ({ id: 'drums', title: 'Drum kit', license: 'CC0-1.0', - credit: 'a test', bytes: flac.length, + credit: 'a test', bytes: flac.length, version, files: [{ name: 'drums/k_36_1.wav', url: 'drums/k_36_1.flac', bytes: flac.length }], - }; + }); await ctx.route('**/packs/index.json', (r) => r.fulfill({ json: [{ id: 'drums', title: 'Drum kit', license: 'CC0-1.0', - credit: 'a test', bytes: flac.length, files: 1 }] })); + credit: 'a test', bytes: flac.length, files: 1, + version }] })); await ctx.route('**/packs/drums/pack.json', - (r) => r.fulfill({ json: manifest })); + (r) => r.fulfill({ json: manifest() })); await ctx.route('**/packs/drums/k_36_1.flac', (r) => r.fulfill({ body: flac })); @@ -2706,8 +2712,10 @@ try pg.on('pageerror', (e) => errors.push(e.message)); await pg.goto(url); await pg.click('#start'); - await pg.waitForFunction(() => window.solo?.settled, null, - { timeout: 20000 }); + /* Started, which the page says by dropping data-unstarted. */ + await pg.waitForFunction(() => window.solo?.settled && + !('unstarted' in document.body.dataset), + null, { timeout: 20000, polling: 100 }); await pg.selectOption('#mode', 'patch'); await pg.evaluate(() => window.solo.settled()); await pg.selectOption('#patch', 'sampled_drums.dsp'); @@ -2745,6 +2753,22 @@ try check(await quiet(again), 'a page opened later finds the pack kept and offers nothing'); + /* A deploy that rebuilt the pack: what is kept is not it. */ + version = '2'; + + const rebuilt = await open(); + + const reoffered = await offered(rebuilt); + const label = await rebuilt.$eval('#packlist button', + (b) => b.getAttribute('aria-label')); + + check(reoffered && label === 'Update Drum kit', + 'a pack rebuilt since it was kept is offered again, to update: ' + + `${reoffered ? 'offered' : 'not offered'}, ${label}`); + + await rebuilt.close(); + version = '1'; + const button = (pg, text) => pg.waitForFunction( (t) => document.querySelector('#packlist button') ?.textContent === t, text, { timeout: 10000, polling: 100 }) From 2913380105f4692b7be5df7f8f0e0b8b178dcd3f Mon Sep 17 00:00:00 2001 From: Misha Nasledov Date: Sun, 4 Oct 2026 14:09:55 -0700 Subject: [PATCH 14/17] packs: the workflow checks the tag; the readme names every platform's directory --- .github/workflows/packs.yml | 6 ++++-- README.md | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/packs.yml b/.github/workflows/packs.yml index a9da5ab8..2cf06f4a 100644 --- a/.github/workflows/packs.yml +++ b/.github/workflows/packs.yml @@ -33,13 +33,15 @@ jobs: sudo apt-get update -q sudo apt-get install -y --no-install-recommends ffmpeg + # The tag, not only the release: a bare tag left behind would pass a + # release check and the release would be made on that old commit. - name: The release this would make, not made already env: GH_TOKEN: ${{ github.token }} run: | tag=$(python3 scripts/packs.py tag) - if gh release view "$tag" > /dev/null 2>&1; then - echo "::error::$tag is published already; bump RELEASE in scripts/packs.py" + if gh api "repos/$GITHUB_REPOSITORY/git/ref/tags/$tag" > /dev/null 2>&1; then + echo "::error::the tag $tag exists already; bump RELEASE in scripts/packs.py" exit 1 fi echo "TAG=$tag" >> "$GITHUB_ENV" diff --git a/README.md b/README.md index a3b8dcb5..64349942 100644 --- a/README.md +++ b/README.md @@ -186,8 +186,10 @@ The sampled instruments (`sampled_*.dsp`: strings, horn, trumpet, flute, upright piano, a drum kit) play public-domain recordings that are not in this repository: they are published as a release of their own, built by the `sample packs` workflow. `scripts/packs.py install` (it needs ffmpeg) -downloads it into `~/.local/share/thinksynth`, where thinksynth looks; the -page offers each pack for download when something in play needs it. +downloads it into this user's data directory, where thinksynth looks — +`~/.local/share/thinksynth` on Linux, `~/Library/Application Support/thinksynth` +on macOS, `%LOCALAPPDATA%\thinksynth` on Windows; the page offers each pack +for download when something in play needs it. If you want JACK on Linux, start `jackd` before thinksynth — RtAudio will use the running server. From b314e4759e4e3616962b56e3a68fb6c06ccfc680 Mon Sep 17 00:00:00 2001 From: Misha Nasledov Date: Sun, 4 Oct 2026 14:55:18 -0700 Subject: [PATCH 15/17] osc: sample files re-read when written over; packs fixes from review thUtil::dataFileChanged names a file a host has written over, and osc::sample reads that file again even though it was found -- a pack updated while it plays. tw_sample names each file it writes. Loading a graph counts as files having changed, so the desktop finds a pack installed since the last load. A plain name with an `@' in it is a file: only a word ending in `@' and a note number is a zone. Inside a Flatpak the user data directory is the host's ~/.local/share, where packs.py puts packs. The page hands samples to the module through the heap rather than ccall's array, which copies onto a 1 MB stack, so a long recording fits. packs.js is in the room page's bundle. --- libthink/thSynth.cpp | 5 +++++ libthink/thUtil.cpp | 22 +++++++++++++++++++ libthink/thUtil.h | 13 +++++++++++- plugins/osc/sampleslot.h | 46 ++++++++++++++++++++++++++++++++++------ scripts/statecheck.cpp | 28 +++++++++++++++++++++++- wasm/web/CMakeLists.txt | 3 ++- wasm/web/engine.js | 13 +++++++++--- wasm/web/thinkweb.cpp | 19 ++++++++++++++--- 8 files changed, 134 insertions(+), 15 deletions(-) diff --git a/libthink/thSynth.cpp b/libthink/thSynth.cpp index 9124dd16..6e63a29e 100644 --- a/libthink/thSynth.cpp +++ b/libthink/thSynth.cpp @@ -1325,6 +1325,11 @@ thSynthTree *thSynth::loadChannel (const string &filename, const string *text, return NULL; } + /* Loading a graph is when somebody expects the files it names to be + found: a sample pack installed since the last load is read now, + rather than the silence cached for it then. */ + thUtil::dataFilesChanged(); + if (text != NULL) { std::lock_guard lock(synthMutex_); diff --git a/libthink/thUtil.cpp b/libthink/thUtil.cpp index 4d3f77ba..8acd8a05 100644 --- a/libthink/thUtil.cpp +++ b/libthink/thUtil.cpp @@ -250,6 +250,9 @@ string thUtil::userDataDir (void) ? (fs::path(base) / PACKAGE_NAME).string() : ""; #else const char *home = getenv("HOME"); + + if (getenv("FLATPAK_ID") != NULL && home != NULL && *home != 0) + return (fs::path(home) / ".local" / "share" / PACKAGE_NAME).string(); #if defined(__APPLE__) return (home != NULL && *home != 0) ? (fs::path(home) / "Library" / "Application Support" / @@ -597,12 +600,31 @@ bool thUtil::findEmbeddedFile (const string &name, const unsigned char *&data, static std::atomic dataGeneration{0}; +/* Each file named to dataFileChanged, and the count after it was. */ +static std::mutex dataLock; +static std::map dataChanged; + void thUtil::dataFilesChanged () { dataGeneration++; } +void thUtil::dataFileChanged (const string &name) +{ + std::lock_guard hold(dataLock); + + dataChanged[name] = ++dataGeneration; +} + unsigned thUtil::dataFilesGeneration () { return dataGeneration.load(); } + +unsigned thUtil::dataFileGeneration (const string &name) +{ + std::lock_guard hold(dataLock); + const auto i = dataChanged.find(name); + + return i == dataChanged.end() ? 0 : i->second; +} diff --git a/libthink/thUtil.h b/libthink/thUtil.h index 279cbd8a..b49972fb 100644 --- a/libthink/thUtil.h +++ b/libthink/thUtil.h @@ -43,7 +43,10 @@ class THINK_API thUtil { * Linux, ~/Library/Application Support/thinksynth on macOS, * %LOCALAPPDATA%\thinksynth on Windows. Where a downloaded sample pack * goes, because an install directory is usually not writable; empty - * if no home directory can be found. */ + * if no home directory can be found. Inside a Flatpak it is the + * host's ~/.local/share/thinksynth, not the sandbox's own data + * directory: that is where scripts/packs.py, run on the host, puts + * them, and the manifest grants the home directory. */ static string userDataDir (void); /* Find a data file that something referred to by bare name. @@ -150,6 +153,14 @@ class THINK_API thUtil { * cached "not found" retries when the count has moved on. */ static void dataFilesChanged (); static unsigned dataFilesGeneration (); + + /* One file, by the name findDataFile is given ("samples/kick.wav"): + * what a host calls once it has written over it. A reader holding the + * old contents reads it again when the file's generation is past the + * count it took. dataFileGeneration is 0 for a file never named, and + * takes a lock, so a reader asks only once the count has moved. */ + static void dataFileChanged (const string &name); + static unsigned dataFileGeneration (const string &name); }; #endif /* TH_UTIL_H */ diff --git a/plugins/osc/sampleslot.h b/plugins/osc/sampleslot.h index 613404d5..b8ae60a2 100644 --- a/plugins/osc/sampleslot.h +++ b/plugins/osc/sampleslot.h @@ -61,7 +61,9 @@ * the same thing the reader returns for an empty file, and both play as * silence -- until the host says new files have arrived * (thUtil::dataFilesChanged), when it is read once more. That is a - * sample pack downloaded while an instrument naming it is loaded. + * sample pack downloaded while an instrument naming it is loaded. A file + * the host says it wrote over (thUtil::dataFileChanged) is read again + * even if it was found: a pack updated while it plays. * * ZONES. A `file' of the form "a.wav@48 b.wav@55 c.wav@62" is a set of * recordings with the MIDI note each was made at; thSampleZoneFor picks @@ -536,14 +538,27 @@ thSampleGet (const thPlugin *plugin, const std::string &name, unsigned rate) if (slot == NULL || slot->table == NULL) return NULL; - std::map::const_iterator i = + std::map::iterator i = slot->table->find(name); const unsigned generation = thUtil::dataFilesGeneration(); - if (i != slot->table->end() && - (!i->second.frames.empty() || i->second.generation == generation)) - return &i->second; + /* Kept unless the count has moved since it was read, and then kept + only if it was found and nothing has written over it since. */ + if (i != slot->table->end()) + { + thSampleData &kept = i->second; + + if (kept.generation == generation) + return &kept; + + if (!kept.frames.empty() && + thUtil::dataFileGeneration("samples/" + name) <= kept.generation) + { + kept.generation = generation; + return &kept; + } + } thSampleData &entry = (*slot->table)[name]; entry.generation = generation; @@ -588,9 +603,28 @@ thSampleGet (const thPlugin *plugin, const std::string &name, unsigned rate) return &entry; } +/* Zoned if some word ends in `@' and a note number: a plain file name may + have an `@' anywhere else. */ static inline bool thSampleIsZoned (const std::string &text) { - return text.find('@') != std::string::npos; + for (size_t at = text.find('@'); at != std::string::npos; + at = text.find('@', at + 1)) + { + const size_t end = text.find_first_of(" \t\n,", at); + const std::string num = + text.substr(at + 1, end == std::string::npos ? std::string::npos + : end - at - 1); + char *stop = NULL; + + if (!num.empty()) + { + strtod(num.c_str(), &stop); + if (*stop == 0) + return true; + } + } + + return false; } /* The zone of a zoned `file' nearest `note', or NULL for a list with no diff --git a/scripts/statecheck.cpp b/scripts/statecheck.cpp index aa72a8c1..3510cd91 100644 --- a/scripts/statecheck.cpp +++ b/scripts/statecheck.cpp @@ -4687,7 +4687,8 @@ static void checkSampleZones (const string &pluginPath) const vector low(4000, 0.25f), high(4000, 0.5f); if (ec || !writeWav(dir + "/samples/low.wav", low, TH_DEFAULT_SAMPLES) || - !writeWav(dir + "/samples/high.wav", high, TH_DEFAULT_SAMPLES)) + !writeWav(dir + "/samples/high.wav", high, TH_DEFAULT_SAMPLES) || + !writeWav(dir + "/samples/take@2.wav", high, TH_DEFAULT_SAMPLES)) { fail("osc::sample zones: could not write the scratch wavs", ""); return; @@ -4707,6 +4708,15 @@ static void checkSampleZones (const string &pluginPath) { 30, 0.25f, "a note under every zone plays the lowest" }, }; + { + string why; + const float got = zoneLevel(pluginPath, "take@2.wav", 60, why); + + okOrFail(fabsf(got - 0.5f) < 1e-3f, + "osc::sample: a plain name with an `@' in it is a file, " + "not zones", why.empty() ? "came out at " + num(got) : why); + } + for (const auto &c : cases) { string why; @@ -4755,6 +4765,22 @@ static void checkSampleZones (const string &pluginPath) okOrFail(fabsf(after - 0.5f) < 1e-3f, "osc::sample: once it does, the file plays", "came out at " + num(after)); + + /* Written over while it plays: the old frames until the host + names the file, the new ones after. */ + const bool rewrote = writeWav(dir + "/samples/late.wav", low, + TH_DEFAULT_SAMPLES); + const float stale = window(); + + thUtil::dataFileChanged("samples/late.wav"); + + const float fresh = window(); + + okOrFail(rewrote && fabsf(stale - 0.5f) < 1e-3f && + fabsf(fresh - 0.25f) < 1e-3f, + "osc::sample: a file written over is read again once " + "the host names it, and not before", + "before " + num(stale) + ", after " + num(fresh)); } } diff --git a/wasm/web/CMakeLists.txt b/wasm/web/CMakeLists.txt index 91f321f8..0b6febae 100644 --- a/wasm/web/CMakeLists.txt +++ b/wasm/web/CMakeLists.txt @@ -229,7 +229,7 @@ target_link_options(thinkweb PRIVATE -sENVIRONMENT=web,node -sALLOW_MEMORY_GROWTH=1 -sSTACK_SIZE=1MB - -sEXPORTED_RUNTIME_METHODS=ccall,UTF8ToString,HEAPF32,HEAPF64,HEAP32,HEAPU32) + -sEXPORTED_RUNTIME_METHODS=ccall,UTF8ToString,HEAPF32,HEAPF64,HEAP32,HEAPU32,HEAPU8) # --------------------------------------------------------------------------- # The site @@ -436,6 +436,7 @@ set(BUNDLE_SOURCES jam.js chat.js room.js mesh.js editor.js doc.js clock.js midi.js midiout.js midioutui.js ../tape.mjs canvasview.js composerview.js nodeview.js rollview.js tapediff.js offline.js layouts.js sourcebox.js seqview.js + packs.js thinklang.js bundle.mjs package-lock.json) set(BUNDLE_DEPENDS "") diff --git a/wasm/web/engine.js b/wasm/web/engine.js index ba49eebf..c69108a3 100644 --- a/wasm/web/engine.js +++ b/wasm/web/engine.js @@ -82,9 +82,16 @@ export function apply (M, m, host = NOWHERE) osc::sample. `array' rather than `string' because a wav has a NUL in its header before it has anything else, and a length beside it for the same reason. */ - M.ccall('tw_sample', 'number', - ['string', 'array', 'number'], - [m.name, m.bytes, m.bytes.length]); + { + /* Through the heap, not ccall's `array', which copies onto + a one-megabyte stack: a recording can be longer. */ + const at = M._tw_alloc(m.bytes.length); + + M.HEAPU8.set(m.bytes, at); + M.ccall('tw_sample', 'number', ['string', 'number', 'number'], + [m.name, at, m.bytes.length]); + M._tw_free(at); + } return true; case 'patch': diff --git a/wasm/web/thinkweb.cpp b/wasm/web/thinkweb.cpp index 2d4986f8..96441bb3 100644 --- a/wasm/web/thinkweb.cpp +++ b/wasm/web/thinkweb.cpp @@ -2604,6 +2604,19 @@ EMSCRIPTEN_KEEPALIVE const char *tw_gens_json (void) return gensJson_.c_str(); } +/* Heap for a file on its way to tw_sample. ccall's `array' copies onto the + stack, which is a megabyte, and a long recording is more than that; so a + caller asks for room here, copies in, and hands the pointer over. */ +EMSCRIPTEN_KEEPALIVE char *tw_alloc (int len) +{ + return len > 0 ? (char *)malloc((size_t)len) : NULL; +} + +EMSCRIPTEN_KEEPALIVE void tw_free (char *p) +{ + free(p); +} + /* And the same for a wav, which is the one shipped file that is not text. * * osc::sample looks a file up with thUtil::findDataFile under `samples/', @@ -2636,10 +2649,10 @@ EMSCRIPTEN_KEEPALIVE int tw_sample (const char *name, const char *bytes, fwrite(bytes, 1, (size_t)len, f) == (size_t)len; const bool ok = fclose(f) == 0 && whole; - /* A pack downloaded while an instrument naming it is loaded: what - osc::sample cached as missing is read again. */ + /* A pack downloaded while an instrument naming it is loaded, or + updated while one plays: osc::sample reads this file again. */ if (ok) - thUtil::dataFilesChanged(); + thUtil::dataFileChanged(name); return ok ? 1 : 0; } From bb28f96a29ac10266aba24b772a7eeda43bed418 Mon Sep 17 00:00:00 2001 From: Misha Nasledov Date: Sun, 4 Oct 2026 14:55:18 -0700 Subject: [PATCH 16/17] web: sample pack downloads fixes from review Files are kept under their pack's version, so an update writes beside the pack that works and clears it only once the new manifest is in; a failed one leaves the old pack playing. One download per pack runs at a time, and the prompt's button waits for it. A finished pack goes into the synth there is then, which may have started during the download. Start no longer waits for kept packs to decode. The room page loads kept packs. Offline, the pane lists what is kept. Names that would leave their pack's directory are not handed to the synth. pagetest checks the decoded WAV a pack becomes; pwatest checks the offline index and that the packs' cache outlives a new version. --- wasm/web/jam.js | 6 ++ wasm/web/main.js | 28 +++++-- wasm/web/packs.js | 169 +++++++++++++++++++++++++++--------------- wasm/web/pagetest.mjs | 30 ++++++++ wasm/web/pwatest.mjs | 21 +++++- 5 files changed, 187 insertions(+), 67 deletions(-) diff --git a/wasm/web/jam.js b/wasm/web/jam.js index cec50367..e21aab29 100644 --- a/wasm/web/jam.js +++ b/wasm/web/jam.js @@ -63,6 +63,7 @@ import { MidiOutControls } from './midioutui.js'; import { keepOffline } from './offline.js'; import { moveLayouts } from './layouts.js'; import * as patch from './patch.js'; +import * as packs from './packs.js'; import { createRollView, showClock } from './rollview.js'; import { Room } from './room.js'; import { createSeqView } from './seqview.js'; @@ -2165,6 +2166,11 @@ async function start () kit.forEach((n, i) => synth.sample(n, new Uint8Array(wavs[i]))); + /* And the sample packs this browser has kept (packs.js), so a + sampled instrument in a room plays as it does on the solo page. + Downloading one is the solo page's business. */ + packs.loadInstalled(synth).catch(() => {}); + /* Into the module's own MEMFS, which is where a .patch's `dsp' line is resolved from -- the same handover the solo page does, and the same one a piece's `instrument' block relies on. It diff --git a/wasm/web/main.js b/wasm/web/main.js index 177ef985..b3757ff0 100644 --- a/wasm/web/main.js +++ b/wasm/web/main.js @@ -883,7 +883,7 @@ const packTitle = (id) => packsOffered.find((p) => p.id === id)?.title ?? id; /* Downloaded, and the version the site offers now. */ const packCurrent = (p) => packsHave.has(p.id) && - packsHave.get(p.id) === p.version; + packsHave.get(p.id).version === p.version; const megabytes = (bytes) => `${(bytes / 1e6).toFixed(1)} MB`; async function showPackNeed () @@ -909,16 +909,25 @@ async function showPackNeed () async function getPacks (ids) { + $('packget').disabled = true; + for (const id of ids) { const title = packTitle(id); try { - const m = await packs.download(id, synth, (done, total) => + const m = await packs.download(id, (done, total) => status(`Downloading ${title}: ` + `${Math.round(100 * done / total)}%`)); - packsHave.set(id, m.version); + + packsHave.set(id, m); + + /* Into the synth there is now, which may have started while + this was downloading. */ + if (synth !== null) + await packs.load(m, synth); + status(`${title} downloaded.`); } catch (e) @@ -928,6 +937,7 @@ async function getPacks (ids) } } + $('packget').disabled = false; drawPacks(); showPackNeed(); } @@ -2364,10 +2374,12 @@ async function start () kit.forEach((name, i) => synth.sample(name, new Uint8Array(wavs[i]))); - /* And any sample pack somebody downloaded before. A pack that + /* And any sample pack somebody downloaded before, without holding + Start for it: osc::sample reads a file once it arrives, so an + instrument plays its pack as soon as the pack is in. A pack that will not read is a missing instrument, not a page that will not start. */ - await packs.loadInstalled(synth).catch((e) => log(e.message)); + packs.loadInstalled(synth).catch((e) => log(e.message)); } catch (e) { @@ -3337,6 +3349,12 @@ async function init () [packsOffered, packsHave] = await Promise.all([packs.available(), packs.installed() .catch(() => new Map())]); + + /* Offline the site offers nothing, but what is kept is still listed, + to be seen and removed. */ + if (packsOffered.length === 0) + packsOffered = [...packsHave.values()]; + drawPacks(); /* What is in play may have been chosen before the list came. */ diff --git a/wasm/web/packs.js b/wasm/web/packs.js index a992c72e..2ef0e95a 100644 --- a/wasm/web/packs.js +++ b/wasm/web/packs.js @@ -14,11 +14,14 @@ * and a pack downloaded once plays offline from then on. * * A PACK IS INSTALLED WHEN ITS pack.json IS IN THE CACHE, and that goes - * in last: a download that stopped half way leaves wavs and no manifest, - * which reads as not installed and is cleared by the next attempt. The - * manifest carries the pack's `version', a hash of its files, and a kept - * pack whose version is not the one offered is one to download again -- - * an instrument may name recordings the old one does not have. + * in last. The manifest carries the pack's `version', a hash of its files, + * and every file is kept under its version (`?v='), so an update writes + * the new files beside the old ones, puts the new manifest over the old + * only once all of them are in, and only then clears the old files: an + * update that fails half way leaves the pack that worked. A kept pack + * whose version is not the one offered is one to download again -- an + * instrument may name recordings the old one does not have. One download + * per pack at a time; asking again while one runs joins it. * * SERVED AS FLAC, about half the size of the WAV it decodes to, and kept * that way. Each file is listed under the .wav name its instrument plays and @@ -26,21 +29,30 @@ * samples when it is handed to the synth. * * Into the synth the way the shipped kit goes: as WAV bytes, through - * synth.sample, under samples//. osc::sample reads a file it once - * found missing again once new ones arrive (thUtil::dataFilesChanged), so - * an instrument that was loaded before its pack is heard as soon as the - * pack is in, without loading anything again. + * synth.sample, under samples//, once a pack is complete -- into + * whichever synth there is then, which may have started while it was + * downloading. osc::sample reads a file again once the host names it as + * written (thUtil::dataFileChanged), so an instrument loaded before its + * pack, or playing an older version, is heard with the new files without + * loading anything again. */ const CACHE = 'thinksynth-packs'; const manifestUrl = (id) => `packs/${id}/pack.json`; -const fileUrl = (f) => `packs/${f.url ?? f.name}`; + +/* A file's key in the cache: where it is served, and the version of the + pack it belongs to. */ +const fileKey = (m, f) => `packs/${f.url ?? f.name}?v=${m.version ?? ''}`; + +/* A name to hand the synth: one under its pack, going nowhere else. */ +const safe = (m, f) => f.name.startsWith(`${m.id}/`) && + !f.name.split('/').includes('..'); /* A file as the WAV osc::sample reads: as it came if it is one, decoded if it is FLAC, at the rate the packs are built at so nothing is resampled twice. */ -async function wavBytes (f, bytes) +export async function wavBytes (f, bytes) { if (!/\.flac$/.test(f.url ?? '')) return new Uint8Array(bytes); @@ -90,32 +102,50 @@ export async function available () } } -async function manifests () +/* `has' before `open', which would make the cache: a page that never + downloads a pack leaves no trace of the feature in its storage. */ +async function cache () { - /* `has' before `open', which would make the cache: a page that never - downloads a pack leaves no trace of the feature in its storage. */ - if (!('caches' in self) || !await caches.has(CACHE)) - return []; + return ('caches' in self) && await caches.has(CACHE) + ? caches.open(CACHE) : null; +} + +/* The packs downloaded so far, id to manifest. */ +export async function installed () +{ + const c = await cache(); + const out = new Map(); - const cache = await caches.open(CACHE); - const out = []; + if (c === null) + return out; - for (const req of await cache.keys()) + for (const req of await c.keys()) if (/\/packs\/[^/]+\/pack\.json$/.test(new URL(req.url).pathname)) - out.push(await (await cache.match(req)).json()); + { + const m = await (await c.match(req)).json(); + + out.set(m.id, m); + } return out; } -/* The packs downloaded so far, id to version. */ -export async function installed () +/* The downloads under way, by pack. */ +const running = new Map(); + +/* Downloads a pack into the cache, `progress(done, total)' in bytes along + the way, and resolves to its manifest once it is complete. A second ask + for the same pack while this runs gets this one. */ +export function download (id, progress = () => {}) { - return new Map((await manifests()).map((m) => [m.id, m.version])); + if (!running.has(id)) + running.set(id, fetchPack(id, progress) + .finally(() => running.delete(id))); + + return running.get(id); } -/* Downloads a pack into the cache, `progress(done, total)' in bytes along - the way, and hands it to `synth' if there is one. */ -export async function download (id, synth, progress = () => {}) +async function fetchPack (id, progress) { const r = await fetch(manifestUrl(id), { cache: 'no-cache' }); @@ -123,83 +153,102 @@ export async function download (id, synth, progress = () => {}) throw new Error(`${manifestUrl(id)}: ${r.status} ${r.statusText}`); const m = await r.json(); - const cache = await caches.open(CACHE); + const c = await caches.open(CACHE); + const kept = (await installed()).get(id); + const queue = m.files.filter((f) => safe(m, f)); let done = 0; - await remove(id); - /* Four at a time: enough to fill a connection, few enough that a progress bar moves. */ - const queue = [...m.files]; - const worker = async () => { for (let f; (f = queue.shift()) !== undefined;) { - const url = fileUrl(f); + const url = `packs/${f.url ?? f.name}`; const got = await fetch(url, { cache: 'no-cache' }); if (!got.ok) throw new Error(`${url}: ${got.status} ${got.statusText}`); - const bytes = await got.arrayBuffer(); - - /* A copy into the cache: decoding detaches what it is given. */ - await cache.put(url, new Response(bytes.slice(0))); - - if (synth) - synth.sample(`samples/${f.name}`, await wavBytes(f, bytes)); + await c.put(fileKey(m, f), new Response(await got.arrayBuffer())); done += f.bytes; progress(done, m.bytes); } }; - /* Every worker settled before any failure is reported: one that is - still fetching would otherwise go on writing the cache and the - progress under a retry that has already started. */ + /* Every worker settled before any failure is reported, so nothing is + still writing when the failure is. */ const ends = await Promise.allSettled([worker(), worker(), worker(), worker()]); const failed = ends.find((e) => e.status === 'rejected'); if (failed) + { + /* What this attempt wrote, unless it is the version that is kept, + whose files are the same ones. */ + if (kept?.version !== m.version) + await clear(c, id, (v) => v === m.version); + throw failed.reason; + } - await cache.put(manifestUrl(id), new Response(JSON.stringify(m))); + await c.put(manifestUrl(id), new Response(JSON.stringify(m))); + await clear(c, id, (v) => v !== m.version); navigator.storage?.persist?.().catch(() => {}); return m; } +/* A pack's files whose version `which' picks. */ +async function clear (c, id, which) +{ + for (const req of await c.keys()) + { + const url = new URL(req.url); + + if (url.pathname.includes(`/packs/${id}/`) && + !url.pathname.endsWith('/pack.json') && + which(url.searchParams.get('v') ?? '')) + await c.delete(req); + } +} + export async function remove (id) { - if (!await caches.has(CACHE)) - return; + const c = await cache(); - const cache = await caches.open(CACHE); + if (c === null) + return; - for (const req of await cache.keys()) + for (const req of await c.keys()) if (new URL(req.url).pathname.includes(`/packs/${id}/`)) - await cache.delete(req); + await c.delete(req); } -/* Every installed pack into `synth', at Start. */ -export async function loadInstalled (synth) +/* A kept pack into `synth'. */ +export async function load (m, synth) { - if (!('caches' in self) || !await caches.has(CACHE)) + const c = await cache(); + + if (c === null) return; - const cache = await caches.open(CACHE); + for (const f of m.files.filter((f) => safe(m, f))) + { + const hit = await c.match(fileKey(m, f)); - for (const m of await manifests()) - for (const f of m.files) - { - const hit = await cache.match(fileUrl(f)); + if (hit) + synth.sample(`samples/${f.name}`, + await wavBytes(f, await hit.arrayBuffer())); + } +} - if (hit) - synth.sample(`samples/${f.name}`, - await wavBytes(f, await hit.arrayBuffer())); - } +/* Every kept pack into `synth', at Start, side by side. */ +export async function loadInstalled (synth) +{ + await Promise.all([...(await installed()).values()] + .map((m) => load(m, synth))); } /* The packs these graph texts play from: a sampled instrument names its diff --git a/wasm/web/pagetest.mjs b/wasm/web/pagetest.mjs index f49dfbfb..e2570aab 100644 --- a/wasm/web/pagetest.mjs +++ b/wasm/web/pagetest.mjs @@ -2748,6 +2748,36 @@ try check(await pg.$eval('#packlist button', (b) => b.textContent) === 'Remove', 'and the pane says it is there'); + /* What the synth is handed for it: the FLAC decoded to a 16-bit + mono WAV at 44.1 kHz, of the length it was, at its level -- the + fixture is ffmpeg's sine, an eighth of full scale. */ + const decoded = await pg.evaluate(async (b64) => + { + const { wavBytes } = await import('./packs.js'); + const bytes = Uint8Array.from(atob(b64), (c) => c.charCodeAt(0)); + const w = await wavBytes({ url: 'x.flac' }, bytes.buffer); + const v = new DataView(w.buffer); + const tag = (at) => String.fromCharCode(...w.slice(at, at + 4)); + let peak = 0; + + for (let i = 44; i + 1 < w.length; i += 2) + peak = Math.max(peak, Math.abs(v.getInt16(i, true))); + + return { riff: tag(0), wave: tag(8), data: tag(36), + channels: v.getUint16(22, true), + rate: v.getUint32(24, true), + bits: v.getUint16(34, true), + frames: v.getUint32(40, true) / 2, peak }; + }, flac.toString('base64')); + + check(decoded.riff === 'RIFF' && decoded.wave === 'WAVE' && + decoded.data === 'data' && decoded.channels === 1 && + decoded.rate === 44100 && decoded.bits === 16 && + decoded.frames === 221 && + Math.abs(decoded.peak - 32768 / 8) < 64, + 'a pack\'s FLAC is handed over as the WAV osc::sample reads: ' + + JSON.stringify(decoded)); + const again = await open(); check(await quiet(again), diff --git a/wasm/web/pwatest.mjs b/wasm/web/pwatest.mjs index e3567dc0..5b00db85 100644 --- a/wasm/web/pwatest.mjs +++ b/wasm/web/pwatest.mjs @@ -198,6 +198,13 @@ try null, { timeout: 60000 }); check(true, 'offline, the synth starts: worklet, module and mirror'); + /* The sample packs' index is the network's, and offline the worker + answers it with an empty list rather than a failed load. */ + check(await page.evaluate(async () => + JSON.stringify(await (await fetch('packs/index.json')).json()) + === '[]'), + 'offline, the sample packs\' index is an empty list'); + await page.selectOption('#mode', 'patch'); await page.evaluate(() => window.solo.settled()); await page.selectOption('#patch', PATCH); @@ -221,6 +228,11 @@ try await context.setOffline(false); + /* A pack somebody downloaded, which no new version may clear. */ + await page.evaluate(async () => + (await caches.open('thinksynth-packs')) + .put('packs/kept/pack.json', new Response('{"id":"kept"}'))); + const text = fs.readFileSync(path.join(site, 'sw.js'), 'utf8'); const old = /^const VERSION = "(\w+)";$/m.exec(text)[1]; const next = 'f'.repeat(old.length) === old ? '0'.repeat(old.length) @@ -282,11 +294,14 @@ try await again; await loaded(page); - const after = await caches(page); + const all = await caches(page); + const after = all.filter((c) => c !== 'thinksynth-packs'); check(after.length === 1 && after[0] === `thinksynth-${next}`, `the next load activates it and the old cache goes: ` + after.join(', ')); + check(all.includes('thinksynth-packs'), + 'and the sample packs\' cache stays'); check(await page.evaluate(async () => (await navigator.serviceWorker.getRegistration()).waiting @@ -308,7 +323,9 @@ try the page as its source, without the variables around it. */ const only = (v) => until(page, async (want) => { - const keys = await caches.keys(); + /* The site's caches; the sample packs' outlives every version. */ + const keys = (await caches.keys()) + .filter((k) => k !== 'thinksynth-packs'); return keys.length === 1 && keys[0] === `thinksynth-${want}`; }, 60000, v); From 4b08fd8e93f60ad2be1b463f45184af3444bb2b2 Mon Sep 17 00:00:00 2001 From: Misha Nasledov Date: Sun, 4 Oct 2026 14:55:18 -0700 Subject: [PATCH 17/17] packs: install and the workflow, fixes from review install refuses manifest names that leave the samples directory, downloads only the packs named, says what to do before the release exists, and extracts with tarfile's data filter where Python has it and the same check by hand where it does not (macOS ships 3.9.6). The workflow runs only on master, takes only a 404 as a free tag, and runs the ci workflow on master once the release is made, since the deploy that came with the merge ran before it. --- .github/workflows/packs.yml | 22 ++++++++++++++--- README.md | 6 +++-- scripts/packs.py | 49 ++++++++++++++++++++++++++++--------- 3 files changed, 59 insertions(+), 18 deletions(-) diff --git a/.github/workflows/packs.yml b/.github/workflows/packs.yml index 2cf06f4a..19e6ef12 100644 --- a/.github/workflows/packs.yml +++ b/.github/workflows/packs.yml @@ -8,9 +8,12 @@ # # By hand, and apart from a thinksynth release: the packs change when a pack # is added or rebuilt, not when the program does. To publish, bump RELEASE, -# merge, and run this. It refuses a tag that already exists -- a release the -# deploy points at is not replaced under it -- and marks its release as not -# the latest, so the program's own releases stay what /releases/latest is. +# merge, and run this on master. It refuses a tag that already exists -- a +# release the deploy points at is not replaced under it -- marks its +# release as not the latest, so the program's own releases stay what +# /releases/latest is, and then runs the ci workflow on master, whose +# deploy is what puts the release on the site: the deploy that came with +# the merge ran before the release existed. name: sample packs @@ -19,10 +22,12 @@ on: permissions: contents: write + actions: write jobs: publish: name: build and publish the sample packs + if: github.ref == 'refs/heads/master' runs-on: ubuntu-latest steps: @@ -40,9 +45,13 @@ jobs: GH_TOKEN: ${{ github.token }} run: | tag=$(python3 scripts/packs.py tag) - if gh api "repos/$GITHUB_REPOSITORY/git/ref/tags/$tag" > /dev/null 2>&1; then + # Free only on a 404: any other failure is not an answer. + if out=$(gh api "repos/$GITHUB_REPOSITORY/git/ref/tags/$tag" 2>&1); then echo "::error::the tag $tag exists already; bump RELEASE in scripts/packs.py" exit 1 + elif ! grep -q "HTTP 404" <<< "$out"; then + echo "::error::could not look up the tag $tag: $out" + exit 1 fi echo "TAG=$tag" >> "$GITHUB_ENV" @@ -61,3 +70,8 @@ jobs: --target "$GITHUB_SHA" --latest=false \ --title "Sample packs ($TAG)" \ --notes "Recorded instruments for the sampled_*.dsp graphs, built by scripts/packs.py at $GITHUB_SHA. Public domain (CC0): VSCO 2 Community Edition (Versilian Studios / Sam Gossner) and Big Rusty Drums (Karoryfer Samples)." + + - name: The site, deployed with it + env: + GH_TOKEN: ${{ github.token }} + run: gh workflow run ci.yml --ref master diff --git a/README.md b/README.md index 64349942..d2ada5a1 100644 --- a/README.md +++ b/README.md @@ -188,8 +188,10 @@ repository: they are published as a release of their own, built by the `sample packs` workflow. `scripts/packs.py install` (it needs ffmpeg) downloads it into this user's data directory, where thinksynth looks — `~/.local/share/thinksynth` on Linux, `~/Library/Application Support/thinksynth` -on macOS, `%LOCALAPPDATA%\thinksynth` on Windows; the page offers each pack -for download when something in play needs it. +on macOS, `%LOCALAPPDATA%\thinksynth` on Windows, and the first of those +for the Flatpak too. A running thinksynth finds a newly installed pack the +next time it loads an instrument; the page offers each pack for download +when something in play needs it. If you want JACK on Linux, start `jackd` before thinksynth — RtAudio will use the running server. diff --git a/scripts/packs.py b/scripts/packs.py index bc0f5c6d..928cfaf5 100755 --- a/scripts/packs.py +++ b/scripts/packs.py @@ -305,16 +305,42 @@ def archive(outdir): t.add(os.path.join(outdir, p["id"]), arcname=p["id"]) -def fetch(outdir, missing_ok=False): - """RELEASE's packs into outdir; the index, or None if there is none.""" +def inside(root, name): + """root/name, if it stays under root; None if it would leave it.""" + path = os.path.realpath(os.path.join(root, name)) + top = os.path.realpath(root) + return path if os.path.commonpath([path, top]) == top else None + + +def extract(t, outdir): + """A tar's files into outdir and nowhere else. tarfile's own `data' + filter where this Python has it (3.12, and backported to 3.8.17, + 3.9.17, 3.10.12, 3.11.4); otherwise the same refusal by hand, since + the python3 macOS ships is 3.9.6.""" + if hasattr(tarfile, "data_filter"): + t.extractall(outdir, filter="data") + return + for m in t.getmembers(): + if not (m.isfile() or m.isdir()) or inside(outdir, m.name) is None: + sys.exit(f"refusing {m.name} in a pack") + t.extractall(outdir) + + +def fetch(outdir, missing_ok=False, names=()): + """RELEASE's packs into outdir -- those named, or all of them; the + index, or None if there is none.""" try: with urllib.request.urlopen(asset("index.json")) as r: index = json.load(r) except urllib.error.HTTPError as e: - if e.code == 404 and missing_ok: + if e.code != 404: + raise + if missing_ok: print(f"::warning::{RELEASE} is not published; no sample packs") return None - raise + sys.exit(f"{RELEASE} is not published yet: run the `sample packs' " + "workflow, or build from the sources with `packs.py build'") + index = [p for p in index if not names or p["id"] in names] os.makedirs(outdir, exist_ok=True) for p in index: with urllib.request.urlopen(asset(p["id"] + ".tar")) as r, \ @@ -322,7 +348,7 @@ def fetch(outdir, missing_ok=False): tmp.write(r.read()) tmp.seek(0) with tarfile.open(fileobj=tmp) as t: - t.extractall(outdir, filter="data") + extract(t, outdir) print(f"{p['id']}: {p['bytes'] / 1e6:.1f} MB") with open(os.path.join(outdir, "index.json"), "w") as f: json.dump(index, f, indent=1) @@ -333,17 +359,16 @@ def install(names): """RELEASE, decoded to .wav, into the user's samples directory.""" dest = user_samples() with tempfile.TemporaryDirectory() as tmp: - index = fetch(tmp) - for p in index: - if names and p["id"] not in names: - continue + for p in fetch(tmp, names=names): with open(os.path.join(tmp, p["id"], "pack.json")) as f: m = json.load(f) for f in m["files"]: - out = os.path.join(dest, f["name"]) + src = inside(tmp, f.get("url", f["name"])) + out = inside(dest, f["name"]) + if src is None or out is None: + sys.exit(f"refusing {f['name']} in {p['id']}") os.makedirs(os.path.dirname(out), exist_ok=True) - subprocess.run(["ffmpeg", "-v", "error", "-y", "-i", - os.path.join(tmp, f.get("url", f["name"])), + subprocess.run(["ffmpeg", "-v", "error", "-y", "-i", src, out], check=True) print(f"{p['id']}: installed in {dest}")