From 71afb03d85663bf9fc00e1af1e587f54e29a709c Mon Sep 17 00:00:00 2001 From: Miklos Szel Date: Thu, 13 Aug 2026 09:26:29 +0200 Subject: [PATCH 1/2] FIX: rclone - directory listing broken by rclone log messages rclone writes log messages to stderr, e.g. the SFTP backend notice NOTICE: het: No host key validation is being performed. Set known_hosts_file to enable it. but RunCommand ran the child process with poStderrToOutPut, so those lines were prepended to the stdout of "lsjson". The resulting text is not valid JSON, GetJSON raised an exception, ParseLsJson swallowed it and returned an empty list, and FsFindFirstW ended up reporting an empty/unreadable directory. Capture stdout and stderr on separate pipes instead, draining both in one loop so neither pipe buffer can fill up and block the child. As a side effect LastError now holds the actual rclone error message rather than a mix of both streams. RunCommandWithProgress keeps poStderrToOutPut on purpose, that is where --progress writes its output. Also make the parsers tolerant of unexpected output: ParseLsJson skips anything printed before the JSON array, and ParseListRemotes only accepts lines ending with a colon, which is the format listremotes prints. --- wfx/rclone/src/urclonecli.pas | 100 ++++++++++++++++++++------------- wfx/rclone/src/urclonejson.pas | 28 ++++++--- 2 files changed, 81 insertions(+), 47 deletions(-) diff --git a/wfx/rclone/src/urclonecli.pas b/wfx/rclone/src/urclonecli.pas index 51cfd88..9287f9d 100644 --- a/wfx/rclone/src/urclonecli.pas +++ b/wfx/rclone/src/urclonecli.pas @@ -27,7 +27,7 @@ interface uses - Classes, SysUtils, Process, WfxPlugin, uRcloneJson; + Classes, SysUtils, Process, Pipes, WfxPlugin, uRcloneJson; type TProgressCallback = function(SourceName, TargetName: PWideChar; @@ -40,7 +40,7 @@ TRcloneCli = class FProgressProc: TProgressCallback; FPluginNr: Integer; function RunCommand(const Args: array of AnsiString; out Output: AnsiString; - out ExitCode: Integer): Boolean; + out ErrorOutput: AnsiString; out ExitCode: Integer): Boolean; function RunCommandWithProgress(const Args: array of AnsiString; const SourceName, TargetName: UnicodeString; out ExitCode: Integer): Boolean; @@ -159,23 +159,44 @@ function GetRcloneConfigPath: AnsiString; Result := GetHomeDir + '/.config/rclone/rclone.conf'; end; +{ Read everything currently buffered in a pipe without blocking. + Returns the number of bytes appended to AStream. } +function DrainPipe(APipe: TInputPipeStream; AStream: TStringStream): Integer; +var + Buffer: array[0..4095] of Byte; + Avail, BytesRead: Integer; +begin + Result := 0; + if APipe = nil then Exit; + repeat + Avail := APipe.NumBytesAvailable; + if Avail <= 0 then Break; + if Avail > SizeOf(Buffer) then + Avail := SizeOf(Buffer); + BytesRead := APipe.Read(Buffer, Avail); + if BytesRead <= 0 then Break; + AStream.Write(Buffer, BytesRead); + Inc(Result, BytesRead); + until False; +end; + function TRcloneCli.RunCommand(const Args: array of AnsiString; - out Output: AnsiString; out ExitCode: Integer): Boolean; + out Output: AnsiString; out ErrorOutput: AnsiString; out ExitCode: Integer): Boolean; var AProcess: TProcess; I: Integer; - OutputStream: TStringStream; - BytesRead: Integer; - Buffer: array[0..4095] of Byte; + OutputStream, ErrorStream: TStringStream; ConfigPath: AnsiString; begin Result := False; Output := ''; + ErrorOutput := ''; ExitCode := -1; FLastError := ''; AProcess := TProcess.Create(nil); OutputStream := TStringStream.Create(''); + ErrorStream := TStringStream.Create(''); try AProcess.Executable := FRclonePath; @@ -194,30 +215,27 @@ function TRcloneCli.RunCommand(const Args: array of AnsiString; AProcess.Environment.Add('PATH=/usr/local/bin:/opt/homebrew/bin:/usr/bin:/bin:/usr/sbin:/sbin'); AProcess.Environment.Add('HOME=' + GetHomeDir); - AProcess.Options := [poUsePipes, poStderrToOutPut, poNoConsole]; + // Keep stderr separate: rclone writes log messages (e.g. the SFTP + // "No host key validation is being performed" NOTICE) to stderr, and + // merging them into stdout would corrupt the JSON of lsjson & friends. + AProcess.Options := [poUsePipes, poNoConsole]; try AProcess.Execute; - // Read output - while AProcess.Running or (AProcess.Output.NumBytesAvailable > 0) do - begin - BytesRead := AProcess.Output.Read(Buffer, SizeOf(Buffer)); - if BytesRead > 0 then - OutputStream.Write(Buffer, BytesRead) - else - Sleep(10); - end; - - // Read any remaining output + // Drain both pipes until the process is gone and nothing is left repeat - BytesRead := AProcess.Output.Read(Buffer, SizeOf(Buffer)); - if BytesRead > 0 then - OutputStream.Write(Buffer, BytesRead); - until BytesRead = 0; + if DrainPipe(AProcess.Output, OutputStream) + + DrainPipe(AProcess.Stderr, ErrorStream) = 0 then + begin + if not AProcess.Running then Break; + Sleep(10); + end; + until False; ExitCode := AProcess.ExitCode; Output := OutputStream.DataString; + ErrorOutput := ErrorStream.DataString; Result := True; except on E: Exception do @@ -227,6 +245,7 @@ function TRcloneCli.RunCommand(const Args: array of AnsiString; end; end; finally + ErrorStream.Free; OutputStream.Free; AProcess.Free; end; @@ -378,27 +397,30 @@ function TRcloneCli.RunCommandWithProgress(const Args: array of AnsiString; function TRcloneCli.ListRemotes: TStringList; var - Output: AnsiString; + Output, ErrorOutput: AnsiString; ExitCode: Integer; begin Result := nil; - if RunCommand(['listremotes'], Output, ExitCode) and (ExitCode = 0) then + if RunCommand(['listremotes'], Output, ErrorOutput, ExitCode) and (ExitCode = 0) then Result := ParseListRemotes(Output) else + begin + FLastError := ErrorOutput; Result := TStringList.Create; + end; end; function TRcloneCli.ListDirectory(const RemotePath: AnsiString): TRcloneFileList; var - Output: AnsiString; + Output, ErrorOutput: AnsiString; ExitCode: Integer; begin Result := nil; - if RunCommand(['lsjson', RemotePath], Output, ExitCode) and (ExitCode = 0) then + if RunCommand(['lsjson', RemotePath], Output, ErrorOutput, ExitCode) and (ExitCode = 0) then Result := ParseLsJson(Output) else begin - FLastError := Output; + FLastError := ErrorOutput; Result := TRcloneFileList.Create(True); end; end; @@ -441,42 +463,42 @@ function TRcloneCli.CopyToRemote(const LocalPath, RemotePath: AnsiString): Integ function TRcloneCli.DeleteFile(const RemotePath: AnsiString): Boolean; var - Output: AnsiString; + Output, ErrorOutput: AnsiString; ExitCode: Integer; begin - Result := RunCommand(['delete', RemotePath], Output, ExitCode) and (ExitCode = 0); + Result := RunCommand(['delete', RemotePath], Output, ErrorOutput, ExitCode) and (ExitCode = 0); if not Result then - FLastError := Output; + FLastError := ErrorOutput; end; function TRcloneCli.DeleteDir(const RemotePath: AnsiString): Boolean; var - Output: AnsiString; + Output, ErrorOutput: AnsiString; ExitCode: Integer; begin - Result := RunCommand(['rmdir', RemotePath], Output, ExitCode) and (ExitCode = 0); + Result := RunCommand(['rmdir', RemotePath], Output, ErrorOutput, ExitCode) and (ExitCode = 0); if not Result then - FLastError := Output; + FLastError := ErrorOutput; end; function TRcloneCli.MakeDir(const RemotePath: AnsiString): Boolean; var - Output: AnsiString; + Output, ErrorOutput: AnsiString; ExitCode: Integer; begin - Result := RunCommand(['mkdir', RemotePath], Output, ExitCode) and (ExitCode = 0); + Result := RunCommand(['mkdir', RemotePath], Output, ErrorOutput, ExitCode) and (ExitCode = 0); if not Result then - FLastError := Output; + FLastError := ErrorOutput; end; function TRcloneCli.MoveFile(const OldPath, NewPath: AnsiString): Boolean; var - Output: AnsiString; + Output, ErrorOutput: AnsiString; ExitCode: Integer; begin - Result := RunCommand(['moveto', OldPath, NewPath], Output, ExitCode) and (ExitCode = 0); + Result := RunCommand(['moveto', OldPath, NewPath], Output, ErrorOutput, ExitCode) and (ExitCode = 0); if not Result then - FLastError := Output; + FLastError := ErrorOutput; end; function RcloneExitToWfx(ExitCode: Integer; const ErrorOutput: AnsiString): Integer; diff --git a/wfx/rclone/src/urclonejson.pas b/wfx/rclone/src/urclonejson.pas index 5592f31..a82dc97 100644 --- a/wfx/rclone/src/urclonejson.pas +++ b/wfx/rclone/src/urclonejson.pas @@ -67,16 +67,27 @@ function ParseLsJson(const JsonOutput: AnsiString): TRcloneFileList; JsonData: TJSONData; JsonArray: TJSONArray; JsonObj: TJSONObject; - I: Integer; + I, StartPos: Integer; RcloneFile: TRcloneFile; + Text: AnsiString; begin Result := TRcloneFileList.Create(True); - if Trim(JsonOutput) = '' then + Text := Trim(JsonOutput); + if Text = '' then Exit; + // Be tolerant of anything printed before the array (log lines, warnings) + if Text[1] <> '[' then + begin + StartPos := Pos('[', Text); + if StartPos = 0 then + Exit; + Text := Copy(Text, StartPos, Length(Text) - StartPos + 1); + end; + try - JsonData := GetJSON(JsonOutput); + JsonData := GetJSON(Text); try if not (JsonData is TJSONArray) then Exit; @@ -134,11 +145,12 @@ function ParseListRemotes(const Output: AnsiString): TStringList; for I := 0 to Lines.Count - 1 do begin RemoteName := Trim(Lines[I]); - // rclone listremotes outputs "remotename:" - remove the trailing colon - if (Length(RemoteName) > 0) and (RemoteName[Length(RemoteName)] = ':') then - RemoteName := Copy(RemoteName, 1, Length(RemoteName) - 1); - if RemoteName <> '' then - Result.Add(RemoteName); + // rclone listremotes outputs "remotename:" - anything without the + // trailing colon is not a remote (e.g. a stray log line) + if (Length(RemoteName) < 2) or (RemoteName[Length(RemoteName)] <> ':') then + Continue; + RemoteName := Copy(RemoteName, 1, Length(RemoteName) - 1); + Result.Add(RemoteName); end; finally Lines.Free; From 3395d4607248a7c9e4d240b217cfceb1b6e9c453 Mon Sep 17 00:00:00 2001 From: Miklos Szel Date: Thu, 13 Aug 2026 13:46:11 +0200 Subject: [PATCH 2/2] UPD: rclone - update author contact email address --- wfx/rclone/build/build_linux.sh | 2 +- wfx/rclone/build/build_macos.sh | 2 +- wfx/rclone/build/build_windows.bat | 2 +- wfx/rclone/src/rclone.lpr | 2 +- wfx/rclone/src/urclonecli.pas | 2 +- wfx/rclone/src/urclonefunc.pas | 2 +- wfx/rclone/src/urclonejson.pas | 2 +- wfx/rclone/src/urcloneutil.pas | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/wfx/rclone/build/build_linux.sh b/wfx/rclone/build/build_linux.sh index 3fa45f9..a00f944 100755 --- a/wfx/rclone/build/build_linux.sh +++ b/wfx/rclone/build/build_linux.sh @@ -1,6 +1,6 @@ #!/bin/bash # Build rclone WFX plugin on Debian 12 (x86_64) -# Copyright (C) 2026 Miklos Mukka Szel +# Copyright (C) 2026 Miklos Mukka Szel # # Usage: # sudo ./build_linux.sh Build the plugin (installs dependencies) diff --git a/wfx/rclone/build/build_macos.sh b/wfx/rclone/build/build_macos.sh index 2d4cd24..39e3a46 100755 --- a/wfx/rclone/build/build_macos.sh +++ b/wfx/rclone/build/build_macos.sh @@ -1,6 +1,6 @@ #!/bin/bash # Build rclone WFX plugin on macOS using Free Pascal -# Copyright (C) 2026 Miklos Mukka Szel +# Copyright (C) 2026 Miklos Mukka Szel # # Usage: # ./build_macos.sh Build the plugin diff --git a/wfx/rclone/build/build_windows.bat b/wfx/rclone/build/build_windows.bat index 73ec47a..bc07ed6 100644 --- a/wfx/rclone/build/build_windows.bat +++ b/wfx/rclone/build/build_windows.bat @@ -2,7 +2,7 @@ setlocal enabledelayedexpansion :: Build rclone WFX plugin on Windows -:: Copyright (C) 2026 Miklos Mukka Szel +:: Copyright (C) 2026 Miklos Mukka Szel :: Requires Lazarus IDE installed :: :: Usage: diff --git a/wfx/rclone/src/rclone.lpr b/wfx/rclone/src/rclone.lpr index 17e7812..fb34130 100644 --- a/wfx/rclone/src/rclone.lpr +++ b/wfx/rclone/src/rclone.lpr @@ -5,7 +5,7 @@ ------------------------------------------------------------------------- WFX plugin for working with rclone remotes - Copyright (C) 2026 Miklos Mukka Szel + Copyright (C) 2026 Miklos Mukka Szel This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public diff --git a/wfx/rclone/src/urclonecli.pas b/wfx/rclone/src/urclonecli.pas index 9287f9d..66df89d 100644 --- a/wfx/rclone/src/urclonecli.pas +++ b/wfx/rclone/src/urclonecli.pas @@ -3,7 +3,7 @@ ------------------------------------------------------------------------- WFX plugin for working with rclone remotes - CLI wrapper - Copyright (C) 2026 Miklos Mukka Szel + Copyright (C) 2026 Miklos Mukka Szel This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public diff --git a/wfx/rclone/src/urclonefunc.pas b/wfx/rclone/src/urclonefunc.pas index 8138521..77bd324 100644 --- a/wfx/rclone/src/urclonefunc.pas +++ b/wfx/rclone/src/urclonefunc.pas @@ -3,7 +3,7 @@ ------------------------------------------------------------------------- WFX plugin for working with rclone remotes - Copyright (C) 2026 Miklos Mukka Szel + Copyright (C) 2026 Miklos Mukka Szel This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public diff --git a/wfx/rclone/src/urclonejson.pas b/wfx/rclone/src/urclonejson.pas index a82dc97..f5fad51 100644 --- a/wfx/rclone/src/urclonejson.pas +++ b/wfx/rclone/src/urclonejson.pas @@ -3,7 +3,7 @@ ------------------------------------------------------------------------- WFX plugin for working with rclone remotes - JSON parsing - Copyright (C) 2026 Miklos Mukka Szel + Copyright (C) 2026 Miklos Mukka Szel This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public diff --git a/wfx/rclone/src/urcloneutil.pas b/wfx/rclone/src/urcloneutil.pas index 50ccdf9..5c68e8b 100644 --- a/wfx/rclone/src/urcloneutil.pas +++ b/wfx/rclone/src/urcloneutil.pas @@ -3,7 +3,7 @@ ------------------------------------------------------------------------- WFX plugin for working with rclone remotes - Copyright (C) 2026 Miklos Mukka Szel + Copyright (C) 2026 Miklos Mukka Szel This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public