Add support for convert sd cpp api

.github/scripts/download_ci_models.py

@@ -22,6 +22,10 @@
         "city96/umt5-xxl-encoder-gguf",
         "umt5-xxl-encoder-Q3_K_S.gguf",
     ),
+    "MODEL_TO_CONVERT": (
+        "black-forest-labs/FLUX.2-small-decoder",
+        "diffusion_pytorch_model.safetensors",
+    ),
 }
 
 # for non-huggingface sources

.pre-commit-config.yaml

@@ -54,3 +54,4 @@ repos:
       - id: end-of-file-fixer
       - id: requirements-txt-fixer
       - id: trailing-whitespace
+        args: [--markdown-linebreak-ext=md]

README.md

@@ -8,18 +8,19 @@ High-performance diffusion model inference in pure Go.
 [![Linux](https://github.com/l8bloom/gosd/actions/workflows/linux.yaml/badge.svg)](https://github.com/l8bloom/gosd/actions/workflows/linux.yaml)
 [![Windows](https://github.com/l8bloom/gosd/actions/workflows/windows.yaml/badge.svg)](https://github.com/l8bloom/gosd/actions/workflows/windows.yaml)
 [![macOS](https://github.com/l8bloom/gosd/actions/workflows/macos.yaml/badge.svg)](https://github.com/l8bloom/gosd/actions/workflows/macos.yaml)
-[![stable-diffusion.cpp](https://img.shields.io/badge/sd.cpp-6614334-yellow)](https://github.com/leejet/stable-diffusion.cpp/releases/tag/master-593-3d6064b)
+[![stable-diffusion.cpp](https://img.shields.io/badge/sd.cpp-6614334-yellow)](https://github.com/leejet/stable-diffusion.cpp/releases/tag/master-596-90e87bc)
 [![Coverage](https://img.shields.io/badge/code%20coverage-80%25-purple)](https://github.com/l8bloom/gosd/actions)
 
 
 ## Features
 
 - Image and video generation
 - Image editing
-- Resolution upscaling via Neural upscaling (ESRGAN)
+- High-resolution upscaling (Neural ESRGAN models + Latent-space methods)
 - Callback support for progressive previews during inference
+- Model conversion (to SafeTensors / GGUF, optional VAE merging, tensor type rules)
+- Hardware-accelerated inference (CUDA, Metal, Vulkan, ROCm and CPU)
 - Minimal performance overhead compared to C/C++
-- GPU + CPU support
 
 ## Quick start
 

examples/image_gen/image_gen.go

@@ -33,6 +33,9 @@ func main() {
 	ctxParams.DiffusionFlashAttn = true // potential hardware optimizations
 	// ctxParams.KeepClipOnCPU = true // in case of lower vram
 
+	// optionally set maximum VRAM budget in GiB(enables inference with larger models)
+	// ctxParams.MaxVRAM = 2.0
+
 	fmt.Printf("\nContext values:\n%s", sd.CtxParamsToStr(ctxParams))
 
 	ctx := sd.NewContext(ctxParams)

examples/system/convert/convert.go

@@ -0,0 +1,26 @@
+// example how to convert model tensor with gosd and stable-diffusion.cpp
+
+package main
+
+import (
+	"os"
+
+	sd "github.com/l8bloom/gosd/pkg/gosd"
+)
+
+func main() {
+	// load dynamic libs of stable_diffusion.cpp and its deps
+	if err := sd.Load(); err != nil {
+		panic(err.Error())
+	}
+
+	modelPath := os.Getenv("MODEL_TO_CONVERT")
+	vaePath := ""
+	outputPath := "converted_model.gguf"
+	outputType := sd.TypeQ2_K
+	tensorTypeRules := ""
+	convertName := false
+
+	sd.Convert(modelPath, vaePath, outputPath, sd.SDType(outputType), tensorTypeRules, convertName)
+
+}

pkg/gosd/context.go

@@ -128,6 +128,7 @@ type contextParams struct {
 	ChromaUseT5Mask             uint8             // bool chroma_use_t5_mask;
 	ChromaT5MaskPad             int32             // int chroma_t5_mask_pad;
 	QwenImageZeroCond           uint8             // bool qwen_image_zero_cond_t;
+	MaxVRAM                     float32           // float max_vram;
 }
 
 func (ctx *contextParams) toGo() *ContextParams {
@@ -177,6 +178,7 @@ func (ctx *contextParams) toGo() *ContextParams {
 		ChromaUseT5Mask:             byteToBool(ctx.ChromaUseT5Mask),
 		ChromaT5MaskPad:             ctx.ChromaT5MaskPad,
 		QwenImageZeroCond:           byteToBool(ctx.QwenImageZeroCond),
+		MaxVRAM:                     ctx.MaxVRAM,
 	}
 }
 
@@ -222,6 +224,7 @@ type ContextParams struct {
 	ChromaUseT5Mask             bool
 	ChromaT5MaskPad             int32
 	QwenImageZeroCond           bool
+	MaxVRAM                     float32
 }
 
 func (ctx *ContextParams) toC() *contextParams {
@@ -271,6 +274,7 @@ func (ctx *ContextParams) toC() *contextParams {
 		ChromaUseT5Mask:             boolToByte(ctx.ChromaUseT5Mask),
 		ChromaT5MaskPad:             ctx.ChromaT5MaskPad,
 		QwenImageZeroCond:           boolToByte(ctx.QwenImageZeroCond),
+		MaxVRAM:                     ctx.MaxVRAM,
 	}
 }
 

pkg/gosd/system.go

@@ -66,6 +66,9 @@ var (
 
 	// SD_API enum sd_hires_upscaler_t str_to_sd_hires_upscaler(const char* str);
 	strToHiresUpscaler ffi.Fun
+
+	// SD_API bool convert(const char* input_path, const char* vae_path, const char* output_path, enum sd_type_t output_type, const char* tensor_type_rules, bool convert_name);
+	convert ffi.Fun
 )
 
 func loadSystemRoutines(lib ffi.Lib) error {
@@ -150,6 +153,10 @@ func loadSystemRoutines(lib ffi.Lib) error {
 		return loadError("str_to_sd_hires_upscaler", err)
 	}
 
+	if convert, err = lib.Prep("convert", &ffi.TypeUint8, &ffi.TypePointer, &ffi.TypePointer, &ffi.TypePointer, &ffi.TypeSint32, &ffi.TypePointer, &ffi.TypeUint8); err != nil {
+		return loadError("convert", err)
+	}
+
 	return nil
 }
 
@@ -312,3 +319,25 @@ func StrToHiresUpscaler(typeName string) HiresUpscalerType {
 	strToHiresUpscaler.Call(unsafe.Pointer(&hiresMode), unsafe.Pointer(&name))
 	return hiresMode
 }
+
+// CPU-bound API
+func Convert(modelPath string, vaePath string, outputPath string, outputType SDType, tensorTypeRules string, convertName bool) bool {
+	mp := stringToChar(modelPath)
+	vp := stringToChar(vaePath)
+	op := stringToChar(outputPath)
+	ttr := stringToChar(tensorTypeRules)
+	cn := boolToByte(convertName)
+
+	res := uint8(0)
+
+	convert.Call(
+		unsafe.Pointer(&res),
+		unsafe.Pointer(&mp),
+		unsafe.Pointer(&vp),
+		unsafe.Pointer(&op),
+		unsafe.Pointer(&outputType),
+		unsafe.Pointer(&ttr),
+		unsafe.Pointer(&cn),
+	)
+	return byteToBool(res)
+}

pkg/gosd/system_test.go

@@ -1,6 +1,8 @@
 package gosd
 
 import (
+	"errors"
+	"os"
 	"testing"
 )
 
@@ -285,3 +287,24 @@ func TestStrToHiresUpscaler(t *testing.T) {
 		t.Errorf("expected `%d` for `bicubic antialiased`, got  %d", HiresUpscalerLatentBicubic, hiresMode)
 	}
 }
+
+func TestConvert(t *testing.T) {
+	modelPath := os.Getenv("MODEL_TO_CONVERT")
+	vaePath := ""
+	outputPath := "converted_model.gguf"
+	outputType := TypeQ2_K
+	tensorTypeRules := ""
+	convertName := false
+
+	res := Convert(modelPath, vaePath, outputPath, SDType(outputType), tensorTypeRules, convertName)
+
+	if !res {
+		t.Error("Conversion failed.")
+	}
+
+	_, err := os.Stat(outputPath)
+	if errors.Is(err, os.ErrNotExist) {
+		t.Error("converted model not saved.")
+	}
+	os.Remove(outputPath)
+}

stable_diffusion.release

@@ -1 +1 @@
-master-593-3d6064b
+master-596-90e87bc

stable_diffusion_api_not_covered

@@ -1,14 +1,6 @@
-SD_API bool convert(const char* input_path,
-                    const char* vae_path,
-                    const char* output_path,
-                    enum sd_type_t output_type,
-                    const char* tensor_type_rules,
-                    bool convert_name);
-
 SD_API bool preprocess_canny(sd_image_t image,
                              float high_threshold,
                              float low_threshold,
                              float weak,
                              float strong,
                              bool inverse);
-