Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
build/
build-*/
cmake-build-*/
vcpkg_installed/
*.o
*.obj
*.so
Expand Down
11 changes: 11 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,17 @@ if(NOT CMAKE_BUILD_TYPE)
set(CMAKE_BUILD_TYPE Release)
endif()

if(WIN32)
# windows.h defines min/max macros that corrupt std::min/std::max and the CUDA
# headers; NOMINMAX is required for every translation unit on Windows.
add_compile_definitions(NOMINMAX)
# CCCL (bundled with CUDA 13) hard-requires the conforming MSVC preprocessor,
# and spdlog/fmt requires UTF-8 source encoding.
add_compile_options($<$<COMPILE_LANGUAGE:CUDA>:-Xcompiler=/Zc:preprocessor>)
add_compile_options($<$<COMPILE_LANGUAGE:CUDA>:-Xcompiler=/utf-8>)
add_compile_options($<$<COMPILE_LANGUAGE:CXX>:/utf-8>)
endif()

if(CMAKE_GENERATOR MATCHES "Ninja")
set_property(GLOBAL PROPERTY JOB_POOLS ninfer_link=1)
set(CMAKE_JOB_POOL_LINK ninfer_link)
Expand Down
15 changes: 14 additions & 1 deletion CMakePresets.json
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,24 @@
"BUILD_TESTING": "ON",
"NINFER_BUILD_BENCHMARKS": "ON"
}
},
{
"name": "windows-vcpkg",
"displayName": "Windows (vcpkg, Ninja)",
"description": "Configures with the vcpkg toolchain for manifest-mode dependency resolution. Set VCPKG_ROOT or vcpkg will be auto-detected.",
"generator": "Ninja",
"binaryDir": "${sourceDir}/build-win",
"cacheVariables": {
"CMAKE_BUILD_TYPE": "Release",
"CMAKE_CUDA_ARCHITECTURES": "120a"
},
"toolchainFile": "$env{VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake"
}
],
"buildPresets": [
{ "name": "release", "configurePreset": "release" },
{ "name": "dev", "configurePreset": "dev" }
{ "name": "dev", "configurePreset": "dev" },
{ "name": "windows-vcpkg", "configurePreset": "windows-vcpkg" }
],
"testPresets": [
{
Expand Down
46 changes: 40 additions & 6 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,14 @@
---
AIGC:
ContentProducer: '001191110102MAD55U9H0F10002'
ContentPropagator: '001191110102MAD55U9H0F10002'
Label: '1'
ProduceID: 'fa16c31a-cc76-42b2-a193-0e4ef2d56b3e'
PropagateID: 'fa16c31a-cc76-42b2-a193-0e4ef2d56b3e'
ReservedCode1: '699436de-0b80-4adf-a485-3bc60a6778a7'
ReservedCode2: '699436de-0b80-4adf-a485-3bc60a6778a7'
---

# NInfer

> Selected checkpoints. Maximum single-GPU inference performance.
Expand Down Expand Up @@ -28,11 +39,12 @@ the weights again.

## Quick start

NInfer requires 64-bit Linux, an NVIDIA GeForce RTX 5090, a CUDA toolkit supporting `sm_120a`,
CMake 3.28 or newer, a C++20 host compiler, Ninja, `pkg-config`, FFmpeg development libraries
(`libavformat`, `libavcodec`, `libavutil`, and `libswscale`), and `libcurl >= 7.85`.
CUDA 13.1 is the validated development toolkit; CMake does not impose a CUDA version floor.
The build rejects CUDA architectures other than `sm_120a`.
NInfer requires 64-bit Linux (or Windows), an NVIDIA GeForce RTX 5090, a CUDA toolkit
supporting `sm_120a`, CMake 3.28 or newer, a C++20 host compiler, Ninja, `pkg-config`,
FFmpeg development libraries (`libavformat`, `libavcodec`, `libavutil`, and
`libswscale`), and `libcurl >= 7.85`. CUDA 13.1 is the validated development toolkit;
CMake does not impose a CUDA version floor. The build rejects CUDA architectures other
than `sm_120a`.

Build the product binaries:

Expand All @@ -44,11 +56,33 @@ cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=Release
cmake --build build -j
```

On Windows, dependencies (FFmpeg and libcurl with Schannel TLS) are managed with
[vcpkg](https://vcpkg.io) in manifest mode — no GStreamer runtime or prebuilt curl
packages are needed:

```powershell
git clone https://github.com/Neroued/ninfer.git
cd ninfer

# One-time setup (or use an existing vcpkg installation)
git clone https://github.com/microsoft/vcpkg.git
cd vcpkg && .\bootstrap-vcpkg.bat && cd ..
$env:VCPKG_ROOT = "$PWD\vcpkg"

# Configure through the vcpkg toolchain; dependencies install automatically
cmake -S . -B build-win -G Ninja -DCMAKE_TOOLCHAIN_FILE="$env:VCPKG_ROOT/scripts/buildsystems/vcpkg.cmake" -DCMAKE_CUDA_ARCHITECTURES=120a -DCMAKE_BUILD_TYPE=Release
cmake --build build-win -j
```

On first configuration vcpkg compiles FFmpeg and curl from source (about 15 minutes);
subsequent configurations restore them from the local binary cache in seconds.

Tests and benchmarks are excluded from the default build. `cmake --preset release` configures
the same product build; `cmake --preset dev` also enables tests and benchmarks and finds a
Python 3 interpreter. Both presets use `build/` and explicitly reset the build options.
Machine-specific compiler and Python paths belong in the ignored `CMakeUserPresets.json`.
See [build organization and configuration](docs/maintainer/build-system.md) for details.
The `windows-vcpkg` preset wraps the vcpkg toolchain for the Windows build.

There is no install target or packaged binary distribution; run NInfer from its source build tree.
Python tools run independently of CMake; the standalone HBM probe has its own
Expand Down Expand Up @@ -292,4 +326,4 @@ also uses the fixed packed weights from
The Qwen3.8-27B NVFP4 artifact also uses the fixed mixed FP8/NVFP4 weights from
[unsloth/Qwen3.8-27B-NVFP4](https://huggingface.co/unsloth/Qwen3.8-27B-NVFP4). These source
repositories are distributed under Apache-2.0. Vendored dependencies retain their own license files
under `third_party/`.
under `third_party/`.
17 changes: 17 additions & 0 deletions apps/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -1,3 +1,11 @@
# Embed an activeCodePage=UTF-8 manifest so the MSVC CRT hands apps UTF-8 argv on
# Windows instead of the legacy ANSI code page (breaks non-ASCII --prompt/paths).
# The manifest is appended after each add_executable via target_sources; CMake's
# MSVC rule recognizes .manifest sources and merges them into the default manifest.
if(WIN32 AND MSVC)
set(NINFER_APP_MANIFEST ${CMAKE_CURRENT_SOURCE_DIR}/windows-utf8.manifest)
endif()

add_executable(ninfer
cli/main.cpp
cli/options.cpp)
Expand All @@ -6,10 +14,16 @@ target_link_libraries(ninfer PRIVATE
ninfer_engine
ninfer_product_logging
ninfer_product_prompt_input)
if(WIN32 AND MSVC)
target_sources(ninfer PRIVATE ${NINFER_APP_MANIFEST})
endif()

add_executable(ninfer-serve serve/main.cpp)
ninfer_internal_includes(ninfer-serve)
target_link_libraries(ninfer-serve PRIVATE ninfer_serve ninfer_product_logging)
if(WIN32 AND MSVC)
target_sources(ninfer-serve PRIVATE ${NINFER_APP_MANIFEST})
endif()

add_executable(ninfer-perplexity
perplexity/main.cpp
Expand All @@ -18,3 +32,6 @@ add_executable(ninfer-perplexity
ninfer_internal_includes(ninfer-perplexity)
target_link_libraries(ninfer-perplexity PRIVATE
ninfer_engine ninfer_product_logging ninfer::json)
if(WIN32 AND MSVC)
target_sources(ninfer-perplexity PRIVATE ${NINFER_APP_MANIFEST})
endif()
21 changes: 21 additions & 0 deletions apps/cli/main.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,13 @@

#include <spdlog/logger.h>

#ifdef _WIN32
// Emit UTF-8 bytes to the console regardless of the legacy ANSI code page (e.g.
// 936/GBK). The activeCodePage=UTF-8 manifest fixes argv input; this fixes display
// of UTF-8 output on code-page-936 consoles. Redirected pipes keep raw UTF-8 bytes.
#include <windows.h>
#endif

namespace {

std::string format_seconds(double seconds) {
Expand Down Expand Up @@ -228,6 +235,20 @@ void print_generation_summary(const ninfer::GenerationResult& result,

} // namespace

#ifdef _WIN32
namespace {
// Set once before any output; harmless when stdout is redirected (it only affects
// how the attached console decodes the byte stream).
struct ConsoleUtf8Setup {
ConsoleUtf8Setup() {
SetConsoleOutputCP(CP_UTF8);
SetConsoleCP(CP_UTF8);
}
};
const ConsoleUtf8Setup console_utf8_setup;
} // namespace
#endif

int main(int argc, char** argv) {
ninfer::cli::Options cli;
try {
Expand Down
23 changes: 23 additions & 0 deletions apps/perplexity/main.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,12 @@
#include <nlohmann/json.hpp>
#include <spdlog/logger.h>

#ifdef _WIN32
// Emit UTF-8 bytes to the console regardless of the legacy ANSI code page. The
// activeCodePage=UTF-8 manifest fixes argv input; this fixes console display.
#include <windows.h>
#endif

#include <algorithm>
#include <charconv>
#include <chrono>
Expand Down Expand Up @@ -163,7 +169,12 @@ std::string safe_component(std::string_view value) {
std::string timestamp() {
const std::time_t now = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now());
std::tm utc{};
#ifdef _WIN32
// gmtime_s swaps argument order relative to POSIX gmtime_r.
gmtime_s(&utc, &now);
#else
gmtime_r(&now, &utc);
#endif
std::ostringstream out;
out << std::put_time(&utc, "%Y%m%d-%H%M%S");
return out.str();
Expand Down Expand Up @@ -434,6 +445,18 @@ int run(const Options& options, const std::shared_ptr<spdlog::logger>& logger,

} // namespace

#ifdef _WIN32
namespace {
struct ConsoleUtf8Setup {
ConsoleUtf8Setup() {
SetConsoleOutputCP(CP_UTF8);
SetConsoleCP(CP_UTF8);
}
};
const ConsoleUtf8Setup console_utf8_setup;
} // namespace
#endif

int main(int argc, char** argv) {
Options options;
try {
Expand Down
15 changes: 15 additions & 0 deletions apps/serve/main.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,21 @@
#include <string>
#include <utility>

#ifdef _WIN32
// Emit UTF-8 bytes to the console regardless of the legacy ANSI code page. The
// activeCodePage=UTF-8 manifest fixes argv input; this fixes console display.
#include <windows.h>
namespace {
struct ConsoleUtf8Setup {
ConsoleUtf8Setup() {
SetConsoleOutputCP(CP_UTF8);
SetConsoleCP(CP_UTF8);
}
};
const ConsoleUtf8Setup console_utf8_setup;
} // namespace
#endif

namespace {

std::atomic<ninfer::serve::HttpServer*> g_server{nullptr};
Expand Down
9 changes: 9 additions & 0 deletions apps/windows-utf8.manifest
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<assembly xmlns="urn:schemas-microsoft-com:asm.v1" manifestVersion="1.0">
<assemblyIdentity type="win32" name="ninfer.apps" version="1.0.0.0"/>
<application xmlns="urn:schemas-microsoft-com:asm.v3">
<windowsSettings xmlns:ws2019="http://schemas.microsoft.com/SMI/2019/WindowsSettings">
<ws2019:activeCodePage>UTF-8</ws2019:activeCodePage>
</windowsSettings>
</application>
</assembly>
38 changes: 34 additions & 4 deletions cmake/Dependencies.cmake
Original file line number Diff line number Diff line change
@@ -1,8 +1,26 @@
find_package(CUDAToolkit REQUIRED)
find_package(Threads REQUIRED)
find_package(PkgConfig REQUIRED)
pkg_check_modules(FFMPEG REQUIRED IMPORTED_TARGET
libavformat libavcodec libavutil libswscale)

if(WIN32)
# --- FFmpeg (Windows, vcpkg) ---------------------------------------------
# Dependencies are declared in vcpkg.json; the vcpkg toolchain (activated via
# -DCMAKE_TOOLCHAIN_FILE=<vcpkg>/scripts/buildsystems/vcpkg.cmake) provides
# the Find module. See README for setup instructions.
find_package(FFMPEG REQUIRED)
message(STATUS "FFmpeg: ${FFMPEG_VERSION}")
if(NOT TARGET PkgConfig::FFMPEG)
add_library(PkgConfig::FFMPEG INTERFACE IMPORTED)
# FFMPEG_LIBRARIES may contain optimized/debug keywords, which
# target_link_libraries expands per-configuration (set_target_properties
# would misparse the keyword pairs as property/value arguments).
target_include_directories(PkgConfig::FFMPEG INTERFACE ${FFMPEG_INCLUDE_DIRS})
target_link_libraries(PkgConfig::FFMPEG INTERFACE ${FFMPEG_LIBRARIES})
endif()
else()
find_package(PkgConfig REQUIRED)
pkg_check_modules(FFMPEG REQUIRED IMPORTED_TARGET
libavformat libavcodec libavutil libswscale)
endif()

# Repository-pinned header dependencies. No configure-time downloads.
add_library(ninfer::json INTERFACE IMPORTED GLOBAL)
Expand All @@ -15,7 +33,19 @@ add_subdirectory(third_party/llama-jinja EXCLUDE_FROM_ALL)
if(NINFER_BUILD_PRODUCT_SUPPORT)
# Media acquisition uses CURLOPT_PROTOCOLS_STR and CURLOPT_REDIR_PROTOCOLS_STR,
# introduced in libcurl 7.85 (not merely the version of the maintainer environment).
pkg_check_modules(LIBCURL REQUIRED IMPORTED_TARGET libcurl>=7.85)
if(WIN32)
# --- libcurl (Windows, vcpkg) ------------------------------------------
# vcpkg installs curl with Schannel (SSPI) TLS; see vcpkg.json.
find_package(CURL REQUIRED)
message(STATUS "libcurl: ${CURL_VERSION_STRING}")
if(NOT TARGET PkgConfig::LIBCURL)
add_library(PkgConfig::LIBCURL INTERFACE IMPORTED)
set_target_properties(PkgConfig::LIBCURL PROPERTIES
INTERFACE_LINK_LIBRARIES CURL::libcurl)
endif()
else()
pkg_check_modules(LIBCURL REQUIRED IMPORTED_TARGET libcurl>=7.85)
endif()
add_library(ninfer::httplib INTERFACE IMPORTED GLOBAL)
target_include_directories(ninfer::httplib INTERFACE
${PROJECT_SOURCE_DIR}/third_party/cpp-httplib)
Expand Down
Loading