{ "schema_version": "1.2", "site": "https://cerinamroth.com", "generated_by": "scripts/build_ml_site.py", "generated_at": "2026-09-06T04:07:55Z", "scope": "The generated ML documentation surfaces of this site are derived from these repositories at the pinned commits; each repository's docs/claims.json at that commit is the only source for numbers. Names, aliases, ownership, the relationship, compatibility floors and the evidence vocabulary come from docs/system-manifest.json (identical in both repositories).", "manifest": { "url": "https://cerinamroth.com/ml/system-manifest.json", "sha256": "196cb55a41e890842f258305f307b873d55639cc617cd4ae61c771a063204918", "provenance_url": "https://cerinamroth.com/ml/system-manifest.provenance.json" }, "system": { "name": "experts4bit-qlora + grouped-nf4-gemm", "umbrella_name": null, "relationship": "one system, two independently released packages: the runtime package depends on the kernel package through its optional `fast` extra; the kernel package never depends on the runtime package", "dependency_direction": [ "experts4bit-qlora -> grouped-nf4-gemm" ] }, "compatibility": { "current": { "consumer": "experts4bit-qlora", "consumer_versions": ">=0.35.0", "kernel": "grouped-nf4-gemm", "floor": ">=0.30.0", "extra": "fast", "since": "2026-09-04", "why": "combine_rows (grouped-nf4-gemm#335) with the swiglu_rows / reduce_partials decode glue that experts4bit-qlora#385 integrates, and gptq_pack for the calibrated int4 expert store (experts4bit-qlora#384)", "consumer_ci_pin": "the consumer's CI installs the kernel package from the git commit of the v0.30.1 release", "line": "experts4bit-qlora >=0.35.0 → grouped-nf4-gemm >=0.30.0 (extra: fast)" }, "historical": [ { "consumer": "experts4bit-qlora", "consumer_versions": "0.34.x", "kernel": "grouped-nf4-gemm", "floor": ">=0.28.0", "extra": "fast", "since": "2026-09-04", "why": "rope_heads (the rotary-only fold for attention without a head norm, grouped-nf4-gemm#330) and gemv_mxfp4_b32 (the decode-grade MXFP4 GEMV)", "line": "experts4bit-qlora 0.34.x → grouped-nf4-gemm >=0.28.0 (extra: fast)" } ], "rule": "the current record is the one whose consumer_versions range contains the rendered runtime version; floors are validated against pyproject.toml at the pin, never trusted from the manifest" }, "evidence_vocabulary": { "confirmed": "a public receipt plus an independent confirmation cell (kernel register)", "verified": "a public receipt plus an independent check (runtime register)", "measured": "the run happened and the receipt is public in the repository", "measured-private": "the run happened and the receipt lives outside the repository; NOT publicly reproducible from the repository alone", "projected": "arithmetic from measured inputs, not a run", "open": "a statement the documents make that has no evidence either way yet; it backs no capability", "superseded": "replaced by a later claim that names it; kept as the record, never current", "retired": "withdrawn; kept as the record, never current" }, "sources": [ { "repository": "https://github.com/pjordanandrsn/experts4bit-qlora", "commit": "0c2a256dcdc2cb0a83cf7692224a8aa716f61ecd", "commit_date": "2026-09-05", "package": "experts4bit-qlora", "package_version": "0.35.3", "published_version": "0.35.3", "publication": "published", "documents_used": [ "docs/capabilities.json", "docs/claims.json", "docs/solutions/bitsandbytes-moe-load-in-4bit-still-ooms.md", "docs/solutions/mxfp4-moe-training-and-residency.md", "docs/solutions/offload-moe-experts-to-cpu-or-nvme.md", "docs/solutions/qlora-fused-moe-experts.md", "docs/solutions/run-moe-larger-than-vram.md", "docs/solutions/serve-large-moe-on-a-consumer-gpu.md", "docs/system-manifest.json", "pyproject.toml" ], "registers_used": [ "capabilities", "claims" ], "synced_at": "2026-09-06T04:07:55Z", "pinned_tree": "https://github.com/pjordanandrsn/experts4bit-qlora/tree/0c2a256dcdc2cb0a83cf7692224a8aa716f61ecd", "current_main": "https://github.com/pjordanandrsn/experts4bit-qlora", "current_main_label": "latest on main — unpinned; not the source of any fact rendered here", "system_manifest": { "url": "https://cerinamroth.com/ml/system-manifest.json", "provenance_url": "https://cerinamroth.com/ml/system-manifest.provenance.json", "sha256": "196cb55a41e890842f258305f307b873d55639cc617cd4ae61c771a063204918", "pinned": "https://github.com/pjordanandrsn/experts4bit-qlora/blob/0c2a256dcdc2cb0a83cf7692224a8aa716f61ecd/docs/system-manifest.json" } }, { "repository": "https://github.com/pjordanandrsn/grouped-nf4-gemm", "commit": "e2af4cfb91b2c29968989c5254bff55aa7da3347", "commit_date": "2026-09-05", "package": "grouped-nf4-gemm", "package_version": "0.30.2", "published_version": "0.30.2", "publication": "published", "documents_used": [ "docs/capabilities.json", "docs/claims.json", "docs/solutions/fp8-paged-attention-for-moe-serving.md", "docs/solutions/int4-decode-gemv.md", "docs/solutions/native-mxfp4-moe-inference.md", "docs/solutions/nf4-grouped-gemm-without-bf16-materialization.md", "docs/solutions/stream-moe-experts-from-host-or-nvme.md", "docs/solutions/verify-quantized-checkpoint-provenance.md", "docs/system-manifest.json", "pyproject.toml" ], "registers_used": [ "capabilities", "claims" ], "synced_at": "2026-09-06T00:07:26Z", "pinned_tree": "https://github.com/pjordanandrsn/grouped-nf4-gemm/tree/e2af4cfb91b2c29968989c5254bff55aa7da3347", "current_main": "https://github.com/pjordanandrsn/grouped-nf4-gemm", "current_main_label": "latest on main — unpinned; not the source of any fact rendered here", "system_manifest": { "url": "https://cerinamroth.com/ml/system-manifest.json", "provenance_url": "https://cerinamroth.com/ml/system-manifest.provenance.json", "sha256": "196cb55a41e890842f258305f307b873d55639cc617cd4ae61c771a063204918", "pinned": "https://github.com/pjordanandrsn/grouped-nf4-gemm/blob/e2af4cfb91b2c29968989c5254bff55aa7da3347/docs/system-manifest.json" } } ], "packages": [ { "name": "experts4bit-qlora", "canonical_package": "experts4bit-qlora", "aliases": [ "e4b", "e4b-qlora", "experts4bit", "expertsnbit", "experts-mxfp4" ], "import_names": [ "experts4bit_qlora" ], "role": "runtime", "owns": [ "model loading (streaming loader, fused expert stacks, Experts4bit/ExpertsNbit bases)", "quantization orchestration (NF4/MXFP4 routes, int4 expert store, calibration drivers)", "adapters (ExpertsLoRA)", "training (trainer, fast/batched/hybrid paths, checkpointing)", "residency integration (host-RAM and NVMe engines bound to a loaded model)", "serving (paged runner, decode stack, HTTP shim)", "model-level measurement, quality gates and the claims register for model-level numbers" ], "site_url": "https://cerinamroth.com/ml/experts4bit-qlora/", "github": "https://github.com/pjordanandrsn/experts4bit-qlora", "pypi": "https://pypi.org/project/experts4bit-qlora/", "source_commit": "0c2a256dcdc2cb0a83cf7692224a8aa716f61ecd", "commit_date": "2026-09-05", "repository_version": "0.35.3", "pypi_version": "0.35.3", "publication": "published", "freshness": { "source": { "repository": "pjordanandrsn/experts4bit-qlora", "commit": "0c2a256dcdc2cb0a83cf7692224a8aa716f61ecd", "version": "0.35.3" }, "publication": { "latest_pypi_version": "0.35.3", "fresh": true, "state": "CURRENT", "checked_at": "2026-09-06T04:07:55Z" }, "generated_at": "2026-09-06T04:07:55Z" }, "synced_at": "2026-09-06T04:07:55Z", "claim_tiers": { "measured": 161, "measured-private": 6, "open": 3, "superseded": 7, "retired": 6 }, "status_url": "https://cerinamroth.com/ml/status/#experts4bit-qlora", "compatibility_url": "https://cerinamroth.com/ml/compatibility/#experts4bit-qlora", "related": [ "grouped-nf4-gemm" ], "relationship": "one system, two independently released packages: the runtime package depends on the kernel package through its optional `fast` extra; the kernel package never depends on the runtime package", "dependency_direction": [ "experts4bit-qlora -> grouped-nf4-gemm" ], "capabilities": "https://cerinamroth.com/ml/experts4bit-qlora/index.json", "capability_status": { "quantize-fused-moe-experts": "supported", "qlora-fused-moe-experts": "supported", "run-moe-larger-than-vram": "supported", "offload-moe-experts-to-cpu-or-nvme": "supported", "serve-moe-on-consumer-gpu": "supported", "mxfp4-moe-training-and-residency": "experimental" }, "install": { "primary": { "purpose": "streaming loader + trainer", "command": "pip install \"experts4bit-qlora[train]\"", "capabilities": [ "quantize-fused-moe-experts", "qlora-fused-moe-experts", "run-moe-larger-than-vram", "offload-moe-experts-to-cpu-or-nvme", "mxfp4-moe-training-and-residency" ] }, "alternatives": [ { "purpose": "accelerated grouped-kernel path", "command": "pip install \"experts4bit-qlora[fast]\"", "capabilities": [ "qlora-fused-moe-experts", "run-moe-larger-than-vram", "offload-moe-experts-to-cpu-or-nvme", "serve-moe-on-consumer-gpu", "mxfp4-moe-training-and-residency" ] }, { "purpose": "HTTP shim (optional)", "command": "pip install \"experts4bit-qlora[serve]\"", "capabilities": [ "serve-moe-on-consumer-gpu" ] } ], "environment": { "os": [ "Linux" ], "python": [ ">=3.11 tested in CI (pyproject says >=3.9; 3.9/3.10 are not tested)" ], "accelerator": [ "NVIDIA CUDA GPU (bitsandbytes 4-bit)", "NVIDIA CUDA GPU, sm_80 or newer (Triton kernels via grouped-nf4-gemm)" ], "special_requirements": [ "an expert arena baked with grouped-nf4-gemm (nvme_bake_nf4 for NF4, the MXFP4 relocation bake for native MXFP4)", "bitsandbytes>=0.43", "grouped-nf4-gemm>=0.30.0 ([fast] extra; the floor is pyproject.toml's, validated by CI)", "local NVMe or a fast block device; pinned host RAM for the hot tier", "torch>=2.2", "transformers>=5.0 for the streaming loader ([train] extra)", "triton>=3.4 (Linux-only distribution)" ] } }, "pinned_urls": { "commit": "https://github.com/pjordanandrsn/experts4bit-qlora/commit/0c2a256dcdc2cb0a83cf7692224a8aa716f61ecd", "tree": "https://github.com/pjordanandrsn/experts4bit-qlora/tree/0c2a256dcdc2cb0a83cf7692224a8aa716f61ecd", "capabilities": "https://github.com/pjordanandrsn/experts4bit-qlora/blob/0c2a256dcdc2cb0a83cf7692224a8aa716f61ecd/docs/capabilities.json", "claims": "https://github.com/pjordanandrsn/experts4bit-qlora/blob/0c2a256dcdc2cb0a83cf7692224a8aa716f61ecd/docs/claims.json", "status": "https://github.com/pjordanandrsn/experts4bit-qlora/blob/0c2a256dcdc2cb0a83cf7692224a8aa716f61ecd/docs/STATUS.md", "solutions_index": "https://github.com/pjordanandrsn/experts4bit-qlora/blob/0c2a256dcdc2cb0a83cf7692224a8aa716f61ecd/docs/SOLUTIONS.md", "system_manifest": "https://github.com/pjordanandrsn/experts4bit-qlora/blob/0c2a256dcdc2cb0a83cf7692224a8aa716f61ecd/docs/system-manifest.json" }, "current_main_urls": { "repository": "https://github.com/pjordanandrsn/experts4bit-qlora", "capabilities": "https://github.com/pjordanandrsn/experts4bit-qlora/blob/main/docs/capabilities.json", "claims": "https://github.com/pjordanandrsn/experts4bit-qlora/blob/main/docs/claims.json", "status": "https://github.com/pjordanandrsn/experts4bit-qlora/blob/main/docs/STATUS.md" }, "current_main_label": "latest on main — unpinned; not the source of any fact rendered here", "solutions": [ { "slug": "bitsandbytes-moe-load-in-4bit-still-ooms", "title": "Why does `load_in_4bit` still OOM on a Mixture-of-Experts model?", "summary": "bitsandbytes' 4-bit walker replaces only nn.Linear and skips a fused MoE's 3-D expert stacks, so load_moe_4bit_streaming quantises exactly those stacks and verify_moe_4bit proves it.", "url": "https://cerinamroth.com/ml/solutions/bitsandbytes-moe-load-in-4bit-still-ooms/" }, { "slug": "mxfp4-moe-training-and-residency", "title": "How do I train and serve MoE models released in MXFP4 (gpt-oss, DeepSeek-V4)?", "summary": "Choose between the convenient QLoRA path, which decodes MXFP4 and re-quantises to NF4, and the native-byte path, which keeps the released blocks and scales in an arena.", "url": "https://cerinamroth.com/ml/solutions/mxfp4-moe-training-and-residency/" }, { "slug": "offload-moe-experts-to-cpu-or-nvme", "title": "How do I offload MoE experts to host RAM, or serve and train them from an NVMe arena?", "summary": "Bind pinned host RAM or a baked NVMe arena to a real model with the streaming loader and the nvme_residency, mxfp4_nvme_residency and nvme_train_residency engines.", "url": "https://cerinamroth.com/ml/solutions/offload-moe-experts-to-cpu-or-nvme/" }, { "slug": "qlora-fused-moe-experts", "title": "How do I QLoRA-train the fused experts of a MoE (per-expert LoRA on 4-bit experts)?", "summary": "ExpertsLoRA adds a trainable per-expert low-rank delta before each routed expert's activation over a frozen NF4 stack PEFT cannot target; enable_fast_train runs it on grouped kernels.", "url": "https://cerinamroth.com/ml/solutions/qlora-fused-moe-experts/" }, { "slug": "run-moe-larger-than-vram", "title": "How can I run a Mixture-of-Experts model larger than my GPU's VRAM?", "summary": "Choose a residency path by workload and by the memory tier that ran out: pinned-host expert streaming, dense offload, profiled hot residency, or an NVMe arena.", "url": "https://cerinamroth.com/ml/solutions/run-moe-larger-than-vram/" }, { "slug": "serve-large-moe-on-a-consumer-gpu", "title": "How do I serve a large MoE on a consumer GPU (RTX 5090 class)?", "summary": "The paged decode runner with experts on grouped-nf4-gemm's kernels is the measured serving path on one RTX 5090 class; the HTTP shim is a separate reference-path deployment.", "url": "https://cerinamroth.com/ml/solutions/serve-large-moe-on-a-consumer-gpu/" } ], "sources": [ { "repository": "https://github.com/pjordanandrsn/experts4bit-qlora", "commit": "0c2a256dcdc2cb0a83cf7692224a8aa716f61ecd", "commit_date": "2026-09-05", "package": "experts4bit-qlora", "package_version": "0.35.3", "published_version": "0.35.3", "publication": "published", "documents_used": [ "docs/capabilities.json", "docs/claims.json", "docs/solutions/bitsandbytes-moe-load-in-4bit-still-ooms.md", "docs/solutions/mxfp4-moe-training-and-residency.md", "docs/solutions/offload-moe-experts-to-cpu-or-nvme.md", "docs/solutions/qlora-fused-moe-experts.md", "docs/solutions/run-moe-larger-than-vram.md", "docs/solutions/serve-large-moe-on-a-consumer-gpu.md", "docs/system-manifest.json", "pyproject.toml" ], "registers_used": [ "capabilities", "claims" ], "synced_at": "2026-09-06T04:07:55Z", "pinned_tree": "https://github.com/pjordanandrsn/experts4bit-qlora/tree/0c2a256dcdc2cb0a83cf7692224a8aa716f61ecd", "current_main": "https://github.com/pjordanandrsn/experts4bit-qlora", "current_main_label": "latest on main — unpinned; not the source of any fact rendered here", "system_manifest": { "url": "https://cerinamroth.com/ml/system-manifest.json", "provenance_url": "https://cerinamroth.com/ml/system-manifest.provenance.json", "sha256": "196cb55a41e890842f258305f307b873d55639cc617cd4ae61c771a063204918", "pinned": "https://github.com/pjordanandrsn/experts4bit-qlora/blob/0c2a256dcdc2cb0a83cf7692224a8aa716f61ecd/docs/system-manifest.json" } } ] }, { "name": "grouped-nf4-gemm", "canonical_package": "grouped-nf4-gemm", "aliases": [ "nf4gemm", "gnf4", "grouped-mxfp4-gemm" ], "import_names": [ "nf4_grouped", "mxfp4_grouped", "int4_b32", "fp8_paged_attn", "fp8_kv", "nvme_arena", "nvme_reader", "nvme_residency", "nvme_bake_nf4", "mxfp4_loader", "gptq_pack", "nf4_pack_ref", "mxfp4_pack_ref", "verify_provenance" ], "role": "kernels", "owns": [ "grouped packed GEMM (NF4 and native MXFP4) and its dgrad", "decode GEMV (int4-b32, MXFP4) and split-K reduce", "FP8 paged attention and the fp8 KV appends", "decode glue kernels (rope, norm and residual folds, swiglu, combine)", "pack and pure-torch reference operations, GPTQ packing, provenance verification", "host/NVMe primitives (arena, reader, row tier, bake, residency primitives)", "kernel-level measurement and the claims register for kernel-level numbers" ], "site_url": "https://cerinamroth.com/ml/grouped-nf4-gemm/", "github": "https://github.com/pjordanandrsn/grouped-nf4-gemm", "pypi": "https://pypi.org/project/grouped-nf4-gemm/", "source_commit": "e2af4cfb91b2c29968989c5254bff55aa7da3347", "commit_date": "2026-09-05", "repository_version": "0.30.2", "pypi_version": "0.30.2", "publication": "published", "freshness": { "source": { "repository": "pjordanandrsn/grouped-nf4-gemm", "commit": "e2af4cfb91b2c29968989c5254bff55aa7da3347", "version": "0.30.2" }, "publication": { "latest_pypi_version": "0.30.2", "fresh": true, "state": "CURRENT", "checked_at": "2026-09-06T04:07:55Z" }, "generated_at": "2026-09-06T04:07:55Z" }, "synced_at": "2026-09-06T00:07:26Z", "claim_tiers": { "confirmed": 11, "measured": 9, "measured-private": 3, "projected": 1, "open": 3, "superseded": 1, "retired": 2 }, "status_url": "https://cerinamroth.com/ml/status/#grouped-nf4-gemm", "compatibility_url": "https://cerinamroth.com/ml/compatibility/#grouped-nf4-gemm", "related": [ "experts4bit-qlora" ], "relationship": "one system, two independently released packages: the runtime package depends on the kernel package through its optional `fast` extra; the kernel package never depends on the runtime package", "dependency_direction": [ "experts4bit-qlora -> grouped-nf4-gemm" ], "capabilities": "https://cerinamroth.com/ml/grouped-nf4-gemm/index.json", "capability_status": { "grouped-nf4-gemm": "supported", "native-mxfp4-moe-inference": "supported", "int4-decode-gemv": "measured-private", "fp8-paged-attention-fp8-compute": "supported", "fp8-paged-attention-f32-compute": "unsupported", "decode-glue-kernels": "measured-private", "stream-experts-from-host-or-nvme": "supported", "verify-checkpoint-provenance": "supported" }, "install": { "primary": { "purpose": "kernel package", "command": "pip install grouped-nf4-gemm", "capabilities": [ "grouped-nf4-gemm", "native-mxfp4-moe-inference", "int4-decode-gemv", "fp8-paged-attention-fp8-compute", "fp8-paged-attention-f32-compute", "decode-glue-kernels", "stream-experts-from-host-or-nvme", "verify-checkpoint-provenance" ] }, "alternatives": [ { "purpose": "through the model consumer", "command": "pip install \"experts4bit-qlora[fast]\"", "capabilities": [ "grouped-nf4-gemm", "native-mxfp4-moe-inference", "int4-decode-gemv", "fp8-paged-attention-fp8-compute", "fp8-paged-attention-f32-compute", "decode-glue-kernels", "stream-experts-from-host-or-nvme" ] } ], "environment": { "os": [ "Linux" ], "python": [ ">=3.11 tested in CI (pyproject says >=3.9; 3.9/3.10 are not tested)" ], "accelerator": [ "NVIDIA CUDA GPU for the serving tiers and for the default NF4 quantise bake (nvme_bake_nf4.bake_nf4 with quantize_fn=None); the relocation bake and verify (nvme_arena.bake / bake_expert_tensors / verify) need no GPU", "NVIDIA CUDA GPU, sm_80 or newer (sm_120 is the primary serving target)", "NVIDIA CUDA GPU, sm_80 or newer (the composition was measured on the RTX 5090, sm_120)", "NVIDIA CUDA GPU, sm_80 or newer; this is the path sm_80-sm_88 take by default. Its reference tests were run on an RTX 5090 (sm_120) with the mode forced; no registered cell exists on sm_80-sm_88", "NVIDIA CUDA GPU, sm_89 or newer (the fp8 tensor-core dot); measured on the RTX 5090 (sm_120) only -- sm_89 and sm_90 meet the requirement but no registered cell was run there", "none: pure torch (a CUDA GPU is optional)" ], "special_requirements": [ "a local NVMe or fast block device for the arena; O_DIRECT reads", "bitsandbytes and CUDA for the NF4 quantise bake (nvme_bake_nf4.bake_nf4 with the default quantiser); the geometry/manifest path is pure torch when a quantize_fn is injected", "macOS/Windows: the pure-torch surface (pack references, dequant, provenance, arena bake/verify) imports and runs without triton via _triton_shim; the Triton kernels need a CUDA GPU; macOS and Windows are not exercised by CI", "pinned host RAM sized from measured free memory (capacity_for_bytes)", "shape constraints, all from fp8_paged_attn.fp8_compute_unsupported (the single predicate the default selector and the path's own asserts share): v_groups == 1; k_groups in (1, 2, 4, 8, 16); head_dim // k_groups >= 32; q in bf16 or fp16; split branch: ktile >= 32 when the caller supplies it; packed branch (pack_heads=True): block_tokens * n_kv_heads >= 32. A call that fails any of these with GNF4_ATTN_COMPUTE unset takes the f32 path (its own entry)", "the open item is on torch 2.8.0+cu128 / triton 3.4.0 -- the package floor -- so no torch/Triton pair inside the supported range is registered as passing this path's reference tests", "torch>=2.8", "torch>=2.8 (pre-releases accepted)", "triton>=3.4 (Linux-only distribution)" ] } }, "pinned_urls": { "commit": "https://github.com/pjordanandrsn/grouped-nf4-gemm/commit/e2af4cfb91b2c29968989c5254bff55aa7da3347", "tree": "https://github.com/pjordanandrsn/grouped-nf4-gemm/tree/e2af4cfb91b2c29968989c5254bff55aa7da3347", "capabilities": "https://github.com/pjordanandrsn/grouped-nf4-gemm/blob/e2af4cfb91b2c29968989c5254bff55aa7da3347/docs/capabilities.json", "claims": "https://github.com/pjordanandrsn/grouped-nf4-gemm/blob/e2af4cfb91b2c29968989c5254bff55aa7da3347/docs/claims.json", "status": "https://github.com/pjordanandrsn/grouped-nf4-gemm/blob/e2af4cfb91b2c29968989c5254bff55aa7da3347/docs/STATUS.md", "solutions_index": "https://github.com/pjordanandrsn/grouped-nf4-gemm/blob/e2af4cfb91b2c29968989c5254bff55aa7da3347/docs/SOLUTIONS.md", "system_manifest": "https://github.com/pjordanandrsn/grouped-nf4-gemm/blob/e2af4cfb91b2c29968989c5254bff55aa7da3347/docs/system-manifest.json" }, "current_main_urls": { "repository": "https://github.com/pjordanandrsn/grouped-nf4-gemm", "capabilities": "https://github.com/pjordanandrsn/grouped-nf4-gemm/blob/main/docs/capabilities.json", "claims": "https://github.com/pjordanandrsn/grouped-nf4-gemm/blob/main/docs/claims.json", "status": "https://github.com/pjordanandrsn/grouped-nf4-gemm/blob/main/docs/STATUS.md" }, "current_main_label": "latest on main — unpinned; not the source of any fact rendered here", "solutions": [ { "slug": "fp8-paged-attention-for-moe-serving", "title": "How do I run paged decode attention over an FP8 KV cache for a quantized MoE serving path, with sliding windows, attention sinks and a custom scale?", "summary": "fp8_paged_decode_attention runs paged flash-decode over an E4M3 KV cache with windows, sinks and a custom scale; the fp8 compute path is supported on sm_89+, the f32 path is open under #319.", "url": "https://cerinamroth.com/ml/solutions/fp8-paged-attention-for-moe-serving/" }, { "slug": "int4-decode-gemv", "title": "How do I run a single-token INT4 decode GEMV over routed MoE experts, and pack calibrated (GPTQ) weights for it?", "summary": "gemv_int4_b32 runs the batch-1 int4-b32 decode GEMV with exact integer accumulation, and gptq_pack_int4_b32 packs calibrated weights onto the same bytes; its numbers are measured-private.", "url": "https://cerinamroth.com/ml/solutions/int4-decode-gemv/" }, { "slug": "native-mxfp4-moe-inference", "title": "How do I run MoE expert inference natively on MXFP4 (e2m1 + e8m0) weights, straight from the released checkpoint bytes?", "summary": "gemm_mxfp4_grouped multiplies gpt-oss and Kimi-class experts on their released e2m1 blocks and e8m0 scales, with no requantization to NF4 and no bf16 materialization.", "url": "https://cerinamroth.com/ml/solutions/native-mxfp4-moe-inference/" }, { "slug": "nf4-grouped-gemm-without-bf16-materialization", "title": "How do I run a grouped GEMM directly on NF4 packed MoE expert weights, without dequantizing to bf16 first?", "summary": "gemm_4bit_grouped runs the routed MoE expert GEMM on bitsandbytes NF4 packed weights in one Triton launch, decoding nibbles in registers with fp32 accumulation and no bf16 expert tensor.", "url": "https://cerinamroth.com/ml/solutions/nf4-grouped-gemm-without-bf16-materialization/" }, { "slug": "stream-moe-experts-from-host-or-nvme", "title": "How do I run a MoE whose experts do not fit in VRAM, streaming them from pinned host RAM or an NVMe arena?", "summary": "The kernel-side storage primitives for experts that do not fit VRAM: the NVMe arena bake and O_DIRECT reader, the pinned-DRAM row tier and low-level residency, and the GPU-driven host gather.", "url": "https://cerinamroth.com/ml/solutions/stream-moe-experts-from-host-or-nvme/" }, { "slug": "verify-quantized-checkpoint-provenance", "title": "How do I verify that the quantized expert bytes I serve or train on are the released checkpoint bytes?", "summary": "file_tensor_sha256, provenance_table and verify_arena_matches hash safetensors byte ranges against the loaded and baked bytes, and verify_provenance re-derives a run's hash table.", "url": "https://cerinamroth.com/ml/solutions/verify-quantized-checkpoint-provenance/" } ], "sources": [ { "repository": "https://github.com/pjordanandrsn/grouped-nf4-gemm", "commit": "e2af4cfb91b2c29968989c5254bff55aa7da3347", "commit_date": "2026-09-05", "package": "grouped-nf4-gemm", "package_version": "0.30.2", "published_version": "0.30.2", "publication": "published", "documents_used": [ "docs/capabilities.json", "docs/claims.json", "docs/solutions/fp8-paged-attention-for-moe-serving.md", "docs/solutions/int4-decode-gemv.md", "docs/solutions/native-mxfp4-moe-inference.md", "docs/solutions/nf4-grouped-gemm-without-bf16-materialization.md", "docs/solutions/stream-moe-experts-from-host-or-nvme.md", "docs/solutions/verify-quantized-checkpoint-provenance.md", "docs/system-manifest.json", "pyproject.toml" ], "registers_used": [ "capabilities", "claims" ], "synced_at": "2026-09-06T00:07:26Z", "pinned_tree": "https://github.com/pjordanandrsn/grouped-nf4-gemm/tree/e2af4cfb91b2c29968989c5254bff55aa7da3347", "current_main": "https://github.com/pjordanandrsn/grouped-nf4-gemm", "current_main_label": "latest on main — unpinned; not the source of any fact rendered here", "system_manifest": { "url": "https://cerinamroth.com/ml/system-manifest.json", "provenance_url": "https://cerinamroth.com/ml/system-manifest.provenance.json", "sha256": "196cb55a41e890842f258305f307b873d55639cc617cd4ae61c771a063204918", "pinned": "https://github.com/pjordanandrsn/grouped-nf4-gemm/blob/e2af4cfb91b2c29968989c5254bff55aa7da3347/docs/system-manifest.json" } } ] } ] }