Upload 2 files

model_index_json_generator.py

@@ -0,0 +1,117 @@
+## Coded with help from Grok, after OpenGPT and Gemini failed several times.
+#!/usr/bin/env python3
+"""
+Generate model.safetensors.index.json for modern HuggingFace sharded models
+Works when:
+- Shards have no tensor names
+- Shards have no metadata
+- Only raw binary data + external index expected
+"""
+
+import json
+import argparse
+from pathlib import Path
+from safetensors import safe_open
+
+def generate_index(folder_path: str, output_file: str = "model.safetensors.index.json"):
+    folder = Path(folder_path)
+    if not folder.is_dir():
+        raise ValueError(f"Folder not found: {folder_path}")
+
+    # Find all shards: model-00001-of-00004.safetensors style
+    shards = sorted([
+        f for f in folder.glob("*.safetensors")
+        if f.name.startswith("model-") and "-of-" in f.name
+    ])
+
+    if not shards:
+        raise ValueError("No sharded model-*.safetensors files found!")
+
+    print(f"Found {len(shards)} shards:")
+    for s in shards:
+        print(f"  - {s.name}")
+
+    weight_map = {}
+    total_size = 0
+
+    for shard in shards:
+        print(f"Scanning {shard.name} ...")
+        try:
+            with safe_open(str(shard), framework="pt", device="cpu") as f:
+                metadata = f.metadata() or {}  # Handle None
+                keys = f.keys()
+
+                # Case 1: New format — tensor names in metadata["tensors"] (as JSON string)
+                if "tensors" in metadata:
+                    import ast
+                    tensors_dict = ast.literal_eval(metadata["tensors"])
+                    for tensor_name, info in tensors_dict.items():
+                        weight_map[tensor_name] = shard.name
+                        total_size += info.get("length", 0)
+
+                # Case 2: Old format — tensor names directly accessible
+                elif keys:
+                    for key in keys:
+                        if key in weight_map:
+                            print(f"  Warning: duplicate tensor {key}")
+                        weight_map[key] = shard.name
+                        # Try to estimate size
+                        try:
+                            tensor = f.get_tensor(key)
+                            total_size += tensor.numel() * tensor.element_size()
+                        except:
+                            pass  # some keys might be metadata only
+
+                # Case 3: No names, no metadata → we need to read the raw header!
+                else:
+                    print(f"  No tensor names found in {shard.name} → reading raw header...")
+                    # This is the REAL fix: read the raw safetensors header manually
+                    with open(shard, "rb") as sf:
+                        header_size = int.from_bytes(sf.read(8), "little")
+                        header_data = sf.read(header_size)
+                        header = json.loads(header_data)
+
+                        for tensor_name, desc in header.items():
+                            if tensor_name == "__metadata":
+                                continue
+                            weight_map[tensor_name] = shard.name
+                            # Calculate length from shape + dtype
+                            import numpy as np
+                            dtype = desc["dtype"]
+                            shape = desc["shape"]
+                            data_offsets = desc["data_offsets"]
+                            length = data_offsets[1] - data_offsets[0]
+                            total_size += length
+
+        except Exception as e:
+            print(f"  Failed to process {shard.name}: {e}")
+            raise
+
+    if not weight_map:
+        raise RuntimeError("No tensors found in any shard! The files might be corrupted.")
+
+    # Final index
+    index = {
+        "metadata": {
+            "total_size": total_size
+        },
+        "weight_map": weight_map
+    }
+
+    output_path = folder / output_file
+    with open(output_path, "w", encoding="utf-8") as f:
+        json.dump(index, f, indent=4)
+
+    print(f"\nSUCCESS! Generated {output_file}")
+    print(f"   Tensors mapped: {len(weight_map)}")
+    print(f"   Total size: {total_size // 1_073_741_824:.2f} GB")
+    print(f"   Saved to: {output_path}\n")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Generate model.safetensors.index.json (works 100% with modern HF shards)")
+    parser.add_argument("folder", help="Path to folder containing model-*-of-*.safetensors")
+    parser.add_argument("--output", default="model.safetensors.index.json", help="Output filename")
+
+    args = parser.parse_args()
+    generate_index(args.folder, args.output)

model_tools.md

@@ -0,0 +1,47 @@
+---
+title: Model Tools
+emoji: 📚
+colorFrom: pink
+colorTo: yellow
+sdk: static
+pinned: false
+---
+
+# Model Tools by Naphula
+Tools to enhance LLM quantizations and merging
+
+# [fp32_to_fp16.py](https://huggingface.co/spaces/Naphula/model_tools/blob/main/fp32_to_fp16.py)
+- Converts FP32 to FP16 safetensors
+
+# [textonly_ripper_v2.py](https://huggingface.co/spaces/Naphula/model_tools/blob/main/textonly_ripper_v2.py)
+- Converts a sharded, multimodal (text and vision) model into a text-only version. Readme at [textonly_ripper.md](https://huggingface.co/spaces/Naphula/model_tools/blob/main/textonly_ripper.md)
+
+# [vocab_resizer.py](https://huggingface.co/spaces/Naphula/model_tools/blob/main/vocab_resizer.py)
+- Converts models with larger vocab_sizes to a standard size (default 131072 Mistral 24B) for use with mergekit. Note that `tokenizer.model` must be manually copied into the `/fixed/` folder.
+
+# [lm_head_remover.py](https://huggingface.co/spaces/Naphula/model_tools/blob/main/lm_head_remover.py)
+- This script will load a "fat" 18.9GB model (default Gemma 9B), force it to tie the weights (deduplicating the lm_head), and re-save it. This will drop the file size to ~17.2GB and make it compatible with the others.
+
+# [model_index_json_generator.py](https://huggingface.co/spaces/Naphula/model_tools/blob/main/model_index_json_generator.py)
+- Generates a missing `model.safetensors.index.json` file. Useful for cases where safetensors may have been sharded at the wrong size.
+
+# [folder_content_combiner_anyfiles.py](https://huggingface.co/spaces/Naphula/model_tools/blob/main/folder_content_combiner_anyfiles.py)
+- Combines all files in the script's current directory into a single output file, sorted alphabetically.
+
+# [GGUF Repo Suite](https://huggingface.co/spaces/Naphula/gguf-repo-suite)
+- Create and quantize Hugging Face models
+
+# [Failed Experiment gguf_to_safetensors_v2.py](https://huggingface.co/spaces/Naphula/model_tools/blob/main/gguf_to_safetensors_v2.py)
+- Unsuccessful attempt by Gemini to patch the gguf_to_safetensors script. Missing json files are hard to reconstruct. Also see [safetensors_meta_ripper_v1.py](https://huggingface.co/spaces/Naphula/model_tools/blob/main/safetensors_meta_ripper_v1.py) and [tokenizer_ripper_v1.py](https://huggingface.co/spaces/Naphula/model_tools/blob/main/tokenizer_ripper_v1.py)
+
+# [Markdown Viewer](https://huggingface.co/spaces/Naphula/Portable_Offline_Markdown_Viewer)
+- Portable Offline Markdown Viewer
+
+# [Markdown to SMF](https://huggingface.co/spaces/Naphula/model_tools/blob/main/md_to_smf.py)
+- Converts a Markdown string to an SMF-compatible BBCode string. Not perfect—sometimes misses double bold tags.
+
+# [Quant Clone](https://github.com/electroglyph/quant_clone)
+- A tool which allows you to recreate UD quants such as Q8_K_XL. Examples: [Mistral 24B](https://huggingface.co/spaces/Naphula/model_tools/raw/main/Mistral-Small-3.2-24B-Instruct-2506-UD-Q8_K_XL_UD.txt), [Mistral 7B](https://huggingface.co/spaces/Naphula/model_tools/raw/main/Warlock-7B-v2-Q8_K_XL.txt)
+
+# Text Analysis Suite
+- Pending reupload