Spaces:
Build error
Build error
File size: 5,028 Bytes
9b4ef96 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 |
#!/usr/bin/env python3
"""
Script untuk download model yang benar-benar open source dan mudah diakses
"""
import os
import sys
import subprocess
from pathlib import Path
def check_huggingface_token():
"""Check if HuggingFace token is available"""
token = os.getenv('HUGGINGFACE_TOKEN')
if not token:
print("❌ HUGGINGFACE_TOKEN tidak ditemukan!")
print("Silakan set environment variable:")
print("export HUGGINGFACE_TOKEN='your_token_here'")
return False
return True
def download_model(model_name, model_path):
"""Download model menggunakan huggingface-cli"""
print(f"📥 Downloading model: {model_name}")
print(f"📁 Target directory: {model_path}")
try:
cmd = [
"huggingface-cli", "download",
model_name,
"--local-dir", str(model_path),
"--local-dir-use-symlinks", "False"
]
result = subprocess.run(cmd, capture_output=True, text=True)
if result.returncode == 0:
print("✅ Model berhasil didownload!")
return True
else:
print(f"❌ Error downloading model: {result.stderr}")
return False
except FileNotFoundError:
print("❌ huggingface-cli tidak ditemukan!")
print("Silakan install dengan: pip install huggingface_hub")
return False
def create_model_config(model_name, model_path):
"""Create model configuration file"""
config_dir = Path("configs")
config_dir.mkdir(exist_ok=True)
config_content = f"""# Model Configuration for {model_name}
model_name: "{model_name}"
model_path: "{model_path}"
max_length: 2048
temperature: 0.7
top_p: 0.9
top_k: 40
repetition_penalty: 1.1
# LoRA Configuration
lora_config:
r: 16
lora_alpha: 32
lora_dropout: 0.1
target_modules: ["q_proj", "v_proj", "k_proj", "o_proj", "gate_proj", "up_proj", "down_proj"]
# Training Configuration
training_config:
learning_rate: 2e-4
batch_size: 4
gradient_accumulation_steps: 4
num_epochs: 3
warmup_steps: 100
save_steps: 500
eval_steps: 500
"""
config_file = config_dir / f"{model_name.split('/')[-1].lower().replace('-', '_')}_config.yaml"
with open(config_file, 'w') as f:
f.write(config_content)
print(f"✅ Model config created: {config_file}")
return str(config_file)
def main():
print("🚀 Download Open Source Models")
print("=" * 50)
if not check_huggingface_token():
sys.exit(1)
# Model options - truly open source
models = [
{
"name": "microsoft/DialoGPT-medium",
"path": "models/dialogpt-medium",
"description": "DialoGPT Medium - Conversational AI model (355M parameters)"
},
{
"name": "distilgpt2",
"path": "models/distilgpt2",
"description": "DistilGPT2 - Lightweight GPT-2 model (82M parameters)"
},
{
"name": "gpt2",
"path": "models/gpt2",
"description": "GPT-2 - Original GPT-2 model (124M parameters)"
},
{
"name": "EleutherAI/gpt-neo-125M",
"path": "models/gpt-neo-125m",
"description": "GPT-Neo 125M - Small but capable model (125M parameters)"
}
]
print("📋 Pilih model yang ingin didownload:")
for i, model in enumerate(models, 1):
print(f"{i}. {model['name']}")
print(f" {model['description']}")
print()
try:
choice = int(input("Pilihan (1-4): ").strip())
if choice < 1 or choice > len(models):
print("❌ Pilihan tidak valid")
return
selected_model = models[choice - 1]
print(f"\n🎯 Model yang dipilih: {selected_model['name']}")
print(f"📝 Deskripsi: {selected_model['description']}")
# Confirm download
confirm = input("\nLanjutkan download? (y/n): ").strip().lower()
if confirm not in ['y', 'yes']:
print("❌ Download dibatalkan")
return
# Download model
print(f"\n1️⃣ Downloading model...")
if download_model(selected_model['name'], selected_model['path']):
print(f"\n2️⃣ Creating model configuration...")
config_file = create_model_config(selected_model['name'], selected_model['path'])
print("\n3️⃣ Setup selesai!")
print(f"\n📋 Langkah selanjutnya:")
print(f"1. Model tersimpan di: {selected_model['path']}")
print(f"2. Config tersimpan di: {config_file}")
print("3. Jalankan: python scripts/finetune_lora.py")
print("4. Atau gunakan Novita AI: python scripts/novita_ai_setup.py")
except ValueError:
print("❌ Input tidak valid")
except KeyboardInterrupt:
print("\n👋 Download dibatalkan")
if __name__ == "__main__":
main()
|