TurkSpeak — Komutlar

cd ~/Speech
bash setup.sh

model:
  name: turkspeak_tts
  size: mid                # small | mid | large
  codec: snac
  vocoder: snac_24khz
  hidden_dim: 1024
  num_layers: 24
  num_heads: 16

dataset:
  path: data/omersaidd__tts_nisan_kumru_tur
  sample_rate: 24000
  text_normalize: turkish_v1

training:
  batch_size: 16           # per-GPU
  grad_accum: 2            # effective = 16 × 2 × num_gpus = 64
  num_workers: 8
  max_steps: 50000
  warmup_steps: 1000
  lr: 3e-4
  lr_min: 1e-5
  weight_decay: 0.01
  grad_clip: 1.0
  precision: bf16-mixed
  compile: true

checkpoint:
  save_every: 5000
  keep_last: 3
  dir: runs/mid_5090dual_${TIMESTAMP}

cd ~/Speech
tmux new -s train -d
tmux send-keys -t train "bash train.sh configs/tts_mid.yaml 2>&1 | tee runs/train_$(date +%Y%m%d_%H%M%S).log" Enter
tmux attach -t train

BATCH_SIZE=24 MAX_STEPS=100000 LR=2e-4 bash train.sh configs/tts_mid.yaml

tail -f ~/Speech/runs/*/train_*.log | grep -E "step=|loss=|Saved"

watch -n 1 nvidia-smi

model:
  name: turkspeak_stt
  base: facebook/wav2vec2-xls-r-300m
  task: ctc
  vocab: turkish_chars          # a-zçğıöşü + boşluk + kesme işareti
  freeze_feature_encoder: true

dataset:
  path: data/omersaidd__tts_nisan_kumru_tur
  sample_rate: 16000            # STT 16 kHz (TTS 24 kHz'den farklı)
  text_normalize: turkish_v1

training:
  batch_size: 8                 # per-GPU
  grad_accum: 4                 # effective = 8 × 4 × num_gpus
  num_workers: 8
  max_steps: 40000
  warmup_steps: 1000
  lr: 1e-4
  lr_min: 1e-6
  weight_decay: 0.0
  grad_clip: 1.0
  precision: bf16-mixed
  ctc_zero_infinity: true

checkpoint:
  save_every: 5000
  keep_last: 3
  dir: runs/stt_xlsr_${TIMESTAMP}

cd ~/Speech
tmux new -s train_stt -d
tmux send-keys -t train_stt "bash train.sh configs/stt_xlsr.yaml 2>&1 | tee runs/stt_$(date +%Y%m%d_%H%M%S).log" Enter
tmux attach -t train_stt

BATCH_SIZE=12 MAX_STEPS=60000 LR=8e-5 bash train.sh configs/stt_xlsr.yaml

tail -f ~/Speech/runs/stt_*/stt_*.log | grep -E "step=|loss=|wer=|Saved"

tmux new -s eval -d 2>/dev/null || tmux attach -t eval

export PROJECT_DIR=$HOME/Speech
export CKPT_DIR=runs/mid_5090dual_20260508_135918

cd "$PROJECT_DIR"
mkdir -p eval_out

nvidia-smi | head -20

python -c "
import torch
print('torch', torch.__version__, 'cuda', torch.cuda.is_available(), 'devs', torch.cuda.device_count())
for i in range(torch.cuda.device_count()):
    print(f'  [{i}] {torch.cuda.get_device_name(i)}')"

ls -lh "$CKPT_DIR/"
grep -E "step=|Saved|Training done" "$CKPT_DIR"/*.log | tail -10

pip install -q matplotlib pandas

python - <<'PYEOF'
import re, glob, os, pandas as pd, matplotlib
matplotlib.use("Agg"); import matplotlib.pyplot as plt

logs = sorted(glob.glob(os.path.join(os.environ["CKPT_DIR"], "*.log")))
pat = re.compile(r"step=(\d+)/\d+\s+loss=([\d.]+).*?grad=([\d.]+).*?lr=([\d.eE+-]+)")
rows = [list(map(float, m.groups())) for line in open(logs[-1], encoding="utf-8", errors="ignore") if (m := pat.search(line))]
df = pd.DataFrame(rows, columns=["step","loss","grad","lr"]).astype({"step": int}).drop_duplicates("step")
print(df.describe())

fig, ax = plt.subplots(1, 3, figsize=(15, 4))
ax[0].plot(df.step, df.loss); ax[0].set_title(f"loss (final {df.loss.iloc[-1]:.4f})"); ax[0].set_yscale("log"); ax[0].grid(True, alpha=.3)
ax[1].plot(df.step, df.grad); ax[1].set_title(f"grad (final {df.grad.iloc[-1]:.4f})"); ax[1].set_yscale("log"); ax[1].grid(True, alpha=.3)
ax[2].plot(df.step, df.lr);   ax[2].set_title(f"lr (final {df.lr.iloc[-1]:.2e})"); ax[2].grid(True, alpha=.3)
plt.tight_layout(); plt.savefig("eval_out/loss_curve.png", dpi=120, bbox_inches="tight")
print("📈 eval_out/loss_curve.png")
PYEOF

cat > eval_prompts.txt <<'PROMPTS'
Bugün hava çok güzel, kahvemi alıp sahile yürüyeceğim.
Bin dokuz yüz yirmi üç yılında Cumhuriyet ilan edildi.
Atatürk'ün gençliğe hitabesini hatırlıyor musun?
Bu dosyayı download edip bilgisayarına atabilirsin.
Acaba bu sayı yedi yüz seksen dokuz mu, yoksa sekiz yüz mü?
İstanbul'da yaşıyorum ama ailem Erzurum'da kaldı.
Soru cümlesi mi bu? Yoksa sıradan bir ifade mi?
Vay canına! Bu gerçekten harika bir sonuç!
Yarın sabah saat dokuzda toplantımız var, unutma.
Üç nokta üst üste, virgül, ünlem, soru işareti.
PROMPTS
echo "📝 $(wc -l < eval_prompts.txt) prompts"

pip install -q soundfile numpy

CKPT=$(ls -1t "$CKPT_DIR"/checkpoint_*.pt | head -1)
echo "Checkpoint: $CKPT"

python -m turkspeak.tts.cli synthesize \
  --checkpoint "$CKPT" \
  --prompts eval_prompts.txt \
  --output-dir eval_out \
  --sample-rate 24000 \
  --device cuda

ls -lh eval_out/sample_*.wav

pip install -q onnxruntime-gpu librosa requests

python - <<'PYEOF'
import os, json, glob, librosa, numpy as np, onnxruntime as ort, requests
from pathlib import Path

# DNSMOS modelini indir (Microsoft DNS Challenge)
MODEL_URL = "https://github.com/microsoft/DNS-Challenge/raw/master/DNSMOS/DNSMOS/sig_bak_ovr.onnx"
MODEL_PATH = Path.home() / ".cache" / "dnsmos.onnx"
MODEL_PATH.parent.mkdir(parents=True, exist_ok=True)
if not MODEL_PATH.exists():
    print("Downloading DNSMOS ...")
    MODEL_PATH.write_bytes(requests.get(MODEL_URL).content)

sess = ort.InferenceSession(str(MODEL_PATH), providers=["CUDAExecutionProvider", "CPUExecutionProvider"])

def dnsmos(wav_path):
    audio, sr = librosa.load(wav_path, sr=16000)
    audio = audio.astype(np.float32)
    if len(audio) < 16000 * 9: audio = np.pad(audio, (0, 16000*9 - len(audio)))
    audio = audio[:16000*9].reshape(1, -1)
    out = sess.run(None, {"input_1": audio})[0][0]
    return {"SIG": float(out[1]), "BAK": float(out[2]), "OVRL": float(out[0])}

wavs = sorted(glob.glob("eval_out/sample_*.wav"))
prompts = [l.strip() for l in open("eval_prompts.txt", encoding="utf-8") if l.strip()]
rows = []
for i, w in enumerate(wavs):
    s = dnsmos(w)
    s["i"] = i; s["wav"] = w; s["text"] = prompts[i] if i < len(prompts) else ""
    rows.append(s)
    print(f"[{i:02d}] OVRL={s['OVRL']:.2f}  SIG={s['SIG']:.2f}  BAK={s['BAK']:.2f}  | {s['text'][:50]}")

mean_ovrl = sum(r["OVRL"] for r in rows) / len(rows)
mean_sig  = sum(r["SIG"]  for r in rows) / len(rows)
mean_bak  = sum(r["BAK"]  for r in rows) / len(rows)

print(f"\n{'='*60}")
print(f"  TurkSpeak TTS  → MEAN OVRL={mean_ovrl:.2f}  SIG={mean_sig:.2f}  BAK={mean_bak:.2f}")
print(f"{'='*60}")

Path("eval_out/tts_eval.json").write_text(
    json.dumps({"mean_OVRL": mean_ovrl, "mean_SIG": mean_sig, "mean_BAK": mean_bak, "samples": rows},
               ensure_ascii=False, indent=2),
    encoding="utf-8")
print("📊 eval_out/tts_eval.json")
PYEOF

python - <<'PYEOF'
import librosa, librosa.display, matplotlib
matplotlib.use("Agg"); import matplotlib.pyplot as plt
import glob, numpy as np
from pathlib import Path

for w in sorted(glob.glob("eval_out/sample_*.wav")):
    y, sr = librosa.load(w, sr=24000)
    fig, ax = plt.subplots(2, 1, figsize=(12, 5))
    ax[0].plot(np.arange(len(y))/sr, y, lw=0.5); ax[0].set_title(f"waveform — {Path(w).name}")
    ax[0].set_xlabel("s"); ax[0].set_xlim(0, len(y)/sr)
    S = librosa.feature.melspectrogram(y=y, sr=sr, n_mels=80, fmax=12000)
    librosa.display.specshow(librosa.power_to_db(S, ref=np.max), sr=sr, x_axis="time", y_axis="mel", ax=ax[1])
    ax[1].set_title("mel-spectrogram (dB)")
    plt.tight_layout()
    out = w.replace(".wav", ".png")
    plt.savefig(out, dpi=110, bbox_inches="tight"); plt.close()
    print(f"  {out}")
print("📈 spectrograms saved")
PYEOF

[ -f eval_prompts.txt ] && cp eval_prompts.txt eval_out/eval_prompts.txt

STAMP=$(date +%Y%m%d_%H%M%S)
tar -czf "eval_out_${STAMP}.tar.gz" eval_out/
ls -lh "eval_out_${STAMP}.tar.gz"
tar -tzf "eval_out_${STAMP}.tar.gz" | head -30

scp $USER@<itu_host>:~/Speech/eval_out_*.tar.gz ~/Downloads/
tar -xzf ~/Downloads/eval_out_*.tar.gz -C ~/Downloads/

cd ~/Speech
find . -type f -name "*.py" \
  | xargs grep -ln "def.*synthesize\|class.*Synthesizer\|cli\s*[:=]" 2>/dev/null | head

python - <<'PYEOF'
import torch, sys
ckpt = sys.argv[1] if len(sys.argv) > 1 else "runs/mid_5090dual_20260508_135918/checkpoint_00050000.pt"
state = torch.load(ckpt, map_location="cpu", weights_only=False)
print("Keys:", list(state.keys())[:10])
if "model" in state: print("model param count:", sum(p.numel() for p in state["model"].values() if torch.is_tensor(p)))
if "step" in state: print("step:", state["step"])
PYEOF

nvidia-smi
fuser -v /dev/nvidia*

pkill -9 -f "train.py\|python.*train\|python.*synthesize"

BATCH_SIZE=1 python -m turkspeak.tts.cli synthesize ...

df -h
du -sh ~/.cache/huggingface ~/Speech/runs ~/Speech/eval_out

ls -t ~/Speech/runs/*/checkpoint_*.pt | tail -n +4 | xargs rm -f

find ~/Speech /workspace /mnt -name "*.parquet" 2>/dev/null | head

tmux ls
tmux attach -t train       # veya eval

cd ~/Speech
rm -rf venv
python3 -m venv venv && source venv/bin/activate
pip install -U pip
pip install -r requirements.txt

head -20 ~/Speech/runs/*/train_*.log
grep "step=" ~/Speech/runs/*/train_*.log | head -3

mkdir -p ~/.cache
curl -L -o ~/.cache/dnsmos.onnx \
  https://github.com/microsoft/DNS-Challenge/raw/master/DNSMOS/DNSMOS/sig_bak_ovr.onnx