import librosa
import numpy as np

def generate_whisper_mel(audio_path: str):
	# 1. 加載音頻並統一採樣率為16kHz
	waveform, _ = librosa.load(audio_path, sr=16000)
	# 2. 按Whisper標準參數生成80通道梅爾譜
	mel_spec = librosa.feature.melspectrogram(
		y=waveform, sr=16000, n_fft=400, hop_length=160, n_mels=80, fmax=8000
	)
	# 3. 對數轉換+歸一化（Whisper必填步驟）
	log_mel = np.log(np.clip(mel_spec, a_min=1e-10, a_max=None))
	return (log_mel - log_mel.mean()) / (log_mel.std() + 1e-6)

mel_feature = generate_whisper_mel("sample_audio.wav")
print(f"梅爾譜形狀: {mel_feature.shape}") # 輸出(80, 時間步長)

import torch
import torch.nn as nn

class WhisperPreprocessStem(nn.Module):
	"""Whisper Transformer前的核心預處理模塊"""
	def __init__(self, out_dim: int = 512):
		super().__init__()
		# 兩次卷積+GELU特徵提取
		self.conv_block = nn.Sequential(
			nn.Conv2d(1, 16, 3, padding=1), # 1→16通道（輸入為單通道梅爾譜）
			nn.GELU(),
			nn.Conv2d(16, 32, 3, padding=1, stride=(2,1)), # 16→32通道，時間降維1/2
			nn.GELU()
		)
		# 映射到Transformer輸入維度
		self.fc = nn.Linear(32 * 80 * 500, out_dim) # 假設輸入時間步1000，降維後500
	
	def forward(self, mel_spec: torch.Tensor):
		# 梅爾譜形狀：(batch, 時間步, 80) → (batch, 1, 時間步, 80)（適配卷積）
		mel_spec = mel_spec.unsqueeze(1)
		conv_out = self.conv_block(mel_spec).flatten(1) # 展平卷積結果
		return self.fc(conv_out).permute(1, 0, 2) # 轉成Transformer需要的(seq_len, batch, d_model)

fake_mel = torch.randn(1, 1000, 80) # 1個batch、1000時間步、80通道
preprocessor = WhisperPreprocessStem()
print(f"預處理後輸出形狀: {preprocessor(fake_mel).shape}")

import torch
import math

class PositionalEncoding(nn.Module):
	def __init__(self, d_model: int, max_len: int = 5000):
		super().__init__()
		# 預計算位置編碼（不參與梯度更新）
		pe = torch.zeros(max_len, d_model)
		position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)
		div_term = torch.exp(torch.arange(0, d_model, 2) * (-math.log(10000.0) / d_model))
		pe[:, 0::2] = torch.sin(position * div_term)
		pe[:, 1::2] = torch.cos(position * div_term)
		self.register_buffer('pe', pe)
	
	def forward(self, x: torch.Tensor):
		return x + self.pe[:x.size(0)] # 按輸入序列長度截取編碼

git clone https://github.com/openai/whisper.git
python -m venv whisper-env
source whisper-env/bin/activate

pip install torch torchaudio librosa

import whisper

# 加載模型，以Windows為例，下載路徑C:\Users\johnny\.cache\whisper
model = whisper.load_model("base")
# load audio and pad/trim it to fit 30 seconds
audio = whisper.load_audio("ok.wav")
audio = whisper.pad_or_trim(audio)
# make log-Mel spectrogram and move to the same device as the model
mel = whisper.log_mel_spectrogram(audio).to(model.device)
# detect the spoken language
_, probs = model.detect_language(mel)
print(f"Detected language: {max(probs, key=probs.get)}")

model = whisper.load_model("tiny")
result = model.transcribe("ok.wav")
print(result["text"])

import torch
torch.cuda.is_available()
DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
model = whisper.load_model("tiny", device=DEVICE)
result = model.transcribe("ok.wav")
print(result["text"])

import whisper
import time

# 轉錄
def basic_transcribe(audio_path: str, model_size: str = "base"):
	model = whisper.load_model(model_size)
	start = time.time()
	# 核心轉錄（內部自動執行梅爾譜+卷積+GELU預處理）
	result = model.transcribe(
		audio_path, language="zh", task="transcribe", word_timestamps=True
	)
	return {
		"text": result["text"],
		"word_level": [{"word": w["word"], "time": (w["start"], w["end"])} for w in result["segments"][0]["words"]],
		"耗時": f"{time.time()-start:.2f}秒"
	}

print(basic_transcribe("meeting.wav", "large-v3-turbo"))

from pyannote.audio import Pipeline
import whisper

def meeting_transcribe(audio_path: str, hf_token: str):
	# 1. 説話人分離
	diarize = Pipeline.from_pretrained("pyannote/speaker-diarization-3.1", use_auth_token=hf_token)
	speaker_segments = diarize(audio_path)
	# 2. Whisper轉錄全量音頻
	model = whisper.load_model("medium")
	result = model.transcribe(audio_path, word_timestamps=True)
	# 3. 文本與説話人對齊
	meeting_notes = []
	for text_seg in result["segments"]:
		# 匹配該時間段的説話人
		speaker = next((spk for turn, _, spk in speaker_segments.itertracks() 
	                  if turn.start <= text_seg["start"] <= turn.end), "未知")
		meeting_notes.append({"speaker": speaker, "text": text_seg["text"]})
	return meeting_notes

print(meeting_transcribe("team_meeting.wav", "xxx"))

import torch
from transformers import AutoModelForSpeechSeq2Seq, AutoProcessor

def industrial_whisper_infer(audio_tensor: torch.Tensor):
	# 加載8-bit量化模型，顯存佔用降低45%
	processor = AutoProcessor.from_pretrained("openai/whisper-large-v3-turbo")
	model = AutoModelForSpeechSeq2Seq.from_pretrained(
		"openai/whisper-large-v3-turbo", load_in_8bit=True, device_map="auto", torch_dtype=torch.float16
	)
	# 低延遲推理參數
	inputs = processor(audio_tensor, sampling_rate=16000, return_tensors="pt").to("cuda")
	with torch.no_grad():
		gen_ids = model.generate(**inputs, max_new_tokens=256, do_sample=False, temperature=0.0)
	return processor.batch_decode(gen_ids, skip_special_tokens=True)[0]

import torch
from transformers import AutoModelForSpeechSeq2Seq, AutoProcessor, pipeline
device = "cuda:0" if torch.cuda.is_available() else "cpu"
torch_dtype = torch.float16 if torch.cuda.is_available() else torch.float32
model_id = "openai/whisper-large-v2"
model = AutoModelForSpeechSeq2Seq.from_pretrained(
	model_id, torch_dtype=torch_dtype, low_cpu_mem_usage=True, use_safetensors=True
)
model.to(device)
processor = AutoProcessor.from_pretrained(model_id)
pipe = pipeline(
	"automatic-speech-recognition",
	model=model,
	tokenizer=processor.tokenizer,
	feature_extractor=processor.feature_extractor,
	torch_dtype=torch_dtype,
	device=device,
)
generate_kwargs = {
    "max_new_tokens": 448
}

# 使用medium模型識別音頻，生成json、srt、tsv、txt、vtt文本文件
whisper --model medium dataset/9s.wav

import openai
CHATGPT_KEY = "OpenAI API Key"
openai.api_key = CHATGPT_KEY
audio_file = open("9s.wav", "rb")
transcription = openai.audio.transcriptions.create(
  model="whisper-1", 
  file=audio_file, 
  response_format="text"
)
print(transcription)