Add audio input support and make prompt optional

app.py

@@ -31,9 +31,13 @@ USAGE_TIPS = """
 ### 💡 使用技巧
 
 1. **视频质量**: 使用清晰、光线良好的视频
-2. **提示词**: 具体描述想要的音频类型
-3. **时长**: 建议1-30秒效果最佳
+2. **三种模式**:
+   - 纯视频：让AI根据画面自动生成音频
+   - 视频+文本：指定想要的音频类型
+   - 视频+音频+文本：使用参考音频的音色风格
+3. **时长**: 建议1-15秒效果最佳
 4. **CFG强度**: 数值越高越贴合提示词，但可能降低质量
+5. **参考音频**: 可提供音色、节奏、风格参考
 """
 
 # Check and install missing dependencies
@@ -205,7 +209,7 @@ class AudioFoleyModel:
     
     def generate_audio(self, video_file, prompt: str, negative_prompt: str = "", 
                       duration: float = 8.0, cfg_strength: float = 4.5, 
-                      seed: int = 42) -> Tuple[Optional[str], str]:
+                      seed: int = 42, reference_audio: str = None) -> Tuple[Optional[str], str]:
         """Generate audio from video and text prompt"""
         try:
             # Validation checks
@@ -216,13 +220,30 @@ class AudioFoleyModel:
                 return None, "❌ Model not loaded. Please load the model first."
             
             if video_file is None:
-                return None, "❌ Please upload a video file."
-            
-            if not prompt.strip():
-                return None, "❌ Please provide a text prompt describing the desired audio."
+                return None, "❌ 请上传视频文件。"
             
             log.info(f'🎬 Processing video: {video_file}')
-            log.info(f'📝 Prompt: "{prompt}"')
+            if prompt.strip():
+                log.info(f'📝 Prompt: "{prompt}"')
+            else:
+                log.info('📝 No prompt provided - will generate based on video content')
+            if reference_audio:
+                log.info(f'🎵 Reference audio: {reference_audio}')
+            
+            # Load and process reference audio if provided
+            reference_audio_tensor = None
+            if reference_audio and os.path.exists(reference_audio):
+                try:
+                    # Use the same Audio class from hf_AC
+                    SAMPLE_RATE = 44100
+                    audio_processor = Audio([reference_audio], SAMPLE_RATE)
+                    audio_list = audio_processor.load_audio()
+                    if audio_list:
+                        reference_audio_tensor = audio_list[0]
+                        log.info(f'🎵 Reference audio loaded: {reference_audio_tensor.shape}')
+                except Exception as e:
+                    log.warning(f"Failed to load reference audio: {e}")
+                    reference_audio_tensor = None
             
             # Load and process video
             try:
@@ -247,7 +268,12 @@ class AudioFoleyModel:
             # Update model sequence configuration
             try:
                 self.model.seq_cfg.duration = duration_sec
-                self.model.seq_cfg.audio_num_sample = 89088  # Default for 44kHz
+                # Set audio sample count based on reference audio or default
+                if reference_audio_tensor is not None:
+                    self.model.seq_cfg.audio_num_sample = reference_audio_tensor.shape[0]
+                else:
+                    self.model.seq_cfg.audio_num_sample = 89088  # Default for 44kHz
+                    
                 self.net.update_seq_lengths(
                     self.model.seq_cfg.latent_seq_len, 
                     self.model.seq_cfg.clip_seq_len, 
@@ -267,7 +293,7 @@ class AudioFoleyModel:
                         clip_frames,
                         sync_frames, 
                         [prompt], 
-                        None,  # No reference audio
+                        reference_audio_tensor,  # Use reference audio if provided
                         negative_text=[negative_prompt] if negative_prompt.strip() else None,
                         feature_utils=self.feature_utils,
                         net=self.net,
@@ -352,7 +378,7 @@ def initialize_model():
     else:
         return "✅ 模型已加载"
 
-def generate_audio_interface(video_file, prompt, duration, cfg_strength):
+def generate_audio_interface(video_file, audio_file, prompt, duration, cfg_strength):
     """Interface function for generating audio"""
     global audio_model, model_loading_status
     
@@ -365,7 +391,7 @@ def generate_audio_interface(video_file, prompt, duration, cfg_strength):
     negative_prompt = ""  # Simplified interface
     
     audio_path, message = audio_model.generate_audio(
-        video_file, prompt, negative_prompt, duration, cfg_strength, seed
+        video_file, prompt, negative_prompt, duration, cfg_strength, seed, audio_file
     )
     return audio_path, message
 
@@ -379,7 +405,11 @@ with gr.Blocks(title="hf_AC Audio Foley Generator", theme=gr.themes.Soft()) as d
     gr.Markdown("""
     # 🎵 hf_AC Audio Foley Generator
     
-    基于AI的视频音频生成工具。上传视频并提供��本描述，模型将生成匹配的音频内容。
+    基于AI的视频音频生成工具，支持三种生成模式：
+    
+    1. **纯视频模式**: 仅上传视频，根据视觉内容自动生成匹配音频
+    2. **视频+文本模式**: 上传视频 + 文本描述，生成指定类型的音频
+    3. **视频+音频+文本模式**: 上传视频 + 参考音频 + 文本，生成具有特定音色风格的音频
     
     **注意**: 模型会在启动时自动加载，首次使用需要下载约3GB的模型文件。
     """)
@@ -401,14 +431,21 @@ with gr.Blocks(title="hf_AC Audio Foley Generator", theme=gr.themes.Soft()) as d
     with gr.Row():
         with gr.Column():
             video_input = gr.Video(
-                label="上传视频",
+                label="上传视频 (必需)",
                 format="mp4"
             )
             
+            audio_input = gr.Audio(
+                label="参考音频 (可选) - 提供音色/风格参考",
+                type="filepath",
+                sources=["upload"]
+            )
+            
             prompt_input = gr.Textbox(
-                label="音频描述",
-                placeholder="描述你想要生成的音频 (例如: '脚步声', '鸟叫声', '汽车引擎声')",
-                lines=3
+                label="音频描述 (可选) - 留空则根据视频内容自动生成",
+                placeholder="可选：描述想要的音频类型 (例如: '脚步声', '鸟叫声', '汽车引擎声')",
+                lines=2,
+                value=""
             )
             
             with gr.Row():
@@ -452,7 +489,7 @@ with gr.Blocks(title="hf_AC Audio Foley Generator", theme=gr.themes.Soft()) as d
     generate_btn.click(
         fn=generate_audio_interface,
         inputs=[
-            video_input, prompt_input, duration_slider, cfg_strength_slider
+            video_input, audio_input, prompt_input, duration_slider, cfg_strength_slider
         ],
         outputs=[audio_output, generation_status]
     )