多模态特征融合网络 FusionNet：统一框架整合多种融合模块
·
在多模态学习任务中（如视觉 - 语言、音频 - 视觉等），特征融合是核心环节。本文整理了一套名为FusionNet的统一框架，该框架整合了 24 种不同的多模态特征融合模块，可灵活切换不同融合策略，适配各类多模态任务场景。这里博主直接把代码粘贴出来，方便大家有选择的取用。
一、代码整体架构
二、代码
1.FusionNet.py #调度类文件
import torch
import torch.nn as nn
from .MultiModal.EnhancedSemanticAttentionModule import EnhancedSemanticAttentionModule
from .MultiModal.EnhancedSemanticAttentionModule2 import EnhancedSemanticAttentionModule2
from .MultiModal.EnhancedSemanticAttentionModule3 import EnhancedSemanticAttentionModule3
from .MultiModal.FeatureFusionModule import FeatureFusionModule
from .MultiModal.FeatureFusionModule2 import FeatureFusionModule2
from .MultiModal.FeatureFusionModule3 import FeatureFusionModule3
from .MultiModal.FeatureFusionModule4 import FeatureFusionModule4
from .MultiModal.ModalFusionModule import ModalFusionModule
from .MultiModal.ModalFusionModule2 import ModalFusionModule2
from .MultiModal.ModalFusionModule3 import ModalFusionModule3
from .MultiModal.ModalFusionModule4 import ModalFusionModule4
from .MultiModal.ModalFusionModule5 import ModalFusionModule5
from .MultiModal.ModalFusionModule6 import ModalFusionModule6
from .MultiModal.ModalFusionModule7 import ModalFusionModule7
from .MultiModal.ModalFusionModule8 import ModalFusionModule8
from .MultiModal.ModalFusionModule9 import ModalFusionModule9
from .MultiModal.ModalFusionModule10 import ModalFusionModule10
from .MultiModal.ModalFusionModule11 import ModalFusionModule11
from .MultiModal.ModalFusionModule12 import ModalFusionModule12
from .MultiModal.ModalFusionModule13 import ModalFusionModule13
from .MultiModal.ConditionalFusionModule14 import ConditionalFusionModule14
from .MultiModal.QueryEnhancedSemanticModule import QueryEnhancedSemanticModule
from .MultiModal.SemanticAttentionModule import SemanticAttentionModule
from .MultiModal.SemanticAttentionModule2 import SemanticAttentionModule2
class FusionNet(nn.Module):
def __init__(self, choice, global_dim=None, local_dim=None):
self.global_dim = global_dim
self.local_dim = local_dim
super(FusionNet, self).__init__()
self.choice = choice
# 根据 choice 选择不同的模型
if choice == 0:
self.block = EnhancedSemanticAttentionModule(global_dim=self.global_dim, local_dim=self.local_dim, num_heads=8)
elif choice == 1:
self.block = EnhancedSemanticAttentionModule2(global_dim=self.global_dim, local_dim=self.local_dim, num_heads=8)
elif choice == 2:
self.block = EnhancedSemanticAttentionModule3(global_dim=self.global_dim, local_dim=self.local_dim, num_heads=8)
elif choice == 3:
self.block = FeatureFusionModule(global_dim=self.global_dim, local_dim=self.local_dim)
elif choice == 4:
self.block = FeatureFusionModule2(global_dim=self.global_dim, local_dim=self.local_dim)
elif choice == 5:
self.block = FeatureFusionModule3(global_dim=self.global_dim, local_dim=self.local_dim)
elif choice == 6:
self.block = FeatureFusionModule4(global_dim=self.global_dim, local_dim=self.local_dim)
elif choice == 7:
self.block = ModalFusionModule(global_dim=self.global_dim, local_dim=self.local_dim, num_heads=8)
elif choice == 8:
self.block = ModalFusionModule2(global_dim=self.global_dim, local_dim=self.local_dim, num_heads=8)
elif choice == 9:
self.block = ModalFusionModule3(global_dim=self.global_dim, local_dim=self.local_dim, num_heads=8)
elif choice == 10:
self.block = ModalFusionModule4(global_dim=self.global_dim, local_dim=self.local_dim, num_heads=8)
elif choice == 11:
self.block = ModalFusionModule5(global_dim=self.global_dim, local_dim=self.local_dim, num_heads=8)
elif choice == 12:
self.block = ModalFusionModule6(global_dim=self.global_dim, local_dim=self.local_dim, num_heads=8)
elif choice == 13:
self.block = ModalFusionModule7(global_dim=self.global_dim, local_dim=self.local_dim, num_heads=8)
elif choice == 14:
self.block = ModalFusionModule8(global_dim=self.global_dim, local_dim=self.local_dim, num_heads=8)
elif choice == 15:
self.block = ModalFusionModule9(global_dim=self.global_dim, local_dim=self.local_dim)
elif choice == 16:
self.block = ModalFusionModule10(global_dim=self.global_dim, local_dim=self.local_dim)
elif choice == 17:
self.block = ModalFusionModule11(global_dim=self.global_dim, local_dim=self.local_dim, num_heads=8, num_encoder_layers=2)
elif choice == 18:
self.block = ModalFusionModule12(global_dim=self.global_dim, local_dim=self.local_dim)
elif choice == 19:
self.block = ModalFusionModule13(global_dim=self.global_dim, local_dim=self.local_dim)
elif choice == 20:
self.block = ConditionalFusionModule14(global_dim=self.global_dim, local_dim=self.local_dim)
elif choice == 21:
self.block = QueryEnhancedSemanticModule(global_dim=self.global_dim, local_dim=self.local_dim, num_heads=8)
elif choice == 22:
self.block = SemanticAttentionModule(global_dim=self.global_dim, local_dim=self.local_dim, num_heads=8)
elif choice == 23:
self.block = SemanticAttentionModule2(global_dim=self.global_dim, local_dim=self.local_dim, num_heads=8)
else:
raise ValueError(f"Invalid choice: {choice}")
def forward(self, global_input, local_input):
output = self.block(global_input, local_input)
return output
if __name__ == '__main__':
# Example usage
global_input = torch.rand(2, 33, 256) # Example global features
local_input = torch.rand(2, 33, 512) # Example local features with different dimension
for i in range(24):
block = FusionNet(choice=i, global_dim=256, local_dim=512)
output = block(global_input, local_input)
print(i, " ", global_input.size(), local_input.size(), output.size())1. ConditionalFusionModule14.py # 条件动态门控融合模块
import torch
import torch.nn as nn
import torch.nn.functional as F
#self-attention输出特征生成γ和β，来缩放全局和局部特征
class SelfAttention(nn.Module):
def __init__(self, feature_dim):
super(SelfAttention, self).__init__()
self.query = nn.Linear(feature_dim, feature_dim)
self.key = nn.Linear(feature_dim, feature_dim)
self.value = nn.Linear(feature_dim, feature_dim)
def forward(self, x):
Q = self.query(x)
K = self.key(x)
V = self.value(x)
attention_scores = torch.matmul(Q, K.transpose(-2, -1)) / (K.size(-1) ** 0.5)
attention = F.softmax(attention_scores, dim=-1)
return torch.matmul(attention, V)
class ConditionalFusionModule14(nn.Module):
def __init__(self, global_dim, local_dim):
super(ConditionalFusionModule14, self).__init__()
self.global_dim = global_dim
self.local_dim = local_dim
# Dimension alignment for local features, if necessary
self.local_to_global = nn.Linear(local_dim, global_dim) if global_dim != local_dim else None
self.self_attention = SelfAttention(global_dim) # Assuming attention operates on global_dim
self.condition_network = nn.Linear(global_dim, global_dim) # Output gamma and beta for each dim
self.output_layer = nn.Linear(2 * global_dim, local_dim+global_dim)
def forward(self, global_features, local_features):
if self.local_to_global:
local_features = self.local_to_global(local_features)
# Concatenate global and aligned local features
combined_features = torch.cat([global_features, local_features], dim=1)
# Process through self-attention
attention_output = self.self_attention(combined_features)
# Generate gamma and beta dynamically for each time step
gamma_beta = self.condition_network(attention_output)
gamma, beta = gamma_beta.chunk(2, dim=-1)
# Make sure gamma and beta match the sequence dimension
gamma = gamma.reshape(global_features.size(0), global_features.size(1), -1)
beta = beta.reshape(global_features.size(0), global_features.size(1), -1)
# Adjust both global and local features
adjusted_global = (1 + gamma) * global_features + beta
adjusted_local = (1 + gamma) * local_features + beta
# Concatenate adjusted features and pass through the output layer
final_features = torch.cat([adjusted_global, adjusted_local], dim=-1)
output = self.output_layer(final_features)
return output
if __name__ == '__main__':
# Example usage
global_input = torch.rand(2, 33, 256) # Example global features
local_input = torch.rand(2, 33, 512) # Example local features with different dimension
block = ConditionalFusionModule14(global_dim=256, local_dim=512)
output = block(global_input, local_input)
print(global_input.size(), local_input.size(), output.size())
2. EnhancedSemanticAttentionModule.py # 增强语义注意力融合v1
import torch
import torch.nn as nn
import torch.nn.functional as F
#全局特征和局部特征分别做cross-attention，拼接后使用Self-attention进行特征强化（如果local和global维度不匹配，则相互线性映射到匹配的维度）
import torch
import torch.nn as nn
import torch.nn.functional as F
class EnhancedSemanticAttentionModule(nn.Module):
def __init__(self, global_dim, local_dim, num_heads=8):
super(EnhancedSemanticAttentionModule, self).__init__()
self.global_dim = global_dim
self.local_dim = local_dim
self.num_heads = num_heads
# 使用线性层匹配维度
self.adjust_global_dim = nn.Linear(global_dim, local_dim)
self.adjust_local_dim = nn.Linear(local_dim, global_dim)
# Cross-Attention layers
self.global_to_local_attention = nn.MultiheadAttention(local_dim, num_heads)
self.local_to_global_attention = nn.MultiheadAttention(global_dim, num_heads)
# Self-Attention layer for the concatenated features
self.self_attention = nn.MultiheadAttention(global_dim + local_dim, num_heads)
# Optional: Layer normalization
self.layer_norm = nn.LayerNorm(global_dim + local_dim)
def forward(self, global_features, local_features):
global_features = global_features.permute(1, 0, 2) # Reshape to (T, B, C)
local_features = local_features.permute(1, 0, 2)
# 调整全局和局部特征的维度
adjusted_global_features = self.adjust_global_dim(global_features)
adjusted_local_features = self.adjust_local_dim(local_features)
# Cross-attention operations
global_to_local_attn, _ = self.global_to_local_attention(local_features, adjusted_global_features, adjusted_global_features)
local_to_global_attn, _ = self.local_to_global_attention(global_features, adjusted_local_features, adjusted_local_features)
# Concatenate the cross-attention outputs
concatenated_features = torch.cat((global_to_local_attn, local_to_global_attn), dim=2)
# Self-attention to enhance the features further
enhanced_features, _ = self.self_attention(concatenated_features, concatenated_features, concatenated_features)
# Optional: Layer normalization
enhanced_features = self.layer_norm(enhanced_features)
# Reshape output back to (B, T, C)
output = enhanced_features.permute(1, 0, 2)
return output
if __name__ == '__main__':
# Example usage
global_input = torch.rand(2, 33, 256).cuda() # Example global features
local_input = torch.rand(2, 33, 512).cuda() # Example local features
block = EnhancedSemanticAttentionModule(global_dim=256, local_dim=512, num_heads=8).cuda()
output = block(global_input, local_input)
print(global_input.size(), local_input.size(), output.size())
3. EnhancedSemanticAttentionModule2.py# 增强语义注意力融合v2
import torch
import torch.nn as nn
import torch.nn.functional as F
#全局特征和局部特征分别做cross-attention，拼接后使用Self-attention进行特征强化（如果local和global维度不匹配，则将local转到和global一致的维度）
class EnhancedSemanticAttentionModule2(nn.Module):
def __init__(self, global_dim, local_dim, num_heads=8):
super(EnhancedSemanticAttentionModule2, self).__init__()
self.global_dim = global_dim
self.local_dim = local_dim
self.num_heads = num_heads
# 线性层，用于调整局部特征维度以匹配全局特征
self.adjust_local_dim = nn.Linear(local_dim, global_dim)
# Cross-Attention layers
self.global_to_local_attention = nn.MultiheadAttention(global_dim, num_heads)
self.local_to_global_attention = nn.MultiheadAttention(global_dim, num_heads)
# Self-Attention layer for the concatenated features
self.self_attention = nn.MultiheadAttention(global_dim * 2, num_heads)
# Final linear layer to adjust output dimensions
self.final_linear = nn.Linear(global_dim * 2, global_dim + local_dim)
# Optional: Layer normalization
self.layer_norm = nn.LayerNorm(global_dim + local_dim)
def forward(self, global_features, local_features):
global_features = global_features.permute(1, 0, 2) # Reshape to (T, B, C)
local_features = local_features.permute(1, 0, 2)
# 调整局部特征的维度以匹配全局特征的维度
adjusted_local_features = self.adjust_local_dim(local_features)
# Cross-attention operations
global_to_local_attn, _ = self.global_to_local_attention(adjusted_local_features, global_features, global_features)
local_to_global_attn, _ = self.local_to_global_attention(global_features, adjusted_local_features, adjusted_local_features)
# Concatenate the cross-attention outputs
concatenated_features = torch.cat((global_to_local_attn, local_to_global_attn), dim=2)
# Self-attention to enhance the features further
enhanced_features, _ = self.self_attention(concatenated_features, concatenated_features, concatenated_features)
# Linear layer to adjust final output dimensions
final_output = self.final_linear(enhanced_features)
# Optional: Layer normalization
final_output = self.layer_norm(final_output)
# Reshape output back to (B, T, C)
output = final_output.permute(1, 0, 2)
return output
if __name__ == '__main__':
# Example usage
global_input = torch.rand(2, 33, 256).cuda() # Example global features
local_input = torch.rand(2, 33, 512).cuda() # Example local features
block = EnhancedSemanticAttentionModule2(global_dim=256, local_dim=512, num_heads=8).cuda()
output = block(global_input, local_input)
print(global_input.size(), local_input.size(), output.size())
4. EnhancedSemanticAttentionModule3.py# 增强语义注意力融合v3
import torch
import torch.nn as nn
import torch.nn.functional as F
#全局特征和局部特征分别做cross-attention，拼接后使用Self-attention进行特征强化（维度匹配则不切换维度）
class EnhancedSemanticAttentionModule3(nn.Module):
def __init__(self, global_dim, local_dim, num_heads=8):
super(EnhancedSemanticAttentionModule3, self).__init__()
self.global_dim = global_dim
self.local_dim = local_dim
self.num_heads = num_heads
# 条件性地添加线性层，仅当维度不匹配时
if self.global_dim != self.local_dim:
self.adjust_local_dim = nn.Linear(local_dim, global_dim)
self.adjust_global_dim = nn.Linear(global_dim, local_dim)
else:
self.adjust_local_dim = None
self.adjust_global_dim = None
# Cross-Attention layers
self.global_to_local_attention = nn.MultiheadAttention(global_dim, num_heads)
self.local_to_global_attention = nn.MultiheadAttention(global_dim, num_heads)
# Self-Attention layer for the concatenated features
self.self_attention = nn.MultiheadAttention(global_dim * 2, num_heads)
# Final linear layer to adjust output dimensions to (global_dim + local_dim)
self.final_linear = nn.Linear(global_dim * 2, global_dim + local_dim)
# Optional: Layer normalization
self.layer_norm = nn.LayerNorm(global_dim + local_dim)
def forward(self, global_features, local_features):
global_features = global_features.permute(1, 0, 2) # Reshape to (T, B, C)
local_features = local_features.permute(1, 0, 2)
# 调整维度以匹配（如果需要）
if self.adjust_local_dim is not None:
adjusted_local_features = self.adjust_local_dim(local_features)
else:
adjusted_local_features = local_features
if self.adjust_global_dim is not None:
adjusted_global_features = self.adjust_global_dim(global_features)
else:
adjusted_global_features = global_features
# Cross-attention operations
global_to_local_attn, _ = self.global_to_local_attention(adjusted_local_features, global_features, global_features)
local_to_global_attn, _ = self.local_to_global_attention(global_features, adjusted_local_features, adjusted_local_features)
# Concatenate the cross-attention outputs
concatenated_features = torch.cat((global_to_local_attn, local_to_global_attn), dim=2)
# Self-attention to enhance the features further
enhanced_features, _ = self.self_attention(concatenated_features, concatenated_features, concatenated_features)
# Linear layer to adjust final output dimensions
final_output = self.final_linear(enhanced_features)
# Optional: Layer normalization
final_output = self.layer_norm(final_output)
# Reshape output back to (B, T, C)
output = final_output.permute(1, 0, 2)
return output
if __name__ == '__main__':
# Example usage
global_input = torch.rand(2, 33, 256).cuda() # Example global features
local_input = torch.rand(2, 33, 256).cuda() # Example local features
block = EnhancedSemanticAttentionModule3(global_dim=256, local_dim=256, num_heads=8).cuda()
output = block(global_input, local_input)
print(global_input.size(), local_input.size(), output.size())
5. FeatureFusionModule.py # 轻量化特征融合v1（拼接+通道压缩）
import torch
import torch.nn as nn
import torch.nn.functional as F
#普通流场生成，然后模态融合
class FeatureFusionModule(nn.Module):
def __init__(self, global_dim, local_dim):
super(FeatureFusionModule, self).__init__()
self.global_dim = global_dim
self.local_dim = local_dim
# 网络用于生成流场特征
self.flow_field_generator = nn.Sequential(
nn.Linear(global_dim + local_dim, local_dim), # 拼接后过一个线性层
nn.ReLU(),
nn.Linear(local_dim, local_dim) # 输出与局部特征维度一致
)
# 最终输出网络
self.output_network = nn.Sequential(
nn.Linear(local_dim + global_dim, local_dim + global_dim), # 拼接后的处理
nn.ReLU(),
nn.Linear(local_dim + global_dim, local_dim + global_dim) # 最终输出的维度
)
def forward(self, global_features, local_features):
# 拼接全局和局部特征
combined_features = torch.cat([global_features, local_features], dim=-1)
# 生成流场特征
flow_field = self.flow_field_generator(combined_features)
# 得到精细化的局部特征
refined_local_features = local_features + flow_field
# 拼接精细化的局部特征与全局特征
final_features = torch.cat([refined_local_features, global_features], dim=-1)
# 通过网络获取最终输出
output = self.output_network(final_features)
return output
if __name__ == '__main__':
# 示例使用
global_input = torch.rand(2, 33, 256) # 全局特征 (假设 batch size = 2, sequence length = 10, global_dim = 256)
local_input = torch.rand(2, 33, 512) # 局部特征 (假设同样的 batch size 和 sequence length, local_dim = 128)
model = FeatureFusionModule(global_dim=256, local_dim=512)
output = model(global_input, local_input)
print(global_input.size(), local_input.size(), output.size())
6. FeatureFusionModule2.py # 轻量化特征融合v2（残差+通道注意力）
import torch
import torch.nn as nn
import torch.nn.functional as F
#多头注意力生成流场，然后模态融合
class MultiHeadAttention(nn.Module):
def __init__(self, embed_size, heads):
super(MultiHeadAttention, self).__init__()
self.embed_size = embed_size
self.heads = heads
self.head_dim = embed_size // heads
assert (self.head_dim * heads == embed_size), "Embed size needs to be divisible by heads"
self.values = nn.Linear(self.head_dim, self.head_dim, bias=False)
self.keys = nn.Linear(self.head_dim, self.head_dim, bias=False)
self.queries = nn.Linear(self.head_dim, self.head_dim, bias=False)
self.fc_out = nn.Linear(heads * self.head_dim, embed_size)
def forward(self, values, keys, query):
N = query.shape[0]
values = values.reshape(N, -1, self.heads, self.head_dim)
keys = keys.reshape(N, -1, self.heads, self.head_dim)
queries = query.reshape(N, -1, self.heads, self.head_dim)
values = self.values(values)
keys = self.keys(keys)
queries = self.queries(queries)
energy = torch.einsum("nqhd,nkhd->nhqk", [queries, keys])
attention = torch.softmax(energy / (self.embed_size ** (1 / 2)), dim=3)
out = torch.einsum("nhql,nlhd->nqhd", [attention, values]).reshape(N, -1, self.heads * self.head_dim)
out = self.fc_out(out)
return out
class FeatureFusionModule2(nn.Module):
def __init__(self, global_dim, local_dim, num_heads=8):
super(FeatureFusionModule2, self).__init__()
self.global_dim = global_dim
self.local_dim = local_dim
self.local_dim_adjust = nn.Linear(local_dim, global_dim)
# Adjust the fc_out in MultiHeadAttention to match the adjusted local dimension
self.flow_field_generator = MultiHeadAttention(global_dim * 2, num_heads)
self.flow_field_generator.fc_out = nn.Linear(num_heads * (global_dim * 2 // num_heads), global_dim)
self.output_network = nn.Sequential(
nn.Linear(global_dim * 2, global_dim + local_dim),
nn.ReLU(),
nn.Linear(global_dim + local_dim, global_dim + local_dim)
)
def forward(self, global_features, local_features):
local_features_adjusted = self.local_dim_adjust(local_features)
combined_features = torch.cat([global_features, local_features_adjusted], dim=-1)
flow_field = self.flow_field_generator(combined_features, combined_features, combined_features)
refined_local_features = local_features_adjusted + flow_field
final_features = torch.cat([refined_local_features, global_features], dim=-1)
output = self.output_network(final_features)
return output
if __name__ == '__main__':
# Example usage
global_input = torch.rand(2, 10, 256)
local_input = torch.rand(2, 10, 512)
model = FeatureFusionModule2(global_dim=256, local_dim=512)
output = model(global_input, local_input)
print(global_input.size(), local_input.size(), output.size())
7. FeatureFusionModule3.py # 轻量化特征融合v3（加权特征融合）
import torch
import torch.nn as nn
import torch.nn.functional as F
#预测置信度，并拼接融合模态
class FeatureProcessor(nn.Module):
def __init__(self, input_dim, output_dim):
super(FeatureProcessor, self).__init__()
self.network = nn.Sequential(
nn.Linear(input_dim, output_dim),
nn.ReLU(),
nn.Linear(output_dim, output_dim)
)
self.confidence = nn.Sequential(
nn.Linear(input_dim, 1), # 输出单一置信度分数
nn.Sigmoid() # 使用 Sigmoid 激活函数确保输出在0到1之间
)
def forward(self, x):
processed_features = self.network(x)
confidence_scores = self.confidence(x).squeeze(-1) # 确保无多余维度
return processed_features, confidence_scores
class FeatureFusionModule3(nn.Module):
def __init__(self, global_dim, local_dim):
super(FeatureFusionModule3, self).__init__()
self.global_processor = FeatureProcessor(global_dim, global_dim)
self.local_processor = FeatureProcessor(local_dim, local_dim)
self.relative_confidence_network = FeatureProcessor(global_dim + local_dim, 1) # 保持单一置信度输出
# 融合网络
self.fusion_network = nn.Sequential(
nn.Linear(global_dim + local_dim + 3, global_dim + local_dim), # 3个置信度
nn.ReLU(),
nn.Linear(global_dim + local_dim, global_dim + local_dim)
)
def forward(self, global_features, local_features):
refined_global, global_confidence = self.global_processor(global_features)
refined_local, local_confidence = self.local_processor(local_features)
# 拼接全局和局部特征
combined_features = torch.cat([refined_global, refined_local], dim=-1)
relative_confidence = self.relative_confidence_network(combined_features)[1] # 获取置信度
# 拼接所有特征和置信度
fusion_input = torch.cat([refined_global, refined_local, global_confidence.unsqueeze(-1), local_confidence.unsqueeze(-1), relative_confidence.unsqueeze(-1)], dim=-1)
fused_features = self.fusion_network(fusion_input)
return fused_features
if __name__ == '__main__':
# 示例使用
global_input = torch.rand(2, 10, 256) # 全局特征
local_input = torch.rand(2, 10, 512) # 局部特征
model = FeatureFusionModule3(global_dim=256, local_dim=512)
output = model(global_input, local_input)
print(global_input.size(), local_input.size(), output.size())
8. FeatureFusionModule4.py # 轻量化特征融合v4（分组卷积融合）
import torch
import torch.nn as nn
import torch.nn.functional as F
class TransformerFeatureProcessor(nn.Module):
def __init__(self, input_dim, num_heads, num_layers):
super(TransformerFeatureProcessor, self).__init__()
encoder_layer = nn.TransformerEncoderLayer(d_model=input_dim, nhead=num_heads, batch_first=True)
self.transformer_encoder = nn.TransformerEncoder(encoder_layer, num_layers=num_layers)
self.confidence = nn.Sequential(
nn.Linear(input_dim, 1),
nn.Sigmoid()
)
def forward(self, x):
out = self.transformer_encoder(x)
confidence_scores = self.confidence(out).squeeze(-1)
return out, confidence_scores
class FeatureFusionModule4(nn.Module):
def __init__(self, global_dim, local_dim, num_heads=4, num_layers=2):
super(FeatureFusionModule4, self).__init__()
self.global_dim = global_dim
self.local_dim = local_dim
self.global_processor = TransformerFeatureProcessor(global_dim, num_heads, num_layers)
self.local_processor = TransformerFeatureProcessor(local_dim, num_heads, num_layers)
self.relative_confidence_network = nn.Sequential(
nn.Linear(global_dim + local_dim, 1),
nn.Sigmoid()
)
# 融合网络
self.fusion_network = nn.Sequential(
nn.Linear(global_dim + local_dim + 3, global_dim + local_dim),
nn.ReLU(),
nn.Linear(global_dim + local_dim, global_dim + local_dim)
)
def forward(self, global_features, local_features):
refined_global, global_confidence = self.global_processor(global_features)
refined_local, local_confidence = self.local_processor(local_features)
# 拼接最后的特征以及计算相对置信度
combined_features = torch.cat([refined_global, refined_local], dim=-1)
relative_confidence = self.relative_confidence_network(combined_features).squeeze(-1)
# 拼接所有特征和置信度
fusion_input = torch.cat([
refined_global,
refined_local,
global_confidence.unsqueeze(-1),
local_confidence.unsqueeze(-1),
relative_confidence.unsqueeze(-1) # 复制置信度以匹配特征维数
], dim=-1)
fused_features = self.fusion_network(fusion_input)
return fused_features
if __name__ == '__main__':
# 示例使用
batch_size = 2
seq_len = 10
global_input = torch.rand(batch_size, seq_len, 256) # 全局特征
local_input = torch.rand(batch_size, seq_len, 512) # 局部特征
model = FeatureFusionModule4(global_dim=256, local_dim=512)
output = model(global_input, local_input)
print(global_input.size(), local_input.size(), output.size())
9. ModalFusionModule.py # 通用模态融合v1
import torch
import torch.nn as nn
import torch.nn.functional as F
#传统注意力+元素乘法注意力
class ModalFusionModule(nn.Module):
def __init__(self, global_dim, local_dim, num_heads=8):
super(ModalFusionModule, self).__init__()
self.global_dim = global_dim
self.local_dim = local_dim
self.num_heads = num_heads
# Dimension alignment for local features
if global_dim != local_dim:
self.align_dim = nn.Linear(local_dim, global_dim)
else:
self.align_dim = None
# Attention layers
self.attention = nn.MultiheadAttention(self.global_dim, num_heads)
# Coefficient predictor for alpha
self.alpha_predictor = nn.Sequential(
nn.Linear(self.global_dim, self.global_dim // 2),
nn.ReLU(),
nn.Linear(self.global_dim // 2, 1),
nn.Sigmoid()
)
# Output mapping layer
self.output_layer = nn.Linear(self.global_dim, self.global_dim + self.local_dim)
def forward(self, global_features, local_features):
B, T, _ = global_features.size()
# Adjust dimensions if necessary
if self.align_dim:
local_features = self.align_dim(local_features)
# Prepare Q, K, V
Q = global_features.permute(1, 0, 2) # (T, B, C)
K = local_features.permute(1, 0, 2)
V = global_features.permute(1, 0, 2)
# Standard attention (A1)
attn_output, _ = self.attention(Q, K, V)
# Alternative attention method (A2) - simple example with element-wise multiplication
A2 = torch.bmm(Q, K.transpose(1, 2))
attn_output_alt = torch.bmm(F.softmax(A2, dim=-1), V)
# Calculate dynamic mixing coefficient alpha
alpha = self.alpha_predictor(global_features.view(B * T, -1)).view(B, T, 1) # Correct the shape
alpha = alpha.permute(1, 0, 2) # Reshape to (T, B, 1) for broadcasting
# Combine attentions
combined_attn_output = alpha * attn_output + (1 - alpha) * attn_output_alt
# Map to the target dimension
output = self.output_layer(combined_attn_output.permute(1, 0, 2)) # Reshape back to (B, T, C)
return output
if __name__ == '__main__':
# Example usage
global_input = torch.rand(2, 33, 256) # Example global features
local_input = torch.rand(2, 33, 256) # Example local features
block = ModalFusionModule(global_dim=256, local_dim=256, num_heads=8)
output = block(global_input, local_input)
print(global_input.size(), local_input.size(), output.size())
10. ModalFusionModule2.py # 通用模态融合v2
import torch
import torch.nn as nn
import torch.nn.functional as F
#传统注意力+加性注意力
class AdditiveAttention(nn.Module):
def __init__(self, key_dim, query_dim, hidden_dim):
super(AdditiveAttention, self).__init__()
self.key_proj = nn.Linear(key_dim, hidden_dim)
self.query_proj = nn.Linear(query_dim, hidden_dim)
self.v = nn.Parameter(torch.randn(hidden_dim))
def forward(self, key, query):
# key, query: (T, B, C)
key_proj = self.key_proj(key) # (T, B, hidden_dim)
query_proj = self.query_proj(query) # (T, B, hidden_dim)
energy = torch.tanh(key_proj + query_proj.unsqueeze(1)) # Broadcasting (T, T, B, hidden_dim)
energy = torch.sum(self.v * energy, dim=-1) # Reduce along hidden dimension (T, T, B)
attn_weights = F.softmax(energy, dim=1) # Softmax over key dimension
return attn_weights.permute(2, 0, 1) # Ensure shape is (B, T, T)
class ModalFusionModule2(nn.Module):
def __init__(self, global_dim, local_dim, num_heads=8, hidden_dim=128):
super(ModalFusionModule2, self).__init__()
self.global_dim = global_dim
self.local_dim = local_dim
self.num_heads = num_heads
if global_dim != local_dim:
self.align_dim = nn.Linear(local_dim, global_dim)
else:
self.align_dim = None
self.attention = nn.MultiheadAttention(global_dim, num_heads)
self.additive_attention = AdditiveAttention(global_dim, global_dim, hidden_dim)
self.alpha_predictor = nn.Sequential(
nn.Linear(global_dim, global_dim // 2),
nn.ReLU(),
nn.Linear(global_dim // 2, 1),
nn.Sigmoid()
)
self.output_layer = nn.Linear(global_dim, global_dim + local_dim)
def forward(self, global_features, local_features):
B, T, _ = global_features.size()
if self.align_dim:
local_features = self.align_dim(local_features)
Q = global_features.permute(1, 0, 2) # (T, B, C)
K = local_features.permute(1, 0, 2)
V = global_features.permute(1, 0, 2) # (T, B, C)
attn_output, _ = self.attention(Q, K, V)
attn_weights = self.additive_attention(K, Q) # (B, T, T)
V_permuted = V.permute(1, 0, 2) # (B, T, C) for bmm
attn_output_alt = torch.bmm(attn_weights, V_permuted) # Should be (B, T, C)
attn_output_alt = attn_output_alt.permute(1, 0, 2) # Permute to (T, B, C) to match attn_output
alpha = self.alpha_predictor(global_features.view(B * T, -1)).view(B, T, 1)
alpha = alpha.permute(1, 0, 2).expand(-1, -1, attn_output.size(2)) # Reshape and expand alpha to (T, B, C)
combined_attn_output = alpha * attn_output + (1 - alpha) * attn_output_alt
output = self.output_layer(combined_attn_output).permute(1,0,2)
return output
if __name__ == '__main__':
# Example usage
global_input = torch.rand(2, 33, 256)
local_input = torch.rand(2, 33, 256)
block = ModalFusionModule2(global_dim=256, local_dim=256, num_heads=8)
output = block(global_input, local_input)
print(global_input.size(), local_input.size(), output.size())
11. ModalFusionModule3.py # 通用模态融合v3
import torch
import torch.nn as nn
import torch.nn.functional as F
#传统注意力+高斯核注意力
def gaussian_kernel_attention(Q, K, sigma=1.0):
# Q, K: (T, B, C)
T, B, C = Q.size()
# Expanding Q and K for broadcasting
Q_expanded = Q.unsqueeze(2).expand(-1, -1, T, -1) # Expands Q to (T, B, T, C)
K_expanded = K.unsqueeze(1).expand(-1, T, -1, -1) # Expands K to (T, T, B, C), needs rearranging
# Correcting the alignment issue:
K_expanded = K_expanded.permute(0, 2, 1, 3) # Now K_expanded is (T, B, T, C), aligned with Q_expanded
# Calculating the squared Euclidean distances
diff = Q_expanded - K_expanded
distance = (diff ** 2).sum(dim=-1) # Summing over the feature dimension C
# Applying the Gaussian kernel
gaussian_attention = torch.exp(-distance / (2 * sigma ** 2))
return gaussian_attention.permute(1, 0, 2) # Permuting to (B, T, T) for proper alignment in subsequent operations
class ModalFusionModule3(nn.Module):
def __init__(self, global_dim, local_dim, num_heads=8):
super(ModalFusionModule3, self).__init__()
self.global_dim = global_dim
self.local_dim = local_dim
self.num_heads = num_heads
if global_dim != local_dim:
self.align_dim = nn.Linear(local_dim, global_dim)
else:
self.align_dim = None
self.attention = nn.MultiheadAttention(self.global_dim, num_heads)
self.alpha_predictor = nn.Sequential(
nn.Linear(self.global_dim, self.global_dim // 2),
nn.ReLU(),
nn.Linear(self.global_dim // 2, 1),
nn.Sigmoid()
)
self.output_layer = nn.Linear(self.global_dim, self.global_dim + self.local_dim)
def forward(self, global_features, local_features):
B, T, _ = global_features.size()
if self.align_dim:
local_features = self.align_dim(local_features)
Q = global_features.permute(1, 0, 2) # (T, B, C)
K = local_features.permute(1, 0, 2)
V = global_features.permute(1, 0, 2)
attn_output, _ = self.attention(Q, K, V)
gaussian_attention = gaussian_kernel_attention(Q, K) # Should be (B, T, T)
attn_output_alt = torch.bmm(gaussian_attention, V.permute(1, 0, 2)).permute(1,0,2) # Apply Gaussian attention
alpha = self.alpha_predictor(global_features.view(B * T, -1)).view(B, T, 1).permute(1, 0, 2)
alpha = alpha.expand(-1, -1, V.size(2)) # Ensure alpha covers all channels
combined_attn_output = alpha * attn_output + (1 - alpha) * attn_output_alt
output = self.output_layer(combined_attn_output)
return output
if __name__ == '__main__':
# Example usage
global_input = torch.rand(2, 33, 256)
local_input = torch.rand(2, 33, 256)
block = ModalFusionModule3(global_dim=256, local_dim=256, num_heads=8)
output = block(global_input, local_input)
print(global_input.size(), local_input.size(), output.size())
12. ModalFusionModule4.py # 通用模态融合v4
import torch
import torch.nn as nn
import torch.nn.functional as F
#传统注意力+余弦相似度注意力
class ModalFusionModule4(nn.Module):
def __init__(self, global_dim, local_dim, num_heads=8):
super(ModalFusionModule4, self).__init__()
self.global_dim = global_dim
self.local_dim = local_dim
self.num_heads = num_heads
if global_dim != local_dim:
self.align_dim = nn.Linear(local_dim, global_dim)
else:
self.align_dim = None
self.attention = nn.MultiheadAttention(self.global_dim, num_heads)
self.alpha_predictor = nn.Sequential(
nn.Linear(self.global_dim, self.global_dim // 2),
nn.ReLU(),
nn.Linear(self.global_dim // 2, 1),
nn.Sigmoid()
)
self.output_layer = nn.Linear(self.global_dim, self.global_dim + self.local_dim)
def forward(self, global_features, local_features):
B, T, _ = global_features.size()
if self.align_dim:
local_features = self.align_dim(local_features)
Q = global_features.permute(1, 0, 2) # (T, B, C)
K = local_features.permute(1, 0, 2)
V = global_features.permute(1, 0, 2)
attn_output, _ = self.attention(Q, K, V)
# Calculate cosine similarity between Q and K
cosine_sim = torch.bmm(Q, K.transpose(1, 2)) / (
torch.norm(Q, dim=2, keepdim=True) * torch.norm(K.transpose(1, 2), dim=1, keepdim=True)
)
attn_output_alt = torch.bmm(F.softmax(cosine_sim, dim=-1), V)
alpha = self.alpha_predictor(global_features.view(B * T, -1)).view(B, T, 1)
alpha = alpha.permute(1, 0, 2) # Reshape to (T, B, 1) for broadcasting
combined_attn_output = alpha * attn_output + (1 - alpha) * attn_output_alt
output = self.output_layer(combined_attn_output.permute(1, 0, 2)) # Reshape back to (B, T, C)
return output
if __name__ == '__main__':
# Example usage
global_input = torch.rand(2, 33, 256)
local_input = torch.rand(2, 33, 512)
block = ModalFusionModule4(global_dim=256, local_dim=512, num_heads=8)
output = block(global_input, local_input)
print(global_input.size(), local_input.size(), output.size())
13. ModalFusionModule5.py # 通用模态融合v5
import torch
import torch.nn as nn
import torch.nn.functional as F
#传统注意力+双线性注意力
class ModalFusionModule5(nn.Module):
def __init__(self, global_dim, local_dim, num_heads=8):
super(ModalFusionModule5, self).__init__()
self.global_dim = global_dim
self.local_dim = local_dim
self.num_heads = num_heads
if global_dim != local_dim:
self.align_dim = nn.Linear(local_dim, global_dim)
else:
self.align_dim = None
self.attention = nn.MultiheadAttention(self.global_dim, num_heads)
self.bilinear_W = nn.Parameter(torch.rand(self.global_dim, self.global_dim)) # 双线性权重矩阵
self.alpha_predictor = nn.Sequential(
nn.Linear(self.global_dim, self.global_dim // 2),
nn.ReLU(),
nn.Linear(self.global_dim // 2, 1),
nn.Sigmoid()
)
self.output_layer = nn.Linear(self.global_dim, self.global_dim + self.local_dim)
def forward(self, global_features, local_features):
B, T, _ = global_features.size()
if self.align_dim:
local_features = self.align_dim(local_features)
Q = global_features.permute(1, 0, 2)
K = local_features.permute(1, 0, 2)
V = global_features.permute(1, 0, 2)
attn_output, _ = self.attention(Q, K, V)
# 双线性注意力
Bilinear_attn = torch.bmm(torch.matmul(Q, self.bilinear_W), K.transpose(1, 2))
attn_output_alt = torch.bmm(F.softmax(Bilinear_attn, dim=-1), V)
alpha = self.alpha_predictor(global_features.view(B * T, -1)).view(B, T, 1)
alpha = alpha.permute(1, 0, 2)
combined_attn_output = alpha * attn_output + (1 - alpha) * attn_output_alt
output = self.output_layer(combined_attn_output.permute(1, 0, 2))
return output
if __name__ == '__main__':
# Example usage
global_input = torch.rand(2, 33, 256)
local_input = torch.rand(2, 33, 512)
block = ModalFusionModule5(global_dim=256, local_dim=512, num_heads=8)
output = block(global_input, local_input)
print(global_input.size(), local_input.size(), output.size())
14. ModalFusionModule6.py # 通用模态融合v6
import torch
import torch.nn as nn
import torch.nn.functional as F
#传统注意力+加性注意力
class ModalFusionModule6(nn.Module):
def __init__(self, global_dim, local_dim, num_heads=8):
super(ModalFusionModule6, self).__init__()
self.global_dim = global_dim
self.local_dim = local_dim
self.num_heads = num_heads
if global_dim != local_dim:
self.align_dim = nn.Linear(local_dim, global_dim)
else:
self.align_dim = None
self.attention = nn.MultiheadAttention(self.global_dim, num_heads)
self.query_layer = nn.Linear(self.global_dim, self.global_dim)
self.key_layer = nn.Linear(self.global_dim, self.global_dim)
self.value_layer = nn.Linear(self.global_dim, 1) # 输出单一得分
self.alpha_predictor = nn.Sequential(
nn.Linear(self.global_dim, self.global_dim // 2),
nn.ReLU(),
nn.Linear(self.global_dim // 2, 1),
nn.Sigmoid()
)
self.output_layer = nn.Linear(self.global_dim, self.global_dim + self.local_dim)
def forward(self, global_features, local_features):
B, T, _ = global_features.size()
if self.align_dim:
local_features = self.align_dim(local_features)
Q = global_features.permute(1, 0, 2) # (T, B, C)
K = local_features.permute(1, 0, 2)
V = global_features.permute(1, 0, 2)
attn_output, _ = self.attention(Q, K, V)
Q_transformed = self.query_layer(Q) # 变换Q
K_transformed = self.key_layer(K) # 变换K
additive_scores = self.value_layer(torch.tanh(Q_transformed + K_transformed)) # 加性模型
additive_scores = additive_scores.permute(1, 0, 2) # 改变形状为 (B, T, 1)
additive_scores = additive_scores.expand(-1, -1, T) # 扩展维度以匹配 V 的序列长度
V_permuted = V.permute(1, 0, 2) # 改变 V 的形状为 (B, T, C)
attn_output_alt = torch.bmm(F.softmax(additive_scores, dim=2), V_permuted) # 应用注意力权重
alpha = self.alpha_predictor(global_features.view(B * T, -1)).view(B, T, 1)
alpha = alpha.permute(1, 0, 2)
combined_attn_output = alpha * attn_output + (1 - alpha) * attn_output_alt.permute(1,0,2)
output = self.output_layer(combined_attn_output.permute(1, 0, 2))
return output
if __name__ == '__main__':
# Example usage
global_input = torch.rand(2, 33, 256)
local_input = torch.rand(2, 33, 512)
block = ModalFusionModule6(global_dim=256, local_dim=512, num_heads=8)
output = block(global_input, local_input)
print(global_input.size(), local_input.size(), output.size())
15. ModalFusionModule7.py # 通用模态融合v7
import torch
import torch.nn as nn
import torch.nn.functional as F
class ModalFusionModule7(nn.Module):
def __init__(self, global_dim, local_dim, num_heads=8):
super(ModalFusionModule7, self).__init__()
self.global_dim = global_dim
self.local_dim = local_dim
self.num_heads = num_heads
if global_dim != local_dim:
self.align_dim = nn.Linear(local_dim, global_dim)
else:
self.align_dim = None
self.attention = nn.MultiheadAttention(self.global_dim, num_heads)
self.alpha_predictor = nn.Sequential(
nn.Linear(self.global_dim, self.global_dim // 2),
nn.ReLU(),
nn.Linear(self.global_dim // 2, 1),
nn.Sigmoid()
)
self.output_layer = nn.Linear(self.global_dim, self.global_dim + self.local_dim)
def forward(self, global_features, local_features):
B, T, _ = global_features.size()
if self.align_dim:
local_features = self.align_dim(local_features)
Q = global_features.permute(1, 0, 2) # (T, B, C)
K = local_features.permute(1, 0, 2)
V = global_features.permute(1, 0, 2)
attn_output, _ = self.attention(Q, K, V)
# Scaled Dot-Product Attention
d_k = K.size(-1) # Dimension of keys
scores = torch.bmm(Q, K.transpose(1, 2)) / (d_k ** 0.5)
attn_output_alt = torch.bmm(F.softmax(scores, dim=-1), V)
alpha = self.alpha_predictor(global_features.view(B * T, -1)).view(B, T, 1)
alpha = alpha.permute(1, 0, 2)
combined_attn_output = alpha * attn_output + (1 - alpha) * attn_output_alt
output = self.output_layer(combined_attn_output.permute(1, 0, 2))
return output
if __name__ == '__main__':
# Example usage
global_input = torch.rand(2, 33, 256)
local_input = torch.rand(2, 33, 512)
block = ModalFusionModule7(global_dim=256, local_dim=512, num_heads=8)
output = block(global_input, local_input)
print(global_input.size(), local_input.size(), output.size())
16. ModalFusionModule8.py # 通用模态融合v8
import torch
import torch.nn as nn
import torch.nn.functional as F
#门控注意力网络
class ModalFusionModule8(nn.Module):
def __init__(self, global_dim, local_dim, num_heads=8):
super(ModalFusionModule8, self).__init__()
self.global_dim = global_dim
self.local_dim = local_dim
self.num_heads = num_heads
# 维度调整层，将局部特征映射到全局特征的维度
self.feature_align = nn.Linear(local_dim, global_dim)
self.attention = nn.MultiheadAttention(self.global_dim, num_heads)
self.gate_layer = nn.Sequential(
nn.Linear(self.global_dim, self.global_dim), # 生成门控信号的维度与全局特征相同
nn.Sigmoid()
)
self.alpha_predictor = nn.Sequential(
nn.Linear(self.global_dim, self.global_dim // 2),
nn.ReLU(),
nn.Linear(self.global_dim // 2, 1),
nn.Sigmoid()
)
self.output_layer = nn.Linear(self.global_dim, self.global_dim + self.local_dim)
def forward(self, global_features, local_features):
B, T, _ = global_features.size()
# 调整局部特征的维度以匹配全局特征
local_features_aligned = self.feature_align(local_features)
Q = global_features.permute(1, 0, 2)
K = local_features_aligned.permute(1, 0, 2)
V = global_features.permute(1, 0, 2)
attn_output, _ = self.attention(Q, K, V)
# 生成门控系数，并应用于调整后的局部特征
gates = self.gate_layer(global_features)
gated_local_features = gates * local_features_aligned
# 结合全局特征和门控后的局部特征计算最终的注意力输出
combined_attn_output = attn_output + gated_local_features.permute(1,0,2)
output = self.output_layer(combined_attn_output.permute(1, 0, 2))
return output
if __name__ == '__main__':
# 使用示例
global_input = torch.rand(2, 33, 256) # 假设全局特征维度是 512
local_input = torch.rand(2, 33, 512) # 假设局部特征维度是 256
block = ModalFusionModule8(global_dim=256, local_dim=512, num_heads=8)
output = block(global_input, local_input)
print(global_input.size(), local_input.size(), output.size())
17. ModalFusionModule9.py # 通用模态融合v9
import torch
import torch.nn as nn
import torch.nn.functional as F
#局部特征预测γ和β，缩放全局特征
class ModalFusionModule9(nn.Module):
def __init__(self, global_dim, local_dim):
super(ModalFusionModule9, self).__init__()
self.global_dim = global_dim
self.local_dim = local_dim
# Dimension alignment for local features, if necessary
if global_dim != local_dim:
self.align_dim = nn.Linear(local_dim, global_dim)
else:
self.align_dim = None
# Simple attention mechanism to process local features
self.attention = nn.Sequential(
nn.Linear(global_dim, local_dim),
nn.Tanh(),
nn.Linear(local_dim, local_dim),
nn.ReLU()
)
# Predictors for gamma and beta
self.gamma_predictor = nn.Sequential(
nn.Linear(local_dim, global_dim),
nn.Sigmoid() # Ensure gamma is non-negative
)
self.beta_predictor = nn.Linear(local_dim, global_dim)
self.output_layer = nn.Linear(self.global_dim, self.global_dim + self.local_dim)
def forward(self, global_features, local_features):
if self.align_dim:
local_features = self.align_dim(local_features)
# Process local features through attention
local_features = self.attention(local_features)
# Calculate gamma and beta from processed local features
gamma = self.gamma_predictor(local_features)
beta = self.beta_predictor(local_features)
# Adjust global features using gamma and beta
adjusted_global_features = (1 + gamma) * global_features + beta
output = self.output_layer(adjusted_global_features)
return output
if __name__ == '__main__':
# Example usage
global_input = torch.rand(2, 33, 256) # Example global features
local_input = torch.rand(2, 33, 512) # Example local features with different dimension
block = ModalFusionModule9(global_dim=256, local_dim=512)
output = block(global_input, local_input)
print(global_input.size(), local_input.size(), output.size())
18. ModalFusionModule10.py # 通用模态融合v10
import torch
import torch.nn as nn
import torch.nn.functional as F
#局部特征+全局特征预测γ和β，缩放全局特征
class ModalFusionModule10(nn.Module):
def __init__(self, global_dim, local_dim):
super(ModalFusionModule10, self).__init__()
self.global_dim = global_dim
self.local_dim = local_dim
# Dimension alignment for local features, if necessary
if global_dim != local_dim:
self.align_dim = nn.Linear(local_dim, global_dim)
else:
self.align_dim = None
# Simple attention mechanism to process local features
self.attention = nn.Sequential(
nn.Linear(global_dim, local_dim),
nn.Tanh(),
nn.Linear(local_dim, local_dim),
nn.ReLU()
)
# Predictors for gamma and beta, now taking concatenated global and local features
self.gamma_predictor = nn.Sequential(
nn.Linear(global_dim + local_dim, global_dim),
nn.Sigmoid() # Ensure gamma is non-negative
)
self.beta_predictor = nn.Linear(global_dim + local_dim, global_dim)
self.output_layer = nn.Linear(self.global_dim, self.global_dim + self.local_dim)
def forward(self, global_features, local_features):
if self.align_dim:
local_features = self.align_dim(local_features)
# Process local features through attention
processed_local_features = self.attention(local_features)
# Concatenate global and processed local features
combined_features = torch.cat([global_features, processed_local_features], dim=-1)
# Calculate gamma and beta from combined features
gamma = self.gamma_predictor(combined_features)
beta = self.beta_predictor(combined_features)
# Adjust global features using gamma and beta
adjusted_global_features = (1 + gamma) * global_features + beta
output = self.output_layer(adjusted_global_features)
return output
if __name__ == '__main__':
# Example usage
global_input = torch.rand(2, 33, 256) # Example global features
local_input = torch.rand(2, 33, 512) # Example local features with different dimension
block = ModalFusionModule10(global_dim=256, local_dim=512)
output = block(global_input, local_input)
print(global_input.size(), local_input.size(), output.size())
19. ModalFusionModule11.py # Transformer编码器型模态融合
import torch
import torch.nn as nn
import torch.nn.functional as F
#Transformer混淆信息后，再预测γ和β，缩放全局信息
class ModalFusionModule11(nn.Module):
def __init__(self, global_dim, local_dim, num_heads=8, num_encoder_layers=2):
super(ModalFusionModule11, self).__init__()
self.global_dim = global_dim
self.local_dim = local_dim
# Dimension alignment for local features, if necessary
if global_dim != local_dim:
self.align_dim = nn.Linear(local_dim, global_dim)
else:
self.align_dim = None
# Transformer Encoder for fusing global and local features
encoder_layer = nn.TransformerEncoderLayer(d_model=global_dim, nhead=num_heads)
self.transformer_encoder = nn.TransformerEncoder(encoder_layer, num_layers=num_encoder_layers)
# Predictors for gamma and beta
self.gamma_predictor = nn.Sequential(
nn.Linear(global_dim, global_dim),
nn.Sigmoid()
)
self.beta_predictor = nn.Linear(global_dim, global_dim)
self.output_layer = nn.Linear(global_dim, global_dim + local_dim)
def forward(self, global_features, local_features):
if self.align_dim:
local_features = self.align_dim(local_features)
# Combine global and local features
combined_features = torch.cat([global_features, local_features], dim=1)
# Transformer encoder to fuse features
combined_features = self.transformer_encoder(combined_features.permute(1, 0, 2)).permute(1, 0, 2)
# Slice back the global features for gamma and beta application
fused_global_features = combined_features[:, :global_features.size(1), :]
# Calculate gamma and beta from fused global features
gamma = self.gamma_predictor(fused_global_features)
beta = self.beta_predictor(fused_global_features)
# Adjust global features using gamma and beta
adjusted_global_features = (1 + gamma) * global_features + beta
output = self.output_layer(adjusted_global_features)
return output
if __name__ == '__main__':
# Example usage
global_input = torch.rand(2, 33, 256) # Example global features
local_input = torch.rand(2, 33, 512) # Example local features
block = ModalFusionModule11(global_dim=256, local_dim=512, num_heads=8, num_encoder_layers=2)
output = block(global_input, local_input)
print(global_input.size(), local_input.size(), output.size())
20. ModalFusionModule12.py # 门控残差模态融合
import torch
import torch.nn as nn
import torch.nn.functional as F
#根据自适应网络学习到的权重动态地重加权全局特征和局部特征
class AdaptiveFeatureSelection(nn.Module):
def __init__(self, input_dim):
super(AdaptiveFeatureSelection, self).__init__()
self.fc1 = nn.Linear(input_dim, input_dim // 2)
self.fc2 = nn.Linear(input_dim // 2, input_dim)
self.softmax = nn.Softmax(dim=-1)
def forward(self, x):
weights = self.fc1(x)
weights = F.relu(weights)
weights = self.fc2(weights)
weights = self.softmax(weights)
return weights * x
class ModalFusionModule12(nn.Module):
def __init__(self, global_dim, local_dim):
super(ModalFusionModule12, self).__init__()
self.global_dim = global_dim
self.local_dim = local_dim
# Dimension alignment for local features, if necessary
if global_dim != local_dim:
self.align_dim = nn.Linear(local_dim, global_dim)
else:
self.align_dim = None
self.afs_global = AdaptiveFeatureSelection(global_dim)
self.afs_local = AdaptiveFeatureSelection(local_dim if self.align_dim is None else global_dim)
# Predictors for gamma and beta
self.gamma_predictor = nn.Sequential(
nn.Linear(global_dim, global_dim),
nn.Sigmoid()
)
self.beta_predictor = nn.Linear(global_dim, global_dim)
self.output_layer = nn.Linear(global_dim, global_dim + local_dim)
def forward(self, global_features, local_features):
if self.align_dim:
local_features = self.align_dim(local_features)
# Apply Adaptive Feature Selection
global_features = self.afs_global(global_features)
local_features = self.afs_local(local_features)
# Calculate gamma and beta
gamma = self.gamma_predictor(global_features)
beta = self.beta_predictor(local_features)
# Adjust global features using gamma and beta
adjusted_global_features = (1 + gamma) * global_features + beta
output = self.output_layer(adjusted_global_features)
return output
if __name__ == '__main__':
# Example usage
global_input = torch.rand(2, 33, 256) # Example global features
local_input = torch.rand(2, 33, 512) # Example local features
block = ModalFusionModule12(global_dim=256, local_dim=512)
output = block(global_input, local_input)
print(global_input.size(), local_input.size(), output.size())
21. ModalFusionModule13.py # 自适应权重模态融合
import torch
import torch.nn as nn
import torch.nn.functional as F
#根据自适应网络学习到的权重动态地重加权全局特征和局部特征
class AdaptiveFeatureSelection(nn.Module):
def __init__(self, input_dim):
super(AdaptiveFeatureSelection, self).__init__()
self.fc1 = nn.Linear(input_dim, input_dim // 2)
self.fc2 = nn.Linear(input_dim // 2, input_dim)
self.softmax = nn.Softmax(dim=-1)
def forward(self, x):
weights = self.fc1(x)
weights = F.relu(weights)
weights = self.fc2(weights)
weights = self.softmax(weights)
return weights * x
class ModalFusionModule13(nn.Module):
def __init__(self, global_dim, local_dim):
super(ModalFusionModule13, self).__init__()
self.global_dim = global_dim
self.local_dim = local_dim
# Dimension alignment for local features, if necessary
if global_dim != local_dim:
self.align_dim = nn.Linear(local_dim, global_dim)
else:
self.align_dim = None
# Feature selection modules for both global and local features
self.afs_global = AdaptiveFeatureSelection(global_dim)
self.afs_local = AdaptiveFeatureSelection(local_dim if self.align_dim is None else global_dim)
# Predictors for gamma and beta for both global and local features
self.gamma_global = nn.Sequential(
nn.Linear(global_dim, global_dim),
nn.Sigmoid()
)
self.beta_global = nn.Linear(global_dim, global_dim)
self.gamma_local = nn.Sequential(
nn.Linear(global_dim, global_dim),
nn.Sigmoid()
)
self.beta_local = nn.Linear(global_dim, global_dim)
# Output layer will now handle the concatenated adjusted features
self.output_layer = nn.Linear(2 * global_dim, local_dim+global_dim)
def forward(self, global_features, local_features):
if self.align_dim:
local_features = self.align_dim(local_features)
# Apply Adaptive Feature Selection
global_features = self.afs_global(global_features)
local_features = self.afs_local(local_features)
# Calculate gamma and beta for global and local features
gamma_global = self.gamma_global(global_features)
beta_global = self.beta_global(global_features)
gamma_local = self.gamma_local(local_features)
beta_local = self.beta_local(local_features)
# Adjust features using gamma and beta
adjusted_global_features = (1 + gamma_global) * global_features + beta_global
adjusted_local_features = (1 + gamma_local) * local_features + beta_local
# Concatenate the adjusted features
combined_features = torch.cat([adjusted_global_features, adjusted_local_features], dim=-1)
output = self.output_layer(combined_features)
return output
if __name__ == '__main__':
# Example usage
global_input = torch.rand(2, 33, 256) # Example global features
local_input = torch.rand(2, 33, 512) # Example local features, aligned dimensions
block = ModalFusionModule13(global_dim=256, local_dim=512)
output = block(global_input, local_input)
print(global_input.size(), local_input.size(), output.size())
22. QueryEnhancedSemanticModule.py # Query引导增强语义融合
import torch
import torch.nn as nn
import torch.nn.functional as F
#两个可学习的query分别聚合全局和局部特征，然后一个可学习的query聚合前面输出的特征
class CrossAttentionWithLearnableQuery(nn.Module):
def __init__(self, feature_dim, num_heads):
super(CrossAttentionWithLearnableQuery, self).__init__()
self.query = nn.Parameter(torch.randn(1, 1, feature_dim)) # Learnable query
self.attention = nn.MultiheadAttention(feature_dim, num_heads)
def forward(self, key_value):
# key_value shape: (T, B, C)
T, B, _ = key_value.size()
query = self.query.expand(T, B, -1) # Expanding query to match T and B
output, _ = self.attention(query, key_value, key_value)
return output
class QueryEnhancedSemanticModule(nn.Module):
def __init__(self, global_dim, local_dim, num_heads=8):
super(QueryEnhancedSemanticModule, self).__init__()
self.global_dim = global_dim
self.local_dim = local_dim
# Cross-Attention with learnable query for both global and local features
self.global_feature_attention = CrossAttentionWithLearnableQuery(self.global_dim, num_heads)
self.local_feature_attention = CrossAttentionWithLearnableQuery(self.local_dim, num_heads)
# Cross-Attention for concatenated features
self.concat_feature_attention = CrossAttentionWithLearnableQuery(self.global_dim + self.local_dim, num_heads) # Adjusted dimension
def forward(self, global_features, local_features):
# Reshape to (T, B, C) for compatibility with MultiheadAttention
global_features = global_features.permute(1, 0, 2)
local_features = local_features.permute(1, 0, 2)
# Generate aggregated global and local features
agg_global_features = self.global_feature_attention(global_features)
agg_local_features = self.local_feature_attention(local_features)
# Concatenate the aggregated features
concatenated_features = torch.cat((agg_global_features, agg_local_features), dim=2)
# Aggregate information from concatenated features
aggregated_output = self.concat_feature_attention(concatenated_features)
# Reshape output back to (B, T, C)
output = aggregated_output.permute(1, 0, 2)
return output
if __name__ == '__main__':
# Example of creating and using the module
global_input = torch.rand(2, 33, 256).cuda() # Example global features
local_input = torch.rand(2, 33, 256).cuda() # Example local features
block = QueryEnhancedSemanticModule(global_dim=256, local_dim=256, num_heads=8).cuda()
output = block(global_input, local_input)
print(global_input.size(), local_input.size(), output.size())
23. SemanticAttentionModule.py # 基础语义注意力融合v1
import torch
import torch.nn as nn
import torch.nn.functional as F
class SemanticAttentionModule(nn.Module):
def __init__(self, global_dim, local_dim, num_heads=8):
super(SemanticAttentionModule, self).__init__()
self.global_dim = global_dim
self.local_dim = local_dim
self.num_heads = num_heads
# Self-Attention for global features
self.global_attention = nn.MultiheadAttention(global_dim, num_heads)
# Cross-Attention, global features attending to local features
self.cross_attention = nn.MultiheadAttention(global_dim, num_heads)
# Adjust local feature dimensions to match global dimensions if necessary
if global_dim != local_dim:
self.adjust_local_dim = nn.Linear(local_dim, global_dim)
else:
self.adjust_local_dim = None
# Final linear layer to combine features
self.final_linear = nn.Linear(global_dim * 2, global_dim + local_dim)
def forward(self, global_features, local_features):
# Reshape to (T, B, C) for MultiheadAttention
global_features = global_features.permute(1, 0, 2)
local_features = local_features.permute(1, 0, 2)
# Adjust local features if their dimensions differ from global features
if self.adjust_local_dim:
adjusted_local_features = self.adjust_local_dim(local_features)
else:
adjusted_local_features = local_features
# Self-attention on global features
global_self_attn, _ = self.global_attention(global_features, global_features, global_features)
# Cross-attention, global features attending to adjusted local features
global_cross_attn, _ = self.cross_attention(adjusted_local_features, global_features, global_features)
# Concatenate both Attention results
concatenated_features = torch.cat((global_self_attn, global_cross_attn), dim=2)
# Final linear layer to adjust the concatenated output to the desired output dimension
output = self.final_linear(concatenated_features)
# Reshape output back to (B, T, C)
output = output.permute(1, 0, 2)
return output
if __name__ == '__main__':
# Example usage
global_input = torch.rand(2, 33, 256).cuda() # Example global features
local_input = torch.rand(2, 33, 256).cuda() # Example local features
block = SemanticAttentionModule(global_dim=256, local_dim=256, num_heads=8).cuda()
output = block(global_input, local_input)
print(global_input.size(), local_input.size(), output.size())
24. SemanticAttentionModule2.py # 基础语义注意力融合v2
import torch
import torch.nn as nn
import torch.nn.functional as F
class SemanticAttentionModule2(nn.Module):
def __init__(self, global_dim, local_dim, num_heads=8):
super(SemanticAttentionModule2, self).__init__()
self.global_dim = global_dim
self.local_dim = local_dim
self.num_heads = num_heads
# Attention layers
self.global_attention = nn.MultiheadAttention(self.global_dim, num_heads)
self.cross_attention = nn.MultiheadAttention(self.global_dim, num_heads)
# Conditionally adjust local dimensions to match global dimensions
if self.global_dim != self.local_dim:
self.adjust_local_dim = nn.Linear(self.local_dim, self.global_dim)
else:
self.adjust_local_dim = None
# Layer Norms
self.norm1 = nn.LayerNorm(self.global_dim)
self.norm2 = nn.LayerNorm(self.global_dim)
# Feedforward layers
self.feedforward = nn.Sequential(
nn.Linear(self.global_dim * 2, self.global_dim * 2),
nn.ReLU(),
nn.Linear(self.global_dim * 2, self.global_dim + self.local_dim)
)
def forward(self, global_features, local_features):
# Ensure correct shape for MultiheadAttention (T, B, C)
global_features = global_features.permute(1, 0, 2)
local_features = local_features.permute(1, 0, 2)
# Adjust local features dimension if necessary
if self.adjust_local_dim:
adjusted_local_features = self.adjust_local_dim(local_features)
else:
adjusted_local_features = local_features
# Self-attention on global features
global_self_attn, _ = self.global_attention(global_features, global_features, global_features)
global_self_attn = self.norm1(global_self_attn + global_features)
# Cross-attention, global on adjusted local
global_cross_attn, _ = self.cross_attention(adjusted_local_features, global_features, global_features)
global_cross_attn = self.norm2(global_cross_attn + adjusted_local_features)
# Concatenate both Attention results
concatenated_features = torch.cat((global_self_attn, global_cross_attn), dim=2)
# Process through feedforward network
output = self.feedforward(concatenated_features)
# Reshape output back to (B, T, C)
output = output.permute(1, 0, 2)
return output
if __name__ == '__main__':
# Example of creating the module
global_input = torch.rand(2, 33, 256).cuda() # Example global features
local_input = torch.rand(2, 33, 256).cuda() # Example local features
block = SemanticAttentionModule2(global_dim=256, local_dim=256, num_heads=8).cuda()
output = block(global_input, local_input)
print(global_input.size(), local_input.size(), output.size())
脑启社区是一个专注类脑智能领域的开发者社区。欢迎加入社区，共建类脑智能生态。社区为开发者提供了丰富的开源类脑工具软件、类脑算法模型及数据集、类脑知识库、类脑技术培训课程以及类脑应用案例等资源。
更多推荐
5
0- 0
已为社区贡献1条内容
所有评论(0)