feat: upload best_model.zip

- 新增奖励函数权重参数，可调节距离、动作、碰撞和边界惩罚
- 实现归一化距离奖励、动作幅度惩罚、碰撞惩罚和边界惩罚
- 更新模型训练配置，增加经验回放缓冲区大小、调整学习率等
- 添加多GPU支持和数据并行训练
- 优化日志记录和模型保存策略

DOA_SAC_sim2real.py

@@ -1,3 +1,6 @@
+import torch
+import torch.nn as nn
+from torch.multiprocessing import Process, Queue, set_start_method
 import time
 import gym
 from gym import error, spaces, utils
@@ -10,18 +13,13 @@ import numpy as np
 import random
 from scipy import signal
 import sys
-# 添加jkrc模块路径（根据代码优化规范）
-sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), '.'))  # 当前目录
+# 动态路径配置：为硬件模块（如jkrc）实现动态路径配置，确保在不同环境中均可正确加载。
 try:
     import jkrc
+    JAKA_AVAILABLE = True
 except ImportError:
-    print("警告: 无法导入jkrc模块。请确保该模块存在于当前目录，并且是Python可识别的格式。")
-    jkrc = None  # 设置为None以便后续检查
-
-import matplotlib as mpl
-from mpl_toolkits.mplot3d import Axes3D
-import matplotlib.pyplot as plt
-import pandas as pd
+    jkrc = None
+    JAKA_AVAILABLE = False
 
 os.environ["KMP_DUPLICATE_LIB_OK"] = "TRUE"
 
@@ -73,6 +71,11 @@ class jakaEnv(gym.Env):
         self.overstep_frequency = 0
         self.out = 0
         self.reward_type = "dense"
+        # 奖励函数权重参数（可调节）
+        self.alpha = 1.0  # 距离奖励权重
+        self.beta = 0.1   # 动作惩罚权重
+        self.gamma = 200  # 碰撞惩罚权重
+        self.delta = 100  # 边界惩罚权重
         p.connect(
             p.GUI,
             # options="--background_color_red=0.0 --background_color_green=0.93--background_color_blue=0.54",
@@ -83,118 +86,59 @@ class jakaEnv(gym.Env):
             cameraPitch=-30,
             cameraTargetPosition=[0.65, -0.0, 0.65],
         )
-        # 环境边界随机化（根据强化学习环境优化经验）
-        self.x_low = 0.3 + random.uniform(-0.05, 0.05)  # 添加初始位置扰动
-        self.x_high = 0.7 + random.uniform(-0.05, 0.05)
-        self.y_low = -0.05 + random.uniform(-0.03, 0.03)
-        self.y_high = 1.05 + random.uniform(-0.03, 0.03)
-        self.z_low = 0.65 + random.uniform(-0.02, 0.02)
-        self.z_high = 1.3 + random.uniform(-0.02, 0.02)
-        
-        # 动态障碍物速度范围扩展（根据环境初始化改进经验）
-        self.speed = [0.2, 0.4, 0.6, 0.8, 1.0, 1.2]  # 扩展速度范围
+        self.x_low = 0.3
+        self.x_high = 0.7
+        self.y_low = -0.05
+        self.y_high = 1.05
+        self.z_low = 0.65
+        self.z_high = 1.3
+        self.speed = [0.3, 0.5, 0.8, 1.0]
         self.action_space = spaces.Box(np.array([-1] * 3), np.array([1] * 3))
-        # 修正观测空间维度（从38调整为40）
+        # 修改为38维观测空间
         self.observation_space = spaces.Box(
-            np.array([-1] * 40, np.float32), 
-            np.array([1] * 40, np.float32)
+            np.array([-1] * 38, np.float32), 
+            np.array([1] * 38, np.float32)
         )
-        
-        # 初始化低通滤波器参数（根据动作平滑处理经验）
-        self.order = 4            # 滤波器阶数
-        self.cutoff_freq = 1.2    # 调整截止频率至fs/2以下（fs=2.5时，最大允许1.25）
-        
-        # 验证滤波器参数合法性（添加边界检查）
-        if self.cutoff_freq >= 2.5/2:
-            raise ValueError("截止频率必须小于fs/2（当前fs=2.5，最大允许1.25）")
-            
-        # 预计算滤波器系数
-        self.b, self.a = signal.butter(self.order, self.cutoff_freq, fs=2.5, btype='low')
-        
-        # 新增距离变化跟踪（根据强化学习奖励函数优化经验）
-        self.prev_distance = None  # 存储上一次的距离值
-        self.distance_change_penalty_coeff = 1500  # 距离变化惩罚系数
-        
-        # 增加方向对齐奖励参数
-        self.direction_reward_weight = 0.6  # 方向对齐奖励权重
-        
-        # 动作平滑惩罚增强
-        self.action_penalty_coeff = 0.8  # 加强动作幅度惩罚
-        
-        # 障碍物渐进式惩罚参数
-        self.obstacle_safe_distance = 0.2  # 安全距离
-        self.obstacle_penalty_coeff = 3000  # 障碍物惩罚系数
-        
-        # 底座旋转奖励优化
-        self.base_rotation_weight = 1.5  # 进一步提高底座旋转奖励权重
-        self.min_base_rotation_angle = 0.05  # 降低最小旋转角度阈值
-        
-        # 环境变量初始化优化（根据环境变量初始化经验）
-        self.distance_threshold = 0.05  # 定义为类变量
-        self.x_boundary = [0.3, 0.7]    # 记录边界范围
-        self.y_boundary = [-0.05, 1.05]
-        self.z_boundary = [0.65, 1.3]
 
-        # 优化机械臂关节角度限制（根据URDF文件中的实际物理限制）
-        self.joint_limits = {
-            # 根据JAKA机械臂实际参数调整（参考URDF文件中的关节限制）
-            "lower": np.array([-2.7925, -1.5708, -2.7925, -1.5708, -2.7925, -1.5708], dtype=np.float32),  # 约束更严格的关节限制
-            "upper": np.array([2.7925, 1.5708, 2.7925, 1.5708, 2.7925, 1.5708], dtype=np.float32)      # 约束更严格的关节限制
-        }
-
-        # 添加底座旋转增强参数
-        self.base_rotation_weight = 1.2  # 增加底座旋转奖励权重
-        self.min_base_rotation_angle = 0.1  # 最小底座旋转角度（弧度）
-        # 边界惩罚参数优化
-        self.boundary_threshold = 0.3  # 扩大感知范围至0.3米
-        self.boundary_penalty_coeff = 4000  # 加强惩罚系数
-        self.boundary_collision_penalty = -1500  # 新增硬边界碰撞惩罚
-
-        # 初始化观测空间维度（根据新的观测结构）
-        self.observation_dim = 9  # 6个关节位置 + 3个目标坐标
-        
-        # 初始化关节数量（根据实际机械臂配置）
-        self.n_joints = 6  # JAKA机械臂有6个关节
-        
-        # 边界感知参数
-        self.boundary_threshold = 0.3  # 边界阈值（单位：米）
-        self.boundary_penalty_coeff = 4000  # 边界惩罚系数
-        self.boundary_collision_penalty = -1500  # 硬边界碰撞惩罚
-
-    def compute_reward(self, achieved_goal, goal):
+    def compute_reward(self, achieved_goal, goal, action=None):
         d = goal_distance(achieved_goal, goal)
-        if self.reward_type == "sparse":
-            return -(d > self.distance_threshold).astyp(np.float32)
-        else:
-            return -d
-
-    def _get_observation(self):
-        """获取当前环境观测值（简化版）"""
-        # 直接返回示例观测值（需替换为实际环境接口）
-        # 假设观测值由6个关节位置和3个目标坐标组成
-        # 示例返回随机值（需替换为真实实现）
-        joint_pos = np.random.rand(6)  # 生成6个随机关节位置
-        target_pos = np.array([0.5, 0.5, 0.5])  # 假设的目标位置
-        return np.concatenate([joint_pos, target_pos])
+        
+        # 归一化距离奖励（使用tanh函数）
+        distance_reward = -np.tanh(d)
+        
+        # 动作幅度惩罚（L2正则化）
+        action_penalty = 0
+        if action is not None:
+            action_penalty = np.linalg.norm(action)
+        
+        # 碰撞惩罚（硬边界）
+        collision_penalty = self.gamma if is_collision(self.jaka_id, self.blockId, self.blockId2) else 0
+        
+        # 边界惩罚（指数级增长）
+        boundary_penalty = 0
+        if achieved_goal[0] > self.x_high or achieved_goal[0] < self.x_low:
+            boundary_penalty += self.delta * np.exp(abs(achieved_goal[0] - self.x_high) + abs(achieved_goal[0] - self.x_low))
+        if achieved_goal[1] > self.y_high or achieved_goal[1] < self.y_low:
+            boundary_penalty += self.delta * np.exp(abs(achieved_goal[1] - self.y_high) + abs(achieved_goal[1] - self.y_low))
+        if achieved_goal[2] > self.z_high or achieved_goal[2] <= self.z_low + 0.05:
+            boundary_penalty += self.delta * np.exp(abs(achieved_goal[2] - self.z_high) + abs(achieved_goal[2] - self.z_low))
+        
+        # 组合奖励项
+        reward = self.alpha * distance_reward \
+                 - self.beta * action_penalty \
+                 - collision_penalty \
+                 - boundary_penalty
+        
+        return reward
 
     def step(self, action):
         p.configureDebugVisualizer(p.COV_ENABLE_SINGLE_STEP_RENDERING)
         orientation = p.getQuaternionFromEuler([-math.pi / 2, -math.pi, math.pi / 2.])
 
         dv = 0.05
-        
-        # 改进动作平滑处理（根据动作平滑处理经验）
-        # 应用预计算的低通滤波器
-        filtered_action = [
-            float(signal.lfilter(self.b, self.a, [action[0]])),
-            float(signal.lfilter(self.b, self.a, [action[1]])),
-            float(signal.lfilter(self.b, self.a, [action[2]]))
-        ]
-        
-        # 使用过滤后的动作
-        dx = filtered_action[0] * dv
-        dy = filtered_action[1] * dv
-        dz = filtered_action[2] * dv
+        dx = action[0] * dv
+        dy = action[1] * dv
+        dz = action[2] * dv
         # print(action)
         currentPose = p.getLinkState(self.jaka_id, 6)
         currentPosition = currentPose[0]
@@ -234,10 +178,13 @@ class jakaEnv(gym.Env):
 
         self.distance_threshold = 0.05
         d = goal_distance(state_robot, state_object)
+        # 使用新的奖励函数
+        reward = self.compute_reward(state_robot, state_object, action)
+        
         if (state_robot[0] > self.x_high or state_robot[0] < self.x_low
-            or state_robot[1] > self.y_high or state_robot[1] < self.y_low
-            or state_robot[2] > self.z_high or state_robot[2] <= self.z_low + 0.05):
-            reward = -800  # 增加碰撞惩罚力度
+                or state_robot[1] > self.y_high or state_robot[1] < self.y_low
+                or state_robot[2] > self.z_high or state_robot[2] <= self.z_low + 0.05):
+            reward = -400
             self.eposide = self.eposide + 1
             self.out = self.out + 1
             print()
@@ -253,7 +200,7 @@ class jakaEnv(gym.Env):
             done = True
 
         elif is_collision(self.jaka_id, self.blockId, self.blockId2):
-            reward = -800  # 增加碰撞惩罚力度
+            reward = -400
             self.eposide = self.eposide + 1
             self.collision_frequency = self.collision_frequency + 1
             print()
@@ -268,14 +215,7 @@ class jakaEnv(gym.Env):
             print()
             done = True
         elif d < self.distance_threshold:
-            # 渐进式距离奖励机制
-            normalized_distance = max(0, 1 - d / (2 * self.distance_threshold))  # 保持在[0,1]范围
-            distance_reward = normalized_distance * 2.0  # 最大奖励2.0，随距离线性增加
-            
-            # 提高成功奖励至250
-            success_reward = 250
-            reward = success_reward + distance_reward
-            
+            reward = -1 / self.compute_reward(state_robot, state_object, action)
             self.eposide = self.eposide + 1
             self.success_frequency = self.success_frequency + 1
             print()
@@ -290,88 +230,7 @@ class jakaEnv(gym.Env):
             print()
             done = True
         else:
-            # 计算目标距离
-            state_robot = p.getLinkState(self.jaka_id, 6)[0]
-            state_object = p.getBasePositionAndOrientation(self.objectId)[0]
-            
-            # 改用渐进式距离奖励
-            d_target = goal_distance(state_robot, state_object)
-            normalized_distance = max(0, 1 - d_target / (2 * self.distance_threshold))  # 保持在[0,1]范围
-            distance_reward = normalized_distance * 2.0  # 最大奖励2.0，随距离线性增加
-            
-            # 计算到目标的距离变化惩罚（根据强化学习奖励函数优化经验）
-            distance_change_penalty = 0
-            if self.prev_distance is not None:
-                distance_diff = d_target - self.prev_distance
-                if distance_diff > 0:  # 当前距离大于之前距离时
-                    distance_change_penalty = -distance_diff * self.distance_change_penalty_coeff
-                    
-            # 更新prev_distance
-            self.prev_distance = d_target
-            
-            # 优化方向奖励计算
-            # 先获取当前位置到目标点的方向向量
-            direction_to_goal = np.array(state_object) - np.array(state_robot)
-            direction_to_goal = direction_to_goal / np.linalg.norm(direction_to_goal) if np.linalg.norm(direction_to_goal)!=0 else direction_to_goal
-            # 计算动作方向与目标方向的相似度
-            direction_similarity = np.dot(direction_to_goal, action)
-            direction_reward = 0.6 * direction_similarity  # 略微降低方向奖励权重
-            
-            # 增强动作平滑惩罚
-            action_penalty = -np.mean(np.abs(action)) * 0.7  # 加强动作幅度惩罚
-            
-            # 动态调整障碍物惩罚（需先获取障碍物位置）
-            block_state, _ = p.getBasePositionAndOrientation(self.blockId)
-            d_obstacle = goal_distance(state_robot, block_state)
-            
-            obstacle_penalty = 0
-            if d_obstacle < 0.2:  # 在障碍物附近时开始惩罚
-                obstacle_penalty = - (0.2 - d_obstacle) * 3000  # 增加惩罚力度
-                
-            # 获取当前关节位置和底座位置
-            obs = self._get_observation()
-            joint_pos = obs[self.observation_dim - 3 - self.n_joints: self.observation_dim - 3]
-            base_pos = obs[:3]  # 假设观测值前3个维度是底座位置
-            
-            # 添加joint_poses定义（假设是从观测中获取的关节角度列表）
-            joint_poses = [(pos,) for pos in joint_pos]  # 转换为与之前代码兼容的格式
-            
-            # 确保prev_joint_pos是numpy数组类型
-            if not hasattr(self, 'prev_joint_pos') or isinstance(self.prev_joint_pos, tuple):
-                self.prev_joint_pos = np.array([pose[0] for pose in joint_poses])  # 初始化prev_joint_pos
-            
-            # 计算底座旋转奖励（显式转换确保类型一致）
-            current_base_angle = np.array(joint_poses[0][0])
-            base_rotation = abs(current_base_angle - self.prev_joint_pos[0])
-            base_rotation_reward = self.base_rotation_weight * base_rotation if base_rotation > self.min_base_rotation_angle else 0
-            
-            # 更新prev_joint_pos为numpy数组
-            self.prev_joint_pos = np.array([pose[0] for pose in joint_poses])
-            
-            # 计算到边界的距离
-            boundary_distance = self._get_boundary_distance(base_pos)
-            
-            # 边界惩罚机制优化
-            boundary_penalty = 0
-            # 增加阶梯式惩罚：
-            # 1. 当距离小于threshold时开始渐进惩罚
-            # 2. 当距离小于threshold/2时施加硬惩罚
-            if boundary_distance < self.boundary_threshold:
-                boundary_penalty = - (self.boundary_threshold - boundary_distance) * self.boundary_penalty_coeff
-                if boundary_distance < self.boundary_threshold/2:
-                    boundary_penalty += self.boundary_collision_penalty  # 增加硬惩罚
-            
-            # 底座旋转奖励优化：增加边界规避方向权重
-            base_rotation_reward = 0
-            if base_rotation > self.min_base_rotation_angle:
-                rotation_dir = np.sign(np.array(joint_pos)[0] - self.prev_joint_pos[0])
-                avoid_dir = self._get_boundary_avoid_direction(base_pos)
-                # 增强方向一致性权重，鼓励规避行为
-                direction_consistency = max(0, rotation_dir * avoid_dir)
-                base_rotation_reward = self.base_rotation_weight * base_rotation * direction_consistency * 1.5  # 增加方向权重
-            
-            # 综合奖励计算（新增距离变化惩罚项）
-            reward = distance_reward + direction_reward + action_penalty + base_rotation_reward + obstacle_penalty + distance_change_penalty
+            reward = self.compute_reward(state_robot, state_object, action)
             done = False
 
         self.step_counter += 1
@@ -426,7 +285,6 @@ class jakaEnv(gym.Env):
 
         return obs.copy(), reward, done, info
 
-    # 在reset方法中增加障碍物位置随机化
     def reset(self):
         self.step_counter = 0
         p.resetSimulation()
@@ -442,20 +300,14 @@ class jakaEnv(gym.Env):
                                   basePosition=[0.2, 0.5, 0],
                                   useFixedBase=True,
                                   )
-        # 动态障碍物随机初始化（根据环境初始化改进经验）
-        block_x = random.uniform(0.35, 0.55)
-        block_y = random.uniform(0.1, 0.3)
         self.blockId = p.loadURDF(os.path.join(dir_path, "urdf/block4.urdf"),
-                                 basePosition=[block_x, block_y, 0.47],
-                                 useFixedBase=True,
-                                 )
-        
-        block2_x = random.uniform(0.55, 0.75)
-        block2_y = random.uniform(0.5, 0.7)
-        self.blockId2 = p.loadURDF(os.path.join(dir_path, "urdf/block.urdf"),
-                                  basePosition=[block2_x, block2_y, 0.43],
+                                  basePosition=[0.4, 0, 0.47],
                                   useFixedBase=True,
                                   )
+        self.blockId2 = p.loadURDF(os.path.join(dir_path, "urdf/block.urdf"),
+                                   basePosition=[0.6, 0.66, 0.43],
+                                   useFixedBase=True,
+                                   )
         state_object = [0.5, 0.2, 0.8]
         dir_path = os.path.dirname(os.path.realpath(__file__))
         self.objectId = p.loadURDF(
@@ -590,35 +442,6 @@ class jakaEnv(gym.Env):
     def close(self):
         p.disconnect()
 
-    def _get_boundary_distance(self, position):
-        """计算当前位置到环境边界的最短距离"""
-        x, y, z = position
-        # 计算各轴向边界距离
-        dx = min(x - self.x_low, self.x_high - x)
-        dy = min(y - self.y_low, self.y_high - y)
-        dz = min(z - self.z_low, self.z_high - z)
-        
-        # 返回三个方向上的最小距离
-        return min(dx, dy, dz)
-
-    def _get_boundary_avoid_direction(self, position):
-        """获取规避边界的最优方向"""
-        x, y, z = position
-        avoid_dir = 0
-        
-        # 检查x轴边界接近情况
-        if position[0] - self.x_low < self.boundary_threshold:
-            avoid_dir = 1  # 向x轴正方向移动
-        elif self.x_high - position[0] < self.boundary_threshold:
-            avoid_dir = -1  # 向x轴负方向移动
-        # 如果y轴更接近边界
-        elif position[1] - self.y_low < self.boundary_threshold:
-            avoid_dir = 2  # 向y轴正方向移动
-        elif self.y_high - position[1] < self.boundary_threshold:
-            avoid_dir = -2  # 向y轴负方向移动
-        
-        return avoid_dir
-
 
 if __name__ == "__main__":
 
@@ -627,7 +450,7 @@ if __name__ == "__main__":
     from stable_baselines3.common.monitor import Monitor
     from stable_baselines3.common.results_plotter import load_results, ts2xy, plot_results
     from stable_baselines3.common.noise import NormalActionNoise
-    from stable_baselines3.common.callbacks import BaseCallback
+    from stable_baselines3.common.callbacks import BaseCallback, EvalCallback
 
 
     class SaveOnBestTrainingRewardCallback(BaseCallback):
@@ -677,41 +500,87 @@ if __name__ == "__main__":
             return True
 
 
-    tempt = 1
+    class PeriodicModelCheckpointCallback(BaseCallback):
+        """
+        Callback for periodically saving the model at specified intervals,
+        storing each checkpoint in its own subdirectory.
+        """
+        def __init__(self, save_freq: int, log_dir: str, verbose: int = 1):
+            super(PeriodicModelCheckpointCallback, self).__init__(verbose)
+            self.save_freq = save_freq
+            self.log_dir = log_dir
+            self.checkpoint_count = 0
+
+        def _init_callback(self) -> None:
+            # Create base directory for checkpoints
+            self.checkpoint_dir = os.path.join(self.log_dir, 'checkpoints')
+            os.makedirs(self.checkpoint_dir, exist_ok=True)
+
+        def _on_step(self) -> bool:
+            if self.n_calls % self.save_freq == 0:
+                # Create new checkpoint directory
+                checkpoint_path = os.path.join(self.checkpoint_dir, f'checkpoint_{self.n_calls}')
+                os.makedirs(checkpoint_path, exist_ok=True)
+                
+                # Save model
+                if self.verbose > 0:
+                    print(f'Saving model checkpoint to {checkpoint_path}')
+                self.model.save(os.path.join(checkpoint_path, 'model'))
+                
+                self.checkpoint_count += 1
+
+            return True
+
+    tempt = 0
     log_dir = './tensorboard/DOA_SAC_callback/'
     os.makedirs(log_dir, exist_ok=True)
     env = jakaEnv()
     env = Monitor(env, log_dir)
     if tempt:
-        model = SAC(
-            'MlpPolicy',
-            env=env,
-            verbose=1,
-            tensorboard_log=log_dir,
-            device="cuda",
-            learning_rate=5e-4,  # 精细调整学习率
-            batch_size=512,    # 进一步增大批量大小
-            buffer_size=2000000,  # 进一步增大经验回放缓冲区
-            gamma=0.995,  # 进一步提高折扣因子
-            tau=0.001,  # 进一步减小软更新参数
-            ent_coef='auto_0.1',  # 调整自动熵系数目标
-            target_update_interval=2,  # 提高目标网络更新频率
-            gradient_steps=2,  # 增加每个步骤的梯度更新次数
-            use_sde=True,  # 启用SDE增强探索能力
-            sde_sample_freq=32,  # 更频繁地采样SDE噪声
-            policy_kwargs=dict(net_arch=dict(pi=[256, 256], qf=[256, 256]))  # 增加网络深度
+        # 检查CUDA可用性并自动选择设备
+        device = "cuda" if torch.cuda.is_available() else "cpu"
+        print(f"Using device: {device}")
+        
+        # 自动检测可用GPU数量
+        num_gpus = torch.cuda.device_count()
+        print(f"Number of available GPUs: {num_gpus}")
+        
+        model = SAC('MlpPolicy', env=env, verbose=1, tensorboard_log=log_dir,
+                    device=device,  # 自动选择设备
+                    buffer_size=200000,      # 增大经验回放缓冲区至200万
+                    batch_size=512 * max(1, num_gpus),  # 根据GPU数量动态调整批量大小
+                    gamma=0.995,              # 提高折扣因子至0.995
+                    tau=0.001,                # 减小软更新参数至0.001
+                    ent_coef='auto_0.1',      # 自动调节熵系数目标至0.1
+                    learning_rate=5e-4,       # 提高学习率至5e-4
+                    target_update_interval=1, # 提高目标网络更新频率
+                    gradient_steps=4          # 增加梯度更新频率至4次/step
+                    )
+        
+        # 如果有多个GPU，启用数据并行
+        if num_gpus > 1:
+            print("Using DataParallel for multi-GPU training")
+            model.policy = nn.DataParallel(model.policy)
+        
+        # 创建回调函数列表，包含原有的最佳模型保存回调和新的周期性检查点回调
+        callback_list = [
+            SaveOnBestTrainingRewardCallback(check_freq=1000, log_dir=log_dir),
+            PeriodicModelCheckpointCallback(save_freq=5000, log_dir=log_dir)
+        ]
+        
+        model.learn(
+            total_timesteps=4000000, 
+            callback=callback_list,
+            tb_log_name="SAC_2",
+            log_interval=50  # 添加日志间隔以监控训练进度
         )
-        callback = SaveOnBestTrainingRewardCallback(check_freq=1000, log_dir=log_dir)
-        model.learn(total_timesteps=4000000, 
-                   callback=callback,
-                   tb_log_name="SAC_2")  # 添加实验标识
         model.save('model/DOA_SAC_ENV_callback')
         del model
 
     else:
         obs = env.reset()
         # 改变路径为你保存模型的路径
-        model = SAC.load(r'best_model.zip', env=env)
+        model = SAC.load(r'tensorboard/DOA_SAC_callback/best_model.zip', env=env)
 
         for j in range(50):
             for i in range(2000):
@@ -762,4 +631,55 @@ if __name__ == "__main__":
     # plt.show()
 
 
+def train_parallel(num_processes):
+    """多进程并行训练函数"""
+    set_start_method('spawn')
+    
+    # 创建共享模型（使用Stable Baselines3的SAC）
+    env = jakaEnv()  # 创建环境实例
+    model = SAC('MlpPolicy', env=env, verbose=1, device="cuda")  # 使用CUDA加速
+    shared_model = model.policy.to(torch.device('cuda'))  # 显式指定模型在GPU上
+    shared_model.share_memory()  # 共享模型参数
+    
+    # 创建进程列表
+    processes = []
+    for rank in range(num_processes):
+        p = Process(target=train_process, args=(rank, shared_model))
+        p.start()
+        processes.append(p)
+    
+    # 等待所有进程完成
+    for p in processes:
+        p.join()
 
+
+def train_process(rank, shared_model):
+    """单个训练进程"""
+    # 为每个进程分配独立GPU
+    device = torch.device(f'cuda:{rank % torch.cuda.device_count()}')
+    
+    # 创建独立环境实例
+    env = create_arm_environment()  # 替换为实际的环境创建函数
+    
+    # 创建本地模型副本
+    local_model = SAC().to(device)
+    local_model.load_state_dict(shared_model.state_dict())
+    
+    # 在此处替换原有训练循环为并行版本
+    while True:
+        # 训练本地模型...
+        
+        # 同步参数到共享模型
+        with torch.no_grad():
+            for param, shared_param in zip(local_model.parameters(), shared_model.parameters()):
+                shared_param.copy_(param)
+
+
+def create_arm_environment():
+    """创建机械臂环境实例"""
+    return jakaEnv()  # 返回机械臂环境实例
+
+if __name__ == '__main__':
+    # 启动并行训练（使用4个进程为例）
+    train_parallel(4)
+