From 9056fbdc79c19e18e58853111f180c3f2d72e0ca Mon Sep 17 00:00:00 2001 From: chenxiaodong Date: Tue, 30 Jul 2024 10:38:39 +0800 Subject: [PATCH] meeting --- PPO.py | 23 +++++++++++++++++++---- environment.py | 1 - llm_connect.py | 2 +- plotDRL.py | 8 ++++---- tools.py | 2 +- 5 files changed, 25 insertions(+), 11 deletions(-) diff --git a/PPO.py b/PPO.py index 1625cdd..7e00889 100644 --- a/PPO.py +++ b/PPO.py @@ -100,6 +100,7 @@ class AgentPPO: self.ratio_clip = 0.2 # ratio.clamp(1 - clip, 1 + clip) self.lambda_entropy = 0.02 # could be 0.01~0.05 + self.gamma = 0.99 self.lambda_gae_adv = 0.98 # could be 0.95~0.99, GAE (Generalized Advantage Estimation. ICLR.2016.) self.get_reward_sum = None # self.get_reward_sum_gae if if_use_gae else self.get_reward_sum_raw self.trajectory_list = None @@ -153,7 +154,7 @@ class AgentPPO: buf_state, buf_action, buf_noise, buf_reward, buf_mask = [ten.to(self.device) for ten in buffer] '''get buf_r_sum, buf_logprob''' - bs = 4096 # set a smaller 'BatchSize' when out of GPU memory: 1024, could change to 4096 + bs = 4096 buf_value = [self.cri_target(buf_state[i:i + bs]) for i in range(0, buf_len, bs)] buf_value = torch.cat(buf_value, dim=0) buf_logprob = self.act.get_old_logprob(buf_action, buf_noise) @@ -210,12 +211,26 @@ class AgentPPO: pre_r_sum = 0 pre_advantage = 0 # advantage value of previous step for i in range(buf_len - 1, -1, -1): - buf_r_sum[i] = ten_reward[i] + ten_mask[i] * pre_r_sum + buf_r_sum[i] = ten_reward[i] + ten_mask[i] * self.gamma * pre_r_sum pre_r_sum = buf_r_sum[i] - buf_advantage[i] = ten_reward[i] + ten_mask[i] * (pre_advantage - ten_value[i]) # fix a bug here - pre_advantage = ten_value[i] + buf_advantage[i] * self.lambda_gae_adv + delta = ten_reward[i] + ten_mask[i] * self.gamma * ten_value[i + 1] - ten_value[i] + buf_advantage[i] = delta + ten_mask[i] * self.gamma * self.lambda_gae_adv * pre_advantage + pre_advantage = buf_advantage[i] return buf_r_sum, buf_advantage + # def get_reward_sum_gae(self, buf_len, ten_reward, ten_mask, ten_value) -> (torch.Tensor, torch.Tensor): + # buf_r_sum = torch.empty(buf_len, dtype=torch.float32, device=self.device) # old policy value + # buf_advantage = torch.empty(buf_len, dtype=torch.float32, device=self.device) # advantage value + # + # pre_r_sum = 0 + # pre_advantage = 0 # advantage value of previous step + # for i in range(buf_len - 1, -1, -1): + # buf_r_sum[i] = ten_reward[i] + ten_mask[i] * pre_r_sum + # pre_r_sum = buf_r_sum[i] + # buf_advantage[i] = ten_reward[i] + ten_mask[i] * (pre_advantage - ten_value[i]) # fix a bug here + # pre_advantage = ten_value[i] + buf_advantage[i] * self.lambda_gae_adv + # return buf_r_sum, buf_advantage + @staticmethod def optim_update(optimizer, objective): optimizer.zero_grad() diff --git a/environment.py b/environment.py index ef9e226..6e73b09 100644 --- a/environment.py +++ b/environment.py @@ -106,7 +106,6 @@ class ESSEnv(gym.Env): self.excess, self.shedding = 0, 0 if unbalance >= 0: # 现在过剩 if unbalance <= self.grid.exchange_ability: - # sell money to grid is little [0.029,0.1] sell_benefit = self.grid.get_cost(price, unbalance) * self.sell_coefficient else: sell_benefit = self.grid.get_cost(price, self.grid.exchange_ability) * self.sell_coefficient diff --git a/llm_connect.py b/llm_connect.py index 0be77c6..728717d 100644 --- a/llm_connect.py +++ b/llm_connect.py @@ -40,7 +40,7 @@ llm = ChatOpenAI( openai_api_key="none", # openai_api_base="http://0.0.0.0:5049/v1/models", openai_api_base="http://0.0.0.0:8501", - model_name="Qwen1.5-32b-int4" + model_name="Qwen1.5-32b" ) prompt = ChatPromptTemplate.from_messages( [ diff --git a/plotDRL.py b/plotDRL.py index d057abc..b3d6811 100644 --- a/plotDRL.py +++ b/plotDRL.py @@ -43,7 +43,7 @@ def plot_optimization_result(datasource, directory): # data source is dataframe # 绘制累计发电量和消耗量 in ax[2] axs[1, 0].cla() - axs[1, 0].set_ylabel('Outputs of Units and Netload (kWh)') + axs[1, 0].set_ylabel('Power (kWh)') axs[1, 0].set_xlabel('Time (h)') # 处理电池充放电数据 battery_positive = np.array(datasource['battery_energy_change']) @@ -69,7 +69,7 @@ def plot_optimization_result(datasource, directory): # data source is dataframe axs[1, 0].legend(loc='upper right', bbox_to_anchor=(1.4, 1), fontsize=12, frameon=False, labelspacing=0.3) # axs[1,0].set_xticks([i for i in range(24)],[i for i in range(1,25)]) - fig.savefig(f"{directory}/optimization_information.svg", format='svg', dpi=600, bbox_inches='tight') + fig.savefig(f"{directory}/rl.svg", format='svg', dpi=600, bbox_inches='tight') print('optimization results have been ploted and saved') @@ -114,7 +114,7 @@ def plot_evaluation_information(datasource, directory): # 绘制发电量和负载量 in ax[2] axs[1, 0].cla() axs[1, 0].set_xlabel('Time (h)') - axs[1, 0].set_ylabel('Outputs of Units and Netload (kWh)') + axs[1, 0].set_ylabel('Power (kWh)') # axs[1,0].set_xticks([i for i in range(24)], [i for i in range(1, 25)]) battery_positive = np.array(eval_data['battery']) battery_negative = np.array(eval_data['battery']) @@ -142,7 +142,7 @@ def plot_evaluation_information(datasource, directory): axs[1, 1].set_xlabel('Time (h)') axs[1, 1].set_ylabel('Costs') axs[1, 1].bar(eval_data['time_step'], eval_data['operation_cost']) - fig.savefig(f"{directory}/evaluation_information.svg", format='svg', dpi=600, bbox_inches='tight') + fig.savefig(f"{directory}/gurobi.svg", format='svg', dpi=600, bbox_inches='tight') print('evaluation figure have been ploted and saved') diff --git a/tools.py b/tools.py index 086ed88..3d7cf4d 100644 --- a/tools.py +++ b/tools.py @@ -61,7 +61,7 @@ def optimization_base_result(env, month, day, initial_soc): pv_voltage = m.addVars(period, vtype=GRB.CONTINUOUS, lb=-1, ub=1, name='pv_voltage') # 计算光伏和风力发电量 - pv = [(0.2 * irradiance[t] + 0.05 * temperature[t] - 9.25) * (1 + pv_voltage[t]) for t in range(period)] + pv = [(0.25 * irradiance[t] + 0.05 * temperature[t] - 9.25) * (1 + pv_voltage[t]) for t in range(period)] wind = [172.265625 * wind_speed[t] ** 3 / 1e3 if 3 <= wind_speed[t] < 8 else (172.265625 * 8 ** 3 / 1e3 if 8 <= wind_speed[t] < 12 else 0) for t in range(period)]