From f27624fc60b5038c69327d52607fbd7a4ef41764 Mon Sep 17 00:00:00 2001 From: Thiago Novaes Date: Mon, 30 Sep 2024 00:07:12 +0200 Subject: [PATCH 1/9] miso --- TimeSeriesHelper.jl/Project.toml | 4 +- TimeSeriesHelper.jl/src/TimeSeriesHelper.jl | 6 ++- TimeSeriesHelper.jl/src/read_open_meteo.jl | 33 ++++++++++++ TimeSeriesHelper.jl/src/read_table_miso.jl | 50 +++++++++++++++++++ .../src/{read_table.jl => read_table_pjm.jl} | 8 +-- 5 files changed, 95 insertions(+), 6 deletions(-) create mode 100644 TimeSeriesHelper.jl/src/read_open_meteo.jl create mode 100644 TimeSeriesHelper.jl/src/read_table_miso.jl rename TimeSeriesHelper.jl/src/{read_table.jl => read_table_pjm.jl} (90%) diff --git a/TimeSeriesHelper.jl/Project.toml b/TimeSeriesHelper.jl/Project.toml index bb5f36f..885d5d7 100644 --- a/TimeSeriesHelper.jl/Project.toml +++ b/TimeSeriesHelper.jl/Project.toml @@ -6,8 +6,10 @@ version = "0.1.0" [deps] CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" +Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" HiddenMarkovModels = "84ca31d5-effc-45e0-bfda-5a68cd981f47" +JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" @@ -15,4 +17,4 @@ Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] -test = ["Test"] \ No newline at end of file +test = ["Test"] diff --git a/TimeSeriesHelper.jl/src/TimeSeriesHelper.jl b/TimeSeriesHelper.jl/src/TimeSeriesHelper.jl index f7c705d..1808b21 100644 --- a/TimeSeriesHelper.jl/src/TimeSeriesHelper.jl +++ b/TimeSeriesHelper.jl/src/TimeSeriesHelper.jl @@ -6,9 +6,13 @@ using LinearAlgebra using Random using CSV using DataFrames +using Dates +using JSON include("structs.jl") -include("read_table.jl") +include("read_table_pjm.jl") +include("read_table_miso.jl") +include("read_open_meteo.jl") include("estimate.jl") end # module TimeSeriesHelper diff --git a/TimeSeriesHelper.jl/src/read_open_meteo.jl b/TimeSeriesHelper.jl/src/read_open_meteo.jl new file mode 100644 index 0000000..1d9f6d4 --- /dev/null +++ b/TimeSeriesHelper.jl/src/read_open_meteo.jl @@ -0,0 +1,33 @@ +""" + read_open_meteo_json(directory::String, + serie::String, + start::Int, + stop::Int + ) + +Returns a dictionary with keys being the node id and values being a vector of wind. +""" +function read_open_meteo_json(directory::String, serie::String, start::Int, stop::Int) + start_string = string(start) + stop_string = string(stop) + start_utc = start_string[1:4] * "-" * start_string[5:6] * "-" * start_string[7:8] * "T00:00" + stop_utc = stop_string[1:4] * "-" * stop_string[5:6] * "-" * stop_string[7:8] * "T23:00" + + dict = Dict{String, Vector{Float64}}() + for file_path in readdir(directory) + if endswith(file_path, ".json") + json_path = joinpath(directory, file_path) + json = JSON.parse(String(read(json_path))) + times = json["hourly"]["time"] + data = json["hourly"][serie] + vec = [data[i] for i in 1:length(times) if times[i] >= start_utc && times[i] <= stop_utc] + key = file_path[1:end - 5] + if !haskey(dict, key) + dict[key] = vec + else + append!(dict[key], vec) + end + end + end + return dict +end \ No newline at end of file diff --git a/TimeSeriesHelper.jl/src/read_table_miso.jl b/TimeSeriesHelper.jl/src/read_table_miso.jl new file mode 100644 index 0000000..8cfe1f8 --- /dev/null +++ b/TimeSeriesHelper.jl/src/read_table_miso.jl @@ -0,0 +1,50 @@ +""" + read_miso_csv(directory::String, + file_pattern::String, + start::Int, + stop::Int + ) + +Returns a dictionary with keys being the node id and values being a vector of prices. +""" +function read_miso_csv(directory::String, file_pattern::String, start::Int, stop::Int) + dict = Dict{String, Vector{Float64}}() + for i in start:stop + file_path = joinpath(directory, string(i) * file_pattern); + df = CSV.read(file_path, DataFrame)[4:end,:] + for row in eachrow(df) + if (row[3] != "LMP") continue end + vec = [parse(Float64, x) for x in values(row[4:27])] + if !haskey(dict, row[1]) + dict[row[1]] = vec + else + append!(dict[row[1]], vec) + end + end + end + return dict +end + +""" + read_miso_da_lmps(directory::String, + start::Int, + stop::Int + ) + +Returns a dictionary with keys being the node id and values being a vector of prices. +""" +function read_miso_da_lmps(directory::String, start::Int, stop::Int) + return read_miso_csv(directory, "_da_lmp_final.csv", start, stop) +end + +""" + read_miso_rt_lmps(directory::String, + start::Int, + stop::Int + ) + +Returns a dictionary with keys being the node id and values being a vector of prices. +""" +function read_miso_rt_lmps(directory::String, start::Int, stop::Int) + return read_miso_csv(directory, "_da_expost_lmp.csv", start, stop) +end diff --git a/TimeSeriesHelper.jl/src/read_table.jl b/TimeSeriesHelper.jl/src/read_table_pjm.jl similarity index 90% rename from TimeSeriesHelper.jl/src/read_table.jl rename to TimeSeriesHelper.jl/src/read_table_pjm.jl index 33d63b3..dca1210 100644 --- a/TimeSeriesHelper.jl/src/read_table.jl +++ b/TimeSeriesHelper.jl/src/read_table_pjm.jl @@ -41,19 +41,19 @@ function read_pjm_csv(path::String, column1::String, column2::String, column3::S end """ - read_da_hrl_lmps(path::String) + read_pjm_da_lmps(path::String) Returns a pivot table of da_hrl_lmps using UTC date as rows, node id as columns and price as values. """ -function read_da_hrl_lmps(path::String) +function read_pjm_da_lmps(path::String) return read_pjm_csv(path, "datetime_beginning_utc", "pnode_name", "total_lmp_da") end """ - read_rt_hrl_lmps(path::String) + read_pjm_rt_lmps(path::String) Returns a pivot table of rt_hrl_lmps using UTC date as rows, node id as columns and price as values. """ -function read_rt_hrl_lmps(path::String) +function read_pjm_rt_lmps(path::String) return read_pjm_csv(path, "datetime_beginning_utc", "pnode_name", "total_lmp_rt") end From fd74b99605e2e640924eedf279816e0aeaa47488 Mon Sep 17 00:00:00 2001 From: Thiago Novaes Date: Tue, 1 Oct 2024 22:13:06 +0200 Subject: [PATCH 2/9] testing miso --- TimeSeriesHelper.jl/src/read_table_miso.jl | 4 +- examples/local_miso.jl | 47 ++++++++++++++++++++++ 2 files changed, 49 insertions(+), 2 deletions(-) create mode 100644 examples/local_miso.jl diff --git a/TimeSeriesHelper.jl/src/read_table_miso.jl b/TimeSeriesHelper.jl/src/read_table_miso.jl index 8cfe1f8..9e0bdbb 100644 --- a/TimeSeriesHelper.jl/src/read_table_miso.jl +++ b/TimeSeriesHelper.jl/src/read_table_miso.jl @@ -34,7 +34,7 @@ end Returns a dictionary with keys being the node id and values being a vector of prices. """ function read_miso_da_lmps(directory::String, start::Int, stop::Int) - return read_miso_csv(directory, "_da_lmp_final.csv", start, stop) + return read_miso_csv(directory, "_da_expost_lmp.csv", start, stop) end """ @@ -46,5 +46,5 @@ end Returns a dictionary with keys being the node id and values being a vector of prices. """ function read_miso_rt_lmps(directory::String, start::Int, stop::Int) - return read_miso_csv(directory, "_da_expost_lmp.csv", start, stop) + return read_miso_csv(directory, "_rt_lmp_final.csv", start, stop) end diff --git a/examples/local_miso.jl b/examples/local_miso.jl new file mode 100644 index 0000000..ee32775 --- /dev/null +++ b/examples/local_miso.jl @@ -0,0 +1,47 @@ +using OptimalEnergyBid +using TimeSeriesHelper + +directory = "C:\\Users\\thiag\\Documents\\Data\\"; +start = 20240810; +stop = 20240824; + +day_ahead = TimeSeriesHelper.read_miso_da_lmps( + directory, start, stop +) + +real_time = TimeSeriesHelper.read_miso_rt_lmps( + directory, start, stop +) + +wind = TimeSeriesHelper.read_open_meteo_json( + directory, "wind_speed_10m", start, stop +) + +wind3 = wind3[(end - 744):end, 4:5] +solar3 = solar3[(end - 744):end, :] +rt3 = rt3[:, 1:2] +da3 = da3[:, 1:2] + +wind4 = vec(wind3) +solar4 = vec(solar3) +rt4 = vec(rt3) +da4 = vec(da3) + +windl, _ = size(wind3) +solarl, _ = size(solar3) +rtl, _ = size(rt3) +dal, _ = size(da3) + +wind5 = [wind4[i:windl:end] for i in 1:windl] +solar5 = [solar4[i:solarl:end] for i in 1:solarl] +rt5 = [rt4[i:rtl:end] for i in 1:rtl] +da5 = [da4[i:dal:end] for i in 1:dal] + +history = TimeSeriesHelper.History() +history.prices_real_time = rt5 +history.prices_day_ahead = da5 +history.inflow = wind5 + +h = TimeSeriesHelper.build_serial_history(history, 745, 24) + +m, o = TimeSeriesHelper.estimate_hmm(h, 5) From 28fdc47ffc7ebcc4284a3aa3676d9a2b10e3bed1 Mon Sep 17 00:00:00 2001 From: Thiago Novaes Date: Wed, 2 Oct 2024 01:49:34 +0200 Subject: [PATCH 3/9] http --- TimeSeriesHelper.jl/Project.toml | 1 + TimeSeriesHelper.jl/src/TimeSeriesHelper.jl | 1 + TimeSeriesHelper.jl/src/read_table_miso.jl | 12 ++++++++++++ examples/local.jl | 4 ++-- examples/local_miso.jl | 4 ++-- 5 files changed, 18 insertions(+), 4 deletions(-) diff --git a/TimeSeriesHelper.jl/Project.toml b/TimeSeriesHelper.jl/Project.toml index 885d5d7..dd387bc 100644 --- a/TimeSeriesHelper.jl/Project.toml +++ b/TimeSeriesHelper.jl/Project.toml @@ -8,6 +8,7 @@ CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" +HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3" HiddenMarkovModels = "84ca31d5-effc-45e0-bfda-5a68cd981f47" JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" diff --git a/TimeSeriesHelper.jl/src/TimeSeriesHelper.jl b/TimeSeriesHelper.jl/src/TimeSeriesHelper.jl index 1808b21..989e74b 100644 --- a/TimeSeriesHelper.jl/src/TimeSeriesHelper.jl +++ b/TimeSeriesHelper.jl/src/TimeSeriesHelper.jl @@ -8,6 +8,7 @@ using CSV using DataFrames using Dates using JSON +using HTTP include("structs.jl") include("read_table_pjm.jl") diff --git a/TimeSeriesHelper.jl/src/read_table_miso.jl b/TimeSeriesHelper.jl/src/read_table_miso.jl index 9e0bdbb..c6e7a34 100644 --- a/TimeSeriesHelper.jl/src/read_table_miso.jl +++ b/TimeSeriesHelper.jl/src/read_table_miso.jl @@ -1,3 +1,10 @@ +function _download_csv(url, filename) + response = HTTP.get(url) + open(filename, "w") do file + write(file, response.body) + end +end + """ read_miso_csv(directory::String, file_pattern::String, @@ -11,6 +18,11 @@ function read_miso_csv(directory::String, file_pattern::String, start::Int, stop dict = Dict{String, Vector{Float64}}() for i in start:stop file_path = joinpath(directory, string(i) * file_pattern); + if !isfile(file_path) + url = joinpath("https://docs.misoenergy.org/marketreports", string(i) * file_pattern) + task = @async _download_csv(url, file_path) + wait(task) + end df = CSV.read(file_path, DataFrame)[4:end,:] for row in eachrow(df) if (row[3] != "LMP") continue end diff --git a/examples/local.jl b/examples/local.jl index 6b05363..0f87dab 100644 --- a/examples/local.jl +++ b/examples/local.jl @@ -8,10 +8,10 @@ solar1, solar2, solar3 = TimeSeriesHelper.read_generation_csv( "C:\\Users\\thiag\\Documents\\Data\\solar_gen_cf_2022.csv" ) -rt1, rt2, rt3 = TimeSeriesHelper.read_rt_hrl_lmps( +rt1, rt2, rt3 = TimeSeriesHelper.read_pjm_rt_lmps( "C:\\Users\\thiag\\Documents\\Data\\rt_hrl_lmps.csv" ) -da1, da2, da3 = TimeSeriesHelper.read_da_hrl_lmps( +da1, da2, da3 = TimeSeriesHelper.read_pjm_da_lmps( "C:\\Users\\thiag\\Documents\\Data\\da_hrl_lmps.csv" ) diff --git a/examples/local_miso.jl b/examples/local_miso.jl index ee32775..8c3c515 100644 --- a/examples/local_miso.jl +++ b/examples/local_miso.jl @@ -3,10 +3,10 @@ using TimeSeriesHelper directory = "C:\\Users\\thiag\\Documents\\Data\\"; start = 20240810; -stop = 20240824; +stop = 20240823; day_ahead = TimeSeriesHelper.read_miso_da_lmps( - directory, start, stop + directory, start, stop + 1 ) real_time = TimeSeriesHelper.read_miso_rt_lmps( From 5089e19a7852f934da744a7cd10708f71546e19d Mon Sep 17 00:00:00 2001 From: Thiago Novaes Date: Wed, 2 Oct 2024 20:36:55 +0200 Subject: [PATCH 4/9] real_tume_steps --- cases/deterministic.json | 2 +- cases/stochastic.json | 2 +- cases/toy.json | 2 +- schemas/problem.json | 4 ++-- src/build.jl | 4 ++-- src/constraints.jl | 8 ++++---- src/graph.jl | 2 +- src/output.jl | 4 ++-- src/preprocess.jl | 4 ++-- src/structs.jl | 2 +- src/variables.jl | 2 +- test/test_preprocess.jl | 2 +- test/test_read_write_json.jl | 6 +++--- 13 files changed, 22 insertions(+), 22 deletions(-) diff --git a/cases/deterministic.json b/cases/deterministic.json index 3ff0ca9..902c3fd 100644 --- a/cases/deterministic.json +++ b/cases/deterministic.json @@ -10,7 +10,7 @@ "period_of_day_ahead_clear": 2, "duration": 6, "day_ahead_steps": 1, - "real_tume_steps": 1 + "real_time_steps": 1 }, "data": { "names": ["unit 1", "unit 2"], diff --git a/cases/stochastic.json b/cases/stochastic.json index 9979441..1fc4334 100644 --- a/cases/stochastic.json +++ b/cases/stochastic.json @@ -10,7 +10,7 @@ "period_of_day_ahead_clear": 2, "duration": 6, "day_ahead_steps": 2, - "real_tume_steps": 2 + "real_time_steps": 2 }, "data": { "names": ["unit 1", "unit 2"], diff --git a/cases/toy.json b/cases/toy.json index 4563d45..5e7d0c0 100644 --- a/cases/toy.json +++ b/cases/toy.json @@ -10,7 +10,7 @@ "period_of_day_ahead_clear": 1, "duration": 1, "day_ahead_steps": 1, - "real_tume_steps": 1 + "real_time_steps": 1 }, "data": { "unit_to_bus": [1], diff --git a/schemas/problem.json b/schemas/problem.json index 0ab88d1..6f7ea19 100644 --- a/schemas/problem.json +++ b/schemas/problem.json @@ -41,7 +41,7 @@ "period_of_day_ahead_clear", "duration", "day_ahead_steps", - "real_tume_steps" + "real_time_steps" ], "properties": { "periods_per_day": { @@ -68,7 +68,7 @@ "day_ahead_steps": { "type": "integer" }, - "real_tume_steps": { + "real_time_steps": { "type": "integer" } } diff --git a/src/build.jl b/src/build.jl index c56a56f..928198a 100644 --- a/src/build.jl +++ b/src/build.jl @@ -134,7 +134,7 @@ function _validate_numbers(prb::Problem)::Nothing @assert 1 <= numbers.period_of_day_ahead_clear && numbers.period_of_day_ahead_clear <= numbers.periods_per_day @assert 1 <= numbers.duration - @assert 1 <= numbers.real_tume_steps + @assert 1 <= numbers.real_time_steps @assert 1 <= numbers.day_ahead_steps return nothing @@ -155,7 +155,7 @@ function _validate_data(prb::Problem)::Nothing for t in 1:(numbers.duration) @assert length(data.prices_real_time_curve[t]) >= numbers.units for i in 1:(numbers.units) - @assert length(data.prices_real_time_curve[t][i]) >= numbers.real_tume_steps + @assert length(data.prices_real_time_curve[t][i]) >= numbers.real_time_steps end end diff --git a/src/constraints.jl b/src/constraints.jl index 1e24a28..b850453 100644 --- a/src/constraints.jl +++ b/src/constraints.jl @@ -93,7 +93,7 @@ function _constraint_real_time_bid_bound!(sp::Model, prb::Problem)::Nothing sp, real_time_bid_bound[i=1:(prb.numbers.units)], sp[:volume][i].out - prb.data.volume_min[i] >= - sum(sp[:real_time_bid][k, i].out for k in 1:(prb.numbers.real_tume_steps)) + sum(sp[:real_time_bid][k, i].out for k in 1:(prb.numbers.real_time_steps)) ) return nothing end @@ -104,7 +104,7 @@ function _constraint_ramp_up!(sp::Model, prb::Problem)::Nothing sp, ramp_up[i=1:(prb.numbers.units)], prb.data.ramp_up[i] >= - sum(sp[:real_time_bid][k, i].out for k in 1:(prb.numbers.real_tume_steps)) - + sum(sp[:real_time_bid][k, i].out for k in 1:(prb.numbers.real_time_steps)) - sp[:generation][i].out ) return nothing @@ -129,7 +129,7 @@ function _constraint_real_time_accepted!( sp, real_time_accepted[i=1:(prb.numbers.units)], sp[:generation][i] == sum( - sp[:real_time_bid][k, i].in for k in 1:(prb.numbers.real_tume_steps) if + sp[:real_time_bid][k, i].in for k in 1:(prb.numbers.real_time_steps) if prb.cache.acceptance_real_time[t][markov_state][i, k] ) ) @@ -155,7 +155,7 @@ function _constraint_real_time_accepted_state!( sp, real_time_accepted_state[i=1:(prb.numbers.units)], sp[:generation][i].out == sum( - sp[:real_time_bid][k, i].in for k in 1:(prb.numbers.real_tume_steps) if + sp[:real_time_bid][k, i].in for k in 1:(prb.numbers.real_time_steps) if prb.cache.acceptance_real_time[t][markov_state][i, k] ) ) diff --git a/src/graph.jl b/src/graph.jl index d1f53c3..8a447c6 100644 --- a/src/graph.jl +++ b/src/graph.jl @@ -78,7 +78,7 @@ function plot_real_time_bids( for t in 1:(prb.numbers.duration), i in 1:(prb.numbers.units) prices = prb.data.prices_real_time_curve[t][i] offer = real_time_bid[:, i, t] - for k in 1:(prb.numbers.real_tume_steps - 1) + for k in 1:(prb.numbers.real_time_steps - 1) offer[k + 1] += offer[k] end plot( diff --git a/src/output.jl b/src/output.jl index 163cd06..4b038df 100644 --- a/src/output.jl +++ b/src/output.jl @@ -76,12 +76,12 @@ function _write_real_time_bid!( )::Nothing numbers = prb.numbers S = length(simul) - real_time_bid = zeros(numbers.real_tume_steps, numbers.units, numbers.duration, S) + real_time_bid = zeros(numbers.real_time_steps, numbers.units, numbers.duration, S) for s in 1:S, t in 1:(numbers.duration), i in 1:(numbers.units), - k in 1:(numbers.real_tume_steps) + k in 1:(numbers.real_time_steps) real_time_bid[k, i, t, s] = simul[s][t][:real_time_bid][k, i].out end diff --git a/src/preprocess.jl b/src/preprocess.jl index bad80cb..95b1045 100644 --- a/src/preprocess.jl +++ b/src/preprocess.jl @@ -15,8 +15,8 @@ function _evaluate_acceptance_real_time!(prb::Problem)::Nothing for t in 1:(numbers.duration) temp = [] for n in 1:(size(random.markov_transitions[t])[2]) - matrix = zeros(numbers.units, numbers.real_tume_steps) - for i in 1:(numbers.units), k in 1:(numbers.real_tume_steps) + matrix = zeros(numbers.units, numbers.real_time_steps) + for i in 1:(numbers.units), k in 1:(numbers.real_time_steps) matrix[i, k] = data.prices_real_time_curve[t][i][k] <= random.prices_real_time[t][prb.data.unit_to_bus[i]][n] diff --git a/src/structs.jl b/src/structs.jl index e901f21..1962a65 100644 --- a/src/structs.jl +++ b/src/structs.jl @@ -28,7 +28,7 @@ Base.@kwdef mutable struct Numbers period_of_day_ahead_clear::Int = 0 # Periods of day ahead clear days::Int = 0 # Number of days duration::Int = 0 # Number of periods of time in the horizon - real_tume_steps::Int = 0 # Number of prices in the real time curve + real_time_steps::Int = 0 # Number of prices in the real time curve day_ahead_steps::Int = 0 # Number of prices in the day ahead curve end diff --git a/src/variables.jl b/src/variables.jl index 8e2508d..bd14bf4 100644 --- a/src/variables.jl +++ b/src/variables.jl @@ -25,7 +25,7 @@ end function _variable_real_time_bid!(sp::Model, prb::Problem)::Nothing @variable( sp, - 0.0 <= real_time_bid[1:(prb.numbers.real_tume_steps), 1:(prb.numbers.units)], + 0.0 <= real_time_bid[1:(prb.numbers.real_time_steps), 1:(prb.numbers.units)], SDDP.State, initial_value = 0.0 ) diff --git a/test/test_preprocess.jl b/test/test_preprocess.jl index 60e2a1b..45ec232 100644 --- a/test/test_preprocess.jl +++ b/test/test_preprocess.jl @@ -9,7 +9,7 @@ numbers.first_period = 2 numbers.units = 1 numbers.buses = 1 numbers.duration = 2 -numbers.real_tume_steps = 2 +numbers.real_time_steps = 2 numbers.day_ahead_steps = 2 numbers.period_of_day_ahead_bid = 1 numbers.period_of_day_ahead_clear = 2 diff --git a/test/test_read_write_json.jl b/test/test_read_write_json.jl index 96b54ab..d80a4d7 100644 --- a/test/test_read_write_json.jl +++ b/test/test_read_write_json.jl @@ -10,7 +10,7 @@ numbers.first_period = 1 numbers.units = 1 numbers.buses = 1 numbers.duration = 1 -numbers.real_tume_steps = 1 +numbers.real_time_steps = 1 numbers.day_ahead_steps = 1 numbers.period_of_day_ahead_bid = 1 numbers.period_of_day_ahead_clear = 1 @@ -41,7 +41,7 @@ prb2 = OptimalEnergyBid.create_problem(joinpath(dirname(@__DIR__), "cases", "toy @test numbers.units == prb2.numbers.units @test numbers.buses == prb2.numbers.buses @test numbers.duration == prb2.numbers.duration -@test numbers.real_tume_steps == prb2.numbers.real_tume_steps +@test numbers.real_time_steps == prb2.numbers.real_time_steps @test numbers.day_ahead_steps == prb2.numbers.day_ahead_steps @test numbers.period_of_day_ahead_bid == prb2.numbers.period_of_day_ahead_bid @test numbers.period_of_day_ahead_clear == prb2.numbers.period_of_day_ahead_clear @@ -68,7 +68,7 @@ mktempdir() do path @test numbers.first_period == prb3.numbers.first_period @test numbers.units == prb3.numbers.units @test numbers.duration == prb3.numbers.duration - @test numbers.real_tume_steps == prb3.numbers.real_tume_steps + @test numbers.real_time_steps == prb3.numbers.real_time_steps @test numbers.day_ahead_steps == prb3.numbers.day_ahead_steps @test numbers.period_of_day_ahead_bid == prb3.numbers.period_of_day_ahead_bid @test numbers.period_of_day_ahead_clear == prb3.numbers.period_of_day_ahead_clear From 553a5ef4e300e189da25a3c421d199d963d22149 Mon Sep 17 00:00:00 2001 From: Thiago Novaes Date: Wed, 2 Oct 2024 21:21:45 +0200 Subject: [PATCH 5/9] test running --- TimeSeriesHelper.jl/src/estimate.jl | 6 +- examples/local_miso.jl | 90 ++++++++++++++++++++++------- 2 files changed, 74 insertions(+), 22 deletions(-) diff --git a/TimeSeriesHelper.jl/src/estimate.jl b/TimeSeriesHelper.jl/src/estimate.jl index 53c78dc..312317d 100644 --- a/TimeSeriesHelper.jl/src/estimate.jl +++ b/TimeSeriesHelper.jl/src/estimate.jl @@ -35,7 +35,8 @@ function build_markov_transition( transition_matrix::Matrix{Float64}, T::Int )::Vector{Matrix{Float64}} transitions_matrix = Vector{Matrix{Float64}}(undef, T) - for t in 1:T + transitions_matrix[1] = sum(transition_matrix, dims=1) / size(transition_matrix, 1) + for t in 2:T transitions_matrix[t] = transition_matrix end return transitions_matrix @@ -94,7 +95,8 @@ function build_scenarios( for w in 1:W push!(temp[n], []) for i in 1:I - push!(temp[n][w], samples_inflow[t, n, w, i]) + # TODO abs + push!(temp[n][w], abs(samples_inflow[t, n, w, i])) end end end diff --git a/examples/local_miso.jl b/examples/local_miso.jl index 8c3c515..bac1526 100644 --- a/examples/local_miso.jl +++ b/examples/local_miso.jl @@ -1,5 +1,6 @@ using OptimalEnergyBid using TimeSeriesHelper +using HiGHS directory = "C:\\Users\\thiag\\Documents\\Data\\"; start = 20240810; @@ -17,31 +18,80 @@ wind = TimeSeriesHelper.read_open_meteo_json( directory, "wind_speed_10m", start, stop ) -wind3 = wind3[(end - 744):end, 4:5] -solar3 = solar3[(end - 744):end, :] -rt3 = rt3[:, 1:2] -da3 = da3[:, 1:2] +nodes = ["AECI", "AEP"] -wind4 = vec(wind3) -solar4 = vec(solar3) -rt4 = vec(rt3) -da4 = vec(da3) +prices_real_time = Vector{Vector{Float64}}() +for i in 1:(stop - start + 1) * 24 + push!(prices_real_time, []) + for node in nodes + push!(prices_real_time[i], real_time[node][i]) + end +end -windl, _ = size(wind3) -solarl, _ = size(solar3) -rtl, _ = size(rt3) -dal, _ = size(da3) +prices_day_ahead = Vector{Vector{Float64}}() +for i in 1:(stop - start + 2) * 24 + push!(prices_day_ahead, []) + for node in nodes + push!(prices_day_ahead[i], day_ahead[node][i]) + end +end -wind5 = [wind4[i:windl:end] for i in 1:windl] -solar5 = [solar4[i:solarl:end] for i in 1:solarl] -rt5 = [rt4[i:rtl:end] for i in 1:rtl] -da5 = [da4[i:dal:end] for i in 1:dal] +inflow = Vector{Vector{Float64}}() +for i in 1:(stop - start + 1) * 24 + push!(inflow, []) + for key in keys(wind) + push!(inflow[i], wind[key][i]) + end +end history = TimeSeriesHelper.History() -history.prices_real_time = rt5 -history.prices_day_ahead = da5 -history.inflow = wind5 +history.prices_real_time = prices_real_time +history.prices_day_ahead = prices_day_ahead +history.inflow = inflow -h = TimeSeriesHelper.build_serial_history(history, 745, 24) +h = TimeSeriesHelper.build_serial_history(history, 336, 24) m, o = TimeSeriesHelper.estimate_hmm(h, 5) + +matrix = TimeSeriesHelper.build_markov_transition(m, 48) + +rt, da, inflow = TimeSeriesHelper.build_scenarios(o, 48, 24, 3, 1, 2, 2) + +prb = OptimalEnergyBid.Problem() + +numbers = prb.numbers +random = prb.random +data = prb.data +options = prb.options + +numbers.periods_per_day = 24 +numbers.first_period = 1 +numbers.units = 2 +numbers.buses = 2 +numbers.duration = 48 +numbers.real_time_steps = 2 +numbers.day_ahead_steps = 2 +numbers.period_of_day_ahead_bid = 12 +numbers.period_of_day_ahead_clear = 20 +# TODO +numbers.days = 2 + +random.prices_real_time = rt +random.prices_day_ahead = da +random.inflow = inflow +random.inflow_probability = v = [[[1/3 for k in 1:3] for j in 1:5] for i in 1:48] +random.markov_transitions = matrix + +data.unit_to_bus = [1, 2] +data.volume_max = ones(2) +data.volume_min = zeros(2) +data.volume_initial = zeros(2) +data.prices_real_time_curve = rt +data.prices_day_ahead_curve = da +data.names = ["unit1", "unit2"] + +OptimalEnergyBid.set_parameter!(prb, OptimalEnergyBid.Parameter.Optimizer, HiGHS.Optimizer) + +OptimalEnergyBid.build_model!(prb, true) +OptimalEnergyBid.train!(prb) +OptimalEnergyBid.simulate!(prb) \ No newline at end of file From c24d9c867e6910fff229815181189f3af111f5a5 Mon Sep 17 00:00:00 2001 From: Thiago Novaes Date: Wed, 2 Oct 2024 21:30:46 +0200 Subject: [PATCH 6/9] time --- examples/local_miso.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/local_miso.jl b/examples/local_miso.jl index bac1526..fea651b 100644 --- a/examples/local_miso.jl +++ b/examples/local_miso.jl @@ -93,5 +93,5 @@ data.names = ["unit1", "unit2"] OptimalEnergyBid.set_parameter!(prb, OptimalEnergyBid.Parameter.Optimizer, HiGHS.Optimizer) OptimalEnergyBid.build_model!(prb, true) -OptimalEnergyBid.train!(prb) +OptimalEnergyBid.train!(prb; time_limit=10) OptimalEnergyBid.simulate!(prb) \ No newline at end of file From 8ebb6266ef24ab06b716d5ad55da6c91798cea05 Mon Sep 17 00:00:00 2001 From: Thiago Novaes Date: Sat, 5 Oct 2024 14:32:02 +0200 Subject: [PATCH 7/9] test --- examples/local_miso.jl | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/examples/local_miso.jl b/examples/local_miso.jl index fea651b..713772f 100644 --- a/examples/local_miso.jl +++ b/examples/local_miso.jl @@ -69,8 +69,8 @@ numbers.first_period = 1 numbers.units = 2 numbers.buses = 2 numbers.duration = 48 -numbers.real_time_steps = 2 -numbers.day_ahead_steps = 2 +numbers.real_time_steps = 5 +numbers.day_ahead_steps = 5 numbers.period_of_day_ahead_bid = 12 numbers.period_of_day_ahead_clear = 20 # TODO @@ -94,4 +94,5 @@ OptimalEnergyBid.set_parameter!(prb, OptimalEnergyBid.Parameter.Optimizer, HiGHS OptimalEnergyBid.build_model!(prb, true) OptimalEnergyBid.train!(prb; time_limit=10) -OptimalEnergyBid.simulate!(prb) \ No newline at end of file +OptimalEnergyBid.simulate!(prb) +OptimalEnergyBid.plot_all(prb, 1, "") From f3bdfa64bcab13fa529cbbc70fdbeecd4552bf56 Mon Sep 17 00:00:00 2001 From: Thiago Novaes Date: Sat, 5 Oct 2024 16:27:15 +0200 Subject: [PATCH 8/9] fix --- examples/local_miso.jl | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/examples/local_miso.jl b/examples/local_miso.jl index 713772f..5de1fc3 100644 --- a/examples/local_miso.jl +++ b/examples/local_miso.jl @@ -83,16 +83,34 @@ random.inflow_probability = v = [[[1/3 for k in 1:3] for j in 1:5] for i in 1:48 random.markov_transitions = matrix data.unit_to_bus = [1, 2] -data.volume_max = ones(2) +data.volume_max = ones(2) * 50 data.volume_min = zeros(2) data.volume_initial = zeros(2) -data.prices_real_time_curve = rt -data.prices_day_ahead_curve = da + +rt_sorted = deepcopy(rt) +da_sorted = deepcopy(da) + +for t in 1:48 + for i in 1:2 + sort!(rt_sorted[t][i]) + end +end + +for d in 1:2 + for j in 1:24 + for i in 1:2 + sort!(da_sorted[d][j][i]) + end + end +end + +data.prices_real_time_curve = rt_sorted +data.prices_day_ahead_curve = da_sorted data.names = ["unit1", "unit2"] OptimalEnergyBid.set_parameter!(prb, OptimalEnergyBid.Parameter.Optimizer, HiGHS.Optimizer) OptimalEnergyBid.build_model!(prb, true) -OptimalEnergyBid.train!(prb; time_limit=10) +OptimalEnergyBid.train!(prb; time_limit=60) OptimalEnergyBid.simulate!(prb) OptimalEnergyBid.plot_all(prb, 1, "") From 57209b8cac68a2240ffc7285e7f466b47e6e17b6 Mon Sep 17 00:00:00 2001 From: Thiago Novaes Date: Sat, 5 Oct 2024 17:00:59 +0200 Subject: [PATCH 9/9] ignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 912f1a7..538d0ab 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ revise/ *.log *.lp *.mps +*.png *.ipynb *notebook-checkpoint .DS_Store