Skip to content

Mas i1807 overflowqueues #1809

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
martinsumner merged 12 commits into from
May 11, 2022
Merged

Mas i1807 overflowqueues #1809

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
c4d97b7
Bound reap/erase queues
martinsumner Dec 3, 2021
910fb5a
Create generic behaviour for eraser/reaper
martinsumner Dec 9, 2021
fee56e5
Stop cleaning folder
martinsumner Dec 9, 2021
8ac9e62
Setup folders correctly
martinsumner Dec 9, 2021
0c73afc
Correct log
martinsumner Dec 9, 2021
5e43a23
Further log correction
martinsumner Dec 9, 2021
d3d6923
Correct cleaning of directories for test
martinsumner Dec 9, 2021
232ca17
Switch to action/2
martinsumner Dec 10, 2021
3d24c6b
Update eqc tests for refactoring of reaper/eraser
martinsumner Dec 10, 2021
1973f94
Improve comments/API
martinsumner Dec 14, 2021
be2d9f0
Pause reaper on overload
martinsumner Jan 27, 2022
c815020
Adjustments following review
martinsumner Feb 24, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.org
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ git push mine some-topic-branch
# standard tests
./rebar3 do xref, dialyzer, eunit
# property-based tests
./rebar3 as test eqc
./rebar3 as test eqc --testing_budget 600
#+END_SRC

For a more complete set of tests, update riak_kv in the full Riak application and run any appropriate [[https://github.com/basho/riak_test/tree/develop-3.0/groups][Riak riak_test groups]]
12 changes: 10 additions & 2 deletions eqc/eraser_eqc.erl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,16 @@ start_args(_S) ->
[gen_delete_mode()].

start(DelMode) ->
{ok, Pid} = gen_server:start_link(riak_kv_eraser, [eqc_job, fun erase/2, DelMode], []),
Pid.
FilePath = riak_kv_test_util:get_test_dir("eraser_eqc"),
{ok, Pid} = riak_kv_eraser:start_link(FilePath),
case DelMode of
keep ->
riak_kv_eraser:override_redo(false);
immediate ->
riak_kv_eraser:override_redo(true)
end,
ok = gen_server:call(Pid, {override_action, fun erase/2}),
Pid.

start_next(S, Pid, [DelMode]) ->
S#{ delete_mode => DelMode, pid => Pid }.
Expand Down
8 changes: 5 additions & 3 deletions eqc/reaper_eqc.erl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,10 @@ start_args(_S) ->
[].

start() ->
{ok, Pid} = gen_server:start_link(riak_kv_reaper, [eqc_job, fun reaper/1], []),
Pid.
FilePath = riak_kv_test_util:get_test_dir("reaper_eqc"),
{ok, Pid} = riak_kv_reaper:start_link(FilePath),
ok = gen_server:call(Pid, {override_action, fun reaper/2}),
Pid.

start_next(S, Pid, []) ->
S#{ pid => Pid }.
Expand Down Expand Up @@ -78,7 +80,7 @@ stop_job(Pid) ->
end.


reaper(Ref) ->
reaper(Ref, _) ->
case ets:lookup(?MODULE, Ref) of
[{_, 1}] ->
ets:delete(?MODULE, Ref), true;
Expand Down
12 changes: 12 additions & 0 deletions priv/riak_kv.schema
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,18 @@
{datatype, directory}
]}.

%% @doc A path under which the eraser overload queue will be stored.
{mapping, "eraser_dataroot", "riak_kv.eraser_dataroot", [
{default, "$(platform_data_dir)/kv_eraser"},
{datatype, directory}
]}.

%% @doc A path under which the reaper overload queue will be stored.
{mapping, "reaper_dataroot", "riak_kv.reaper_dataroot", [
{default, "$(platform_data_dir)/kv_reaper"},
{datatype, directory}
]}.

%% @doc Parallel key store type
%% When running in parallel mode, which will be the default if the backend does
%% not support native tictac aae (i.e. is not leveled), what type of parallel
Expand Down
11 changes: 3 additions & 8 deletions rebar.config
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,7 @@
{i, "./_build/default/plugins/gpb/include"},
{d, 'TEST_FS2_BACKEND_IN_RIAK_KV'}]}.

{eunit_opts, [
no_tty, %% This turns off the default output, MUST HAVE
{report, {eunit_progress, [colored, profile]}} %% Use `profile' to see test timing information
%% Uses the progress formatter with ANSI-colored output
]}.
{eunit_opts, [verbose]}.

{xref_checks,[undefined_function_calls,undefined_functions]}.

Expand All @@ -41,8 +37,7 @@
]}.

{profiles, [
{test, [{deps, [meck,
{eunit_formatters, ".*", {git, "https://github.com/seancribbs/eunit_formatters", {tag, "v0.5.0"}}}]}]},
{test, [{deps, [meck]}]},
{gha, [{erl_opts, [{d, 'GITHUBEXCLUDE'}]}]}
]}.

Expand All @@ -57,6 +52,6 @@
{riak_dt, {git, "https://github.com/basho/riak_dt.git", {tag, "riak_kv-3.0.0"}}},
{riak_api, {git, "https://github.com/basho/riak_api.git", {tag, "riak_kv-3.0.9"}}},
{hyper, {git, "https://github.com/basho/hyper", {tag, "1.1.0"}}},
{kv_index_tictactree, {git, "https://github.com/martinsumner/kv_index_tictactree.git", {branch, "develop-3.0"}}},
{kv_index_tictactree, {git, "https://github.com/martinsumner/kv_index_tictactree.git", {branch, "mas-i102-formatstatus"}}},
{riakhttpc, {git, "https://github.com/basho/riak-erlang-http-client", {tag, "3.0.8"}}}
]}.
Loading