diff --git a/src/couch/src/couch_file.erl b/src/couch/src/couch_file.erl index acd4fda781..04ca44ddf3 100644 --- a/src/couch/src/couch_file.erl +++ b/src/couch/src/couch_file.erl @@ -43,7 +43,8 @@ -export([append_raw_chunk/2, assemble_file_chunk/1, assemble_file_chunk/2]). -export([append_term/2, append_term/3, append_term_md5/2, append_term_md5/3]). -export([write_header/2, read_header/1]). --export([delete/2, delete/3, nuke_dir/2, init_delete_dir/1]). +-export([delete/2, delete/3, nuke_dir/2, nuke_dir/3]). +-export([init_delete_dir/1, init_recovery_dir/1]). -export([msec_since_last_read/1]). % gen_server callbacks @@ -265,6 +266,18 @@ rename_file(Original) -> Else -> Else end. +rename_dir(RootDelDir, Original, DbName) -> + DbDir = binary_to_list(DbName) ++ "_design", + RenamedIndexDir = filename:join( + [RootDelDir, ".recovery", DbDir] + ), + Now = calendar:local_time(), + filelib:ensure_dir(RenamedIndexDir), + case file:rename(Original, RenamedIndexDir) of + ok -> file:change_time(RenamedIndexDir, Now); + Else -> Else + end. + deleted_filename(Original) -> {{Y, Mon, D}, {H, Min, S}} = calendar:universal_time(), Suffix = lists:flatten( @@ -273,14 +286,18 @@ deleted_filename(Original) -> ++ filename:extension(Original), [Y, Mon, D, H, Min, S])), filename:rootname(Original) ++ Suffix. -nuke_dir(RootDelDir, Dir) -> +nuke_dir(RootDir, Dir) -> + nuke_dir(RootDir, Dir, []). +nuke_dir(RootDelDir, Dir, Options) -> EnableRecovery = config:get_boolean("couchdb", "enable_database_recovery", false), + Context = couch_util:get_value(context, Options, compaction), case EnableRecovery of - true -> - rename_file(Dir); - false -> - delete_dir(RootDelDir, Dir) + true when Context == delete -> + DbName = couch_util:get_value(db_name, Options), + rename_dir(RootDelDir, Dir, DbName); + true -> rename_file(Dir); + false -> delete_dir(RootDelDir, Dir) end. delete_dir(RootDelDir, Dir) -> @@ -318,6 +335,9 @@ init_delete_dir(RootDir) -> end), ok. +init_recovery_dir(RootDir) -> + Dir = filename:join(RootDir, ".recovery"), + filelib:ensure_dir(filename:join(Dir, "foo")). read_header(Fd) -> case ioq:call(Fd, find_header, erlang:get(io_priority)) of @@ -762,11 +782,25 @@ make_filename_fixtures(DbNames) -> "shards/00000000-1fffffff/~s.1458336317.couch", ".shards/00000000-1fffffff/~s.1458336317_design", ".shards/00000000-1fffffff/~s.1458336317_design" - "/mrview/3133e28517e89a3e11435dd5ac4ad85a.view" + "/mrview/3133e28517e89a3e11435dd5ac4ad85a.view", + ".recovery/1499329402/shards/00000000-1fffffff/~s.1499329402_design", + ".recovery/1499329402/shards/00000000-1fffffff/~s.1499329402_design" + "/mrview/8fabddcb28f501d6764afd7def3bd352.view" ], lists:flatmap(fun(DbName) -> lists:map(fun(Format) -> - filename:join("/srv/data", io_lib:format(Format, [DbName])) + % Count how many times we need to specify the database name + % by splitting on the ~s formatter. + ArgCount = erlang:length(binary:split( + list_to_binary(Format), + <<"~s">>, + [global]) + ) - 1, + Args = case ArgCount of + 1 -> [DbName]; + 2 -> [DbName, DbName] + end, + filename:join("/srv/data/", io_lib:format(Format, Args)) end, Formats) end, DbNames). diff --git a/src/couch/test/couch_file_tests.erl b/src/couch/test/couch_file_tests.erl index c16be16c41..7990533223 100644 --- a/src/couch/test/couch_file_tests.erl +++ b/src/couch/test/couch_file_tests.erl @@ -419,32 +419,44 @@ nuke_dir_test_() -> RootDir = filename:dirname(File0), BaseName = filename:basename(File0), Seed = crypto:rand_uniform(1000000000, 9999999999), - DDocDir = io_lib:format("db.~b_design", [Seed]), + DBName0 = io_lib:format("db.~b", [Seed]), + DBName = iolist_to_binary(DBName0), + DDocDir = io_lib:format("~s_design", [DBName0]), ViewDir = filename:join([RootDir, DDocDir]), file:make_dir(ViewDir), File = filename:join([ViewDir, BaseName]), file:rename(File0, File), ok = couch_file:init_delete_dir(RootDir), ok = file:write_file(File, <<>>), - {RootDir, ViewDir} + {RootDir, ViewDir, DBName} end, - fun({RootDir, ViewDir}) -> + fun({RootDir, ViewDir, _DBName}) -> meck:unload(config), remove_dir(ViewDir), Ext = filename:extension(ViewDir), case filelib:wildcard(RootDir ++ "/*.deleted" ++ Ext) of [DelDir] -> remove_dir(DelDir); _ -> ok - end + end, + RecDirPaths = RootDir ++ "/.recovery" ++ "/*_design", + [remove_dir(Dir) || Dir <- filelib:wildcard(RecDirPaths)] end, [ fun(Cfg) -> - {"enable_database_recovery = false", - make_rename_dir_test_case(Cfg, false)} + {"enable_database_recovery = false, context = delete", + make_rename_dir_test_case(Cfg, false, delete)} + end, + fun(Cfg) -> + {"enable_database_recovery = false, context = compaction", + make_rename_dir_test_case(Cfg, false, compaction)} + end, + fun(Cfg) -> + {"enable_database_recovery = true, context = delete", + make_rename_dir_test_case(Cfg, true, delete)} end, fun(Cfg) -> - {"enable_database_recovery = true", - make_rename_dir_test_case(Cfg, true)} + {"enable_database_recovery = true, context = compaction", + make_rename_dir_test_case(Cfg, true, compaction)} end, fun(Cfg) -> {"delete_after_rename = true", @@ -459,16 +471,26 @@ nuke_dir_test_() -> }. -make_rename_dir_test_case({RootDir, ViewDir}, EnableRecovery) -> +make_rename_dir_test_case({RootDir, ViewDir, DBName}, EnableRecovery, Context) -> meck:expect(config, get_boolean, fun ("couchdb", "enable_database_recovery", _) -> EnableRecovery; ("couchdb", "delete_after_rename", _) -> true end), DirExistsBefore = filelib:is_dir(ViewDir), - couch_file:nuke_dir(RootDir, ViewDir), + + couch_file:nuke_dir( + RootDir, + ViewDir, + [{db_name, DBName}, {context, Context}] + ), DirExistsAfter = filelib:is_dir(ViewDir), - Ext = filename:extension(ViewDir), - RenamedDirs = filelib:wildcard(RootDir ++ "/*.deleted" ++ Ext), + RenamedDirs = if Context =:= delete -> + filelib:wildcard(RootDir ++ "/.recovery" ++ "/*_design"); + true -> + Ext = filename:extension(ViewDir), + filelib:wildcard(RootDir ++ "/*.deleted" ++ Ext) + end, + ExpectRenamedCount = if EnableRecovery -> 1; true -> 0 end, [ ?_assert(DirExistsBefore), @@ -476,7 +498,7 @@ make_rename_dir_test_case({RootDir, ViewDir}, EnableRecovery) -> ?_assertEqual(ExpectRenamedCount, length(RenamedDirs)) ]. -make_delete_dir_test_case({RootDir, ViewDir}, DeleteAfterRename) -> +make_delete_dir_test_case({RootDir, ViewDir, _DBName}, DeleteAfterRename) -> meck:expect(config, get_boolean, fun ("couchdb", "enable_database_recovery", _) -> false; ("couchdb", "delete_after_rename", _) -> DeleteAfterRename diff --git a/src/couch_index/src/couch_index_server.erl b/src/couch_index/src/couch_index_server.erl index 8225a90a33..c16b2a475f 100644 --- a/src/couch_index/src/couch_index_server.erl +++ b/src/couch_index/src/couch_index_server.erl @@ -129,6 +129,7 @@ init([]) -> couch_event:link_listener(?MODULE, handle_db_event, nil, [all_dbs]), RootDir = couch_index_util:root_dir(), couch_file:init_delete_dir(RootDir), + couch_file:init_recovery_dir(RootDir), {ok, #st{root_dir=RootDir}}. @@ -161,14 +162,9 @@ handle_call({async_error, {DbName, _DDocId, Sig}, Error}, _From, State) -> [gen_server:reply(From, Error) || From <- Waiters], ets:delete(?BY_SIG, {DbName, Sig}), {reply, ok, State}; -handle_call({reset_indexes, DbName}, _From, State) -> - reset_indexes(DbName, State#st.root_dir), +handle_call({reset_indexes, DbName, Options}, _From, State) -> + reset_indexes(DbName, State#st.root_dir, Options), {reply, ok, State}. - - -handle_cast({reset_indexes, DbName}, State) -> - reset_indexes(DbName, State#st.root_dir), - {noreply, State}; handle_cast({add_to_ets, [Pid, DbName, DDocId, Sig]}, State) -> % check if Pid still exists case ets:lookup(?BY_PID, Pid) of @@ -179,6 +175,11 @@ handle_cast({add_to_ets, [Pid, DbName, DDocId, Sig]}, State) -> {noreply, State}; handle_cast({rem_from_ets, [DbName, DDocId, Sig]}, State) -> ets:delete_object(?BY_DB, {DbName, {DDocId, Sig}}), + {noreply, State}; + + +handle_cast({reset_indexes, DbName, Options}, State) -> + reset_indexes(DbName, State#st.root_dir, Options), {noreply, State}. handle_info({'EXIT', Pid, Reason}, Server) -> @@ -237,7 +238,7 @@ new_index({Mod, IdxState, DbName, Sig}) -> end. -reset_indexes(DbName, Root) -> +reset_indexes(DbName, Root, Options) -> % shutdown all the updaters and clear the files, the db got changed SigDDocIds = lists:foldl(fun({_, {DDocId, Sig}}, DDict) -> dict:append(Sig, DDocId, DDict) @@ -251,7 +252,8 @@ reset_indexes(DbName, Root) -> end, lists:foreach(Fun, dict:to_list(SigDDocIds)), Path = couch_index_util:index_dir("", DbName), - couch_file:nuke_dir(Root, Path). + Options1 = [{db_name, DbName} | Options], + couch_file:nuke_dir(Root, Path, Options1). add_to_ets(DbName, Sig, DDocId, Pid) -> @@ -269,10 +271,10 @@ rem_from_ets(DbName, Sig, DDocIds, Pid) -> handle_db_event(DbName, created, St) -> - gen_server:cast(?MODULE, {reset_indexes, DbName}), + gen_server:cast(?MODULE, {reset_indexes, DbName, []}), {ok, St}; handle_db_event(DbName, deleted, St) -> - gen_server:cast(?MODULE, {reset_indexes, DbName}), + gen_server:cast(?MODULE, {reset_indexes, DbName, [{context, delete}]}), {ok, St}; handle_db_event(<<"shards/", _/binary>> = DbName, {ddoc_updated, DDocId}, St) ->