From f28d896acf01dc105f7bfecb76d4c5aaeee8be1c Mon Sep 17 00:00:00 2001 From: Jan Lehnardt Date: Fri, 31 Oct 2014 15:37:29 +0100 Subject: [PATCH 01/52] make it so --- README.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000000..83758a53d2 --- /dev/null +++ b/README.md @@ -0,0 +1 @@ +This module implements /_cluster_setup and manages the setting up, duh, of a CouchDB cluster. From 3621725d7c3761bbede21945821aa13508c26d82 Mon Sep 17 00:00:00 2001 From: Jan Lehnardt Date: Fri, 31 Oct 2014 15:41:27 +0100 Subject: [PATCH 02/52] add bootstrap --- src/setup.app.src | 12 ++++++++++++ src/setup_app.erl | 16 ++++++++++++++++ src/setup_sup.erl | 27 +++++++++++++++++++++++++++ 3 files changed, 55 insertions(+) create mode 100644 src/setup.app.src create mode 100644 src/setup_app.erl create mode 100644 src/setup_sup.erl diff --git a/src/setup.app.src b/src/setup.app.src new file mode 100644 index 0000000000..d79e4b4c79 --- /dev/null +++ b/src/setup.app.src @@ -0,0 +1,12 @@ +{application, setup, + [ + {description, ""}, + {vsn, "1"}, + {registered, []}, + {applications, [ + kernel, + stdlib + ]}, + {mod, { setup_app, []}}, + {env, []} + ]}. diff --git a/src/setup_app.erl b/src/setup_app.erl new file mode 100644 index 0000000000..0d43a50d46 --- /dev/null +++ b/src/setup_app.erl @@ -0,0 +1,16 @@ +-module(setup_app). + +-behaviour(application). + +%% Application callbacks +-export([start/2, stop/1]). + +%% =================================================================== +%% Application callbacks +%% =================================================================== + +start(_StartType, _StartArgs) -> + setup_sup:start_link(). + +stop(_State) -> + ok. diff --git a/src/setup_sup.erl b/src/setup_sup.erl new file mode 100644 index 0000000000..eae0eaedee --- /dev/null +++ b/src/setup_sup.erl @@ -0,0 +1,27 @@ +-module(setup_sup). + +-behaviour(supervisor). + +%% API +-export([start_link/0]). + +%% Supervisor callbacks +-export([init/1]). + +%% Helper macro for declaring children of supervisor +-define(CHILD(I, Type), {I, {I, start_link, []}, permanent, 5000, Type, [I]}). + +%% =================================================================== +%% API functions +%% =================================================================== + +start_link() -> + supervisor:start_link({local, ?MODULE}, ?MODULE, []). + +%% =================================================================== +%% Supervisor callbacks +%% =================================================================== + +init([]) -> + {ok, { {one_for_one, 5, 10}, []} }. + From 0f559a9513a299c7883af7687092d545257e6446 Mon Sep 17 00:00:00 2001 From: Jan Lehnardt Date: Fri, 31 Oct 2014 15:41:59 +0100 Subject: [PATCH 03/52] add ignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000..1dbfa4bce8 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +ebin +.rebar From 58c4948bb8f342c3bf2815c413cb6086cae43e6c Mon Sep 17 00:00:00 2001 From: Jan Lehnardt Date: Fri, 31 Oct 2014 15:54:25 +0100 Subject: [PATCH 04/52] add http stub --- src/setup_httpd.erl | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 src/setup_httpd.erl diff --git a/src/setup_httpd.erl b/src/setup_httpd.erl new file mode 100644 index 0000000000..f001492dc2 --- /dev/null +++ b/src/setup_httpd.erl @@ -0,0 +1,19 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(setup_httpd). + +-export([handle_setup_req/1]). + +handle_setup_req(Req) -> + io:format("~nHandle Req: ~p~n", [Req]). + From e8c49669f950b5461ef79dfefd4767010f92c216 Mon Sep 17 00:00:00 2001 From: Jan Lehnardt Date: Fri, 31 Oct 2014 17:06:08 +0100 Subject: [PATCH 05/52] add basic action handling --- src/setup_httpd.erl | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/src/setup_httpd.erl b/src/setup_httpd.erl index f001492dc2..6a4433a150 100644 --- a/src/setup_httpd.erl +++ b/src/setup_httpd.erl @@ -15,5 +15,38 @@ -export([handle_setup_req/1]). handle_setup_req(Req) -> - io:format("~nHandle Req: ~p~n", [Req]). + ok = chttpd:verify_is_server_admin(Req), + % TBD uncomment after devving + %couch_httpd:validate_ctype(Req, "application/json"), + Setup = get_body(Req), + io:format("~nSetup: ~p~n", [Setup]), + Action = binary_to_list(couch_util:get_value(<<"action">>, Setup, <<"missing">>)), + case handle_action(Action, Setup) of + ok -> + chttpd:send_json(Req, 201, {[{ok, true}]}); + {error, Message} -> + couch_httpd:send_error(Req, 400, <<"bad_request">>, Message) + end. + +handle_action("enable_cluster", Setup) -> + io:format("~nenable_cluster: ~p~n", [Setup]); +handle_action("finish_cluster", Setup) -> + io:format("~nfinish_cluster: ~p~n", [Setup]); +handle_action("add_node", Setup) -> + io:format("~nadd_node: ~p~n", [Setup]); +handle_action("remove_node", Setup) -> + io:format("~nremove_node: ~p~n", [Setup]); +handle_action(_, _) -> + io:format("~ninvalid_action: ~n", []), + {error, <<"Invalid Action'">>}. + + +get_body(Req) -> + case catch couch_httpd:json_body_obj(Req) of + {Body} -> + Body; + Else -> + io:format("~nBody Fail: ~p~n", [Else]), + couch_httpd:send_error(Req, 400, <<"bad_request">>, <<"Missing JSON body'">>) + end. From a5213f79b3a82ce4e42d03a26b78bf8ed2126efd Mon Sep 17 00:00:00 2001 From: Jan Lehnardt Date: Fri, 31 Oct 2014 17:06:43 +0100 Subject: [PATCH 06/52] add Apache License stanza everywhere --- src/setup.app.src | 12 ++++++++++++ src/setup_app.erl | 12 ++++++++++++ src/setup_sup.erl | 12 ++++++++++++ 3 files changed, 36 insertions(+) diff --git a/src/setup.app.src b/src/setup.app.src index d79e4b4c79..8c85e14fb3 100644 --- a/src/setup.app.src +++ b/src/setup.app.src @@ -1,3 +1,15 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + {application, setup, [ {description, ""}, diff --git a/src/setup_app.erl b/src/setup_app.erl index 0d43a50d46..330450131e 100644 --- a/src/setup_app.erl +++ b/src/setup_app.erl @@ -1,3 +1,15 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + -module(setup_app). -behaviour(application). diff --git a/src/setup_sup.erl b/src/setup_sup.erl index eae0eaedee..b69733395a 100644 --- a/src/setup_sup.erl +++ b/src/setup_sup.erl @@ -1,3 +1,15 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + -module(setup_sup). -behaviour(supervisor). From 404692f5f0c65e8604ab2c47078abe2c17301a7c Mon Sep 17 00:00:00 2001 From: Jan Lehnardt Date: Fri, 31 Oct 2014 17:14:43 +0100 Subject: [PATCH 07/52] add the plan to readme --- README.md | 144 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 144 insertions(+) diff --git a/README.md b/README.md index 83758a53d2..c067d80365 100644 --- a/README.md +++ b/README.md @@ -1 +1,145 @@ This module implements /_cluster_setup and manages the setting up, duh, of a CouchDB cluster. + +The Plan: + +N. End User Action +- What happens behind the scenes. + + +1. Launch CouchDB with `$ couchdb`, or init.d, or any other way, exactly +like it is done in 1.x.x. +- CouchDB launches and listens on 127.0.0.1:5984 + +From here on, there are two paths, one is via Fauxton (a) the other is +using a HTTP endpoint (b). Fauxton just uses the HTTP endpoint in (b). +(b) can be used to set up a cluster programmatically. + + +2.a. Go to Fauxton. There is a “Cluster Setup” tab in the sidebar. Go +to the tab and get presented with a form that asks you to enter an admin +username, admin password and optionally a bind_address and port to bind +to publicly. Submit the form with the [Enable Cluster] button. + +- POST to /_setup with + { + "action": "enable_cluster", + "admin": { + "user": "username", + "pass": "password" + }, + ["bind_address": "xxxx",] + ["port": yyyy] + } + +This sets up the admin user on the current node and binds to 0.0.0.0:5984 +or the specified ip:port. Logs admin user into Fauxton automatically. + +2.b. POST to /_setup as shown above. + +Repeat on all nodes. +- keep the same username/password everywhere. + + +3. Pick any one node, for simplicity use the first one, to be the +“setup coordination node”. +- this is a “master” node that manages the setup and requires all + other nodes to be able to see it and vice versa. Setup won’t work + with unavailable nodes (duh). The notion of “master” will be gone + once the setup is finished. At that point, the system has no + master node. Ignore I ever said “master”. + +a. Go to Fauxton / Cluster Setup, once we have enabled the cluster, the +UI shows an “Add Node” interface with the fields admin, and node: +- POST to /_setup with + { + "action": "add_node", + "admin": { // should be auto-filled from Fauxton + "user": "username", + "pass": "password" + }, + "node": { + "host": "hostname", + ["port": 5984] + } + } + +b. as in a, but without the Fauxton bits, just POST to /_setup +- this request will do this: + - on the “setup coordination node”: + - check if we have an Erlang Cookie Secret. If not, generate + a UUID and set the erlang cookie to to that UUID. + // TBD: persist the cookie, so it survives restarts + - make a POST request to the node specified in the body above + using the admin credentials in the body above: + POST to http://username:password@node_b:5984/_setup with: + { + "action": "receive_cookie", + "cookie": "", + } + // TBD: persist the cookie on node B, so it survives restarts + + - when the request to node B returns, we know the Erlang-level + inter-cluster communication is enabled and we can start adding + the node on the CouchDB level. To do that, the “setup + coordination node” does this to it’s own HTTP endpoint: + PUT /nodes/node_b:5984 or the same thing with internal APIs. + +- Repeat for all nodes. +- Fauxton keeps a list of all set up nodes for users to see. + + +4.a. When all nodes are added, click the [Finish Cluster Setup] button +in Fauxton. +- this does POST /_setup + { + "action": "finish_setup" + } + +b. Same as in a. + +- this manages the final setup bits, like creating the _users, + _replicator and _db_updates endpoints and whatever else is needed. + // TBD: collect what else is needed. + + +## The Setup Endpoint + +This is not a REST-y endpoint, it is a simple state machine operated +by HTTP POST with JSON bodies that have an `action` field. + +### State 1: No Cluster Enabled + +This is right after starting a node for the first time, and any time +before the cluster is enabled as outlined above. + +GET /_setup +{"state": "cluster_disabled"} + +POST /_setup {"action":"enable_cluster"...} -> Transition to State 2 +POST /_setup {"action":"enable_cluster"...} with empty admin user/pass or invalid host/post or host/port not available -> Error +POST /_setup {"action":"anything_but_enable_cluster"...} -> Error + + +### State 2: Cluster enabled, admin user set, waiting for nodes to be added. + +GET /_setup +{"state":"cluster_enabled","nodes":[]} + +POST /_setup {"action":"enable_cluster"...} -> Error +POST /_setup {"action":"add_node"...} -> Stay in State 2, but return "nodes":["node B"}] on GET +POST /_setup {"action":"add_node"...} -> if target node not available, Error +POST /_setup {"action":"finish_cluster"} with no nodes set up -> Error +POST /_setup {"action":"finish_cluster"} -> Transition to State 3 + + +### State 3: Cluster set up, all nodes operational + +GET /_setup +{"state":"cluster_finished","nodes":["node a", "node b", ...]} + +POST /_setup {"action":"enable_cluster"...} -> Error +POST /_setup {"action":"finish_cluster"...} -> Stay in State 3, do nothing +POST /_setup {"action":"add_node"...} -> Error +POST /_setup?i_know_what_i_am_doing=true {"action":"add_node"...} -> Add node, stay in State 3. + +// TBD: we need to persist the setup state somewhere. \ No newline at end of file From ecf310a69ff9fc86f76513c356b2f23e6d583c23 Mon Sep 17 00:00:00 2001 From: Jan Lehnardt Date: Fri, 31 Oct 2014 17:22:06 +0100 Subject: [PATCH 08/52] add note about skipping a step if the node is already setup --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index c067d80365..715d9878b5 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,11 @@ to the tab and get presented with a form that asks you to enter an admin username, admin password and optionally a bind_address and port to bind to publicly. Submit the form with the [Enable Cluster] button. +If this is a single node install that already has an admin set up, there +is no need to ask for admin credentials here. If the bind_address is != +127.0.0.1, we can skip this entirely and Fauxton can show the add_node +UI right away. + - POST to /_setup with { "action": "enable_cluster", From 38eaa88e6223c518aca1ed90154c59462974e1ea Mon Sep 17 00:00:00 2001 From: Jan Lehnardt Date: Fri, 31 Oct 2014 17:36:21 +0100 Subject: [PATCH 09/52] add delete_node API --- README.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 715d9878b5..f4e84c8253 100644 --- a/README.md +++ b/README.md @@ -58,7 +58,8 @@ UI shows an “Add Node” interface with the fields admin, and node: - POST to /_setup with { "action": "add_node", - "admin": { // should be auto-filled from Fauxton + "admin": { // should be auto-filled from Fauxton, store plaintext PW in + // localStorage until we finish_cluster or timeout. "user": "username", "pass": "password" }, @@ -135,6 +136,8 @@ POST /_setup {"action":"add_node"...} -> Stay in State 2, but return "nodes":["n POST /_setup {"action":"add_node"...} -> if target node not available, Error POST /_setup {"action":"finish_cluster"} with no nodes set up -> Error POST /_setup {"action":"finish_cluster"} -> Transition to State 3 +POST /_setup {"action":"delete_node"...} -> Stay in State 2, but delete node from /nodes, reflect the change in GET /_setup +POST /_setup {"action":"delete_node","node":"unknown"} -> Error Unknown Node ### State 3: Cluster set up, all nodes operational @@ -146,5 +149,7 @@ POST /_setup {"action":"enable_cluster"...} -> Error POST /_setup {"action":"finish_cluster"...} -> Stay in State 3, do nothing POST /_setup {"action":"add_node"...} -> Error POST /_setup?i_know_what_i_am_doing=true {"action":"add_node"...} -> Add node, stay in State 3. +POST /_setup {"action":"delete_node"...} -> Stay in State 3, but delete node from /nodes, reflect the change in GET /_setup +POST /_setup {"action":"delete_node","node":"unknown"} -> Error Unknown Node -// TBD: we need to persist the setup state somewhere. \ No newline at end of file +// TBD: we need to persist the setup state somewhere. From 9f1fa23274ef475945a1b55555936d1353f5d0fe Mon Sep 17 00:00:00 2001 From: Jan Lehnardt Date: Fri, 31 Oct 2014 17:38:44 +0100 Subject: [PATCH 10/52] hack for storing erlang cookie value on new nodes --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f4e84c8253..966ba1bebd 100644 --- a/README.md +++ b/README.md @@ -74,7 +74,7 @@ b. as in a, but without the Fauxton bits, just POST to /_setup - on the “setup coordination node”: - check if we have an Erlang Cookie Secret. If not, generate a UUID and set the erlang cookie to to that UUID. - // TBD: persist the cookie, so it survives restarts + - store the cookie in config.ini, re-set_cookie() on startup. - make a POST request to the node specified in the body above using the admin credentials in the body above: POST to http://username:password@node_b:5984/_setup with: From 068bdf18f0c2c9d30e5fa9fb70fb9c12a0247a7c Mon Sep 17 00:00:00 2001 From: Jan Lehnardt Date: Fri, 31 Oct 2014 17:47:48 +0100 Subject: [PATCH 11/52] add action hints --- src/setup_httpd.erl | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/setup_httpd.erl b/src/setup_httpd.erl index 6a4433a150..a5eb6441b0 100644 --- a/src/setup_httpd.erl +++ b/src/setup_httpd.erl @@ -31,8 +31,19 @@ handle_setup_req(Req) -> handle_action("enable_cluster", Setup) -> io:format("~nenable_cluster: ~p~n", [Setup]); + % if admin.username && admin.password + % create admin + % if bind_address + % set bind_address + % else + % bind_address to 0.0.0.0 + % if port + % set port + % set cluster_state to cluster_enabled handle_action("finish_cluster", Setup) -> io:format("~nfinish_cluster: ~p~n", [Setup]); + % create clustered databases (_users, _replicator, _cassim/_metadata + handle_action("add_node", Setup) -> io:format("~nadd_node: ~p~n", [Setup]); handle_action("remove_node", Setup) -> From 94eab1247fbdd7f87a298f7e00fa9faffec9b5d5 Mon Sep 17 00:00:00 2001 From: Jan Lehnardt Date: Fri, 31 Oct 2014 17:48:02 +0100 Subject: [PATCH 12/52] add license --- LICENSE | 202 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 202 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000000..e06d208186 --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ +Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + From 3ad82e58f4217285038c4ff1829c6ee140d57e32 Mon Sep 17 00:00:00 2001 From: Jan Lehnardt Date: Fri, 31 Oct 2014 17:48:12 +0100 Subject: [PATCH 13/52] remove leftover --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 966ba1bebd..45f15d7fe3 100644 --- a/README.md +++ b/README.md @@ -82,7 +82,6 @@ b. as in a, but without the Fauxton bits, just POST to /_setup "action": "receive_cookie", "cookie": "", } - // TBD: persist the cookie on node B, so it survives restarts - when the request to node B returns, we know the Erlang-level inter-cluster communication is enabled and we can start adding From 317e5a4d37c6ffe09de34875ade286b32d0388f3 Mon Sep 17 00:00:00 2001 From: Jan Lehnardt Date: Fri, 31 Oct 2014 17:52:53 +0100 Subject: [PATCH 14/52] formatting --- README.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 45f15d7fe3..88d1580adc 100644 --- a/README.md +++ b/README.md @@ -117,16 +117,18 @@ by HTTP POST with JSON bodies that have an `action` field. This is right after starting a node for the first time, and any time before the cluster is enabled as outlined above. +``` GET /_setup {"state": "cluster_disabled"} POST /_setup {"action":"enable_cluster"...} -> Transition to State 2 POST /_setup {"action":"enable_cluster"...} with empty admin user/pass or invalid host/post or host/port not available -> Error POST /_setup {"action":"anything_but_enable_cluster"...} -> Error - +``` ### State 2: Cluster enabled, admin user set, waiting for nodes to be added. +``` GET /_setup {"state":"cluster_enabled","nodes":[]} @@ -137,10 +139,11 @@ POST /_setup {"action":"finish_cluster"} with no nodes set up -> Error POST /_setup {"action":"finish_cluster"} -> Transition to State 3 POST /_setup {"action":"delete_node"...} -> Stay in State 2, but delete node from /nodes, reflect the change in GET /_setup POST /_setup {"action":"delete_node","node":"unknown"} -> Error Unknown Node - +``` ### State 3: Cluster set up, all nodes operational +``` GET /_setup {"state":"cluster_finished","nodes":["node a", "node b", ...]} @@ -150,5 +153,6 @@ POST /_setup {"action":"add_node"...} -> Error POST /_setup?i_know_what_i_am_doing=true {"action":"add_node"...} -> Add node, stay in State 3. POST /_setup {"action":"delete_node"...} -> Stay in State 3, but delete node from /nodes, reflect the change in GET /_setup POST /_setup {"action":"delete_node","node":"unknown"} -> Error Unknown Node +``` // TBD: we need to persist the setup state somewhere. From 0145bae5e225e167fbeffbaec6739a1170aef19e Mon Sep 17 00:00:00 2001 From: Jan Lehnardt Date: Fri, 31 Oct 2014 17:54:06 +0100 Subject: [PATCH 15/52] formatting & clarification --- README.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 88d1580adc..ead1f435f4 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,7 @@ Repeat on all nodes. a. Go to Fauxton / Cluster Setup, once we have enabled the cluster, the UI shows an “Add Node” interface with the fields admin, and node: - POST to /_setup with +``` { "action": "add_node", "admin": { // should be auto-filled from Fauxton, store plaintext PW in @@ -68,6 +69,7 @@ UI shows an “Add Node” interface with the fields admin, and node: ["port": 5984] } } +``` b. as in a, but without the Fauxton bits, just POST to /_setup - this request will do this: @@ -78,10 +80,12 @@ b. as in a, but without the Fauxton bits, just POST to /_setup - make a POST request to the node specified in the body above using the admin credentials in the body above: POST to http://username:password@node_b:5984/_setup with: +``` { "action": "receive_cookie", "cookie": "", } +``` - when the request to node B returns, we know the Erlang-level inter-cluster communication is enabled and we can start adding @@ -96,15 +100,17 @@ b. as in a, but without the Fauxton bits, just POST to /_setup 4.a. When all nodes are added, click the [Finish Cluster Setup] button in Fauxton. - this does POST /_setup +``` { "action": "finish_setup" } +``` b. Same as in a. - this manages the final setup bits, like creating the _users, - _replicator and _db_updates endpoints and whatever else is needed. - // TBD: collect what else is needed. + _replicator and _cassim/_metadata, _db_updates endpoints and + whatever else is needed. // TBD: collect what else is needed. ## The Setup Endpoint From bc41677484785f407981c91a2525e2778885f37b Mon Sep 17 00:00:00 2001 From: Jan Lehnardt Date: Fri, 31 Oct 2014 17:54:29 +0100 Subject: [PATCH 16/52] mroe formatting --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index ead1f435f4..4e4e01bcfb 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,7 @@ is no need to ask for admin credentials here. If the bind_address is != UI right away. - POST to /_setup with +``` { "action": "enable_cluster", "admin": { @@ -35,6 +36,7 @@ UI right away. ["bind_address": "xxxx",] ["port": yyyy] } +``` This sets up the admin user on the current node and binds to 0.0.0.0:5984 or the specified ip:port. Logs admin user into Fauxton automatically. From 277ca66441bf56f93132137c0e8d7d1337c99f06 Mon Sep 17 00:00:00 2001 From: Jan Lehnardt Date: Thu, 6 Nov 2014 17:33:43 +0100 Subject: [PATCH 17/52] wip: implement setup handling --- src/setup.erl | 159 ++++++++++++++++++++++++++++++++++++++++++++++++++++ src/setup_httpd.erl | 73 +++++++++++++++++++----- 2 files changed, 219 insertions(+), 13 deletions(-) create mode 100644 src/setup.erl diff --git a/src/setup.erl b/src/setup.erl new file mode 100644 index 0000000000..4a4524c435 --- /dev/null +++ b/src/setup.erl @@ -0,0 +1,159 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(setup). + +-export([enable_cluster/1, finish_cluster/0, add_node/1]). +-include_lib("../couch/include/couch_db.hrl"). + + +require_admins(undefined, {undefined, undefined}) -> + % no admin in CouchDB, no admin in request + throw({error, "Cluster setup requires admin account to be configured"}); +require_admins(_,_) -> + ok. + +error_bind_address() -> + throw({error, "Cluster setup requires bind_addres != 127.0.0.1"}). + +require_bind_address("127.0.0.1", undefined) -> + error_bind_address(); +require_bind_address("127.0.0.1", <<"127.0.0.1">>) -> + error_bind_address(); +require_bind_address(_, _) -> + ok. + +is_cluster_enabled() -> + % bind_address != 127.0.0.1 AND admins != empty + BindAddress = config:get("httpd", "bind_address"), + Admins = config:get("admins"), + case {BindAddress, Admins} of + {"127.0.0.1", _} -> no; + {_,[]} -> no; + {_,_} -> ok + end. + + +has_cluster_system_dbs() -> + % GET /_users /_replicator /_cassim + + Users = fabric:get_db_info("_users"), + Replicator = fabric:get_db_info("_replicator"), + Cassim = fabric:get_db_info("_cassim"), + case {Users, Replicator, Cassim} of + {{ok, _}, {ok, _}, {ok, _}} -> ok; + _Else -> no + end. + +enable_cluster(Options) -> + enable_cluster_int(Options, is_cluster_enabled()). + +enable_cluster_int(_Options, ok) -> + {error, cluster_enabled}; +enable_cluster_int(Options, no) -> + + % if no admin in config and no admin in req -> error + CurrentAdmins = config:get("admins"), + NewCredentials = { + proplists:get_value(username, Options), + proplists:get_value(password, Options) + }, + + % if bind_address == 127.0.0.1 and no bind_address in req -> error + CurrentBindAddress = config:get("httpd","bind_address"), + NewBindAddress = proplists:get_value(bind_address, Options), + ok = require_admins(CurrentAdmins, NewCredentials), + ok = require_bind_address(CurrentBindAddress, NewBindAddress), + + case NewCredentials of + {undefined, undefined} -> + ok; + {Username, Password} -> + % TODO check if this gets hashed + config:set("admins", binary_to_list(Username), binary_to_list(Password)) + end, + + case NewBindAddress of + undefined -> + config:set("httpd", "bind_address", "0.0.0.0"); + NewBindAddress -> + config:set("httpd", "bind_address", binary_to_list(NewBindAddress)) + end, + + Port = proplists:get_value(port, Options), + case Port of + undefined -> + ok; + Port -> + config:set("httpd", "port", integer_to_list(Port)) + end, + io:format("~nEnable Cluster: ~p~n", [Options]). + %cluster_state:set(enabled). + +finish_cluster() -> + finish_cluster_int(has_cluster_system_dbs()). +finish_cluster_int(no) -> + {error, cluster_finished}; +finish_cluster_int(ok) -> + io:format("~nFinish Cluster~n"). + % create clustered databases (_users, _replicator, _cassim/_metadata + % am I in enabled mode, are there nodes? + + +add_node(Options) -> + add_node_int(Options, is_cluster_enabled()). + +add_node_int(_Options, no) -> + {error, cluster_not_enabled}; +add_node_int(Options, ok) -> + io:format("~nadd node: ~p~n", [Options]), + ErlangCookie = erlang:get_cookie(), + + % POST to nodeB/_setup + RequestOptions = [ + {basic_auth, { + proplists:get_value(username, Options), + proplists:get_value(password, Options) + }} + ], + + Body = ?JSON_ENCODE({[ + {<<"action">>, <<"receive_cookie">>}, + {<<"cookie">>, atom_to_binary(ErlangCookie, utf8)} + ]}), + + Headers = [ + {"Content-Type","application/json"} + ], + + Host = proplists:get_value(host, Options), + Port = proplists:get_value(port, Options, <<"5984">>), + Url = binary_to_list(<<"http://", Host/binary, ":", Port/binary, "/_setup">>), + + io:format("~nUrl: ~p~n", [Url]), + io:format("~nBody: ~p~n", [Body]), + + case ibrowse:send_req(Url, Headers, post, Body, RequestOptions) of + {ok, 200, _, _} -> + % when done, PUT :5986/nodes/nodeB + create_node_doc(Host, Port); + Else -> + io:format("~nsend_req: ~p~n", [Else]), + Else + end. + + +create_node_doc(Host, Port) -> + {ok, Db} = couch_db:open_int("nodes"), + Doc = {[{<<"_id">>, <>}]}, + Options = [], + couch_db:update_doc(Db, Doc, Options). diff --git a/src/setup_httpd.erl b/src/setup_httpd.erl index a5eb6441b0..e0751d2107 100644 --- a/src/setup_httpd.erl +++ b/src/setup_httpd.erl @@ -29,25 +29,72 @@ handle_setup_req(Req) -> end. +get_options(Options, Setup) -> + ExtractValues = fun({Tag, Option}, OptionsAcc) -> + case couch_util:get_value(Option, Setup) of + undefined -> OptionsAcc; + Value -> [{Tag, Value} | OptionsAcc] + end + end, + lists:foldl(ExtractValues, [], Options). + handle_action("enable_cluster", Setup) -> - io:format("~nenable_cluster: ~p~n", [Setup]); - % if admin.username && admin.password - % create admin - % if bind_address - % set bind_address - % else - % bind_address to 0.0.0.0 - % if port - % set port - % set cluster_state to cluster_enabled + Options = get_options([ + {username, <<"username">>}, + {password, <<"password">>}, + {bind_address, <<"bind_address">>}, + {port, <<"port">>} + ], Setup), + case setup:enable_cluster(Options) of + {error, cluster_enabled} -> + {error, <<"Cluster is already enabled">>}; + _ -> ok + end; + + handle_action("finish_cluster", Setup) -> - io:format("~nfinish_cluster: ~p~n", [Setup]); - % create clustered databases (_users, _replicator, _cassim/_metadata + io:format("~nfinish_cluster: ~p~n", [Setup]), + case etup:finish_cluster() of + {error, cluster_finished} -> + {error, <<"Cluster is already finished">>}; + _ -> ok + end; handle_action("add_node", Setup) -> - io:format("~nadd_node: ~p~n", [Setup]); + io:format("~nadd_node: ~p~n", [Setup]), + + Options = get_options([ + {username, <<"username">>}, + {password, <<"password">>}, + {host, <<"host">>}, + {port, <<"port">>} + ], Setup), + case setup:add_node(Options) of + {error, cluster_not_enabled} -> + {error, <<"Cluster is not enabled.">>}; + {error, {conn_failed, {error, econnrefused}}} -> + {error, <<"Add node failed. Invalid Host and/or Port.">>}; + {error, wrong_credentials} -> + {error, <<"Add node failed. Invalid admin credentials,">>}; + {error, Message} -> + {error, Message}; + _ -> ok + end; + handle_action("remove_node", Setup) -> io:format("~nremove_node: ~p~n", [Setup]); + +handle_action("receive_cookie", Setup) -> + io:format("~nreceive_cookie: ~p~n", [Setup]), + Options = get_options([ + {cookue, <<"cookie">>} + ], Setup), + case setup:receive_cookie(Options) of + {error, Error} -> + {error, Error}; + _ -> ok + end; + handle_action(_, _) -> io:format("~ninvalid_action: ~n", []), {error, <<"Invalid Action'">>}. From 92da54ed202802e4e8cc8f2e5c5e62fd70ea4dd7 Mon Sep 17 00:00:00 2001 From: Jan Lehnardt Date: Fri, 7 Nov 2014 15:01:29 +0100 Subject: [PATCH 18/52] wip: full receive feature, setup now works yay --- src/setup.erl | 43 ++++++++++++++++++++++++++++++------------- src/setup_httpd.erl | 5 ++--- 2 files changed, 32 insertions(+), 16 deletions(-) diff --git a/src/setup.erl b/src/setup.erl index 4a4524c435..e5afb2a9b8 100644 --- a/src/setup.erl +++ b/src/setup.erl @@ -12,7 +12,7 @@ -module(setup). --export([enable_cluster/1, finish_cluster/0, add_node/1]). +-export([enable_cluster/1, finish_cluster/0, add_node/1, receive_cookie/1]). -include_lib("../couch/include/couch_db.hrl"). @@ -121,8 +121,8 @@ add_node_int(Options, ok) -> % POST to nodeB/_setup RequestOptions = [ {basic_auth, { - proplists:get_value(username, Options), - proplists:get_value(password, Options) + binary_to_list(proplists:get_value(username, Options)), + binary_to_list(proplists:get_value(password, Options)) }} ], @@ -136,14 +136,11 @@ add_node_int(Options, ok) -> ], Host = proplists:get_value(host, Options), - Port = proplists:get_value(port, Options, <<"5984">>), - Url = binary_to_list(<<"http://", Host/binary, ":", Port/binary, "/_setup">>), - - io:format("~nUrl: ~p~n", [Url]), - io:format("~nBody: ~p~n", [Body]), + Port = integer_to_binary(proplists:get_value(port, Options, 5984)), + Url = binary_to_list(<<"http://", Host/binary, ":", Port/binary, "/_cluster_setup">>), case ibrowse:send_req(Url, Headers, post, Body, RequestOptions) of - {ok, 200, _, _} -> + {ok, "201", _, _} -> % when done, PUT :5986/nodes/nodeB create_node_doc(Host, Port); Else -> @@ -151,9 +148,29 @@ add_node_int(Options, ok) -> Else end. - create_node_doc(Host, Port) -> - {ok, Db} = couch_db:open_int("nodes"), - Doc = {[{<<"_id">>, <>}]}, + {ok, Db} = couch_db:open_int(<<"nodes">>, []), + Name = get_name(Port), + Doc = {[{<<"_id">>, <>}]}, Options = [], - couch_db:update_doc(Db, Doc, Options). + CouchDoc = couch_doc:from_json_obj(Doc), + + couch_db:update_doc(Db, CouchDoc, Options). + +get_name(Port) -> + case Port of + % shortcut for easier development + <<"15984">> -> + <<"node1">>; + <<"25984">> -> + <<"node2">>; + <<"35984">> -> + <<"node3">>; + % by default, all nodes have the user `couchdb` + _ -> + <<"couchdb">> + end. + +receive_cookie(Options) -> + Cookie = proplists:get_value(cookie, Options), + erlang:set_cookie(node(), binary_to_atom(Cookie, latin1)). diff --git a/src/setup_httpd.erl b/src/setup_httpd.erl index e0751d2107..550b04aad6 100644 --- a/src/setup_httpd.erl +++ b/src/setup_httpd.erl @@ -16,8 +16,7 @@ handle_setup_req(Req) -> ok = chttpd:verify_is_server_admin(Req), - % TBD uncomment after devving - %couch_httpd:validate_ctype(Req, "application/json"), + couch_httpd:validate_ctype(Req, "application/json"), Setup = get_body(Req), io:format("~nSetup: ~p~n", [Setup]), Action = binary_to_list(couch_util:get_value(<<"action">>, Setup, <<"missing">>)), @@ -87,7 +86,7 @@ handle_action("remove_node", Setup) -> handle_action("receive_cookie", Setup) -> io:format("~nreceive_cookie: ~p~n", [Setup]), Options = get_options([ - {cookue, <<"cookie">>} + {cookie, <<"cookie">>} ], Setup), case setup:receive_cookie(Options) of {error, Error} -> From fc39fab4ed8f1492785d5d510307db9d2ac2a082 Mon Sep 17 00:00:00 2001 From: Jan Lehnardt Date: Fri, 7 Nov 2014 15:01:52 +0100 Subject: [PATCH 19/52] add simple test script --- test/t.sh | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100755 test/t.sh diff --git a/test/t.sh b/test/t.sh new file mode 100755 index 0000000000..7b1ca1b9a5 --- /dev/null +++ b/test/t.sh @@ -0,0 +1,34 @@ +#!/bin/sh -ex + +HEADERS="-HContent-Type:application/json" +# show cluster state: +curl a:b@127.0.0.1:15986/nodes/_all_docs + +# Enable Cluster on node A +curl a:b@127.0.0.1:15984/_cluster_setup -d '{"action":"enable_cluster","username":"foo","password":"baz","bind_address":"0.0.0.0"}' $HEADERS + +# Enable Cluster on node B +curl a:b@127.0.0.1:25984/_cluster_setup -d '{"action":"enable_cluster","username":"foo","password":"baz","bind_address":"0.0.0.0"}' $HEADERS + +# Add node B on node A +curl a:b@127.0.0.1:15984/_cluster_setup -d '{"action":"add_node","username":"foo","password":"baz","host":"127.0.0.1","port":25984}' $HEADERS + +# Show cluster state: +curl a:b@127.0.0.1:15986/nodes/_all_docs + +# Show db doesn’t exist on node A +curl a:b@127.0.0.1:15984/foo + +# Show db doesn’t exist on node B +curl a:b@127.0.0.1:25984/foo + +# Create database (on node A) +curl -X PUT a:b@127.0.0.1:15984/foo + +# Show db does exist on node A +curl a:b@127.0.0.1:15984/foo + +# Show db does exist on node B +curl a:b@127.0.0.1:25984/foo + +echo "YAY ALL GOOD" \ No newline at end of file From 354647bfaef652113ad033763658a42c45ba03d6 Mon Sep 17 00:00:00 2001 From: Jan Lehnardt Date: Fri, 7 Nov 2014 15:34:54 +0100 Subject: [PATCH 20/52] add finish cluster routine --- src/setup.erl | 18 +++++++++--------- src/setup_httpd.erl | 8 +++++--- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/src/setup.erl b/src/setup.erl index e5afb2a9b8..781691e052 100644 --- a/src/setup.erl +++ b/src/setup.erl @@ -46,12 +46,12 @@ is_cluster_enabled() -> has_cluster_system_dbs() -> % GET /_users /_replicator /_cassim - Users = fabric:get_db_info("_users"), - Replicator = fabric:get_db_info("_replicator"), - Cassim = fabric:get_db_info("_cassim"), - case {Users, Replicator, Cassim} of + case catch { + fabric:get_db_info("_users"), + fabric:get_db_info("_replicator"), + fabric:get_db_info("_cassim")} of {{ok, _}, {ok, _}, {ok, _}} -> ok; - _Else -> no + _ -> no end. enable_cluster(Options) -> @@ -101,12 +101,12 @@ enable_cluster_int(Options, no) -> finish_cluster() -> finish_cluster_int(has_cluster_system_dbs()). -finish_cluster_int(no) -> - {error, cluster_finished}; finish_cluster_int(ok) -> - io:format("~nFinish Cluster~n"). + {error, cluster_finished}; +finish_cluster_int(no) -> % create clustered databases (_users, _replicator, _cassim/_metadata - % am I in enabled mode, are there nodes? + Databases = ["_users", "_replicator", "_cassim"], + lists:foreach(fun fabric:create_db/1, Databases). add_node(Options) -> diff --git a/src/setup_httpd.erl b/src/setup_httpd.erl index 550b04aad6..755b951007 100644 --- a/src/setup_httpd.erl +++ b/src/setup_httpd.erl @@ -52,11 +52,13 @@ handle_action("enable_cluster", Setup) -> handle_action("finish_cluster", Setup) -> - io:format("~nfinish_cluster: ~p~n", [Setup]), - case etup:finish_cluster() of + io:format("~nffinish_cluster: ~p~n", [Setup]), + case setup:finish_cluster() of {error, cluster_finished} -> {error, <<"Cluster is already finished">>}; - _ -> ok + Else -> + io:format("~nElse: ~p~n", [Else]), + ok end; handle_action("add_node", Setup) -> From 7c6c3bb649f264b92c9cf2e41dd0545feae12672 Mon Sep 17 00:00:00 2001 From: Jan Lehnardt Date: Fri, 7 Nov 2014 15:35:06 +0100 Subject: [PATCH 21/52] add some more testing --- test/t.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/test/t.sh b/test/t.sh index 7b1ca1b9a5..7d3e1b3253 100755 --- a/test/t.sh +++ b/test/t.sh @@ -31,4 +31,17 @@ curl a:b@127.0.0.1:15984/foo # Show db does exist on node B curl a:b@127.0.0.1:25984/foo +# Finish cluster +curl a:b@127.0.0.1:15984/_cluster_setup -d '{"action":"finish_cluster"}' $HEADERS + +# Show system dbs exist on node A +curl a:b@127.0.0.1:15984/_users +curl a:b@127.0.0.1:15984/_replicator +curl a:b@127.0.0.1:15984/_cassim + +# Show system dbs exist on node B +curl a:b@127.0.0.1:25984/_users +curl a:b@127.0.0.1:25984/_replicator +curl a:b@127.0.0.1:25984/_cassim + echo "YAY ALL GOOD" \ No newline at end of file From 4c423e67fa8e6a4f5af7a5c5db38bbc81642cb27 Mon Sep 17 00:00:00 2001 From: Jan Lehnardt Date: Fri, 7 Nov 2014 15:43:04 +0100 Subject: [PATCH 22/52] s/_cassim/cassim/ for the time being --- src/setup.erl | 2 +- test/t.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/setup.erl b/src/setup.erl index 781691e052..85ebf793d4 100644 --- a/src/setup.erl +++ b/src/setup.erl @@ -105,7 +105,7 @@ finish_cluster_int(ok) -> {error, cluster_finished}; finish_cluster_int(no) -> % create clustered databases (_users, _replicator, _cassim/_metadata - Databases = ["_users", "_replicator", "_cassim"], + Databases = ["_users", "_replicator", "cassim"], lists:foreach(fun fabric:create_db/1, Databases). diff --git a/test/t.sh b/test/t.sh index 7d3e1b3253..ab0b04767a 100755 --- a/test/t.sh +++ b/test/t.sh @@ -37,11 +37,11 @@ curl a:b@127.0.0.1:15984/_cluster_setup -d '{"action":"finish_cluster"}' $HEADER # Show system dbs exist on node A curl a:b@127.0.0.1:15984/_users curl a:b@127.0.0.1:15984/_replicator -curl a:b@127.0.0.1:15984/_cassim +curl a:b@127.0.0.1:15984/cassim # Show system dbs exist on node B curl a:b@127.0.0.1:25984/_users curl a:b@127.0.0.1:25984/_replicator -curl a:b@127.0.0.1:25984/_cassim +curl a:b@127.0.0.1:25984/cassim echo "YAY ALL GOOD" \ No newline at end of file From 7528f5bdfc6fc03a97ccc8d3e20c6b7f409ab923 Mon Sep 17 00:00:00 2001 From: Jan Lehnardt Date: Fri, 7 Nov 2014 15:44:58 +0100 Subject: [PATCH 23/52] add license header --- test/t.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/test/t.sh b/test/t.sh index ab0b04767a..f7c1e7617d 100755 --- a/test/t.sh +++ b/test/t.sh @@ -1,4 +1,15 @@ #!/bin/sh -ex +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. HEADERS="-HContent-Type:application/json" # show cluster state: From 0a676fcdfeee06a03a0bfed7383b5e38384f59cc Mon Sep 17 00:00:00 2001 From: Jan Lehnardt Date: Fri, 7 Nov 2014 17:58:41 +0100 Subject: [PATCH 24/52] add testing instructions to readme --- README.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/README.md b/README.md index 4e4e01bcfb..4929dc2a46 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,23 @@ This module implements /_cluster_setup and manages the setting up, duh, of a CouchDB cluster. +### Testing + +``` +git clone https://git-wip-us.apache.org/repos/asf/couchdb.git +cd couchdb +git checkout setup +./configure +make + +# in dev/run comment out the line `connect_nodes("127.0.0.1", 15984)` + +dev/run --admin a:b + +# in a new terminal +src/setup/test/t.sh + +``` + The Plan: N. End User Action From 3304add80963e265b80a342f650d4bb526a6c755 Mon Sep 17 00:00:00 2001 From: Jan Lehnardt Date: Thu, 13 Nov 2014 18:19:10 +0100 Subject: [PATCH 25/52] hash admin passwords, more resilient port parsing --- src/setup.erl | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/src/setup.erl b/src/setup.erl index 85ebf793d4..6374685104 100644 --- a/src/setup.erl +++ b/src/setup.erl @@ -78,8 +78,7 @@ enable_cluster_int(Options, no) -> {undefined, undefined} -> ok; {Username, Password} -> - % TODO check if this gets hashed - config:set("admins", binary_to_list(Username), binary_to_list(Password)) + maybe_set_admin(Username, Password) end, case NewBindAddress of @@ -99,6 +98,16 @@ enable_cluster_int(Options, no) -> io:format("~nEnable Cluster: ~p~n", [Options]). %cluster_state:set(enabled). +maybe_set_admin(Username, Password) -> + case couch_auth_cache:get_admin(Username) of + nil -> + HashedPassword = couch_passwords:hash_admin_password(Password), + config:set("admins", binary_to_list(Username), binary_to_list(HashedPassword)); + _Else -> + ok + end. + + finish_cluster() -> finish_cluster_int(has_cluster_system_dbs()). finish_cluster_int(ok) -> @@ -136,7 +145,8 @@ add_node_int(Options, ok) -> ], Host = proplists:get_value(host, Options), - Port = integer_to_binary(proplists:get_value(port, Options, 5984)), + Port = get_port(proplists:get_value(port, Options, 5984)), + Url = binary_to_list(<<"http://", Host/binary, ":", Port/binary, "/_cluster_setup">>), case ibrowse:send_req(Url, Headers, post, Body, RequestOptions) of @@ -148,6 +158,16 @@ add_node_int(Options, ok) -> Else end. +get_port(Port) when is_integer(Port) -> + integer_to_binary(Port); +get_port(Port) when is_list(Port) -> + list_to_binary(Port); +get_port(Port) when is_binary(Port) -> + Port; +get_port(Port) -> + {error, <<"invalid type for port">>}. + + create_node_doc(Host, Port) -> {ok, Db} = couch_db:open_int(<<"nodes">>, []), Name = get_name(Port), From 14e0374429b654e0779e7c0c7dd739289728e682 Mon Sep 17 00:00:00 2001 From: Jan Lehnardt Date: Fri, 14 Nov 2014 12:19:25 +0100 Subject: [PATCH 26/52] handle GET cluster state --- src/setup.erl | 2 ++ src/setup_httpd.erl | 12 ++++++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/setup.erl b/src/setup.erl index 6374685104..29576b2fa7 100644 --- a/src/setup.erl +++ b/src/setup.erl @@ -13,6 +13,8 @@ -module(setup). -export([enable_cluster/1, finish_cluster/0, add_node/1, receive_cookie/1]). +-export([is_cluster_enabled/0]). + -include_lib("../couch/include/couch_db.hrl"). diff --git a/src/setup_httpd.erl b/src/setup_httpd.erl index 755b951007..bdb33123dc 100644 --- a/src/setup_httpd.erl +++ b/src/setup_httpd.erl @@ -11,10 +11,11 @@ % the License. -module(setup_httpd). +-include_lib("couch/include/couch_db.hrl"). -export([handle_setup_req/1]). -handle_setup_req(Req) -> +handle_setup_req(#httpd{method='POST'}=Req) -> ok = chttpd:verify_is_server_admin(Req), couch_httpd:validate_ctype(Req, "application/json"), Setup = get_body(Req), @@ -25,9 +26,16 @@ handle_setup_req(Req) -> chttpd:send_json(Req, 201, {[{ok, true}]}); {error, Message} -> couch_httpd:send_error(Req, 400, <<"bad_request">>, Message) + end; +handle_setup_req(#httpd{method='GET'}=Req) -> + ok = chttpd:verify_is_server_admin(Req), + case setup:is_cluster_enabled() of + no -> + chttpd:send_json(Req, 201, {[{state, cluster_disabled}]}); + ok -> + chttpd:send_json(Req, 201, {[{state, cluster_enabled}]}) end. - get_options(Options, Setup) -> ExtractValues = fun({Tag, Option}, OptionsAcc) -> case couch_util:get_value(Option, Setup) of From 9c3eb0a1a332195a1e4f8e85a368bbc8a36469eb Mon Sep 17 00:00:00 2001 From: Jan Lehnardt Date: Sat, 15 Nov 2014 20:25:44 +0100 Subject: [PATCH 27/52] show cluster finished state --- src/setup.erl | 4 ++-- src/setup_httpd.erl | 7 ++++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/setup.erl b/src/setup.erl index 29576b2fa7..c5a38ea2bb 100644 --- a/src/setup.erl +++ b/src/setup.erl @@ -13,7 +13,7 @@ -module(setup). -export([enable_cluster/1, finish_cluster/0, add_node/1, receive_cookie/1]). --export([is_cluster_enabled/0]). +-export([is_cluster_enabled/0, has_cluster_system_dbs/0]). -include_lib("../couch/include/couch_db.hrl"). @@ -51,7 +51,7 @@ has_cluster_system_dbs() -> case catch { fabric:get_db_info("_users"), fabric:get_db_info("_replicator"), - fabric:get_db_info("_cassim")} of + fabric:get_db_info("cassim")} of {{ok, _}, {ok, _}, {ok, _}} -> ok; _ -> no end. diff --git a/src/setup_httpd.erl b/src/setup_httpd.erl index bdb33123dc..aca98e7e51 100644 --- a/src/setup_httpd.erl +++ b/src/setup_httpd.erl @@ -33,7 +33,12 @@ handle_setup_req(#httpd{method='GET'}=Req) -> no -> chttpd:send_json(Req, 201, {[{state, cluster_disabled}]}); ok -> - chttpd:send_json(Req, 201, {[{state, cluster_enabled}]}) + case setup:has_cluster_system_dbs() of + no -> + chttpd:send_json(Req, 201, {[{state, cluster_enabled}]}); + ok -> + chttpd:send_json(Req, 201, {[{state, cluster_finished}]}) + end end. get_options(Options, Setup) -> From be52f7ea1cca9a7b845ac189715724b7e0c06d7e Mon Sep 17 00:00:00 2001 From: Robert Newson Date: Tue, 25 Nov 2014 11:54:28 +0000 Subject: [PATCH 28/52] R14 compatibility --- src/setup.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/setup.erl b/src/setup.erl index c5a38ea2bb..f774608a9b 100644 --- a/src/setup.erl +++ b/src/setup.erl @@ -161,7 +161,7 @@ add_node_int(Options, ok) -> end. get_port(Port) when is_integer(Port) -> - integer_to_binary(Port); + list_to_binary(integer_to_list(Port)); get_port(Port) when is_list(Port) -> list_to_binary(Port); get_port(Port) when is_binary(Port) -> From 9728b342a58b8c59a749f605d17375666871042a Mon Sep 17 00:00:00 2001 From: Robert Newson Date: Tue, 25 Nov 2014 11:55:35 +0000 Subject: [PATCH 29/52] Remove error-handling clause The error tuple returned is not tested by the caller so it serves only to crash the binary comprehension in an obscure way. --- src/setup.erl | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/setup.erl b/src/setup.erl index f774608a9b..c48a616cb5 100644 --- a/src/setup.erl +++ b/src/setup.erl @@ -165,10 +165,7 @@ get_port(Port) when is_integer(Port) -> get_port(Port) when is_list(Port) -> list_to_binary(Port); get_port(Port) when is_binary(Port) -> - Port; -get_port(Port) -> - {error, <<"invalid type for port">>}. - + Port. create_node_doc(Host, Port) -> {ok, Db} = couch_db:open_int(<<"nodes">>, []), From cd7d0ecc05d4eb4d280c45e7f906e1b2d9d23acb Mon Sep 17 00:00:00 2001 From: Alexander Shorin Date: Mon, 22 Dec 2014 18:12:46 +0300 Subject: [PATCH 30/52] Fix LICENSE indention --- LICENSE | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index e06d208186..94ad231b84 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,5 @@ -Apache License + + Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ From deeb073ad16f54286e7fa7f6606b1d353171f6d8 Mon Sep 17 00:00:00 2001 From: Alexander Shorin Date: Thu, 26 Feb 2015 22:29:43 +0300 Subject: [PATCH 31/52] Rename cassim db to _metadata This closes #1 COUCHDB-2619 COUCHDB-2620 --- README.md | 2 +- src/setup.erl | 12 ++++++------ test/t.sh | 6 +++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 4929dc2a46..7043b0f84b 100644 --- a/README.md +++ b/README.md @@ -129,7 +129,7 @@ in Fauxton. b. Same as in a. - this manages the final setup bits, like creating the _users, - _replicator and _cassim/_metadata, _db_updates endpoints and + _replicator and _metadata, _db_updates endpoints and whatever else is needed. // TBD: collect what else is needed. diff --git a/src/setup.erl b/src/setup.erl index c48a616cb5..b04b0bafbc 100644 --- a/src/setup.erl +++ b/src/setup.erl @@ -46,12 +46,12 @@ is_cluster_enabled() -> has_cluster_system_dbs() -> - % GET /_users /_replicator /_cassim + % GET /_users /_replicator /_metadata case catch { fabric:get_db_info("_users"), fabric:get_db_info("_replicator"), - fabric:get_db_info("cassim")} of + fabric:get_db_info("_metadata")} of {{ok, _}, {ok, _}, {ok, _}} -> ok; _ -> no end. @@ -75,7 +75,7 @@ enable_cluster_int(Options, no) -> NewBindAddress = proplists:get_value(bind_address, Options), ok = require_admins(CurrentAdmins, NewCredentials), ok = require_bind_address(CurrentBindAddress, NewBindAddress), - + case NewCredentials of {undefined, undefined} -> ok; @@ -89,7 +89,7 @@ enable_cluster_int(Options, no) -> NewBindAddress -> config:set("httpd", "bind_address", binary_to_list(NewBindAddress)) end, - + Port = proplists:get_value(port, Options), case Port of undefined -> @@ -115,8 +115,8 @@ finish_cluster() -> finish_cluster_int(ok) -> {error, cluster_finished}; finish_cluster_int(no) -> - % create clustered databases (_users, _replicator, _cassim/_metadata - Databases = ["_users", "_replicator", "cassim"], + % create clustered databases (_users, _replicator, _metadata) + Databases = ["_users", "_replicator", "_metadata"], lists:foreach(fun fabric:create_db/1, Databases). diff --git a/test/t.sh b/test/t.sh index f7c1e7617d..e323bbf3fc 100755 --- a/test/t.sh +++ b/test/t.sh @@ -48,11 +48,11 @@ curl a:b@127.0.0.1:15984/_cluster_setup -d '{"action":"finish_cluster"}' $HEADER # Show system dbs exist on node A curl a:b@127.0.0.1:15984/_users curl a:b@127.0.0.1:15984/_replicator -curl a:b@127.0.0.1:15984/cassim +curl a:b@127.0.0.1:15984/_metadata # Show system dbs exist on node B curl a:b@127.0.0.1:25984/_users curl a:b@127.0.0.1:25984/_replicator -curl a:b@127.0.0.1:25984/cassim +curl a:b@127.0.0.1:25984/_metadata -echo "YAY ALL GOOD" \ No newline at end of file +echo "YAY ALL GOOD" From 127e85adf20a363b524bb639e8375d4e1ca6bdde Mon Sep 17 00:00:00 2001 From: Alexander Shorin Date: Sat, 4 Apr 2015 22:52:15 +0300 Subject: [PATCH 32/52] Use _nodes db --- src/setup.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/setup.erl b/src/setup.erl index b04b0bafbc..95b2a41ada 100644 --- a/src/setup.erl +++ b/src/setup.erl @@ -168,7 +168,7 @@ get_port(Port) when is_binary(Port) -> Port. create_node_doc(Host, Port) -> - {ok, Db} = couch_db:open_int(<<"nodes">>, []), + {ok, Db} = couch_db:open_int(<<"_nodes">>, []), Name = get_name(Port), Doc = {[{<<"_id">>, <>}]}, Options = [], From 372dd8be046ee999f6d318ff7016621c6e355a60 Mon Sep 17 00:00:00 2001 From: Robert Kowalski Date: Sun, 5 Apr 2015 02:34:18 +0200 Subject: [PATCH 33/52] fix tests --- test/t.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/t.sh b/test/t.sh index e323bbf3fc..f6e0422a8c 100755 --- a/test/t.sh +++ b/test/t.sh @@ -13,7 +13,7 @@ HEADERS="-HContent-Type:application/json" # show cluster state: -curl a:b@127.0.0.1:15986/nodes/_all_docs +curl a:b@127.0.0.1:15986/_nodes/_all_docs # Enable Cluster on node A curl a:b@127.0.0.1:15984/_cluster_setup -d '{"action":"enable_cluster","username":"foo","password":"baz","bind_address":"0.0.0.0"}' $HEADERS @@ -25,7 +25,7 @@ curl a:b@127.0.0.1:25984/_cluster_setup -d '{"action":"enable_cluster","username curl a:b@127.0.0.1:15984/_cluster_setup -d '{"action":"add_node","username":"foo","password":"baz","host":"127.0.0.1","port":25984}' $HEADERS # Show cluster state: -curl a:b@127.0.0.1:15986/nodes/_all_docs +curl a:b@127.0.0.1:15986/_nodes/_all_docs # Show db doesn’t exist on node A curl a:b@127.0.0.1:15984/foo From ecb601b2004f33f396530a6ae974f2f0bff4d816 Mon Sep 17 00:00:00 2001 From: Alexander Shorin Date: Sat, 18 Apr 2015 22:41:39 +0300 Subject: [PATCH 34/52] Create _global_changes database on cluster setup --- src/setup.erl | 20 +++++++++++--------- test/t.sh | 2 ++ 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/setup.erl b/src/setup.erl index 95b2a41ada..194383556a 100644 --- a/src/setup.erl +++ b/src/setup.erl @@ -43,16 +43,20 @@ is_cluster_enabled() -> {_,[]} -> no; {_,_} -> ok end. + + +cluster_system_dbs() -> + ["_users", "_replicator", "_metadata", "_global_changes"]. has_cluster_system_dbs() -> - % GET /_users /_replicator /_metadata + has_cluster_system_dbs(cluster_system_dbs()). - case catch { - fabric:get_db_info("_users"), - fabric:get_db_info("_replicator"), - fabric:get_db_info("_metadata")} of - {{ok, _}, {ok, _}, {ok, _}} -> ok; +has_cluster_system_dbs([]) -> + ok; +has_cluster_system_dbs([Db|Dbs]) -> + case catch fabric:get_db_info(Db) of + {ok, _} -> has_cluster_system_dbs(Dbs); _ -> no end. @@ -115,9 +119,7 @@ finish_cluster() -> finish_cluster_int(ok) -> {error, cluster_finished}; finish_cluster_int(no) -> - % create clustered databases (_users, _replicator, _metadata) - Databases = ["_users", "_replicator", "_metadata"], - lists:foreach(fun fabric:create_db/1, Databases). + lists:foreach(fun fabric:create_db/1, cluster_system_dbs()). add_node(Options) -> diff --git a/test/t.sh b/test/t.sh index f6e0422a8c..62abb61d7c 100755 --- a/test/t.sh +++ b/test/t.sh @@ -49,10 +49,12 @@ curl a:b@127.0.0.1:15984/_cluster_setup -d '{"action":"finish_cluster"}' $HEADER curl a:b@127.0.0.1:15984/_users curl a:b@127.0.0.1:15984/_replicator curl a:b@127.0.0.1:15984/_metadata +curl a:b@127.0.0.1:15984/_global_changes # Show system dbs exist on node B curl a:b@127.0.0.1:25984/_users curl a:b@127.0.0.1:25984/_replicator curl a:b@127.0.0.1:25984/_metadata +curl a:b@127.0.0.1:25984/_global_changes echo "YAY ALL GOOD" From 616789bac1bcdf9897e6725baaf0249f742389fd Mon Sep 17 00:00:00 2001 From: Robert Kowalski Date: Tue, 26 May 2015 02:33:50 +0200 Subject: [PATCH 35/52] cluster_enable: add remote_node feature this feature makes it easier to setup a cluster for browser applications like fauxton as browsers follow the same-origin policy. Before this PR you had to open the wizard in Fauxton on all three nodes and enter your data there, which was quite confusing and hard to explain. Now you can stay in the same tab at the same address. This PR enables three new params in the body: `remote_node`: ip of the remote node where we want to send the `enable_cluster` request `remote_current_user`: the current admin username of the remote node `remote_current_password`: the current admin password of the remote node To test, I run: ``` rm -rf dev/lib/ && ./dev/run --no-join --admin=a:b ``` and then run the test script: ``` ./src/setup/test/t-frontend-setup.sh ``` COUCHDB-2598 PR: #2 PR-URL: https://github.com/apache/couchdb-setup/pull/2 Reviewed-By: Jan Lehnardt Reviewed-By: Alexander Shorin --- src/setup.erl | 42 ++++++++++++++++++++++++++++++++- src/setup_httpd.erl | 5 +++- test/t-frontend-setup.sh | 60 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 105 insertions(+), 2 deletions(-) create mode 100755 test/t-frontend-setup.sh diff --git a/src/setup.erl b/src/setup.erl index 194383556a..9aa98f945b 100644 --- a/src/setup.erl +++ b/src/setup.erl @@ -61,7 +61,47 @@ has_cluster_system_dbs([Db|Dbs]) -> end. enable_cluster(Options) -> - enable_cluster_int(Options, is_cluster_enabled()). + + case couch_util:get_value(remote_node, Options, undefined) of + undefined -> + enable_cluster_int(Options, is_cluster_enabled()); + _ -> + enable_cluster_http(Options) + end. + +enable_cluster_http(Options) -> + % POST to nodeB/_setup + RequestOptions = [ + {basic_auth, { + binary_to_list(couch_util:get_value(remote_current_user, Options)), + binary_to_list(couch_util:get_value(remote_current_password, Options)) + }} + ], + + Body = ?JSON_ENCODE({[ + {<<"action">>, <<"enable_cluster">>}, + {<<"username">>, couch_util:get_value(username, Options)}, + {<<"password">>, couch_util:get_value(password, Options)}, + {<<"bind_address">>, couch_util:get_value(bind_address, Options)}, + {<<"port">>, couch_util:get_value(port, Options)} + ]}), + + Headers = [ + {"Content-Type","application/json"} + ], + + RemoteNode = couch_util:get_value(remote_node, Options), + Port = get_port(couch_util:get_value(port, Options, 5984)), + + Url = binary_to_list(<<"http://", RemoteNode/binary, ":", Port/binary, "/_cluster_setup">>), + + case ibrowse:send_req(Url, Headers, post, Body, RequestOptions) of + {ok, "201", _, _} -> + ok; + Else -> + io:format("~nsend_req: ~p~n", [Else]), + {error, Else} + end. enable_cluster_int(_Options, ok) -> {error, cluster_enabled}; diff --git a/src/setup_httpd.erl b/src/setup_httpd.erl index aca98e7e51..de1bff5fac 100644 --- a/src/setup_httpd.erl +++ b/src/setup_httpd.erl @@ -55,7 +55,10 @@ handle_action("enable_cluster", Setup) -> {username, <<"username">>}, {password, <<"password">>}, {bind_address, <<"bind_address">>}, - {port, <<"port">>} + {port, <<"port">>}, + {remote_node, <<"remote_node">>}, + {remote_current_user, <<"remote_current_user">>}, + {remote_current_password, <<"remote_current_password">>} ], Setup), case setup:enable_cluster(Options) of {error, cluster_enabled} -> diff --git a/test/t-frontend-setup.sh b/test/t-frontend-setup.sh new file mode 100755 index 0000000000..1c610b6cc9 --- /dev/null +++ b/test/t-frontend-setup.sh @@ -0,0 +1,60 @@ +#!/bin/sh -ex +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +HEADERS="-HContent-Type:application/json" +# show cluster state: +curl a:b@127.0.0.1:15986/_nodes/_all_docs + +# Enable Cluster on node A +curl a:b@127.0.0.1:15984/_cluster_setup -d '{"action":"enable_cluster","username":"foo","password":"baz","bind_address":"0.0.0.0"}' $HEADERS + +# Enable Cluster on node B +curl a:b@127.0.0.1:15984/_cluster_setup -d '{"action":"enable_cluster","remote_node":"127.0.0.1","port":"25984","remote_current_user":"a","remote_current_password":"b","username":"foo","password":"baz","bind_address":"0.0.0.0"}' $HEADERS + +# Add node B on node A +curl a:b@127.0.0.1:15984/_cluster_setup -d '{"action":"add_node","username":"foo","password":"baz","host":"127.0.0.1","port":25984}' $HEADERS + +# Show cluster state: +curl a:b@127.0.0.1:15986/_nodes/_all_docs + +# Show db doesn’t exist on node A +curl a:b@127.0.0.1:15984/foo + +# Show db doesn’t exist on node B +curl a:b@127.0.0.1:25984/foo + +# Create database (on node A) +curl -X PUT a:b@127.0.0.1:15984/foo + +# Show db does exist on node A +curl a:b@127.0.0.1:15984/foo + +# Show db does exist on node B +curl a:b@127.0.0.1:25984/foo + +# Finish cluster +curl a:b@127.0.0.1:15984/_cluster_setup -d '{"action":"finish_cluster"}' $HEADERS + +# Show system dbs exist on node A +curl a:b@127.0.0.1:15984/_users +curl a:b@127.0.0.1:15984/_replicator +curl a:b@127.0.0.1:15984/_metadata +curl a:b@127.0.0.1:15984/_global_changes + +# Show system dbs exist on node B +curl a:b@127.0.0.1:25984/_users +curl a:b@127.0.0.1:25984/_replicator +curl a:b@127.0.0.1:25984/_metadata +curl a:b@127.0.0.1:25984/_global_changes + +echo "YAY ALL GOOD" From f4fd3face65ff2f9c5365cfa96bedd352f2906b8 Mon Sep 17 00:00:00 2001 From: Robert Kowalski Date: Tue, 26 May 2015 02:41:54 +0200 Subject: [PATCH 36/52] whitespace fix PR: #2 PR-URL: https://github.com/apache/couchdb-setup/pull/2 Reviewed-By: Jan Lehnardt Reviewed-By: Alexander Shorin --- src/setup.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/setup.erl b/src/setup.erl index 9aa98f945b..b2278568b0 100644 --- a/src/setup.erl +++ b/src/setup.erl @@ -47,7 +47,7 @@ is_cluster_enabled() -> cluster_system_dbs() -> ["_users", "_replicator", "_metadata", "_global_changes"]. - + has_cluster_system_dbs() -> has_cluster_system_dbs(cluster_system_dbs()). From aa17a557bb6ad207c1d4e42d0e74ef81f1d45f2c Mon Sep 17 00:00:00 2001 From: Robert Kowalski Date: Tue, 23 Jun 2015 11:42:12 +0200 Subject: [PATCH 37/52] use couch_log instead of io:format PR: #2 PR-URL: https://github.com/apache/couchdb-setup/pull/2 Reviewed-By: Jan Lehnardt Reviewed-By: Alexander Shorin --- src/setup.erl | 8 ++++---- src/setup_httpd.erl | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/setup.erl b/src/setup.erl index b2278568b0..2118349286 100644 --- a/src/setup.erl +++ b/src/setup.erl @@ -99,7 +99,7 @@ enable_cluster_http(Options) -> {ok, "201", _, _} -> ok; Else -> - io:format("~nsend_req: ~p~n", [Else]), + couch_log:notice("send_req: ~p~n", [Else]), {error, Else} end. @@ -141,7 +141,7 @@ enable_cluster_int(Options, no) -> Port -> config:set("httpd", "port", integer_to_list(Port)) end, - io:format("~nEnable Cluster: ~p~n", [Options]). + couch_log:notice("Enable Cluster: ~p~n", [Options]). %cluster_state:set(enabled). maybe_set_admin(Username, Password) -> @@ -168,7 +168,7 @@ add_node(Options) -> add_node_int(_Options, no) -> {error, cluster_not_enabled}; add_node_int(Options, ok) -> - io:format("~nadd node: ~p~n", [Options]), + couch_log:notice("add node: ~p~n", [Options]), ErlangCookie = erlang:get_cookie(), % POST to nodeB/_setup @@ -198,7 +198,7 @@ add_node_int(Options, ok) -> % when done, PUT :5986/nodes/nodeB create_node_doc(Host, Port); Else -> - io:format("~nsend_req: ~p~n", [Else]), + couch_log:notice("send_req: ~p~n", [Else]), Else end. diff --git a/src/setup_httpd.erl b/src/setup_httpd.erl index de1bff5fac..f84112b21f 100644 --- a/src/setup_httpd.erl +++ b/src/setup_httpd.erl @@ -19,7 +19,7 @@ handle_setup_req(#httpd{method='POST'}=Req) -> ok = chttpd:verify_is_server_admin(Req), couch_httpd:validate_ctype(Req, "application/json"), Setup = get_body(Req), - io:format("~nSetup: ~p~n", [Setup]), + couch_log:notice("Setup: ~p~n", [Setup]), Action = binary_to_list(couch_util:get_value(<<"action">>, Setup, <<"missing">>)), case handle_action(Action, Setup) of ok -> @@ -68,17 +68,17 @@ handle_action("enable_cluster", Setup) -> handle_action("finish_cluster", Setup) -> - io:format("~nffinish_cluster: ~p~n", [Setup]), + couch_log:notice("finish_cluster: ~p~n", [Setup]), case setup:finish_cluster() of {error, cluster_finished} -> {error, <<"Cluster is already finished">>}; Else -> - io:format("~nElse: ~p~n", [Else]), + couch_log:notice("Else: ~p~n", [Else]), ok end; handle_action("add_node", Setup) -> - io:format("~nadd_node: ~p~n", [Setup]), + couch_log:notice("add_node: ~p~n", [Setup]), Options = get_options([ {username, <<"username">>}, @@ -99,10 +99,10 @@ handle_action("add_node", Setup) -> end; handle_action("remove_node", Setup) -> - io:format("~nremove_node: ~p~n", [Setup]); + couch_log:notice("remove_node: ~p~n", [Setup]); handle_action("receive_cookie", Setup) -> - io:format("~nreceive_cookie: ~p~n", [Setup]), + couch_log:notice("receive_cookie: ~p~n", [Setup]), Options = get_options([ {cookie, <<"cookie">>} ], Setup), @@ -113,7 +113,7 @@ handle_action("receive_cookie", Setup) -> end; handle_action(_, _) -> - io:format("~ninvalid_action: ~n", []), + couch_log:notice("invalid_action: ~n", []), {error, <<"Invalid Action'">>}. @@ -122,6 +122,6 @@ get_body(Req) -> {Body} -> Body; Else -> - io:format("~nBody Fail: ~p~n", [Else]), + couch_log:notice("Body Fail: ~p~n", [Else]), couch_httpd:send_error(Req, 400, <<"bad_request">>, <<"Missing JSON body'">>) end. From 5c0e927c11df8f6b45b9a60f0c8eddaccbf3debe Mon Sep 17 00:00:00 2001 From: Robert Newson Date: Tue, 21 Jul 2015 14:15:23 +0100 Subject: [PATCH 38/52] Use dynamic handlers --- src/setup.app.src | 5 ++++- src/setup_httpd_handlers.erl | 21 +++++++++++++++++++++ src/setup_sup.erl | 5 +++-- 3 files changed, 28 insertions(+), 3 deletions(-) create mode 100644 src/setup_httpd_handlers.erl diff --git a/src/setup.app.src b/src/setup.app.src index 8c85e14fb3..ae685c9718 100644 --- a/src/setup.app.src +++ b/src/setup.app.src @@ -17,7 +17,10 @@ {registered, []}, {applications, [ kernel, - stdlib + stdlib, + couch_epi, + chttpd, + couch_log ]}, {mod, { setup_app, []}}, {env, []} diff --git a/src/setup_httpd_handlers.erl b/src/setup_httpd_handlers.erl new file mode 100644 index 0000000000..2d7d82e0bd --- /dev/null +++ b/src/setup_httpd_handlers.erl @@ -0,0 +1,21 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(setup_httpd_handlers). + +-export([url_handler/1, db_handler/1, design_handler/1]). + +url_handler(<<"_cluster_setup">>) -> fun setup_httpd:handle_setup_req/1. + +db_handler(_) -> no_match. + +design_handler(_) -> no_match. diff --git a/src/setup_sup.erl b/src/setup_sup.erl index b69733395a..c86237dfac 100644 --- a/src/setup_sup.erl +++ b/src/setup_sup.erl @@ -35,5 +35,6 @@ start_link() -> %% =================================================================== init([]) -> - {ok, { {one_for_one, 5, 10}, []} }. - + {ok, { {one_for_one, 5, 10}, [ + chttpd_handlers:provider(setup, setup_httpd_handlers) + ]} }. From ff19be1c1855fcdc5b9d6351c5c40c85a7977195 Mon Sep 17 00:00:00 2001 From: Robert Newson Date: Tue, 21 Jul 2015 14:23:30 +0100 Subject: [PATCH 39/52] add catch-all clause for url_handler --- src/setup_httpd_handlers.erl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/setup_httpd_handlers.erl b/src/setup_httpd_handlers.erl index 2d7d82e0bd..994c217e89 100644 --- a/src/setup_httpd_handlers.erl +++ b/src/setup_httpd_handlers.erl @@ -14,7 +14,8 @@ -export([url_handler/1, db_handler/1, design_handler/1]). -url_handler(<<"_cluster_setup">>) -> fun setup_httpd:handle_setup_req/1. +url_handler(<<"_cluster_setup">>) -> fun setup_httpd:handle_setup_req/1; +url_handler(_) -> no_match. db_handler(_) -> no_match. From 75a7682e3db9ac021053af025fce5eb1cd78cb55 Mon Sep 17 00:00:00 2001 From: Robert Kowalski Date: Fri, 24 Jul 2015 15:38:46 +0200 Subject: [PATCH 40/52] require nodecount on setup when setting up a node, require the nodecount from the user. when setting up a cluster, they will probably know it, if not the ui other interfaces can count it easily for them. this will remove the warning for a non matching nodecount for the user, and it is easy to implement in uis and cli clients (e.g. a wizard for fauxton or cli client like nmo). once clusterwide setup lands this gets obviously superfluous. COUCHDB-2598 This closes COUCHDB-2594 --- src/setup.erl | 13 +++++++++++-- src/setup_httpd.erl | 3 ++- test/t-frontend-setup.sh | 7 +++++-- test/t.sh | 7 +++++-- 4 files changed, 23 insertions(+), 7 deletions(-) diff --git a/src/setup.erl b/src/setup.erl index 2118349286..4816309a60 100644 --- a/src/setup.erl +++ b/src/setup.erl @@ -24,6 +24,11 @@ require_admins(undefined, {undefined, undefined}) -> require_admins(_,_) -> ok. +require_clustersize(undefined) -> + throw({error, "Cluster setup requires node_count to be configured"}); +require_clustersize(_) -> + ok. + error_bind_address() -> throw({error, "Cluster setup requires bind_addres != 127.0.0.1"}). @@ -83,7 +88,8 @@ enable_cluster_http(Options) -> {<<"username">>, couch_util:get_value(username, Options)}, {<<"password">>, couch_util:get_value(password, Options)}, {<<"bind_address">>, couch_util:get_value(bind_address, Options)}, - {<<"port">>, couch_util:get_value(port, Options)} + {<<"port">>, couch_util:get_value(port, Options)}, + {<<"node_count">>, couch_util:get_value(node_count, Options)} ]}), Headers = [ @@ -134,6 +140,10 @@ enable_cluster_int(Options, no) -> config:set("httpd", "bind_address", binary_to_list(NewBindAddress)) end, + NodeCount = couch_util:get_value(node_count, Options), + ok = require_clustersize(NodeCount), + config:set("cluster", "n", integer_to_list(NodeCount)), + Port = proplists:get_value(port, Options), case Port of undefined -> @@ -142,7 +152,6 @@ enable_cluster_int(Options, no) -> config:set("httpd", "port", integer_to_list(Port)) end, couch_log:notice("Enable Cluster: ~p~n", [Options]). - %cluster_state:set(enabled). maybe_set_admin(Username, Password) -> case couch_auth_cache:get_admin(Username) of diff --git a/src/setup_httpd.erl b/src/setup_httpd.erl index f84112b21f..21e81cd047 100644 --- a/src/setup_httpd.erl +++ b/src/setup_httpd.erl @@ -58,7 +58,8 @@ handle_action("enable_cluster", Setup) -> {port, <<"port">>}, {remote_node, <<"remote_node">>}, {remote_current_user, <<"remote_current_user">>}, - {remote_current_password, <<"remote_current_password">>} + {remote_current_password, <<"remote_current_password">>}, + {node_count, <<"node_count">>} ], Setup), case setup:enable_cluster(Options) of {error, cluster_enabled} -> diff --git a/test/t-frontend-setup.sh b/test/t-frontend-setup.sh index 1c610b6cc9..74743bb765 100755 --- a/test/t-frontend-setup.sh +++ b/test/t-frontend-setup.sh @@ -16,10 +16,10 @@ HEADERS="-HContent-Type:application/json" curl a:b@127.0.0.1:15986/_nodes/_all_docs # Enable Cluster on node A -curl a:b@127.0.0.1:15984/_cluster_setup -d '{"action":"enable_cluster","username":"foo","password":"baz","bind_address":"0.0.0.0"}' $HEADERS +curl a:b@127.0.0.1:15984/_cluster_setup -d '{"action":"enable_cluster","username":"foo","password":"baz","bind_address":"0.0.0.0","node_count":2}' $HEADERS # Enable Cluster on node B -curl a:b@127.0.0.1:15984/_cluster_setup -d '{"action":"enable_cluster","remote_node":"127.0.0.1","port":"25984","remote_current_user":"a","remote_current_password":"b","username":"foo","password":"baz","bind_address":"0.0.0.0"}' $HEADERS +curl a:b@127.0.0.1:15984/_cluster_setup -d '{"action":"enable_cluster","remote_node":"127.0.0.1","port":"25984","remote_current_user":"a","remote_current_password":"b","username":"foo","password":"baz","bind_address":"0.0.0.0","node_count":2}' $HEADERS # Add node B on node A curl a:b@127.0.0.1:15984/_cluster_setup -d '{"action":"add_node","username":"foo","password":"baz","host":"127.0.0.1","port":25984}' $HEADERS @@ -57,4 +57,7 @@ curl a:b@127.0.0.1:25984/_replicator curl a:b@127.0.0.1:25984/_metadata curl a:b@127.0.0.1:25984/_global_changes +# Number of nodes is set to 2 +curl a:b@127.0.0.1:25984/_node/node2@127.0.0.1/_config/cluster/n + echo "YAY ALL GOOD" diff --git a/test/t.sh b/test/t.sh index 62abb61d7c..c569caa2bc 100755 --- a/test/t.sh +++ b/test/t.sh @@ -16,10 +16,10 @@ HEADERS="-HContent-Type:application/json" curl a:b@127.0.0.1:15986/_nodes/_all_docs # Enable Cluster on node A -curl a:b@127.0.0.1:15984/_cluster_setup -d '{"action":"enable_cluster","username":"foo","password":"baz","bind_address":"0.0.0.0"}' $HEADERS +curl a:b@127.0.0.1:15984/_cluster_setup -d '{"action":"enable_cluster","username":"foo","password":"baz","bind_address":"0.0.0.0","node_count":2}' $HEADERS # Enable Cluster on node B -curl a:b@127.0.0.1:25984/_cluster_setup -d '{"action":"enable_cluster","username":"foo","password":"baz","bind_address":"0.0.0.0"}' $HEADERS +curl a:b@127.0.0.1:25984/_cluster_setup -d '{"action":"enable_cluster","username":"foo","password":"baz","bind_address":"0.0.0.0","node_count":2}' $HEADERS # Add node B on node A curl a:b@127.0.0.1:15984/_cluster_setup -d '{"action":"add_node","username":"foo","password":"baz","host":"127.0.0.1","port":25984}' $HEADERS @@ -57,4 +57,7 @@ curl a:b@127.0.0.1:25984/_replicator curl a:b@127.0.0.1:25984/_metadata curl a:b@127.0.0.1:25984/_global_changes +# Number of nodes is set to 2 +curl a:b@127.0.0.1:25984/_node/node2@127.0.0.1/_config/cluster/n + echo "YAY ALL GOOD" From dd68945a20c9662f0f8912312c2320195c470a00 Mon Sep 17 00:00:00 2001 From: Robert Kowalski Date: Fri, 24 Jul 2015 16:10:41 +0200 Subject: [PATCH 41/52] use config:setineger/3 --- src/setup.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/setup.erl b/src/setup.erl index 4816309a60..a456a74687 100644 --- a/src/setup.erl +++ b/src/setup.erl @@ -142,7 +142,7 @@ enable_cluster_int(Options, no) -> NodeCount = couch_util:get_value(node_count, Options), ok = require_clustersize(NodeCount), - config:set("cluster", "n", integer_to_list(NodeCount)), + config:set_integer("cluster", "n", NodeCount), Port = proplists:get_value(port, Options), case Port of From b107042a3193047bdf66fa5c5154c7f2f586814a Mon Sep 17 00:00:00 2001 From: Robert Kowalski Date: Fri, 24 Jul 2015 18:46:49 +0200 Subject: [PATCH 42/52] fix wording --- src/setup.erl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/setup.erl b/src/setup.erl index a456a74687..f81abb7b8e 100644 --- a/src/setup.erl +++ b/src/setup.erl @@ -24,9 +24,9 @@ require_admins(undefined, {undefined, undefined}) -> require_admins(_,_) -> ok. -require_clustersize(undefined) -> +require_node_count(undefined) -> throw({error, "Cluster setup requires node_count to be configured"}); -require_clustersize(_) -> +require_node_count(_) -> ok. error_bind_address() -> @@ -141,7 +141,7 @@ enable_cluster_int(Options, no) -> end, NodeCount = couch_util:get_value(node_count, Options), - ok = require_clustersize(NodeCount), + ok = require_node_count(NodeCount), config:set_integer("cluster", "n", NodeCount), Port = proplists:get_value(port, Options), From bdb8a0c19e95316912488b986de8d113690b0cd6 Mon Sep 17 00:00:00 2001 From: Robert Newson Date: Mon, 14 Sep 2015 12:39:49 +0100 Subject: [PATCH 43/52] configure the right http interface --- src/setup.erl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/setup.erl b/src/setup.erl index 2118349286..40679561e1 100644 --- a/src/setup.erl +++ b/src/setup.erl @@ -36,7 +36,7 @@ require_bind_address(_, _) -> is_cluster_enabled() -> % bind_address != 127.0.0.1 AND admins != empty - BindAddress = config:get("httpd", "bind_address"), + BindAddress = config:get("chttpd", "bind_address"), Admins = config:get("admins"), case {BindAddress, Admins} of {"127.0.0.1", _} -> no; @@ -115,7 +115,7 @@ enable_cluster_int(Options, no) -> }, % if bind_address == 127.0.0.1 and no bind_address in req -> error - CurrentBindAddress = config:get("httpd","bind_address"), + CurrentBindAddress = config:get("chttpd","bind_address"), NewBindAddress = proplists:get_value(bind_address, Options), ok = require_admins(CurrentAdmins, NewCredentials), ok = require_bind_address(CurrentBindAddress, NewBindAddress), @@ -129,9 +129,9 @@ enable_cluster_int(Options, no) -> case NewBindAddress of undefined -> - config:set("httpd", "bind_address", "0.0.0.0"); + config:set("chttpd", "bind_address", "0.0.0.0"); NewBindAddress -> - config:set("httpd", "bind_address", binary_to_list(NewBindAddress)) + config:set("chttpd", "bind_address", binary_to_list(NewBindAddress)) end, Port = proplists:get_value(port, Options), @@ -139,7 +139,7 @@ enable_cluster_int(Options, no) -> undefined -> ok; Port -> - config:set("httpd", "port", integer_to_list(Port)) + config:set("chttpd", "port", integer_to_list(Port)) end, couch_log:notice("Enable Cluster: ~p~n", [Options]). %cluster_state:set(enabled). From 647ffbc4a1216239045af1e210863b9086f71cf4 Mon Sep 17 00:00:00 2001 From: Robert Kowalski Date: Wed, 16 Sep 2015 15:05:39 +0200 Subject: [PATCH 44/52] fix enable_cluster_http for admin-party clusters PR: #7 PR-URL: https://github.com/apache/couchdb-setup/pull/7 Reviewed-By: Robert Newson --- src/setup.erl | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/src/setup.erl b/src/setup.erl index 40679561e1..34cbdffcc2 100644 --- a/src/setup.erl +++ b/src/setup.erl @@ -69,14 +69,22 @@ enable_cluster(Options) -> enable_cluster_http(Options) end. +get_remote_request_options(Options) -> + case couch_util:get_value(remote_current_user, Options, undefined) of + undefined -> + []; + _ -> + [ + {basic_auth, { + binary_to_list(couch_util:get_value(remote_current_user, Options)), + binary_to_list(couch_util:get_value(remote_current_password, Options)) + }} + ] + end. + enable_cluster_http(Options) -> % POST to nodeB/_setup - RequestOptions = [ - {basic_auth, { - binary_to_list(couch_util:get_value(remote_current_user, Options)), - binary_to_list(couch_util:get_value(remote_current_password, Options)) - }} - ], + RequestOptions = get_remote_request_options(Options), Body = ?JSON_ENCODE({[ {<<"action">>, <<"enable_cluster">>}, @@ -138,8 +146,10 @@ enable_cluster_int(Options, no) -> case Port of undefined -> ok; - Port -> - config:set("chttpd", "port", integer_to_list(Port)) + Port when is_binary(Port) -> + config:set("chttpd", "port", binary_to_list(Port)); + Port when is_integer(Port) -> + config:set_integer("chttpd", "port", Port) end, couch_log:notice("Enable Cluster: ~p~n", [Options]). %cluster_state:set(enabled). @@ -168,7 +178,7 @@ add_node(Options) -> add_node_int(_Options, no) -> {error, cluster_not_enabled}; add_node_int(Options, ok) -> - couch_log:notice("add node: ~p~n", [Options]), + couch_log:notice("add node_int: ~p~n", [Options]), ErlangCookie = erlang:get_cookie(), % POST to nodeB/_setup From fb61c046649addb82ccb9fc9a3c4f56b5663e5f4 Mon Sep 17 00:00:00 2001 From: ILYA Khlopotov Date: Mon, 28 Sep 2015 10:30:26 -0700 Subject: [PATCH 45/52] Update to new couch_epi API --- src/setup_epi.erl | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ src/setup_sup.erl | 4 +--- 2 files changed, 50 insertions(+), 3 deletions(-) create mode 100644 src/setup_epi.erl diff --git a/src/setup_epi.erl b/src/setup_epi.erl new file mode 100644 index 0000000000..c3f2636f0b --- /dev/null +++ b/src/setup_epi.erl @@ -0,0 +1,49 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + + +-module(setup_epi). + +-behaviour(couch_epi_plugin). + +-export([ + app/0, + providers/0, + services/0, + data_subscriptions/0, + data_providers/0, + processes/0, + notify/3 +]). + +app() -> + setup. + +providers() -> + [ + {chttpd_handlers, setup_httpd_handlers} + ]. + +services() -> + []. + +data_subscriptions() -> + []. + +data_providers() -> + []. + +processes() -> + []. + +notify(_Key, _Old, _New) -> + ok. diff --git a/src/setup_sup.erl b/src/setup_sup.erl index c86237dfac..d8b700852e 100644 --- a/src/setup_sup.erl +++ b/src/setup_sup.erl @@ -35,6 +35,4 @@ start_link() -> %% =================================================================== init([]) -> - {ok, { {one_for_one, 5, 10}, [ - chttpd_handlers:provider(setup, setup_httpd_handlers) - ]} }. + {ok, {{one_for_one, 5, 10}, couch_epi:register_service(setup_epi)}}. From d0a9b722485639fc43ccbfc4267f3a2dd9aa9d5a Mon Sep 17 00:00:00 2001 From: ILYA Khlopotov Date: Tue, 29 Sep 2015 13:05:41 -0700 Subject: [PATCH 46/52] Pass supervisor's children to couch_epi --- src/setup_sup.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/setup_sup.erl b/src/setup_sup.erl index d8b700852e..b81aa3afb6 100644 --- a/src/setup_sup.erl +++ b/src/setup_sup.erl @@ -35,4 +35,4 @@ start_link() -> %% =================================================================== init([]) -> - {ok, {{one_for_one, 5, 10}, couch_epi:register_service(setup_epi)}}. + {ok, {{one_for_one, 5, 10}, couch_epi:register_service(setup_epi, [])}}. From 747144ee259b1fe084ee041f783936a7ee1cf2de Mon Sep 17 00:00:00 2001 From: Alexander Shorin Date: Mon, 12 Oct 2015 18:09:29 +0300 Subject: [PATCH 47/52] Return HTTP 200 on GET --- src/setup_httpd.erl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/setup_httpd.erl b/src/setup_httpd.erl index f84112b21f..32c610470a 100644 --- a/src/setup_httpd.erl +++ b/src/setup_httpd.erl @@ -31,13 +31,13 @@ handle_setup_req(#httpd{method='GET'}=Req) -> ok = chttpd:verify_is_server_admin(Req), case setup:is_cluster_enabled() of no -> - chttpd:send_json(Req, 201, {[{state, cluster_disabled}]}); + chttpd:send_json(Req, 200, {[{state, cluster_disabled}]}); ok -> case setup:has_cluster_system_dbs() of no -> - chttpd:send_json(Req, 201, {[{state, cluster_enabled}]}); + chttpd:send_json(Req, 200, {[{state, cluster_enabled}]}); ok -> - chttpd:send_json(Req, 201, {[{state, cluster_finished}]}) + chttpd:send_json(Req, 200, {[{state, cluster_finished}]}) end end. From b9e1f3b5d5a78a706abb358e17130fb7344567d2 Mon Sep 17 00:00:00 2001 From: Alexander Shorin Date: Mon, 12 Oct 2015 18:10:41 +0300 Subject: [PATCH 48/52] Return HTTP 405 for unsupported request method --- src/setup_httpd.erl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/setup_httpd.erl b/src/setup_httpd.erl index 32c610470a..006ed455b7 100644 --- a/src/setup_httpd.erl +++ b/src/setup_httpd.erl @@ -39,7 +39,10 @@ handle_setup_req(#httpd{method='GET'}=Req) -> ok -> chttpd:send_json(Req, 200, {[{state, cluster_finished}]}) end - end. + end; +handle_setup_req(#httpd{}=Req) -> + chttpd:send_method_not_allowed(Req, "GET,POST"). + get_options(Options, Setup) -> ExtractValues = fun({Tag, Option}, OptionsAcc) -> From d75693ea94de8595b69fcf8e9eb189664e115574 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20Verg=C3=A9?= Date: Thu, 1 Sep 2016 22:31:57 +0200 Subject: [PATCH 49/52] add_node: Don't fail if node name != "couchdb" or "node1" Adding nodes to a cluster fails if the node names (the `name` of `name@hostname` in vm.args) is different from "couchdb". The code currently infers this name from the port: "node1" if 15984, "node2" if 25984, "node3" if 35984, "couchdb" otherwise. No other possibility. This is not suited for a production set-up, where multiple servers could have different names. This patch fixes this problem by adding an optional "name" option to the "add_node" command: POST /_cluster_setup { "action": "add_node", "username": "root", "password": "******", "host": "production-server.com", "port": 5984, "name": "node5" } This fixes: COUCHDB-3119 --- README.md | 3 ++- src/setup.erl | 8 ++++---- src/setup_httpd.erl | 3 ++- test/t-frontend-setup.sh | 2 +- test/t.sh | 2 +- 5 files changed, 10 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 7043b0f84b..7316b25ef1 100644 --- a/README.md +++ b/README.md @@ -86,7 +86,8 @@ UI shows an “Add Node” interface with the fields admin, and node: }, "node": { "host": "hostname", - ["port": 5984] + ["port": 5984], + "name": "node1" // as in “node1@hostname”, same as in vm.args } } ``` diff --git a/src/setup.erl b/src/setup.erl index 34cbdffcc2..144c2c33c1 100644 --- a/src/setup.erl +++ b/src/setup.erl @@ -200,13 +200,14 @@ add_node_int(Options, ok) -> Host = proplists:get_value(host, Options), Port = get_port(proplists:get_value(port, Options, 5984)), + Name = proplists:get_value(name, Options, get_default_name(Port)), Url = binary_to_list(<<"http://", Host/binary, ":", Port/binary, "/_cluster_setup">>), case ibrowse:send_req(Url, Headers, post, Body, RequestOptions) of {ok, "201", _, _} -> % when done, PUT :5986/nodes/nodeB - create_node_doc(Host, Port); + create_node_doc(Host, Name); Else -> couch_log:notice("send_req: ~p~n", [Else]), Else @@ -219,16 +220,15 @@ get_port(Port) when is_list(Port) -> get_port(Port) when is_binary(Port) -> Port. -create_node_doc(Host, Port) -> +create_node_doc(Host, Name) -> {ok, Db} = couch_db:open_int(<<"_nodes">>, []), - Name = get_name(Port), Doc = {[{<<"_id">>, <>}]}, Options = [], CouchDoc = couch_doc:from_json_obj(Doc), couch_db:update_doc(Db, CouchDoc, Options). -get_name(Port) -> +get_default_name(Port) -> case Port of % shortcut for easier development <<"15984">> -> diff --git a/src/setup_httpd.erl b/src/setup_httpd.erl index 006ed455b7..59c9bf8810 100644 --- a/src/setup_httpd.erl +++ b/src/setup_httpd.erl @@ -87,7 +87,8 @@ handle_action("add_node", Setup) -> {username, <<"username">>}, {password, <<"password">>}, {host, <<"host">>}, - {port, <<"port">>} + {port, <<"port">>}, + {name, <<"name">>} ], Setup), case setup:add_node(Options) of {error, cluster_not_enabled} -> diff --git a/test/t-frontend-setup.sh b/test/t-frontend-setup.sh index 1c610b6cc9..7888925029 100755 --- a/test/t-frontend-setup.sh +++ b/test/t-frontend-setup.sh @@ -22,7 +22,7 @@ curl a:b@127.0.0.1:15984/_cluster_setup -d '{"action":"enable_cluster","username curl a:b@127.0.0.1:15984/_cluster_setup -d '{"action":"enable_cluster","remote_node":"127.0.0.1","port":"25984","remote_current_user":"a","remote_current_password":"b","username":"foo","password":"baz","bind_address":"0.0.0.0"}' $HEADERS # Add node B on node A -curl a:b@127.0.0.1:15984/_cluster_setup -d '{"action":"add_node","username":"foo","password":"baz","host":"127.0.0.1","port":25984}' $HEADERS +curl a:b@127.0.0.1:15984/_cluster_setup -d '{"action":"add_node","username":"foo","password":"baz","host":"127.0.0.1","port":25984,"name":"node2"}' $HEADERS # Show cluster state: curl a:b@127.0.0.1:15986/_nodes/_all_docs diff --git a/test/t.sh b/test/t.sh index 62abb61d7c..4fbe7ea32b 100755 --- a/test/t.sh +++ b/test/t.sh @@ -22,7 +22,7 @@ curl a:b@127.0.0.1:15984/_cluster_setup -d '{"action":"enable_cluster","username curl a:b@127.0.0.1:25984/_cluster_setup -d '{"action":"enable_cluster","username":"foo","password":"baz","bind_address":"0.0.0.0"}' $HEADERS # Add node B on node A -curl a:b@127.0.0.1:15984/_cluster_setup -d '{"action":"add_node","username":"foo","password":"baz","host":"127.0.0.1","port":25984}' $HEADERS +curl a:b@127.0.0.1:15984/_cluster_setup -d '{"action":"add_node","username":"foo","password":"baz","host":"127.0.0.1","port":25984,"name":"node2"}' $HEADERS # Show cluster state: curl a:b@127.0.0.1:15986/_nodes/_all_docs From e8d1e32ba3b4f5f3be0e06e5269b12d811f24d52 Mon Sep 17 00:00:00 2001 From: Jan Lehnardt Date: Thu, 15 Sep 2016 10:13:11 +0200 Subject: [PATCH 50/52] feat: cassim is off for now --- src/setup.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/setup.erl b/src/setup.erl index 34cbdffcc2..b27c6c63dc 100644 --- a/src/setup.erl +++ b/src/setup.erl @@ -46,7 +46,7 @@ is_cluster_enabled() -> cluster_system_dbs() -> - ["_users", "_replicator", "_metadata", "_global_changes"]. + ["_users", "_replicator", "_global_changes"]. has_cluster_system_dbs() -> From 54623ce17e49ee9b5a6b69f0a8314c61b870f866 Mon Sep 17 00:00:00 2001 From: Jan Lehnardt Date: Sat, 20 May 2017 14:56:00 +0200 Subject: [PATCH 51/52] fix cluster setup: use same admin pq salt on all nodes --- src/setup.erl | 23 +++++++++++------------ src/setup_httpd.erl | 1 + 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/setup.erl b/src/setup.erl index b27c6c63dc..6657073cec 100644 --- a/src/setup.erl +++ b/src/setup.erl @@ -85,11 +85,13 @@ get_remote_request_options(Options) -> enable_cluster_http(Options) -> % POST to nodeB/_setup RequestOptions = get_remote_request_options(Options), + AdminUsername = couch_util:get_value(username, Options), + AdminPasswordHash = config:get("admins", binary_to_list(AdminUsername)), Body = ?JSON_ENCODE({[ {<<"action">>, <<"enable_cluster">>}, - {<<"username">>, couch_util:get_value(username, Options)}, - {<<"password">>, couch_util:get_value(password, Options)}, + {<<"username">>, AdminUsername}, + {<<"password_hash">>, ?l2b(AdminPasswordHash)}, {<<"bind_address">>, couch_util:get_value(bind_address, Options)}, {<<"port">>, couch_util:get_value(port, Options)} ]}), @@ -119,7 +121,10 @@ enable_cluster_int(Options, no) -> CurrentAdmins = config:get("admins"), NewCredentials = { proplists:get_value(username, Options), - proplists:get_value(password, Options) + case proplists:get_value(password_hash, Options) of + undefined -> proplists:get_value(password, Options); + Pw -> Pw + end }, % if bind_address == 127.0.0.1 and no bind_address in req -> error @@ -132,7 +137,7 @@ enable_cluster_int(Options, no) -> {undefined, undefined} -> ok; {Username, Password} -> - maybe_set_admin(Username, Password) + set_admin(Username, Password) end, case NewBindAddress of @@ -154,14 +159,8 @@ enable_cluster_int(Options, no) -> couch_log:notice("Enable Cluster: ~p~n", [Options]). %cluster_state:set(enabled). -maybe_set_admin(Username, Password) -> - case couch_auth_cache:get_admin(Username) of - nil -> - HashedPassword = couch_passwords:hash_admin_password(Password), - config:set("admins", binary_to_list(Username), binary_to_list(HashedPassword)); - _Else -> - ok - end. +set_admin(Username, Password) -> + config:set("admins", binary_to_list(Username), binary_to_list(Password)). finish_cluster() -> diff --git a/src/setup_httpd.erl b/src/setup_httpd.erl index 006ed455b7..910fcb78d1 100644 --- a/src/setup_httpd.erl +++ b/src/setup_httpd.erl @@ -57,6 +57,7 @@ handle_action("enable_cluster", Setup) -> Options = get_options([ {username, <<"username">>}, {password, <<"password">>}, + {password_hash, <<"password_hash">>}, {bind_address, <<"bind_address">>}, {port, <<"port">>}, {remote_node, <<"remote_node">>}, From 2590fbcc0ecbe854404edae751514d1fd20b07e4 Mon Sep 17 00:00:00 2001 From: Guillaume Belrose Date: Tue, 19 Jan 2016 09:19:45 +0200 Subject: [PATCH 52/52] Fixed some minor errors in the documentation. --- README.md | 97 ++++++++++++++++++++++++++++++++------------------------------- 1 file changed, 50 insertions(+), 47 deletions(-) diff --git a/README.md b/README.md index 7043b0f84b..cde7aae6fe 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,14 @@ From here on, there are two paths, one is via Fauxton (a) the other is using a HTTP endpoint (b). Fauxton just uses the HTTP endpoint in (b). (b) can be used to set up a cluster programmatically. +When using (b) you POST HTTP requests with a JSON request body (the request content type has to be set to application/json). + +If you have already setup a server admin account, you might need to pass the credentials to the HTTP calls using HTTP basic authentication. +Alternativaly, if you use the cURL command you can can add username and password inline, like so: + +``` +curl -X PUT "http://admin:password@127.0.0.1:5984/mydb" +``` 2.a. Go to Fauxton. There is a “Cluster Setup” tab in the sidebar. Go to the tab and get presented with a form that asks you to enter an admin @@ -43,23 +51,21 @@ is no need to ask for admin credentials here. If the bind_address is != 127.0.0.1, we can skip this entirely and Fauxton can show the add_node UI right away. -- POST to /_setup with +- POST a JSON entity to /_cluster_setup, the entity looks like: ``` - { - "action": "enable_cluster", - "admin": { - "user": "username", - "pass": "password" - }, - ["bind_address": "xxxx",] - ["port": yyyy] - } +{ + "action":"enable_cluster", + "username":"username", + "password":"password", + "bind_address":"0.0.0.0", + "port": 5984 +} ``` This sets up the admin user on the current node and binds to 0.0.0.0:5984 or the specified ip:port. Logs admin user into Fauxton automatically. -2.b. POST to /_setup as shown above. +2.b. POST to /_cluster_setup as shown above. Repeat on all nodes. - keep the same username/password everywhere. @@ -75,23 +81,20 @@ Repeat on all nodes. a. Go to Fauxton / Cluster Setup, once we have enabled the cluster, the UI shows an “Add Node” interface with the fields admin, and node: -- POST to /_setup with +- POST a JSON entity to /_cluster_setup, the entity looks like: ``` - { - "action": "add_node", - "admin": { // should be auto-filled from Fauxton, store plaintext PW in - // localStorage until we finish_cluster or timeout. - "user": "username", - "pass": "password" - }, - "node": { - "host": "hostname", - ["port": 5984] - } - } +{ + "action":"add_node", + "username":"username", + "password":"password", + "host":"192.168.1.100", + "port": 5984 +} ``` -b. as in a, but without the Fauxton bits, just POST to /_setup +In the example above, this adds the node with IP address 192.168.1.100 to the cluster. + +b. as in a, but without the Fauxton bits, just POST to /_cluster_setup - this request will do this: - on the “setup coordination node”: - check if we have an Erlang Cookie Secret. If not, generate @@ -99,7 +102,7 @@ b. as in a, but without the Fauxton bits, just POST to /_setup - store the cookie in config.ini, re-set_cookie() on startup. - make a POST request to the node specified in the body above using the admin credentials in the body above: - POST to http://username:password@node_b:5984/_setup with: + POST to http://username:password@node_b:5984/_cluster_setup with: ``` { "action": "receive_cookie", @@ -119,10 +122,10 @@ b. as in a, but without the Fauxton bits, just POST to /_setup 4.a. When all nodes are added, click the [Finish Cluster Setup] button in Fauxton. -- this does POST /_setup +- this does POST /_cluster_setup ``` { - "action": "finish_setup" + "action": "finish_cluster" } ``` @@ -144,41 +147,41 @@ This is right after starting a node for the first time, and any time before the cluster is enabled as outlined above. ``` -GET /_setup +GET /_cluster_setup {"state": "cluster_disabled"} -POST /_setup {"action":"enable_cluster"...} -> Transition to State 2 -POST /_setup {"action":"enable_cluster"...} with empty admin user/pass or invalid host/post or host/port not available -> Error -POST /_setup {"action":"anything_but_enable_cluster"...} -> Error +POST /_cluster_setup {"action":"enable_cluster"...} -> Transition to State 2 +POST /_cluster_setup {"action":"enable_cluster"...} with empty admin user/pass or invalid host/post or host/port not available -> Error +POST /_cluster_setup {"action":"anything_but_enable_cluster"...} -> Error ``` ### State 2: Cluster enabled, admin user set, waiting for nodes to be added. ``` -GET /_setup +GET /_cluster_setup {"state":"cluster_enabled","nodes":[]} -POST /_setup {"action":"enable_cluster"...} -> Error -POST /_setup {"action":"add_node"...} -> Stay in State 2, but return "nodes":["node B"}] on GET -POST /_setup {"action":"add_node"...} -> if target node not available, Error -POST /_setup {"action":"finish_cluster"} with no nodes set up -> Error -POST /_setup {"action":"finish_cluster"} -> Transition to State 3 -POST /_setup {"action":"delete_node"...} -> Stay in State 2, but delete node from /nodes, reflect the change in GET /_setup -POST /_setup {"action":"delete_node","node":"unknown"} -> Error Unknown Node +POST /_cluster_setup {"action":"enable_cluster"...} -> Error +POST /_cluster_setup {"action":"add_node"...} -> Stay in State 2, but return "nodes":["node B"}] on GET +POST /_cluster_setup {"action":"add_node"...} -> if target node not available, Error +POST /_cluster_setup {"action":"finish_cluster"} with no nodes set up -> Error +POST /_cluster_setup {"action":"finish_cluster"} -> Transition to State 3 +POST /_cluster_setup {"action":"delete_node"...} -> Stay in State 2, but delete node from /nodes, reflect the change in GET /_cluster_setup +POST /_cluster_setup {"action":"delete_node","node":"unknown"} -> Error Unknown Node ``` ### State 3: Cluster set up, all nodes operational ``` -GET /_setup +GET /_cluster_setup {"state":"cluster_finished","nodes":["node a", "node b", ...]} -POST /_setup {"action":"enable_cluster"...} -> Error -POST /_setup {"action":"finish_cluster"...} -> Stay in State 3, do nothing -POST /_setup {"action":"add_node"...} -> Error -POST /_setup?i_know_what_i_am_doing=true {"action":"add_node"...} -> Add node, stay in State 3. -POST /_setup {"action":"delete_node"...} -> Stay in State 3, but delete node from /nodes, reflect the change in GET /_setup -POST /_setup {"action":"delete_node","node":"unknown"} -> Error Unknown Node +POST /_cluster_setup {"action":"enable_cluster"...} -> Error +POST /_cluster_setup {"action":"finish_cluster"...} -> Stay in State 3, do nothing +POST /_cluster_setup {"action":"add_node"...} -> Error +POST /_cluster_setup?i_know_what_i_am_doing=true {"action":"add_node"...} -> Add node, stay in State 3. +POST /_cluster_setup {"action":"delete_node"...} -> Stay in State 3, but delete node from /nodes, reflect the change in GET /_cluster_setup +POST /_cluster_setup {"action":"delete_node","node":"unknown"} -> Error Unknown Node ``` // TBD: we need to persist the setup state somewhere.