Servers in cluster have a stale proxy entry in server_status table #5871
Description
I'm currently getting a warning in rethinkdb that not all servers in my cluster are properly connected. The underlying issue seems to be that there is an outdated proxy instance in the cluster config.
I'm getting the following result when running r.db('rethinkdb').table('server_status')
{
"id": "b59f8fa3-b52a-43f3-a85f-2b8ead718a2c",
"name": "rethinkdb_master_1hp",
"network": {
"canonical_addresses": [{
"host": "10.132.0.9",
"port": 29015
}, {
"host": "127.0.0.1",
"port": 29015
}, {
"host": "::1",
"port": 29015
}],
"cluster_port": 29015,
"connected_to": {
"proxy-15f9deff-5fc3-4af8-a337-17eb7a9b2aed": true,
"proxy-267468c9-58ac-4b35-ae56-112fc0994ede": true,
"proxy-2b4ff4b8-0eee-4407-9cd2-e94d06f4fa6b": true,
"proxy-2c3b80eb-4fa8-4a0a-bbc2-13f9744fd37e": true,
"proxy-350af9cc-2f1c-4a01-8885-e21f2365195f": true,
"proxy-3dafde42-dacf-4bb3-8de0-36596fc38bc5": true,
"proxy-465e61d3-102b-423d-aee5-b110c1ba5825": true,
"proxy-4c6c5936-75ae-46fa-b5a7-4db0ff9495df": true,
"proxy-60bf2192-54b1-4b5d-a022-0f5509c2d5b7": true,
"proxy-610dd522-fb21-4612-be87-3fa8806f81f8": true,
"proxy-632c774f-ce2b-4279-aa18-f78b8c460d1e": true,
"proxy-71732ea8-f34d-49d9-9a1c-c57f65dcc785": true,
"proxy-78b529b5-0044-4310-a1d5-c35c471b6980": true,
"proxy-8e7f6962-c652-40e8-a440-cf103d92c13d": true,
"proxy-95213284-daec-4d67-b523-1d27f2ad431f": true,
"proxy-96086ec7-9e4a-4d42-8613-26800ffb09bb": true,
"proxy-a76a41ee-a694-4bc4-acab-b7a2beed7fbe": true,
"proxy-a8015c4a-af03-4cfb-8f6c-67da009fc9f5": true,
"proxy-a886bfc9-18bc-403c-8b74-afd7f9021e83": true,
"proxy-ac5f6eb2-bcef-4ff2-86f8-198ec8b0bbf7": true,
"proxy-b4daae6c-bd5f-4424-a91f-70f933336652": true,
"proxy-b8800df0-cba7-44e2-8409-c78c77507960": true,
"proxy-bf5f64c6-1047-40fc-85b9-9dd6f1a3b8d4": true,
"proxy-c04ab7dc-18c0-4fb1-b2b0-3b7c3fbfe366": true,
"proxy-c2b11fe0-e3e3-4ee6-b26d-6e4c5a7b2f78": true,
"proxy-c2d84f05-dc7d-4221-824b-b3f79fac481b": true,
"proxy-d2f8aed3-a692-4967-879e-6d8374cc8645": true,
"proxy-d59900e8-8bba-498d-b890-5f1715af84e6": true,
"proxy-d83304b6-51ee-4077-a3bb-39a7ec642b32": true,
"proxy-e291269b-d561-4b60-a094-99b4bfa14a55": true,
"proxy-f0eb0dc6-428a-4443-85de-8aa2bcb2b9a6": true,
"proxy-f88898c4-1cf1-4cf7-acd4-bebc526d88ca": true,
"proxy-fbde3042-7003-452b-a91f-b1507aad4224": false,
"rethinkdb_worker1_dxe": true,
"rethinkdb_worker2_uq2": true
},
"hostname": "rethinkdb-master",
"http_admin_port": 8080,
"reql_port": 28015,
"time_connected": Thu Jun 09 2016 19: 03: 55 GMT + 00: 00
},
"process": {
"argv": [
"/usr/bin/rethinkdb",
"serve",
"--config-file",
"/etc/rethinkdb/instances.d/rethinkdb-master.conf",
"--initial-password",
"auto",
"--join",
"10.132.0.10:29015"
],
"cache_size_mb": 6950.21875,
"pid": 421,
"time_started": Thu Jun 09 2016 19: 03: 55 GMT + 00: 00,
"version": "rethinkdb 2.3.4~0jessie (GCC 4.9.2)"
}
} {
"id": "7433bc24-4ea4-4c06-97b9-0f831a24aa09",
"name": "rethinkdb_worker1_dxe",
"network": {
"canonical_addresses": [{
"host": "10.132.0.10",
"port": 29015
}, {
"host": "127.0.0.1",
"port": 29015
}, {
"host": "::1",
"port": 29015
}],
"cluster_port": 29015,
"connected_to": {
"proxy-15f9deff-5fc3-4af8-a337-17eb7a9b2aed": true,
"proxy-267468c9-58ac-4b35-ae56-112fc0994ede": true,
"proxy-2b4ff4b8-0eee-4407-9cd2-e94d06f4fa6b": true,
"proxy-2c3b80eb-4fa8-4a0a-bbc2-13f9744fd37e": true,
"proxy-350af9cc-2f1c-4a01-8885-e21f2365195f": true,
"proxy-3dafde42-dacf-4bb3-8de0-36596fc38bc5": true,
"proxy-465e61d3-102b-423d-aee5-b110c1ba5825": true,
"proxy-4c6c5936-75ae-46fa-b5a7-4db0ff9495df": true,
"proxy-60bf2192-54b1-4b5d-a022-0f5509c2d5b7": true,
"proxy-610dd522-fb21-4612-be87-3fa8806f81f8": true,
"proxy-632c774f-ce2b-4279-aa18-f78b8c460d1e": true,
"proxy-71732ea8-f34d-49d9-9a1c-c57f65dcc785": true,
"proxy-78b529b5-0044-4310-a1d5-c35c471b6980": true,
"proxy-8e7f6962-c652-40e8-a440-cf103d92c13d": true,
"proxy-95213284-daec-4d67-b523-1d27f2ad431f": true,
"proxy-96086ec7-9e4a-4d42-8613-26800ffb09bb": true,
"proxy-a76a41ee-a694-4bc4-acab-b7a2beed7fbe": true,
"proxy-a8015c4a-af03-4cfb-8f6c-67da009fc9f5": true,
"proxy-a886bfc9-18bc-403c-8b74-afd7f9021e83": true,
"proxy-ac5f6eb2-bcef-4ff2-86f8-198ec8b0bbf7": true,
"proxy-b4daae6c-bd5f-4424-a91f-70f933336652": true,
"proxy-b8800df0-cba7-44e2-8409-c78c77507960": true,
"proxy-bf5f64c6-1047-40fc-85b9-9dd6f1a3b8d4": true,
"proxy-c04ab7dc-18c0-4fb1-b2b0-3b7c3fbfe366": true,
"proxy-c2b11fe0-e3e3-4ee6-b26d-6e4c5a7b2f78": true,
"proxy-c2d84f05-dc7d-4221-824b-b3f79fac481b": true,
"proxy-d2f8aed3-a692-4967-879e-6d8374cc8645": true,
"proxy-d59900e8-8bba-498d-b890-5f1715af84e6": true,
"proxy-d83304b6-51ee-4077-a3bb-39a7ec642b32": true,
"proxy-e291269b-d561-4b60-a094-99b4bfa14a55": true,
"proxy-f0eb0dc6-428a-4443-85de-8aa2bcb2b9a6": true,
"proxy-f88898c4-1cf1-4cf7-acd4-bebc526d88ca": true,
"proxy-fbde3042-7003-452b-a91f-b1507aad4224": false,
"rethinkdb_master_1hp": true,
"rethinkdb_worker2_uq2": true
},
"hostname": "rethinkdb-worker1",
"http_admin_port": "<no http admin>",
"reql_port": 28015,
"time_connected": Mon Jun 20 2016 18: 04: 30 GMT + 00: 00
},
"process": {
"argv": [
"/usr/bin/rethinkdb",
"serve",
"--config-file",
"/etc/rethinkdb/instances.d/rethinkdb-worker.conf",
"--initial-password",
"auto",
"--join",
"10.132.0.9:29015"
],
"cache_size_mb": 6950.24609375,
"pid": 424,
"time_started": Mon Jun 20 2016 18: 04: 30 GMT + 00: 00,
"version": "rethinkdb 2.3.4~0jessie (GCC 4.9.2)"
}
} {
"id": "17646b64-cfd7-4feb-9218-d2ea1533c24a",
"name": "rethinkdb_worker2_uq2",
"network": {
"canonical_addresses": [{
"host": "10.132.0.11",
"port": 29015
}, {
"host": "127.0.0.1",
"port": 29015
}, {
"host": "::1",
"port": 29015
}],
"cluster_port": 29015,
"connected_to": {
"proxy-15f9deff-5fc3-4af8-a337-17eb7a9b2aed": true,
"proxy-267468c9-58ac-4b35-ae56-112fc0994ede": true,
"proxy-2b4ff4b8-0eee-4407-9cd2-e94d06f4fa6b": true,
"proxy-2c3b80eb-4fa8-4a0a-bbc2-13f9744fd37e": true,
"proxy-350af9cc-2f1c-4a01-8885-e21f2365195f": true,
"proxy-3dafde42-dacf-4bb3-8de0-36596fc38bc5": true,
"proxy-465e61d3-102b-423d-aee5-b110c1ba5825": true,
"proxy-4c6c5936-75ae-46fa-b5a7-4db0ff9495df": true,
"proxy-60bf2192-54b1-4b5d-a022-0f5509c2d5b7": true,
"proxy-610dd522-fb21-4612-be87-3fa8806f81f8": true,
"proxy-632c774f-ce2b-4279-aa18-f78b8c460d1e": true,
"proxy-71732ea8-f34d-49d9-9a1c-c57f65dcc785": true,
"proxy-78b529b5-0044-4310-a1d5-c35c471b6980": true,
"proxy-8e7f6962-c652-40e8-a440-cf103d92c13d": true,
"proxy-95213284-daec-4d67-b523-1d27f2ad431f": true,
"proxy-96086ec7-9e4a-4d42-8613-26800ffb09bb": true,
"proxy-a76a41ee-a694-4bc4-acab-b7a2beed7fbe": true,
"proxy-a8015c4a-af03-4cfb-8f6c-67da009fc9f5": true,
"proxy-a886bfc9-18bc-403c-8b74-afd7f9021e83": true,
"proxy-ac5f6eb2-bcef-4ff2-86f8-198ec8b0bbf7": true,
"proxy-b4daae6c-bd5f-4424-a91f-70f933336652": true,
"proxy-b8800df0-cba7-44e2-8409-c78c77507960": true,
"proxy-bf5f64c6-1047-40fc-85b9-9dd6f1a3b8d4": true,
"proxy-c04ab7dc-18c0-4fb1-b2b0-3b7c3fbfe366": true,
"proxy-c2b11fe0-e3e3-4ee6-b26d-6e4c5a7b2f78": true,
"proxy-c2d84f05-dc7d-4221-824b-b3f79fac481b": true,
"proxy-d2f8aed3-a692-4967-879e-6d8374cc8645": true,
"proxy-d59900e8-8bba-498d-b890-5f1715af84e6": true,
"proxy-d83304b6-51ee-4077-a3bb-39a7ec642b32": true,
"proxy-e291269b-d561-4b60-a094-99b4bfa14a55": true,
"proxy-f0eb0dc6-428a-4443-85de-8aa2bcb2b9a6": true,
"proxy-f88898c4-1cf1-4cf7-acd4-bebc526d88ca": true,
"proxy-fbde3042-7003-452b-a91f-b1507aad4224": false,
"rethinkdb_master_1hp": true,
"rethinkdb_worker1_dxe": true
},
"hostname": "rethinkdb-worker2",
"http_admin_port": "<no http admin>",
"reql_port": 28015,
"time_connected": Mon Jun 20 2016 18: 02: 12 GMT + 00: 00
},
"process": {
"argv": [
"/usr/bin/rethinkdb",
"serve",
"--config-file",
"/etc/rethinkdb/instances.d/rethinkdb-worker.conf",
"--initial-password",
"auto",
"--join",
"10.132.0.9:29015"
],
"cache_size_mb": 6946.29296875,
"pid": 429,
"time_started": Mon Jun 20 2016 18: 02: 12 GMT + 00: 00,
"version": "rethinkdb 2.3.4~0jessie (GCC 4.9.2)"
}
}
As far as I can tell there is no proxy instance proxy-fbde3042-7003-452b-a91f-b1507aad4224 running in my system.