diff --git a/confluent_server/confluent/collective/manager.py b/confluent_server/confluent/collective/manager.py index 5b71e917..fd13a525 100644 --- a/confluent_server/confluent/collective/manager.py +++ b/confluent_server/confluent/collective/manager.py @@ -209,28 +209,30 @@ async def follow_leader(remote, leader): exitcause = await cfm.follow_channel(remote) newleader = exitcause.get('newleader', None) finally: + handled = False if cleanexit: log.log({'info': 'Previous following cleanly closed', 'subsystem': 'collective'}) - return - if newleader: + handled = True + if not handled and newleader: log.log( {'info': 'Previous leader directed us to join new leader {}'.format(newleader)}) try: if await connect_to_leader(None, get_myname(), newleader): - return + handled = True except Exception: log.log({'error': 'Unknown error attempting to connect to {}, check trace log'.format(newleader), 'subsystem': 'collective'}) cfm.logException() - log.log({'info': 'Current leader ({0}) has disappeared, restarting ' - 'collective membership'.format(leader), 'subsystem': 'collective'}) - # The leader has folded, time to startup again... - follower = None - await cfm.stop_following() - currentleader = None - if retrythread is None: # start a recovery - retrythread = tasks.spawn_task_after( - random.random(), start_collective) + if not handled: + log.log({'info': 'Current leader ({0}) has disappeared, restarting ' + 'collective membership'.format(leader), 'subsystem': 'collective'}) + # The leader has folded, time to startup again... + follower = None + await cfm.stop_following() + currentleader = None + if retrythread is None: # start a recovery + retrythread = tasks.spawn_task_after( + random.random(), start_collective) async def _create_tls_connection(host, port): cloop = asyncio.get_running_loop() diff --git a/confluent_server/confluent/config/configmanager.py b/confluent_server/confluent/config/configmanager.py index 6e743a5a..4e6b5fd8 100644 --- a/confluent_server/confluent/config/configmanager.py +++ b/confluent_server/confluent/config/configmanager.py @@ -782,6 +782,8 @@ async def relay_slaved_requests(name, listener): msg = await lh.get_next_msg() except Exception: msg = None + except Exception: + pass # unexpected loss or misbehavior of the follower ends the relay finally: try: listener.close() @@ -798,10 +800,10 @@ async def relay_slaved_requests(name, listener): *[_push_rpc(cfgstreams[s]['stream'], payload) for s in cfgstreams]) if membership_callback: membership_callback() - if not cfgstreams and not cfgleader: # last one out, set cfgleader to boolean to mark dead collective - await stop_following(True) - return False - return True + if not cfgstreams and not cfgleader: # last one out, set cfgleader to boolean to mark dead collective + await stop_following(True) + return False + return True lastheartbeat = None async def check_leader():