root@debpgstreama:~# systemctl stop pgpool2 root@debpgstreama:~# ps -ef |grep pool root 728 637 0 08:55 pts/0 00:00:00 grep pool root@debpgstreama:~# systemctl start pgpool2 root@debpgstreama:~# ps -ef |grep pool postgres 731 1 0 08:55 ? 00:00:00 /usr/sbin/pgpool -n postgres 734 731 0 08:55 ? 00:00:00 pgpool: wait for connection request postgres 735 731 0 08:55 ? 00:00:00 pgpool: wait for connection request postgres 736 731 0 08:55 ? 00:00:00 pgpool: wait for connection request postgres 737 731 0 08:55 ? 00:00:00 pgpool: wait for connection request postgres 738 731 0 08:55 ? 00:00:00 pgpool: wait for connection request postgres 739 731 0 08:55 ? 00:00:00 pgpool: wait for connection request postgres 740 731 0 08:55 ? 00:00:00 pgpool: wait for connection request postgres 741 731 0 08:55 ? 00:00:00 pgpool: wait for connection request postgres 742 731 0 08:55 ? 00:00:00 pgpool: wait for connection request postgres 743 731 0 08:55 ? 00:00:00 pgpool: wait for connection request postgres 744 731 0 08:55 ? 00:00:00 pgpool: wait for connection request postgres 745 731 0 08:55 ? 00:00:00 pgpool: wait for connection request postgres 746 731 0 08:55 ? 00:00:00 pgpool: wait for connection request postgres 747 731 0 08:55 ? 00:00:00 pgpool: wait for connection request postgres 748 731 0 08:55 ? 00:00:00 pgpool: wait for connection request postgres 749 731 0 08:55 ? 00:00:00 pgpool: wait for connection request postgres 750 731 0 08:55 ? 00:00:00 pgpool: wait for connection request postgres 751 731 0 08:55 ? 00:00:00 pgpool: wait for connection request postgres 752 731 0 08:55 ? 00:00:00 pgpool: wait for connection request postgres 753 731 0 08:55 ? 00:00:00 pgpool: wait for connection request postgres 754 731 0 08:55 ? 00:00:00 pgpool: wait for connection request postgres 755 731 0 08:55 ? 00:00:00 pgpool: wait for connection request postgres 756 731 0 08:55 ? 00:00:00 pgpool: wait for connection request postgres 757 731 0 08:55 ? 00:00:00 pgpool: wait for connection request postgres 758 731 0 08:55 ? 00:00:00 pgpool: wait for connection request postgres 759 731 0 08:55 ? 00:00:00 pgpool: wait for connection request postgres 760 731 0 08:55 ? 00:00:00 pgpool: wait for connection request postgres 761 731 0 08:55 ? 00:00:00 pgpool: wait for connection request postgres 762 731 0 08:55 ? 00:00:00 pgpool: wait for connection request postgres 763 731 0 08:55 ? 00:00:00 pgpool: wait for connection request postgres 764 731 0 08:55 ? 00:00:00 pgpool: wait for connection request postgres 765 731 0 08:55 ? 00:00:00 pgpool: wait for connection request postgres 766 731 0 08:55 ? 00:00:00 pgpool: PCP: wait for connection request postgres 767 731 0 08:55 ? 00:00:00 pgpool: worker process postgres 768 731 0 08:55 ? 00:00:00 pgpool: health check process(0) postgres 769 731 0 08:55 ? 00:00:00 pgpool: health check process(1) root 771 637 0 08:56 pts/0 00:00:00 grep pool root@debpgstreama:~# May 15 08:55:30 debpgstreama pgpool[731]: 2023-05-15 08:55:30.460: main pid 731: LOG: node status[0]: 1 May 15 08:55:30 debpgstreama pgpool[731]: 2023-05-15 08:55:30.460: main pid 731: LOG: node status[1]: 2 May 15 08:57:12 debpgstreama pgpool[766]: 2023-05-15 08:57:12.438: pcp_main pid 766: LOG: forked new pcp worker, pid=773 socket=7 May 15 08:57:12 debpgstreama pgpool[773]: 2023-05-15 08:57:12.440: pcp_child pid 773: FATAL: authentication failed for user "pcpuser" May 15 08:57:12 debpgstreama pgpool[773]: 2023-05-15 08:57:12.440: pcp_child pid 773: DETAIL: username and/or password does not match May 15 08:57:12 debpgstreama pgpool[766]: 2023-05-15 08:57:12.451: pcp_main pid 766: LOG: PCP process with pid: 773 exit with SUCCESS. May 15 08:57:12 debpgstreama pgpool[766]: 2023-05-15 08:57:12.451: pcp_main pid 766: LOG: PCP process with pid: 773 exits with status 256 May 15 08:57:17 debpgstreama pgpool[766]: 2023-05-15 08:57:17.204: pcp_main pid 766: LOG: forked new pcp worker, pid=778 socket=7 May 15 08:57:17 debpgstreama pgpool[766]: 2023-05-15 08:57:17.238: pcp_main pid 766: LOG: PCP process with pid: 778 exit with SUCCESS. May 15 08:57:17 debpgstreama pgpool[766]: 2023-05-15 08:57:17.238: pcp_main pid 766: LOG: PCP process with pid: 778 exits with status 0 May 15 09:03:00 debpgstreama pgpool[768]: 2023-05-15 09:03:00.901: health_check0 pid 768: LOG: failed to connect to PostgreSQL server on "10.0.0.224:5432", getsockopt() failed May 15 09:03:00 debpgstreama pgpool[768]: 2023-05-15 09:03:00.901: health_check0 pid 768: DETAIL: Operation now in progress May 15 09:03:00 debpgstreama pgpool[768]: 2023-05-15 09:03:00.901: health_check0 pid 768: LOG: health check retrying on DB node: 0 (round:1) May 15 09:03:01 debpgstreama pgpool[767]: 2023-05-15 09:03:01.367: sr_check_worker pid 767: LOG: failed to connect to PostgreSQL server on "10.0.0.224:5432", getsockopt() failed May 15 09:03:01 debpgstreama pgpool[767]: 2023-05-15 09:03:01.367: sr_check_worker pid 767: DETAIL: Operation now in progress May 15 09:03:01 debpgstreama pgpool[768]: 2023-05-15 09:03:01.903: health_check0 pid 768: LOG: failed to connect to PostgreSQL server on "10.0.0.224:5432", getsockopt() failed May 15 09:03:01 debpgstreama pgpool[768]: 2023-05-15 09:03:01.903: health_check0 pid 768: DETAIL: Operation now in progress May 15 09:03:01 debpgstreama pgpool[768]: 2023-05-15 09:03:01.903: health_check0 pid 768: LOG: health check retrying on DB node: 0 (round:2) May 15 09:03:02 debpgstreama pgpool[768]: 2023-05-15 09:03:02.905: health_check0 pid 768: LOG: failed to connect to PostgreSQL server on "10.0.0.224:5432", getsockopt() failed May 15 09:03:02 debpgstreama pgpool[768]: 2023-05-15 09:03:02.905: health_check0 pid 768: DETAIL: Operation now in progress May 15 09:03:02 debpgstreama pgpool[768]: 2023-05-15 09:03:02.905: health_check0 pid 768: LOG: health check retrying on DB node: 0 (round:3) May 15 09:03:03 debpgstreama pgpool[768]: 2023-05-15 09:03:03.907: health_check0 pid 768: LOG: failed to connect to PostgreSQL server on "10.0.0.224:5432", getsockopt() failed May 15 09:03:03 debpgstreama pgpool[768]: 2023-05-15 09:03:03.907: health_check0 pid 768: DETAIL: Operation now in progress May 15 09:03:03 debpgstreama pgpool[768]: 2023-05-15 09:03:03.907: health_check0 pid 768: LOG: health check retrying on DB node: 0 (round:4) May 15 09:03:04 debpgstreama pgpool[768]: 2023-05-15 09:03:04.909: health_check0 pid 768: LOG: failed to connect to PostgreSQL server on "10.0.0.224:5432", getsockopt() failed May 15 09:03:04 debpgstreama pgpool[768]: 2023-05-15 09:03:04.909: health_check0 pid 768: DETAIL: Operation now in progress May 15 09:03:04 debpgstreama pgpool[768]: 2023-05-15 09:03:04.909: health_check0 pid 768: LOG: health check retrying on DB node: 0 (round:5) May 15 09:03:05 debpgstreama pgpool[768]: 2023-05-15 09:03:05.911: health_check0 pid 768: LOG: failed to connect to PostgreSQL server on "10.0.0.224:5432", getsockopt() failed May 15 09:03:05 debpgstreama pgpool[768]: 2023-05-15 09:03:05.911: health_check0 pid 768: DETAIL: Operation now in progress May 15 09:03:05 debpgstreama pgpool[768]: 2023-05-15 09:03:05.911: health_check0 pid 768: LOG: health check retrying on DB node: 0 (round:6) May 15 09:03:06 debpgstreama pgpool[768]: 2023-05-15 09:03:06.913: health_check0 pid 768: LOG: failed to connect to PostgreSQL server on "10.0.0.224:5432", getsockopt() failed May 15 09:03:06 debpgstreama pgpool[768]: 2023-05-15 09:03:06.913: health_check0 pid 768: DETAIL: Operation now in progress May 15 09:03:06 debpgstreama pgpool[768]: 2023-05-15 09:03:06.913: health_check0 pid 768: LOG: health check failed on node 0 (timeout:0) May 15 09:03:06 debpgstreama pgpool[768]: 2023-05-15 09:03:06.913: health_check0 pid 768: LOG: received degenerate backend request for node_id: 0 from pid [768] May 15 09:03:06 debpgstreama pgpool[768]: 2023-05-15 09:03:06.913: health_check0 pid 768: LOG: signal_user1_to_parent_with_reason(0) May 15 09:03:06 debpgstreama pgpool[731]: 2023-05-15 09:03:06.914: main pid 731: LOG: Pgpool-II parent process received SIGUSR1 May 15 09:03:06 debpgstreama pgpool[731]: 2023-05-15 09:03:06.914: main pid 731: LOG: Pgpool-II parent process has received failover request May 15 09:03:06 debpgstreama pgpool[731]: 2023-05-15 09:03:06.914: main pid 731: LOG: starting degeneration. shutdown host 10.0.0.224(5432) May 15 09:03:06 debpgstreama pgpool[731]: 2023-05-15 09:03:06.927: main pid 731: LOG: Restart all children May 15 09:03:06 debpgstreama pgpool[731]: 2023-05-15 09:03:06.927: main pid 731: LOG: execute command: /etc/pgpool2/scripts/failover.sh 0 10.0.0.224 5432 /data/pgcluster01 1 10.0.0.225 0 0 5432 /data/pgcluster01 10.0.0.224 5432 > /etc/pgpool2/scripts/failover.sh.log 2>&1 May 15 09:03:11 debpgstreama pgpool[767]: 2023-05-15 09:03:11.382: sr_check_worker pid 767: ERROR: Failed to check replication time lag May 15 09:03:11 debpgstreama pgpool[767]: 2023-05-15 09:03:11.382: sr_check_worker pid 767: DETAIL: No persistent db connection for the node 0 May 15 09:03:11 debpgstreama pgpool[767]: 2023-05-15 09:03:11.382: sr_check_worker pid 767: HINT: check sr_check_user and sr_check_password May 15 09:03:11 debpgstreama pgpool[767]: 2023-05-15 09:03:11.382: sr_check_worker pid 767: CONTEXT: while checking replication time lag May 15 09:03:31 debpgstreama pgpool[731]: 2023-05-15 09:03:31.936: main pid 731: LOG: find_primary_node_repeatedly: waiting for finding a primary node May 15 09:03:31 debpgstreama pgpool[731]: 2023-05-15 09:03:31.944: main pid 731: LOG: find_primary_node: primary node is 1 May 15 09:03:31 debpgstreama pgpool[731]: 2023-05-15 09:03:31.944: main pid 731: LOG: failover: set new primary node: 1 May 15 09:03:31 debpgstreama pgpool[731]: 2023-05-15 09:03:31.944: main pid 731: LOG: failover: set new main node: 1 May 15 09:03:31 debpgstreama pgpool[767]: 2023-05-15 09:03:31.950: sr_check_worker pid 767: LOG: worker process received restart request May 15 09:03:31 debpgstreama pgpool[731]: failover done. shutdown host 10.0.0.224(5432)2023-05-15 09:03:31.951: main pid 731: LOG: failover done. shutdown host 10.0.0.224(5432) May 15 09:03:32 debpgstreama pgpool[766]: 2023-05-15 09:03:32.951: pcp_main pid 766: LOG: restart request received in pcp child process May 15 09:03:32 debpgstreama pgpool[731]: 2023-05-15 09:03:32.954: main pid 731: LOG: PCP child 766 exits with status 0 in failover() May 15 09:03:32 debpgstreama pgpool[731]: 2023-05-15 09:03:32.955: main pid 731: LOG: fork a new PCP child pid 857 in failover() May 15 09:03:32 debpgstreama pgpool[731]: 2023-05-15 09:03:32.955: main pid 731: LOG: child process with pid: 734 exits with status 256 May 15 09:03:32 debpgstreama pgpool[731]: 2023-05-15 09:03:32.955: main pid 731: LOG: child process with pid: 735 exits with status 256 May 15 09:03:32 debpgstreama pgpool[731]: 2023-05-15 09:03:32.955: main pid 731: LOG: child process with pid: 736 exits with status 256 May 15 09:03:32 debpgstreama pgpool[731]: 2023-05-15 09:03:32.955: main pid 731: LOG: child process with pid: 737 exits with status 256 May 15 09:03:32 debpgstreama pgpool[731]: 2023-05-15 09:03:32.955: main pid 731: LOG: child process with pid: 738 exits with status 256 May 15 09:03:32 debpgstreama pgpool[731]: 2023-05-15 09:03:32.955: main pid 731: LOG: child process with pid: 739 exits with status 256 May 15 09:03:32 debpgstreama pgpool[731]: 2023-05-15 09:03:32.955: main pid 731: LOG: child process with pid: 740 exits with status 256 May 15 09:03:32 debpgstreama pgpool[731]: 2023-05-15 09:03:32.955: main pid 731: LOG: child process with pid: 741 exits with status 256 May 15 09:03:32 debpgstreama pgpool[731]: 2023-05-15 09:03:32.955: main pid 731: LOG: child process with pid: 742 exits with status 256 May 15 09:03:32 debpgstreama pgpool[731]: 2023-05-15 09:03:32.955: main pid 731: LOG: child process with pid: 743 exits with status 256 May 15 09:03:32 debpgstreama pgpool[731]: 2023-05-15 09:03:32.955: main pid 731: LOG: child process with pid: 744 exits with status 256 May 15 09:03:32 debpgstreama pgpool[731]: 2023-05-15 09:03:32.955: main pid 731: LOG: child process with pid: 745 exits with status 256 May 15 09:03:32 debpgstreama pgpool[731]: 2023-05-15 09:03:32.955: main pid 731: LOG: child process with pid: 746 exits with status 256 May 15 09:03:32 debpgstreama pgpool[731]: 2023-05-15 09:03:32.955: main pid 731: LOG: child process with pid: 747 exits with status 256 May 15 09:03:32 debpgstreama pgpool[731]: 2023-05-15 09:03:32.955: main pid 731: LOG: child process with pid: 748 exits with status 256 May 15 09:03:32 debpgstreama pgpool[731]: 2023-05-15 09:03:32.956: main pid 731: LOG: child process with pid: 749 exits with status 256 May 15 09:03:32 debpgstreama pgpool[731]: 2023-05-15 09:03:32.956: main pid 731: LOG: child process with pid: 750 exits with status 256 May 15 09:03:32 debpgstreama pgpool[731]: 2023-05-15 09:03:32.956: main pid 731: LOG: child process with pid: 751 exits with status 256 May 15 09:03:32 debpgstreama pgpool[857]: 2023-05-15 09:03:32.956: pcp_main pid 857: LOG: PCP process: 857 started May 15 09:03:32 debpgstreama pgpool[731]: 2023-05-15 09:03:32.956: main pid 731: LOG: child process with pid: 752 exits with status 256 May 15 09:03:32 debpgstreama pgpool[731]: 2023-05-15 09:03:32.956: main pid 731: LOG: child process with pid: 753 exits with status 256 May 15 09:03:32 debpgstreama pgpool[731]: 2023-05-15 09:03:32.956: main pid 731: LOG: child process with pid: 754 exits with status 256 May 15 09:03:32 debpgstreama pgpool[731]: 2023-05-15 09:03:32.956: main pid 731: LOG: child process with pid: 755 exits with status 256 May 15 09:03:32 debpgstreama pgpool[731]: 2023-05-15 09:03:32.956: main pid 731: LOG: child process with pid: 756 exits with status 256 May 15 09:03:32 debpgstreama pgpool[731]: 2023-05-15 09:03:32.956: main pid 731: LOG: child process with pid: 757 exits with status 256 May 15 09:03:32 debpgstreama pgpool[731]: 2023-05-15 09:03:32.956: main pid 731: LOG: child process with pid: 758 exits with status 256 May 15 09:03:32 debpgstreama pgpool[731]: 2023-05-15 09:03:32.956: main pid 731: LOG: child process with pid: 759 exits with status 256 May 15 09:03:32 debpgstreama pgpool[731]: 2023-05-15 09:03:32.956: main pid 731: LOG: child process with pid: 760 exits with status 256 May 15 09:03:32 debpgstreama pgpool[731]: 2023-05-15 09:03:32.956: main pid 731: LOG: child process with pid: 761 exits with status 256 May 15 09:03:32 debpgstreama pgpool[731]: 2023-05-15 09:03:32.956: main pid 731: LOG: child process with pid: 762 exits with status 256 May 15 09:03:32 debpgstreama pgpool[731]: 2023-05-15 09:03:32.956: main pid 731: LOG: child process with pid: 763 exits with status 256 May 15 09:03:32 debpgstreama pgpool[731]: 2023-05-15 09:03:32.956: main pid 731: LOG: child process with pid: 764 exits with status 256 May 15 09:03:32 debpgstreama pgpool[731]: 2023-05-15 09:03:32.956: main pid 731: LOG: child process with pid: 765 exits with status 256 May 15 09:03:32 debpgstreama pgpool[731]: 2023-05-15 09:03:32.956: main pid 731: LOG: worker child process with pid: 767 exits with status 256 May 15 09:03:32 debpgstreama pgpool[731]: 2023-05-15 09:03:32.957: main pid 731: LOG: fork a new worker child process with pid: 858 May 15 09:03:32 debpgstreama pgpool[858]: 2023-05-15 09:03:32.957: sr_check_worker pid 858: LOG: process started