[Pgpool-general] race conditions in master-slave configuration

Fri Apr 9 00:43:26 UTC 2010

> Hello!
> 
> To begin with, kudos and thanks for great tool (also easy to set-up, which is a big plus) for postgres connection pooling :) 
> 
> We have a web application that's going to receive some large traffic (opensocial gadget for the largest social site in Germany), so we've set up 4 application servers and 2 postgres servers.
> We have a master-slave configuration for postgres, so pgpool is configured to direct all "write" queries (insert, update) into master db and split "read" queries (selects) between both machines.
> 
> There's also RubyRep (http://www.rubyrep.org/) running in the background, responsible for replication of all the new stuff from master into slave, so that both dbs are in sync.
> 
> And that's where sometimes bad things happen. According to the logs, sometimes right after insert, when the inserted record is requested, the app server gets nothing and spits some errors.
> 
> Is there any way to prevent such race condition, i.e. make pgpool direct selectss to the master db only for some time right after insert (the perfect one would be per-table), until it's replicated to the slave db?

One idea is, the SELECTing app server retries if target records are
not available. The other idea is using replication mode, instead of
master_slave mode.
--
Tatsuo Ishii
SRA OSS, Inc. Japan
English: http://www.sraoss.co.jp/index_en.php
Japanese: http://www.sraoss.co.jp

> Tomasz Stachewicz
> 
> 
> 
> Here's our pgpool.conf:
> 
> #
> # pgpool-II configuration file sample
> # $Header: /cvsroot/pgpool/pgpool-II/pgpool.conf.sample,v 1.26 2009/02/15 05:26:28 t-ishii Exp $
> 
> # Host name or IP address to listen on: '*' for all, '' for no TCP/IP
> # connections
> listen_addresses = '*'
> 
> # Port number for pgpool
> port = 5433
> 
> # Port number for pgpool communication manager
> pcp_port = 9898
> 
> # Unix domain socket path.  (The Debian package defaults to
> # /var/run/postgresql.)
> socket_dir = '/var/run/postgresql'
> 
> # Unix domain socket path for pgpool communication manager.
> # (Debian package defaults to /var/run/postgresql)
> pcp_socket_dir = '/var/run/postgresql'
> 
> # Unix domain socket path for the backend. Debian package defaults to /var/run/postgresql!
> backend_socket_dir = '/var/run/postgresql'
> 
> # pgpool communication manager timeout. 0 means no timeout, but strongly not recommended!
> pcp_timeout = 10
> 
> # number of pre-forked child process
> num_init_children = 80
> 
> # Number of connection pools allowed for a child process
> max_pool = 8
> 
> # If idle for this many seconds, child exits.  0 means no timeout.
> child_life_time = 300
> 
> # If idle for this many seconds, connection to PostgreSQL closes.
> # 0 means no timeout.
> connection_life_time = 0
> 
> # If child_max_connections connections were received, child exits.
> # 0 means no exit.
> child_max_connections = 0
> 
> # If client_idle_limit is n (n > 0), the client is forced to be
> # disconnected whenever after n seconds idle (even inside an explicit
> # transactions!)
> # 0 means no disconnect.
> client_idle_limit = 0
> 
> # Maximum time in seconds to complete client authentication.
> # 0 means no timeout.
> authentication_timeout = 60
> 
> # Logging directory (more accurately, the directory for the PID file)
> logdir = '/var/run/postgresql'
> 
> # pid file name
> pid_file_name = '/var/run/pgpool/pgpool.pid'
> 
> # Replication mode
> replication_mode = false
> 
> # Load balancing mode, i.e., all SELECTs are load balanced.
> # This is ignored if replication_mode is false.
> load_balance_mode = true
> 
> # if there's a data mismatch between master and secondary
> # start degeneration to stop replication mode
> replication_stop_on_mismatch = false
> 
> # If true, replicate SELECT statement when load balancing is disabled.
> # If false, it is only sent to the master node.
> replicate_select = false
> 
> # Semicolon separated list of queries to be issued at the end of a session
> reset_query_list = 'ABORT; RESET ALL; SET SESSION AUTHORIZATION DEFAULT'
> # for 8.3 or newer PostgreSQL versions DISCARD ALL can be used as
> # follows. However beware that DISCARD ALL holds exclusive lock on
> # pg_listener so it will be a serious performance problem if there are
> # lots of concurrent sessions.
> # reset_query_list = 'ABORT; DISCARD ALL'
> 
> # If true print timestamp on each log line.
> print_timestamp = true
> 
> # If true, operate in master/slave mode.
> master_slave_mode = true
> 
> # If true, cache connection pool.
> connection_cache = true
> 
> # Health check timeout.  0 means no timeout.
> health_check_timeout = 20
> 
> # Health check period.  0 means no health check.
> health_check_period = 0
> 
> # Health check user
> health_check_user = 'nobody'
> 
> # Execute command by failover.
> # special values:  %d = node id
> #                  %h = host name
> #                  %p = port number
> #                  %D = database cluster path
> #                  %m = new master node id
> #                  %M = old master node id
> #                  %% = '%' character
> #
> failover_command = ''
> 
> # Execute command by failback.
> # special values:  %d = node id
> #                  %h = host name
> #                  %p = port number
> #                  %D = database cluster path
> #                  %m = new master node id
> #                  %M = old master node id
> #                  %% = '%' character
> #
> failback_command = ''
> 
> # If true, automatically locks a table with INSERT statements to keep
> # SERIAL data consistency.  If the data does not have SERIAL data
> # type, no lock will be issued. An /*INSERT LOCK*/ comment has the
> # same effect.  A /NO INSERT LOCK*/ comment disables the effect.
> insert_lock = true
> 
> # If true, ignore leading white spaces of each query while pgpool judges
> # whether the query is a SELECT so that it can be load balanced.  This
> # is useful for certain APIs such as DBI/DBD which is known to adding an
> # extra leading white space.
> ignore_leading_white_space = true
> 
> # If true, print all statements to the log.  Like the log_statement option
> # to PostgreSQL, this allows for observing queries without engaging in full
> # debugging.
> log_statement = false
> 
> # If true, incoming connections will be printed to the log.
> log_connections = false
> 
> # If true, hostname will be shown in ps status. Also shown in
> # connection log if log_connections = true.
> # Be warned that this feature will add overhead to look up hostname.
> log_hostname = false
> 
> # if non 0, run in parallel query mode
> parallel_mode = false
> 
> # if non 0, use query cache
> enable_query_cache = false
> 
> #set pgpool2 hostname 
> pgpool2_hostname = ''
> 
> # system DB info
> system_db_hostname = 'localhost'
> system_db_port = 5432
> system_db_dbname = 'pgpool'
> system_db_schema = 'pgpool_catalog'
> system_db_user = 'pgpool'
> system_db_password = ''
> 
> # backend_hostname, backend_port, backend_weight
> # here are examples
> backend_hostname0 = '10.248.218.14'
> backend_port0 = 5432
> backend_weight0 = 1
> backend_data_directory0 = '/data'
> backend_hostname1 = '10.248.218.15'
> backend_port1 = 5432
> backend_weight1 = 1
> backend_data_directory1 = '/data1'
> 
> # - HBA -
> 
> # If true, use pool_hba.conf for client authentication. In pgpool-II
> # 1.1, the default value is false. The default value will be true in
> # 1.2.
> enable_pool_hba = false
> 
> # - online recovery -
> # online recovery user
> recovery_user = 'nobody'
> 
> # online recovery password
> recovery_password = ''
> 
> # execute a command in first stage.
> recovery_1st_stage_command = ''
> 
> # execute a command in second stage.
> recovery_2nd_stage_command = ''
> 
> # maximum time in seconds to wait for the recovering node's postmaster
> # start-up. 0 means no wait.
> # this is also used as a timer waiting for clients disconnected before
> # starting 2nd stage
> recovery_timeout = 90
> 
> # If client_idle_limit_in_recovery is n (n > 0), the client is forced
> # to be disconnected whenever after n seconds idle (even inside an
> # explicit transactions!)  0 means no disconnect. This parameter only
> # takes effect in recovery 2nd stage.
> client_idle_limit_in_recovery = 0
> _______________________________________________
> Pgpool-general mailing list
> Pgpool-general at pgfoundry.org
> http://pgfoundry.org/mailman/listinfo/pgpool-general