Implement cluster killing in tap tests.

Author: kvap

Cluster.pm

@@ -3,6 +3,7 @@ package Cluster;
 use strict;
 use warnings;
 
+use Proc::ProcessTable;
 use PostgresNode;
 use TestLib;
 use Test::More;
@@ -135,7 +136,12 @@ sub stopnode
 	$mode = 'fast' unless defined $mode;
 	diag("stopping node $name ${mode}ly at $pgdata port $port");
 	next unless defined $node->{_pid};
-	my $ret = TestLib::system_log('pg_ctl', '-D', $pgdata, '-m', 'fast', 'stop');
+	my $ret = 0;
+	if ($mode eq 'kill') {
+		killtree($node->{_pid});
+	} else {
+		$ret = TestLib::system_log('pg_ctl', '-D', $pgdata, '-m', 'fast', 'stop');
+	}
 	$node->{_pid} = undef;
 	$node->_update_pid;
 
@@ -147,6 +153,45 @@ sub stopnode
 	return 1;
 }
 
+sub killtree
+{
+	my $root = shift;
+	diag("killtree $root\n");
+
+	my $t = new Proc::ProcessTable;
+
+	my %parent = ();
+	#my %cmd = ();
+	foreach my $p (@{$t->table}) {
+		$parent{$p->pid} = $p->ppid;
+	#	$cmd{$p->pid} = $p->cmndline;
+	}
+
+	if (!defined $root) {
+		return;
+	}
+	my @queue = ($root);
+	my @killist = ();
+
+	while (scalar @queue) {
+		my $victim = shift @queue;
+		while (my ($pid, $ppid) = each %parent) {
+			if ($ppid == $victim) {
+				push @queue, $pid;
+			}
+		}
+		diag("SIGSTOP to $victim");
+		kill 'STOP', $victim;
+		unshift @killist, $victim;
+	}
+
+	diag("SIGKILL to " . join(' ', @killist));
+	kill 'KILL', @killist;
+	#foreach my $victim (@killist) {
+	#	print("kill $victim " . $cmd{$victim} . "\n");
+	#}
+}
+
 sub stop
 {
 	my ($self, $mode) = @_;
@@ -155,8 +200,8 @@ sub stop
 
 	my $ok = 1;
 	diag("stopping cluster ${mode}ly");
-	foreach my $node (@$nodes)
-	{
+	
+	foreach my $node (@$nodes) {
 		if (!stopnode($node, $mode)) {
 			$ok = 0;
 			if (!stopnode($node, 'immediate')) {

t/000_deadlock.pl

@@ -93,5 +93,5 @@ sub query_exec_async
 
 query_row($conns[0], "select * from t where k = 1");
 
-ok($cluster->stop(), "cluster stops");
+ok($cluster->stop('kill'), "cluster stops");
 1;

t/001_basic_recovery.pl

@@ -108,5 +108,5 @@
 
 is($psql_out, '90', "Check replication after failed node recovery.");
 
-ok($cluster->stop(), "cluster stops");
+ok($cluster->stop('kill'), "cluster stops");
 1;

t/002_cross.pl

@@ -150,5 +150,5 @@ sub parse_state
 
 is($anomalies, 0, "no cross anomalies after $selects selects");
 
-ok($cluster->stop(), "cluster stops");
+ok($cluster->stop('kill'), "cluster stops");
 1;

t/003_pgbench.pl

@@ -110,5 +110,5 @@ sub writer
 ($rc, $out, $err) = $cluster->psql(0, 'postgres', "select count(*) from reader_log where v = 0;");
 isnt($out, 0, "there are some zeros in reader_log");
 
-ok($cluster->stop(), "cluster stops");
+ok($cluster->stop('kill'), "cluster stops");
 1;