bugGNU Parallel - Bugs: bug #41964, --controlmaster not sems to reuse...

 
 

bug #41964: --controlmaster not sems to reuse OpenSSH connections to the same host

Submitter:  None
Submitted:  Wed 26 Mar 2014 02:58:40 PM UTC
   
 
Category:  None Severity:  3 - Normal
Item Group:  None Status:  Fixed
Privacy:  Public Assigned to:  tange
Open/Closed:  Closed
* Mandatory Fields

Add a New Comment Rich Markup
   

Jump to the original submission

Mon 12 May 2014 07:30:25 AM UTC, comment #24: 

No feedback within a month. Closing bug.

Re-open bug when you can provide info that can reproduce the bug.

Ole Tange <tange>
Group administrator
Tue 08 Apr 2014 01:34:06 AM UTC, comment #23: 

I need to be able to reproduce the bug. So you need to be able to reproduce the bug on vm1/vm2. Without that I cannot help.

Ole Tange <tange>
Group administrator
Mon 07 Apr 2014 01:09:38 AM UTC, comment #22: 

Thanks for coorectthe 'foo' bug.
I confirm that ui don't have a ~/.shh/config file.
It's seems that reproduce the issue, no poblem without ontrolmster:
cat /myfind/all.txt | time parallel -j+0 --controlmaster -S :,peritmp1,peritmp2,peritmp3,peritmp4 true
[root@virtualdev01 ~]# wc -l /myfind/all.txt
546360 /myfind/all.txt
[root@virtualdev01 ~]#


If it's can help you to understand, i provide some verbatim:


[root@virtualdev01 ~]# pstree -Ap 63698
peristats.ksh(63698)---time(64031)---perl(64032)-+-ssh(64034)
                                                 |-ssh(64147)
                                                 |-ssh(64245)
                                                 `-ssh(64311)
[root@virtualdev01 ~]# ps -ef | grep perl

root     64032 64031 40 17:11 pts/0    00:24:07 perl /usr/bin/parallel -j+0 --controlmaster -S :,peritmp1,peritmp2,peritmp3,peritmp4 /usr/local/bin/identify -depth 8 -limit thread 1 -format "%b|%c|%d|%e|%f|%g|%h|%k|%l|%m|%n|%o|%p|%q|%r|%s|%t|%u|%w|%x|%y|%z|%A|%C|%D|%G|%H|%O|%P|%Q|%S|%T|%W|%X|%Y|%Z|%@|%#|%[base]|%[channels]|%[colorspace]|%[depth]|%[deskew:angle]|%[distortion]|%[gamma]|%[group]|%[kurtosis]|%[label]|%[max]|%[mean]|%[min]|%[name]|%[opaque]|%[orientation]|%[page]|%[scene]|%[size]|%[skewness]|%[standarddeviation]|%[type]|%[unique]|%[units]|%[version]|%[zero]|%[date:modify]|\n" {}
root     64034 64032  1 17:11 pts/0    00:00:36 ssh -tt -MTS /root/.parallel/tmp/control_path_dir-C5oE/ssh-%r@%h:%p peritmp4 perl -e \$\|=1\;while\(1\)\{sleep\ 1\;print\ \"foo\\n\"\}
root     64147 64032  0 17:11 pts/0    00:00:26 ssh -tt -MTS /root/.parallel/tmp/control_path_dir-rTYX/ssh-%r@%h:%p peritmp1 perl -e \$\|=1\;while\(1\)\{sleep\ 1\;print\ \"foo\\n\"\}
root     64245 64032  0 17:11 pts/0    00:00:26 ssh -tt -MTS /root/.parallel/tmp/control_path_dir-woAg/ssh-%r@%h:%p peritmp3 perl -e \$\|=1\;while\(1\)\{sleep\ 1\;print\ \"foo\\n\"\}
root     64311 64032  0 17:11 pts/0    00:00:25 ssh -tt -MTS /root/.parallel/tmp/control_path_dir-9Y2p/ssh-%r@%h:%p peritmp2 perl -e \$\|=1\;while\(1\)\{sleep\ 1\;print\ \"foo\\n\"\}
root     70982 63789  0 18:11 pts/1    00:00:00 grep perl
[root@virtualdev01 ~]#


Output strace against  perl parallel process:

[root@virtualdev01 ~]# strace -p 64032
Process 64032 attached - interrupt to quit
select(0, NULL, NULL, NULL, {1, 67523}) = 0 (Timeout)
write(437, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"..., 4096) = 4096
write(437, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"..., 4096) = 4096
write(437, "x", 1)                      = 1
ftruncate(437, 5806395486)              = 0
wait4(-1, 0x7fff43936e4c, WNOHANG, NULL) = 0
select(0, NULL, NULL, NULL, {1, 92400}) = 0 (Timeout)
write(437, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"..., 4096) = 4096
write(437, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"..., 4096) = 4096
write(437, "x", 1)                      = 1
ftruncate(437, 5806403679)              = 0
wait4(-1, 0x7fff43936e4c, WNOHANG, NULL) = 0
select(0, NULL, NULL, NULL, {1, 92400}) = 0 (Timeout)
write(437, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"..., 4096) = 4096
write(437, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"..., 4096) = 4096
write(437, "x", 1)                      = 1
ftruncate(437, 5806411872)              = 0
wait4(-1, 0x7fff43936e4c, WNOHANG, NULL) = 0
select(0, NULL, NULL, NULL, {1, 92400}) = 0 (Timeout)
write(437, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"..., 4096) = 4096
write(437, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"..., 4096) = 4096
write(437, "x", 1)                      = 1
ftruncate(437, 5806420065)              = 0
wait4(-1, 0x7fff43936e4c, WNOHANG, NULL) = 0
select(0, NULL, NULL, NULL, {1, 92400}) = 0 (Timeout)
write(437, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"..., 4096) = 4096
write(437, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"..., 4096) = 4096
write(437, "x", 1)                      = 1
ftruncate(437, 5806428258)              = 0
wait4(-1, 0x7fff43936e4c, WNOHANG, NULL) = 0
select(0, NULL, NULL, NULL, {1, 92400}) = 0 (Timeout)
write(437, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"..., 4096) = 4096
write(437, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"..., 4096) = 4096
write(437, "x", 1)                      = 1
ftruncate(437, 5806436451)              = 0
wait4(-1, 0x7fff43936e4c, WNOHANG, NULL) = 0
select(0, NULL, NULL, NULL, {1, 92400}) = 0 (Timeout)
write(437, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"..., 4096) = 4096
write(437, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"..., 4096) = 4096
write(437, "x", 1)                      = 1
ftruncate(437, 5806444644)              = 0
wait4(-1, 0x7fff43936e4c, WNOHANG, NULL) = 0
select(0, NULL, NULL, NULL, {1, 92400}) = 0 (Timeout)
write(437, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"..., 4096) = 4096
write(437, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"..., 4096) = 4096
write(437, "x", 1)                      = 1
ftruncate(437, 5806452837)              = 0
wait4(-1, 0x7fff43936e4c, WNOHANG, NULL) = 0
select(0, NULL, NULL, NULL, {1, 92400}) = 0 (Timeout)
write(437, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"..., 4096) = 4096
write(437, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"..., 4096) = 4096
write(437, "x", 1)                      = 1
ftruncate(437, 5806461030)              = 0
wait4(-1, 0x7fff43936e4c, WNOHANG, NULL) = 0
select(0, NULL, NULL, NULL, {1, 92400}) = 0 (Timeout)
write(437, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"..., 4096) = 4096
write(437, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"..., 4096) = 4096
write(437, "x", 1)                      = 1
ftruncate(437, 5806469223)              = 0
wait4(-1, 0x7fff43936e4c, WNOHANG, NULL) = 0
select(0, NULL, NULL, NULL, {1, 92400}) = 0 (Timeout)
write(437, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"..., 4096) = 4096
write(437, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"..., 4096) = 4096
write(437, "x", 1)                      = 1
ftruncate(437, 5806477416)              = 0
wait4(-1, 0x7fff43936e4c, WNOHANG, NULL) = 0
select(0, NULL, NULL, NULL, {1, 92400}) = 0 (Timeout)
write(437, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"..., 4096) = 4096
write(437, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"..., 4096) = 4096
write(437, "x", 1)                      = 1
ftruncate(437, 5806485609)              = 0
wait4(-1, 0x7fff43936e4c, WNOHANG, NULL) = 0
select(0, NULL, NULL, NULL, {1, 92400}) = 0 (Timeout)
write(437, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"..., 4096) = 4096
write(437, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"..., 4096) = 4096
write(437, "x", 1)                      = 1
ftruncate(437, 5806493802)              = 0
wait4(-1, 0x7fff43936e4c, WNOHANG, NULL) = 0
select(0, NULL, NULL, NULL, {1, 92400}) = 0 (Timeout)
write(437, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"..., 4096) = 4096
write(437, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"..., 4096) = 4096
write(437, "x", 1)                      = 1
ftruncate(437, 5806501995)              = 0
wait4(-1, 0x7fff43936e4c, WNOHANG, NULL) = 0
select(0, NULL, NULL, NULL, {1, 92400}) = 0 (Timeout)
write(437, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"..., 4096) = 4096
write(437, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"..., 4096) = 4096
write(437, "x", 1)                      = 1
ftruncate(437, 5806510188)              = 0
wait4(-1, 0x7fff43936e4c, WNOHANG, NULL) = 0
select(0, NULL, NULL, NULL, {1, 92400}) = 0 (Timeout)
write(437, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"..., 4096) = 4096
write(437, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"..., 4096) = 4096
write(437, "x", 1)                      = 1
ftruncate(437, 5806518381)              = 0
wait4(-1, 0x7fff43936e4c, WNOHANG, NULL) = 0
select(0, NULL, NULL, NULL, {1, 92400}) = 0 (Timeout)
write(437, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"..., 4096) = 4096
write(437, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"..., 4096) = 4096
write(437, "x", 1)                      = 1
ftruncate(437, 5806526574)              = 0
wait4(-1, 0x7fff43936e4c, WNOHANG, NULL) = 0
select(0, NULL, NULL, NULL, {1, 92400}) = 0 (Timeout)
write(437, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"..., 4096) = 4096
write(437, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"..., 4096) = 4096
write(437, "x", 1)                      = 1
ftruncate(437, 5806534767)              = 0
wait4(-1, 0x7fff43936e4c, WNOHANG, NULL) = 0
select(0, NULL, NULL, NULL, {1, 92400}) = 0 (Timeout)
write(437, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"..., 4096) = 4096
write(437, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"..., 4096) = 4096
write(437, "x", 1)                      = 1
ftruncate(437, 5806542960)              = 0
wait4(-1, 0x7fff43936e4c, WNOHANG, NULL) = 0
select(0, NULL, NULL, NULL, {1, 92400}) = 0 (Timeout)
write(437, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"..., 4096) = 4096
write(437, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"..., 4096) = 4096
write(437, "x", 1)                      = 1
ftruncate(437, 5806551153)              = 0
wait4(-1, 0x7fff43936e4c, WNOHANG, NULL) = 0
select(0, NULL, NULL, NULL, {1, 92400}) = 0 (Timeout)
write(437, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"..., 4096) = 4096
write(437, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"..., 4096) = 4096
write(437, "x", 1)                      = 1
ftruncate(437, 5806559346)              = 0
wait4(-1, 0x7fff43936e4c, WNOHANG, NULL) = 0
select(0, NULL, NULL, NULL, {1, 92400}) = 0 (Timeout)
write(437, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"..., 4096) = 4096
write(437, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"..., 4096) = 4096
write(437, "x", 1)                      = 1
ftruncate(437, 5806567539)              = 0
wait4(-1, 0x7fff43936e4c, WNOHANG, NULL) = 0
select(0, NULL, NULL, NULL, {1, 92400}) = 0 (Timeout)
write(437, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"..., 4096) = 4096
write(437, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"..., 4096) = 4096
write(437, "x", 1)                      = 1
ftruncate(437, 5806575732)              = 0
wait4(-1, 0x7fff43936e4c, WNOHANG, NULL) = 0
select(0, NULL, NULL, NULL, {1, 92400}) = 0 (Timeout)
write(437, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"..., 4096) = 4096
write(437, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"..., 4096) = 4096
write(437, "x", 1)                      = 1
ftruncate(437, 5806583925)              = 0
wait4(-1, 0x7fff43936e4c, WNOHANG, NULL) = 0
select(0, NULL, NULL, NULL, {1, 92400}) = 0 (Timeout)
write(437, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"..., 4096) = 4096
write(437, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"..., 4096) = 4096
write(437, "x", 1)                      = 1
ftruncate(437, 5806592118)              = 0
wait4(-1, 0x7fff43936e4c, WNOHANG, NULL) = 0
select(0, NULL, NULL, NULL, {1, 92400}) = 0 (Timeout)
write(437, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"..., 4096) = 4096
write(437, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"..., 4096) = 4096
write(437, "x", 1)                      = 1
ftruncate(437, 5806600311)              = 0
wait4(-1, 0x7fff43936e4c, WNOHANG, NULL) = 0
select(0, NULL, NULL, NULL, {1, 92400}^C <unfinished ...>
Process 64032 detached
[root@virtualdev01 ~]#


Output from one one strace agains ssh ContolSocket:

[root@virtualdev01 ~]# strace -p 64034
Process 64034 attached - interrupt to quit
select(8, [3 7], [], NULL, NULL)        = 1 (in [3])
read(3, "\270\312\225\304\0ls\361P\33\226\300\230\206/\327Q\272L\322\373\26!\243\177\230$^\367\265\242\267"..., 8192) = 48
select(8, [3 7], [5], NULL, NULL)       = 1 (out [5])
write(5, "foo\n", 4)                    = 4
select(8, [3 7], [], NULL, NULL)        = 1 (in [3])
read(3, "\362\233Ez\332\255\32\".h\304\352\247s,\214\202\351\263C /\345\250\336\265\335u\327\255^\212"..., 8192) = 48
select(8, [3 7], [5], NULL, NULL)       = 1 (out [5])
write(5, "foo\n", 4)                    = 4
select(8, [3 7], [], NULL, NULL)        = 1 (in [3])
read(3, "\351\25h\313\272\242\340\360:,\7\307\226\365\220\204\322b\352G\272\320^\315\310EROJ*\374\355"..., 8192) = 48
select(8, [3 7], [5], NULL, NULL)       = 1 (out [5])
write(5, "foo\n", 4)                    = 4
select(8, [3 7], [], NULL, NULL)        = 1 (in [3])
read(3, "\33]I\252\233\340H\307,\360\214\2674yo\301\16\231/,\31\367\206A\374\303V\254\220;\321c"..., 8192) = 48
select(8, [3 7], [5], NULL, NULL)       = 1 (out [5])
write(5, "foo\n", 4)                    = 4
select(8, [3 7], [], NULL, NULL)        = 1 (in [3])
read(3, "g\204\245}\247\0228\200\251H\0,\304\202K# \37\232X\215\274\1\200\20\226\220=I3\242\17"..., 8192) = 48
select(8, [3 7], [5], NULL, NULL)       = 1 (out [5])
write(5, "foo\n", 4)                    = 4
select(8, [3 7], [], NULL, NULL)        = 1 (in [3])
read(3, "\1\207 0\304\351\3477x\r;jY\236\275\334f\314I\331I\177\200\345\344\205\\\fl\2\221\250"..., 8192) = 48
select(8, [3 7], [5], NULL, NULL)       = 1 (out [5])
write(5, "foo\n", 4)                    = 4
select(8, [3 7], [], NULL, NULL)        = 1 (in [3])
read(3, "\215*\343N\216\373.\320\177)}A\314\367\2JB\303\323u\231\202H\215XA\334\351\353\247C\211"..., 8192) = 48
select(8, [3 7], [5], NULL, NULL)       = 1 (out [5])
write(5, "foo\n", 4)                    = 4
select(8, [3 7], [], NULL, NULL)        = 1 (in [3])
read(3, "L\233_>\347\224\310\6\375px\273\300\202\r\255\r\22\341\266\351\375\213\220\262\216\266\1&\261V\255"..., 8192) = 48
select(8, [3 7], [5], NULL, NULL)       = 1 (out [5])
write(5, "foo\n", 4)                    = 4
select(8, [3 7], [], NULL, NULL^C <unfinished ...>
Process 64034 detached
[root@virtualdev01 ~]#


Output from node1:

[root@peritmp1 ~]# ps -ef | grep notty
root      4245  4211  0 18:14 pts/0    00:00:00 grep notty
root     18853  1779  2 17:10 ?        00:01:46 sshd: root@notty
[root@peritmp1 ~]# pstree -Ap 18853
sshd(18853)---perl(18857)
[root@peritmp1 ~]# ps -ef | grep 18857
root      4248  4211  0 18:14 pts/0    00:00:00 grep 18857
root     18857 18853  0 17:10 ?        00:00:00 perl -e $|=1;while(1){sleep 1;print "foo\n"}
[root@peritmp1 ~]#


Output from node2:

[root@peritmp2 ~]# ps -ef | grep notty
root     12908 12892  0 18:14 pts/0    00:00:00 grep notty
root     21537  1874  2 17:10 ?        00:01:40 sshd: root@notty
[root@peritmp2 ~]# pstree -Ap 21537
sshd(21537)---perl(21541)
[root@peritmp2 ~]# ps -ef | grep 21541
root     12911 12892  0 18:14 pts/0    00:00:00 grep 21541
root     21541 21537  0 17:10 ?        00:00:00 perl -e $|=1;while(1){sleep 1;print "foo\n"}
[root@peritmp2 ~]#


Output from node3:
+verbatiom+
[root@peritmp3 ~]# ps -ef | grep notty
root      3499  1878  2 17:09 ?        00:01:49 sshd: root@notty
root     15617 15601  0 18:14 pts/0    00:00:00 grep notty
[root@peritmp3 ~]# pstree -Ap 3499
sshd(3499)---perl(3503)
[root@peritmp3 ~]# ps -ef | grep 3503
root      3503  3499  0 17:09 ?        00:00:00 perl -e $|=1;while(1){sleep 1;print "foo\n"}
root     15620 15601  0 18:14 pts/0    00:00:00 grep 3503
[root@peritmp3 ~]#
-verbatim-

Output from node4:

[root@peritmp4 ~]# ps -ef | grep 25665
root     25665 25661  0 17:09 ?        00:00:00 perl -e $|=1;while(1){sleep 1;print "foo\n"}
root     58407 58386  0 18:16 pts/0    00:00:00 grep 25665
[root@peritmp4 ~]#


phoebus <frphoebus>
Sat 05 Apr 2014 02:03:45 PM UTC, comment #21: 

Found the bug for the 'foo'. Fixed in: [f2c1f65]

You still see the ControlSocket already exists. Is that due to something in ~/.ssh/config?

I can reproduce it if I put this in ~/.ssh/config:

Host *
ControlMaster auto

Ole Tange <tange>
Group administrator
Wed 02 Apr 2014 01:33:14 PM UTC, comment #20: 

Sorry for get back again about controlmaster, but the string 'foo' is included in my command output. So i must to display all lines not matching the specified pattern 'foo' to give the expect data.

phoebus <frphoebus>
Tue 01 Apr 2014 03:55:10 PM UTC, comment #19: 

I confirm that ssh-sleep is now fixed.
I note you use the Metasyntactic string 'foo'. Which meaningful name do you use to replace foo?


[root@virtualdev01 ~]# seq 100 | time parallel --controlmaster -S :,peritmp1,peritmp2,peritmp3,peritmp4 true
When using programs that use GNU Parallel to process data for publication please cite:

  O. Tange (2011): GNU Parallel - The Command-Line Power Tool,
  ;login: The USENIX Magazine, February 2011:42-47.

This helps funding further development; and it won't cost you a cent.

To silence this citation notice run 'parallel --bibtex' once or use '--no-notice'.

foo
foo
0.50user 0.53system 0:01.85elapsed 55%CPU (0avgtext+0avgdata 45024maxresident)k
0inputs+7032outputs (0major+217513minor)pagefaults 0swaps
[root@virtualdev01 ~]#


I don't need to have ControlMaster in my ~/.ssh/config. Concerning the test to use --controlmaster and in same time to have ControlMaster into ~/.ssh/config the issue is not fixed.


[root@virtualdev01 ~]# seq 100 | time parallel --controlmaster -S :,peritmp1,peritmp2,peritmp3,peritmp4 true
When using programs that use GNU Parallel to process data for publication please cite:

  O. Tange (2011): GNU Parallel - The Command-Line Power Tool,
  ;login: The USENIX Magazine, February 2011:42-47.

This helps funding further development; and it won't cost you a cent.

To silence this citation notice run 'parallel --bibtex' once or use '--no-notice'.

ControlSocket /root/.parallel/tmp/control_path_dir-hY1Y/ssh-root@peritmp4:22 already exists, disabling multiplexing
ControlSocket /root/.parallel/tmp/control_path_dir-MiD1/ssh-root@peritmp1:22 already exists, disabling multiplexing
foo
foo
foo
foo
0.58user 0.58system 0:02.31elapsed 50%CPU (0avgtext+0avgdata 45024maxresident)k
0inputs+11224outputs (0major+223075minor)pagefaults 0swaps
[root@virtualdev01 ~]#


phoebus <frphoebus>
Tue 01 Apr 2014 08:46:46 AM UTC, comment #18: 

ssh-sleep is now fixed in [d11e647] by replacing the sleep with a perl script that prints every second: It dies if the connection closes (at most 1 second later).

I cannot explain why ssh complains about the ControlSocket, but it is pretty clear that it is unhappy about it. But if you have ControlSocket in your .config why would you ever use --controlsocket? SSH ought to do that on its own.

Please [67a8743] and see if all issues are now fixed.

Ole Tange <tange>
Group administrator
Tue 01 Apr 2014 12:18:03 AM UTC, comment #17: 

i cloned vm1 to create vm3 and vm4.
I can't reproduce the issue (ControlSocket already exists, disabling multiplexing)from vm1 against :,vm2 or :,vm2,vm3,vm4.
I reproduce the issue concerning ssh-sleep.

vm1 -> ':,vm2':

[root@vm1 ~]# rm -rf /root/.parallel
[root@vm1 ~]# ssh vm2 rm -rf /root/.parallel
[root@vm1 ~]#
[root@vm1 ~]# seq 100 | time parallel --controlmaster -S :,vm2 true
When using programs that use GNU Parallel to process data for publication please cite:

  O. Tange (2011): GNU Parallel - The Command-Line Power Tool,
  ;login: The USENIX Magazine, February 2011:42-47.

This helps funding further development; and it won't cost you a cent.

To silence this citation notice run 'parallel --bibtex' once or use '--no-notice'.

2.28user 4.61system 0:08.68elapsed 79%CPU (0avgtext+0avgdata 31664maxresident)k
0inputs+0outputs (0major+105830minor)pagefaults 0swaps
[root@vm1 ~]#
[root@vm1 ~]# ps -ef | grep ssh
root      2124     1  0 22:26 ?        00:00:00 /usr/sbin/sshd
root      2642  2606  0 23:21 ?        00:00:00 /usr/bin/ssh-agent /bin/sh -c exec -l /bin/bash -c "/usr/bin/dbus-launch --exit-with-session /etc/X11/xinit/Xclients"
root      3113     1  0 23:50 pts/1    00:00:00 sh -c ssh -tt -MTS /root/.parallel/tmp/control_path_dir-1x03/ssh-%r@%h:%p vm2 sleep 1000000000 2>/dev/null </dev/null
root      3114  3113  0 23:50 pts/1    00:00:00 ssh -tt -MTS /root/.parallel/tmp/control_path_dir-1x03/ssh-%r@%h:%p vm2 sleep 1000000000
root      3223  2887  0 23:51 pts/1    00:00:00 grep ssh
[root@vm1 ~]# ssh vm2 ps -ef | grep sleep
root      3116  3112  0 23:53 ?        00:00:00 sleep 1000000000
[root@vm1 ~]#


vm1 -> ':,vm2,vm3,vm4':

[root@vm1 .ssh]# rm -rf /root/.parallel
[root@vm1 .ssh]# ssh vm2 rm -rf /root/.parallel
[root@vm1 .ssh]# ssh vm3 rm -rf /root/.parallel
[root@vm1 .ssh]# ssh vm4 rm -rf /root/.parallel
[root@vm1 .ssh]#
[root@vm1 .ssh]# seq 100 | time parallel --controlmaster -S :,vm2,vm3,vm4 true
When using programs that use GNU Parallel to process data for publication please cite:

  O. Tange (2011): GNU Parallel - The Command-Line Power Tool,
  ;login: The USENIX Magazine, February 2011:42-47.

This helps funding further development; and it won't cost you a cent.

To silence this citation notice run 'parallel --bibtex' once or use '--no-notice'.

3.57user 8.50system 0:20.57elapsed 58%CPU (0avgtext+0avgdata 31696maxresident)k
0inputs+0outputs (19major+127774minor)pagefaults 0swaps
[root@vm1 .ssh]#
[root@vm1 .ssh]# ps -ef  | grep ssh
root      2115     1  0 00:14 ?        00:00:00 /usr/sbin/sshd
root      2419  2383  0 00:18 ?        00:00:00 /usr/bin/ssh-agent /bin/sh -c exec -l /bin/bash -c "/usr/bin/dbus-launch --exit-with-session /etc/X11/xinit/Xclients"
root      2692     1  0 00:31 pts/1    00:00:00 sh -c ssh -tt -MTS /root/.parallel/tmp/control_path_dir-1JnM/ssh-%r@%h:%p vm3 sleep 1000000000 2>/dev/null </dev/null
root      2696  2692  0 00:31 pts/1    00:00:00 ssh -tt -MTS /root/.parallel/tmp/control_path_dir-1JnM/ssh-%r@%h:%p vm3 sleep 1000000000
root      2708     1  0 00:31 pts/1    00:00:00 sh -c ssh -tt -MTS /root/.parallel/tmp/control_path_dir-G43s/ssh-%r@%h:%p vm2 sleep 1000000000 2>/dev/null </dev/null
root      2710  2708  0 00:31 pts/1    00:00:00 ssh -tt -MTS /root/.parallel/tmp/control_path_dir-G43s/ssh-%r@%h:%p vm2 sleep 1000000000
root      2717     1  0 00:31 pts/1    00:00:00 sh -c ssh -tt -MTS /root/.parallel/tmp/control_path_dir-8Xue/ssh-%r@%h:%p vm4 sleep 1000000000 2>/dev/null </dev/null
root      2721  2717  0 00:31 pts/1    00:00:00 ssh -tt -MTS /root/.parallel/tmp/control_path_dir-8Xue/ssh-%r@%h:%p vm4 sleep 1000000000
root      2857  2579  0 00:33 pts/1    00:00:00 grep ssh
[root@vm1 .ssh]# ssh vm2 ps -ef  | grep sleep
root      2690  2686  0 00:31 ?        00:00:00 sleep 1000000000
[root@vm1 .ssh]# ssh vm3 ps -ef  | grep sleep
root      2810  2806  0 00:31 ?        00:00:00 sleep 1000000000
[root@vm1 .ssh]# ssh vm4 ps -ef  | grep sleep
root      2822  2809  0 00:29 ?        00:00:00 sleep 1000000000
[root@vm1 .ssh]#



The issue 'ControlSocket already exists, disabling multiplexing' is a problem with my configuration. Since there are no stale control socket left by a ssh process (becasue i clean directory before action). I think taht's manifest a race condition somewhere which ssh client.
It seems like:

  •  the socket file is created before the connection is active
  •  one ssh session shuts down the connection before deleting the control socket file, so the next ssh command are disturbed to run as expected by sharing a previous master session.


I reproduced the issue below:

[root@virtualdev01 .ssh]# cat config
Host *
ControlMaster auto
ControlPath /root/tmp/%r@%h:%p
[root@virtualdev01 .ssh]#
[root@virtualdev01 ~]# for i in `seq 10`; do    ssh peritmp1 date& done
[1] 3408
[2] 3409
[3] 3410
[4] 3411
[5] 3412
[6] 3413
[7] 3414
[8] 3415
[9] 3416
[10] 3417
[root@virtualdev01 ~]# ControlSocket /root/tmp/root@peritmp1:22 already exists, disabling multiplexing
ControlSocket /root/tmp/root@peritmp1:22 already exists, disabling multiplexing
ControlSocket /root/tmp/root@peritmp1:22 already exists, disabling multiplexing
ControlSocket /root/tmp/root@peritmp1:22 already exists, disabling multiplexing
ControlSocket /root/tmp/root@peritmp1:22 already exists, disabling multiplexing
ControlSocket /root/tmp/root@peritmp1:22 already exists, disabling multiplexing
ControlSocket /root/tmp/root@peritmp1:22 already exists, disabling multiplexing
ControlSocket /root/tmp/root@peritmp1:22 already exists, disabling multiplexing
ControlSocket /root/tmp/root@peritmp1:22 already exists, disabling multiplexing
Tue Apr  1 01:36:00 CEST 2014
Tue Apr  1 01:36:00 CEST 2014
Tue Apr  1 01:36:00 CEST 2014
Tue Apr  1 01:36:00 CEST 2014
Tue Apr  1 01:36:00 CEST 2014
Tue Apr  1 01:36:00 CEST 2014
Tue Apr  1 01:36:00 CEST 2014
Tue Apr  1 01:36:00 CEST 2014
Tue Apr  1 01:36:00 CEST 2014
Tue Apr  1 01:36:00 CEST 2014

[1]   Done                    ssh peritmp1 date
[2]   Done                    ssh peritmp1 date
[3]   Done                    ssh peritmp1 date
[4]   Done                    ssh peritmp1 date
[5]   Done                    ssh peritmp1 date
[6]   Done                    ssh peritmp1 date
[7]   Done                    ssh peritmp1 date
[8]   Done                    ssh peritmp1 date
[9]-  Done                    ssh peritmp1 date
[10]+  Done                    ssh peritmp1 date
[root@virtualdev01 ~]#


After deleting the ssh client configuration file (~/.ssh/config). The issue is solved.


[root@virtualdev01 .ssh]# seq 100 | time parallel --controlmaster -S :,peritmp1,peritmp2,peritmp3,peritmp4 true
When using programs that use GNU Parallel to process data for publication please cite:

  O. Tange (2011): GNU Parallel - The Command-Line Power Tool,
  ;login: The USENIX Magazine, February 2011:42-47.

This helps funding further development; and it won't cost you a cent.

To silence this citation notice run 'parallel --bibtex' once or use '--no-notice'.

0.48user 0.53system 0:01.87elapsed 54%CPU (0avgtext+0avgdata 45040maxresident)k
0inputs+8216outputs (0major+217942minor)pagefaults 0swaps
[root@virtualdev01 .ssh]# seq 100 | time parallel --controlmaster -S :,peritmp1,peritmp2,peritmp3,peritmp4 true
When using programs that use GNU Parallel to process data for publication please cite:

  O. Tange (2011): GNU Parallel - The Command-Line Power Tool,
  ;login: The USENIX Magazine, February 2011:42-47.

This helps funding further development; and it won't cost you a cent.

To silence this citation notice run 'parallel --bibtex' once or use '--no-notice'.

0.49user 0.52system 0:01.89elapsed 53%CPU (0avgtext+0avgdata 45040maxresident)k
0inputs+8120outputs (0major+219578minor)pagefaults 0swaps
[root@virtualdev01 .ssh]#


To reproduce on vm1 do with OpenSSH client:

[root@vm1 ~]# cat /root/.ssh/config
Host *
ControlMaster auto
ControlPath /root/tmp/socket
oot@vm1 ~]# for i in $(seq 10); do ssh vm2 date & done
[1] 4000
[2] 4001
[3] 4002
[4] 4003
[5] 4004
[6] 4005
[7] 4006
[8] 4007
[9] 4008
[10] 4009
[root@vm1 ~]# ControlSocket /root/tmp/socket already exists
ControlSocket /root/tmp/socket already exists
ControlSocket /root/tmp/socket already exists
ControlSocket /root/tmp/socket already exists
ControlSocket /root/tmp/socket already exists
ControlSocket /root/tmp/socket already exists
Tue Apr  1 01:41:05 CEST 2014
ControlSocket /root/tmp/socket already exists
ControlSocket /root/tmp/socket already exists
Tue Apr  1 01:41:05 CEST 2014

[1]   Done                    ssh vm2 date
[2]   Exit 255                ssh vm2 date
[3]   Exit 255                ssh vm2 date
[4]   Exit 255                ssh vm2 date
[5]   Exit 255                ssh vm2 date
[6]   Exit 255                ssh vm2 date
[7]   Exit 255                ssh vm2 date
[8]   Exit 255                ssh vm2 date
[9]-  Done                    ssh vm2 date
[10]+  Exit 255                ssh vm2 date
[root@vm1 ~]#


With parallel:

[root@vm1 ~]# seq 100 | time parallel --controlmaster -S :,vm2,vm3,vm4 true
When using programs that use GNU Parallel to process data for publication please cite:

  O. Tange (2011): GNU Parallel - The Command-Line Power Tool,
  ;login: The USENIX Magazine, February 2011:42-47.

This helps funding further development; and it won't cost you a cent.

To silence this citation notice run 'parallel --bibtex' once or use '--no-notice'.

ControlSocket /root/.parallel/tmp/control_path_dir-e3U0/ssh-root@vm3:22 already exists
parallel: Warning: Could not figure out number of cpus on vm3 (). Using 1.
ControlSocket /root/.parallel/tmp/control_path_dir-AR5f/ssh-root@vm2:22 already exists
parallel: Warning: Could not figure out number of cpus on vm2 (). Using 1.
ControlSocket /root/.parallel/tmp/control_path_dir-dHeC/ssh-root@vm4:22 already exists
parallel: Warning: Could not figure out number of cpus on vm4 (). Using 1.
3.60user 8.28system 0:16.44elapsed 72%CPU (0avgtext+0avgdata 27504maxresident)k
0inputs+0outputs (0major+120681minor)pagefaults 0swaps
[root@vm1 ~]#


Delete the config file => paralle runs fine:

[root@vm1 ~]# cd .ssh
[root@vm1 .ssh]# rm config
rm: remove regular file `config'? y
[root@vm1 .ssh]# seq 100 | time parallel --controlmaster -S :,vm2,vm3,vm4 true
When using programs that use GNU Parallel to process data for publication please cite:

  O. Tange (2011): GNU Parallel - The Command-Line Power Tool,
  ;login: The USENIX Magazine, February 2011:42-47.

This helps funding further development; and it won't cost you a cent.

To silence this citation notice run 'parallel --bibtex' once or use '--no-notice'.

3.47user 7.60system 0:16.70elapsed 66%CPU (0avgtext+0avgdata 27520maxresident)k
0inputs+0outputs (0major+118420minor)pagefaults 0swaps
[root@vm1 .ssh]#


Could you explain the mismatch with Controlmaster socket used by parallel /root/.parallel/tmp and this from root based on ~/.ssh/config?

phoebus <frphoebus>
Mon 31 Mar 2014 09:21:27 PM UTC, comment #16: 


>> The --filterhosts is probably due to a bug: I never envisioned you would use ':' with --filterhosts, and it should of course never be tested for being down. I believe what we see is a timeout when it tries to run 'ssh :' which obviously fails.


OK, i'll not use --filterhosts and do some to write the list hosts top pass to parallel.


i used version Fixed in git version 8078bda but seems to have always the issue. Please find below details:

[root@virtualdev01 ~]# parallel --version
GNU parallel 20140326
Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014 Ole Tange and Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
GNU parallel comes with no warranty.

Web site: http://www.gnu.org/software/parallel

When using programs that use GNU Parallel to process data for publication please cite:

O. Tange (2011): GNU Parallel - The Command-Line Power Tool,
;login: The USENIX Magazine, February 2011:42-47.
[root@virtualdev01 ~]# grep MTS /usr/bin/parallel
            my $master = "ssh -tt -MTS $control_path $serverlogin sleep 1000000000 2>/dev/null";
[root@virtualdev01 ~]# rm -rf /root/.parallel
[root@virtualdev01 ~]# scp -p /usr/bin/parallel peritmp1:/usr/bin/parallel
parallel                                      100%  200KB 199.8KB/s   00:00
[root@virtualdev01 ~]# scp -p /usr/bin/parallel peritmp2:/usr/bin/parallel
parallel                                      100%  200KB 199.8KB/s   00:00
[root@virtualdev01 ~]# scp -p /usr/bin/parallel peritmp3:/usr/bin/parallel
parallel                                      100%  200KB 199.8KB/s   00:00
[root@virtualdev01 ~]# scp -p /usr/bin/parallel peritmp4:/usr/bin/parallel
parallel                                      100%  200KB 199.8KB/s   00:00
[root@virtualdev01 ~]#
[root@virtualdev01 ~]# seq 100 |  time parallel --controlmaster -S :,peritmp1,peritmp2,peritmp3,peritmp4 true
When using programs that use GNU Parallel to process data for publication please cite:

  O. Tange (2011): GNU Parallel - The Command-Line Power Tool,
  ;login: The USENIX Magazine, February 2011:42-47.

This helps funding further development; and it won't cost you a cent.

To silence this citation notice run 'parallel --bibtex' once or use '--no-notice'.

ControlSocket /root/.parallel/tmp/control_path_dir-pk2Q/ssh-root@peritmp3:22 already exists, disabling multiplexing
0.63user 0.69system 0:02.80elapsed 47%CPU (0avgtext+0avgdata 49216maxresident)k
0inputs+14752outputs (0major+234298minor)pagefaults 0swaps
[root@virtualdev01 ~]# ps -ef | grep ssh
root      4793     1  0 14:54 ?        00:00:00 /usr/sbin/sshd
root     37040  4793  0 20:53 ?        00:00:00 sshd: root@pts/0
root     46439     1  0 21:46 pts/0    00:00:00 sh -c ssh -tt -MTS /root/.parallel/tmp/control_path_dir-ezDc/ssh-%r@%h:%p peritmp4 sleep 1000000000 2>/dev/null </dev/null
root     46442 46439  0 21:46 pts/0    00:00:00 ssh -tt -MTS /root/.parallel/tmp/control_path_dir-ezDc/ssh-%r@%h:%p peritmp4 sleep 1000000000
root     46555     1  0 21:46 pts/0    00:00:00 sh -c ssh -tt -MTS /root/.parallel/tmp/control_path_dir-Xy33/ssh-%r@%h:%p peritmp1 sleep 1000000000 2>/dev/null </dev/null
root     46558 46555  0 21:46 pts/0    00:00:00 ssh -tt -MTS /root/.parallel/tmp/control_path_dir-Xy33/ssh-%r@%h:%p peritmp1 sleep 1000000000
root     46657     1  0 21:46 pts/0    00:00:00 sh -c ssh -tt -MTS /root/.parallel/tmp/control_path_dir-pk2Q/ssh-%r@%h:%p peritmp3 sleep 1000000000 2>/dev/null </dev/null
root     46660 46657  0 21:46 pts/0    00:00:00 ssh -tt -MTS /root/.parallel/tmp/control_path_dir-pk2Q/ssh-%r@%h:%p peritmp3 sleep 1000000000
root     46722     1  0 21:46 pts/0    00:00:00 sh -c ssh -tt -MTS /root/.parallel/tmp/control_path_dir-E0Na/ssh-%r@%h:%p peritmp2 sleep 1000000000 2>/dev/null </dev/null
root     46725 46722  0 21:46 pts/0    00:00:00 ssh -tt -MTS /root/.parallel/tmp/control_path_dir-E0Na/ssh-%r@%h:%p peritmp2 sleep 1000000000
root     46936 37098  0 21:46 pts/0    00:00:00 grep ssh
[root@virtualdev01 ~]# ssh peritmp1 ps -ef | grep sleep                         root     21814 21810  0 23:16 ?        00:00:00 sleep 1000000000
[root@virtualdev01 ~]# ssh peritmp2 ps -ef | grep sleep
root      8352  8348  0 23:16 ?        00:00:00 sleep 1000000000
[root@virtualdev01 ~]# ssh peritmp2 ps -ef | grep sleep
root      8352  8348  0 23:16 ?        00:00:00 sleep 1000000000
[root@virtualdev01 ~]# ssh peritmp3 ps -ef | grep sleep
root     16579 16575  0 23:15 ?        00:00:00 sleep 1000000000
[root@virtualdev01 ~]# ssh peritmp4 ps -ef | grep sleep
root     16519 16515  0 23:16 ?        00:00:00 sleep 1000000000
[root@virtualdev01 ~]#


I'll try to reproduce the issue using vm1 and vm2 and get back to you later.

phoebus <frphoebus>
Mon 31 Mar 2014 07:37:56 PM UTC, comment #15: 

I can reproduce the remaining ssh-sleep. Fix that by adding -tt:

-            my $master = "ssh -MTS $control_path $serverlogin sleep 1000000000 2>/dev/null";
+            my $master = "ssh -MTS -tt $control_path $serverlogin sleep 1000000000 2>/dev/null";

(Fixed in git version 8078bda)

The -tt is needed for the reasons explained earlier: If ssh is killed, we want the other end to die, too.

But I really cannot explain why you get ControlSocket already exists, disabling multiplexing.

Can you reproduce that using vm1 and vm2?

- o -

The --filterhosts is probably due to a bug: I never envisioned you would use ':' with --filterhosts, and it should of course never be tested for being down. I believe what we see is a timeout when it tries to run 'ssh :' which obviously fails.


Ole Tange <tange>
Group administrator
Mon 31 Mar 2014 05:54:24 PM UTC, comment #14: 

That sound like logical a reasonable explanation.

I try to continue with my issue with several hosts in parallel within controlmaster.

I don't understand why it tell me the multiplexing already exist (because i cleaned the directory before to start) and so it disable it.

Execute from parallel sever:
[root@virtualdev01 /]# rm -rf /root/.parallel/
[root@virtualdev01 /]# seq 100 | time  parallel --controlmaster -S :,peritmp1,peritmp2,peritmp3,peritmp4 true
When using programs that use GNU Parallel to process data for publication please cite:

  O. Tange (2011): GNU Parallel - The Command-Line Power Tool,
  ;login: The USENIX Magazine, February 2011:42-47.

This helps funding further development; and it won't cost you a cent.

To silence this citation notice run 'parallel --bibtex' once or use '--no-notice'.

ControlSocket /root/.parallel/tmp/control_path_dir-oZpv/ssh-root@peritmp4:22 already exists, disabling multiplexing
ControlSocket /root/.parallel/tmp/control_path_dir-74Py/ssh-root@peritmp3:22 already exists, disabling multiplexing
ControlSocket /root/.parallel/tmp/control_path_dir-7tUu/ssh-root@peritmp2:22 already exists, disabling multiplexing
0.57user 0.56system 0:02.16elapsed 52%CPU (0avgtext+0avgdata 49216maxresident)k
0inputs+9696outputs (0major+229452minor)pagefaults 0swaps
[root@virtualdev01 /]#

The other point, it's after task finished, i have always some process which stay alive in parallel server and parallel client:
Parallel server:
[root@virtualdev01 /]# ps -ef | grep ssh
root      4793     1  0 14:54 ?        00:00:00 /usr/sbin/sshd
root      5102     1  0 17:52 pts/0    00:00:00 sh -c ssh -MTS /root/.parallel/tmp/control_path_dir-oZpv/ssh-%r@%h:%p peritmp4 sleep 1000000000 2>/dev/null </dev/null
root      5105  5102  0 17:52 pts/0    00:00:00 ssh -MTS /root/.parallel/tmp/control_path_dir-oZpv/ssh-%r@%h:%p peritmp4 sleep 1000000000
root      5241     1  0 17:52 pts/0    00:00:00 sh -c ssh -MTS /root/.parallel/tmp/control_path_dir-w84S/ssh-%r@%h:%p peritmp1 sleep 1000000000 2>/dev/null </dev/null
root      5244  5241  0 17:52 pts/0    00:00:00 ssh -MTS /root/.parallel/tmp/control_path_dir-w84S/ssh-%r@%h:%p peritmp1 sleep 1000000000
root      5343     1  0 17:52 pts/0    00:00:00 sh -c ssh -MTS /root/.parallel/tmp/control_path_dir-74Py/ssh-%r@%h:%p peritmp3 sleep 1000000000 2>/dev/null </dev/null
root      5346  5343  0 17:52 pts/0    00:00:00 ssh -MTS /root/.parallel/tmp/control_path_dir-74Py/ssh-%r@%h:%p peritmp3 sleep 1000000000
root      5412     1  0 17:53 pts/0    00:00:00 sh -c ssh -MTS /root/.parallel/tmp/control_path_dir-7tUu/ssh-%r@%h:%p peritmp2 sleep 1000000000 2>/dev/null </dev/null
root      5415  5412  0 17:53 pts/0    00:00:00 ssh -MTS /root/.parallel/tmp/control_path_dir-7tUu/ssh-%r@%h:%p peritmp2 sleep 1000000000
root      5589  7101  0 17:53 pts/0    00:00:00 grep ssh
root      7099  4793  0 15:06 ?        00:00:00 sshd: root@pts/0
root      7338  4793  0 15:08 ?        00:00:00 sshd: root@notty
root      7340  7338  0 15:08 ?        00:00:00 /usr/libexec/openssh/sftp-server
root     12341  4793  0 15:13 ?        00:00:07 sshd: root@pts/1
[root@virtualdev01 /]#

Parallel client remote host:
[root@virtualdev01 /]# ssh peritmp1 ps -ef | grep sleep
root     19755 19751  0 19:23 ?        00:00:00 sleep 1000000000
[root@virtualdev01 /]# ssh peritmp2 ps -ef | grep sleep
root      6464  6461  0 19:22 ?        00:00:00 sleep 1000000000
[root@virtualdev01 /]# ssh peritmp2 ps -ef | grep sleep
root      6464  6461  0 19:22 ?        00:00:00 sleep 1000000000
[root@virtualdev01 /]# ssh peritmp3 ps -ef | grep sleep
root     14683 14680  0 19:21 ?        00:00:00 sleep 1000000000
[root@virtualdev01 /]# ssh peritmp4 ps -ef | grep sleep
root     13937 13933  0 19:22 ?        00:00:00 sleep 1000000000
[root@virtualdev01 /]#

Parallel client localhost:
[root@virtualdev01 /]#  ps -ef | grep sleep
root      5102     1  0 17:52 pts/0    00:00:00 sh -c ssh -MTS /root/.parallel/tmp/control_path_dir-oZpv/ssh-%r@%h:%p peritmp4 sleep 1000000000 2>/dev/null </dev/null
root      5105  5102  0 17:52 pts/0    00:00:00 ssh -MTS /root/.parallel/tmp/control_path_dir-oZpv/ssh-%r@%h:%p peritmp4 sleep 1000000000
root      5241     1  0 17:52 pts/0    00:00:00 sh -c ssh -MTS /root/.parallel/tmp/control_path_dir-w84S/ssh-%r@%h:%p peritmp1 sleep 1000000000 2>/dev/null </dev/null
root      5244  5241  0 17:52 pts/0    00:00:00 ssh -MTS /root/.parallel/tmp/control_path_dir-w84S/ssh-%r@%h:%p peritmp1 sleep 1000000000
root      5343     1  0 17:52 pts/0    00:00:00 sh -c ssh -MTS /root/.parallel/tmp/control_path_dir-74Py/ssh-%r@%h:%p peritmp3 sleep 1000000000 2>/dev/null </dev/null
root      5346  5343  0 17:52 pts/0    00:00:00 ssh -MTS /root/.parallel/tmp/control_path_dir-74Py/ssh-%r@%h:%p peritmp3 sleep 1000000000
root      5412     1  0 17:53 pts/0    00:00:00 sh -c ssh -MTS /root/.parallel/tmp/control_path_dir-7tUu/ssh-%r@%h:%p peritmp2 sleep 1000000000 2>/dev/null </dev/null
root      5415  5412  0 17:53 pts/0    00:00:00 ssh -MTS /root/.parallel/tmp/control_path_dir-7tUu/ssh-%r@%h:%p peritmp2 sleep 1000000000
root      5769  7101  0 17:54 pts/0    00:00:00 grep sleep
[root@virtualdev01 /]#


If i don't use multiplexing (that's fine):
[root@virtualdev01 /]# seq 100 | time  parallel -S :,peritmp1,peritmp2,peritmp3,peritmp4 true
When using programs that use GNU Parallel to process data for publication please cite:

  O. Tange (2011): GNU Parallel - The Command-Line Power Tool,
  ;login: The USENIX Magazine, February 2011:42-47.

This helps funding further development; and it won't cost you a cent.

To silence this citation notice run 'parallel --bibtex' once or use '--no-notice'.

0.66user 0.67system 0:02.97elapsed 45%CPU (0avgtext+0avgdata 44800maxresident)k
0inputs+17816outputs (0major+224332minor)pagefaults 0swaps
[root@virtualdev01 /]# ssh peritmp1 ps -ef | grep sleep
[root@virtualdev01 /]# ssh peritmp2 ps -ef | grep sleep
[root@virtualdev01 /]# ssh peritmp3 ps -ef | grep sleep
[root@virtualdev01 /]# ssh peritmp4 ps -ef | grep sleep
[root@virtualdev01 /]#  ps -ef | grep sleep
root      7853  7101  0 18:03 pts/0    00:00:00 grep sleep
[root@virtualdev01 /]# ps -ef | grep ssh
root      4793     1  0 14:54 ?        00:00:00 /usr/sbin/sshd
root      7099  4793  0 15:06 ?        00:00:00 sshd: root@pts/0
root      7338  4793  0 15:08 ?        00:00:00 sshd: root@notty
root      7340  7338  0 15:08 ?        00:00:00 /usr/libexec/openssh/sftp-server
root      7855  7101  0 18:03 pts/0    00:00:00 grep ssh
root     12341  4793  0 15:13 ?        00:00:07 sshd: root@pts/1
[root@virtualdev01 /]#


But if i use --filterhosts to check if a host is down, i don't have nice execution time. It seems the execution time is proportional to number of entry (seq 100, seq 200), that's not expected.
[root@virtualdev01 /]# seq 100 | time  parallel --filterhosts -S :,peritmp1,peritmp2,peritmp3,peritmp4 true
When using programs that use GNU Parallel to process data for publication please cite:

  O. Tange (2011): GNU Parallel - The Command-Line Power Tool,
  ;login: The USENIX Magazine, February 2011:42-47.

This helps funding further development; and it won't cost you a cent.

To silence this citation notice run 'parallel --bibtex' once or use '--no-notice'.

parallel: Warning: Removed :
1.33user 0.87system 0:51.24elapsed 4%CPU (0avgtext+0avgdata 42656maxresident)k
0inputs+21800outputs (0major+291233minor)pagefaults 0swaps
[root@virtualdev01 /]#
[root@virtualdev01 /]# seq 200 | time  parallel --filterhosts -S :,peritmp1,peritmp2,peritmp3,peritmp4 true
When using programs that use GNU Parallel to process data for publication please cite:

  O. Tange (2011): GNU Parallel - The Command-Line Power Tool,
  ;login: The USENIX Magazine, February 2011:42-47.

This helps funding further development; and it won't cost you a cent.

To silence this citation notice run 'parallel --bibtex' once or use '--no-notice'.

parallel: Warning: Removed :
2.21user 1.52system 1:15.74elapsed 4%CPU (0avgtext+0avgdata 42656maxresident)k
0inputs+28488outputs (0major+490601minor)pagefaults 0swaps
[root@virtualdev01 /]#


phoebus <frphoebus>
Mon 31 Mar 2014 07:59:31 AM UTC, comment #13: 

I am happy to see that the jobs do not fail and your timings resemble mine: Using --controlmaster is clearly faster. It seems you cannot use 'last' to check whether --controlmaster is used or not.

I think we have found the reason why you see the logins in 'last':

GNU Parallel reserves a tty for each connection.

This is needed to make sure that if you press CTRL-C on your local machine that this propagates to the jobs on the remote machines.

In versions before 20130422 CTRL-C was not propagated and thus if you started a job, that ran on remote machines, and you pressed CTRL-C, it would stop the local jobs, but the remote jobs would continue running.

Does that sound like a reasonable explanation?

Ole Tange <tange>
Group administrator
Mon 31 Mar 2014 01:06:17 AM UTC, comment #12: 

1st the keyboard layout is french into the VM (i added because the VM image provide by default Italian).
Sent in attachment file all output of action done into vm1 and vm2.
I don't have the same behaviour between gnu ssh with control master and parallel (with controlmaster and without controlmaster)respect if i take in reference the system connections logs (/var/log/wtmp, shows by last command).
With parallel (with/without controlmaster, i have one ssh session connection spawn for each sequence piped by parallel.  So i don't use a single SSH-TCP connection for multiple ssh session with the controlmaster => same behaviour that without controlmaster set to parallel (100 sessions in the logs)

My time results are very long seems to be the fact VirtualBox runs into a VM hosted by Vmware ESXi. Seems the controlmaster is faster for the first 2 samples but not the third.

Resume tests:
test1 (reset log before each test): parallel --controlmaster -S root@vm2 echo ::: 1 2 3
Conclusion: 3 SSH-TCP connection for multiple ssh session with the controlmaster (wtmp)

test2a (2m11), test2b (1m48), test2c (3m01) (reset log before each test): seq 100  | time parallel --controlmaster -S root@vm2 true
Conclusion: 100 SSH-TCP connection for multiple ssh session with the controlmaster (wtmp)

test3a (4m22), test3b (3m48) , test3c (2m57) (reset log before each test): seq 100  | time parallel -S root@vm2 true
Conclusion: 100 SSH-TCP connection for multiple ssh session without the controlmaster (wtmp)

test4a, test4b (reset log before 1st test): ssh -v -MTS /tmp/vm2 vm2 sleep 10000 &;ssh -v -S /tmp/vm2 vm2 echo 1;ssh -v -S /tmp/vm2 vm2 echo 2;ssh -v -S /tmp/vm2 vm2 echo 3;ssh -v -O exit -S /tmp/vm2 vm2
Conclusion: No session appears in the connection log (no new entry added into wtmp file).

* It's seems my expectation is to have one SSH connection into wtmp file with Controlmaster in action and and many SSH connection in default case. **

In GNU case no SSH connection​s appearing in wtmp because no tty used because i establish connection to pass command (ssh <hostname> <cmd>) and there don't need to use a tty for open connection since tehre is no login shell. By default, ssh does not log to wtmp for users who do not have a login shell in RHEL/Centos (for other Linxu flavours not tested). The only way to have a wtmp entry written is to allocate a tty (cf paragraph about -t option of ssh man pages).

I don't understand althrough parallel establish a main connection without allocate a tty (cf paragraph about -T option of ssh man pages), there are entry written in wtmp for each TCP-SSH session.



(file #31083)

phoebus <frphoebus>
Sun 30 Mar 2014 08:04:52 PM UTC, comment #11: 

Downloaded. MD5 checked. Opens in VirtualBox, detects settings (OS-type, CPU, RAM) but when I click 'Import' I get:

  Internal inconsistency looking up disk image 'vmdisk1'.

This applies to both images.

I use VirtualBox 4.2.10_ubuntu r84101.

I have then tar xvf'ed the ovas, created 2 new VMs and attached the vmdisks from each of the two ova-files. This boots up fine, though network is wrong.

(I thought the Danish keyboard was weird, but the keymap you use is completely bonkers).

So I changed network to 'bridged' which works for my other VMs.

Then I logged into vm1 and changed /etc/hosts to reflect the new IP-addresses.

Finally I ran from vm1:

    parallel --controlmaster -S root@vm2 echo ::: 1 2 3

and got no warnings at all. I then ran this 3 times:

    seq 100 | time parallel --controlmaster -S root@vm2 true

It took 5.4-5.9 seconds

I then ran this 3 times:

    seq 100 | time parallel -S root@vm2 true

It took 13.2-14.2 seconds - so significantly longer than with --controlmaster.

This proves that --controlmaster does have a clear effect and I cannot reproduce any errors that you see.

This leads me to believe, that what you see is not errors, but the normal situation. Can you test using SSH without GNU Parallel and see if you get a different behaviour? So:

window1_vm1$ ssh -v -MTS /tmp/vm2 vm2 sleep 10000 &
window2_vm1$ ssh -v -S /tmp/vm2 vm2 echo 1
window2_vm1$ ssh -v -S /tmp/vm2 vm2 echo 2
window2_vm1$ ssh -v -S /tmp/vm2 vm2 echo 3

Also can you try and compare the time to run for:

    seq 100 | time parallel --controlmaster -S root@vm2 true
    seq 100 | time parallel -S root@vm2 true

Ole Tange <tange>
Group administrator
Sun 30 Mar 2014 12:26:00 AM UTC, comment #10: 

I send to you both VM and md5sim file to grab from http server and not ftp server.
Note dl.free.fr, it's a free service to share file for short time.

  • VM1 (centos 5.8 x86): tested against Virtual box 4.3 within NAT network

hostname vm1
login / password: root / reverse
expoted to OVA extension (format OVF2 + Manifest): centos5.8_x86_1.ova (size 1.3 GB)
Download from: http://dl.free.fr/msieCS1hE

  • VM2 (centos 5.8 x86): tested against Virtual box 4.3 within NAT network

hostname vm2
login / password: root / reverse
expoted to OVF extension (format OVF2 + Manifest): centos5.8_x86_2.ova (size 1.3 GB)
Download from: http://dl.free.fr/mDwrEtSP1

  • a md5 file to check checksum for ova file

Download from: http://dl.free.fr/mUQA2WBMi

  • I cloned vm1 to create vm2 before done parallel test.
  • /etc/hosts defined on both nodes within vm1 and vm2
  • authorized_keys stored on vm2 to authorize ssh without password from vm1 to vm2.
  • test: ssh from vm1 to vm2 via parallel (command used: parallel --controlmaster -S vm1 echo ::: 1 2 3)


last command done:
vm1:
parallel --controlmaster -S vm1 echo ::: 1 2 3
shutdown -fh now

vm2:
cd /var/log;cp wtmp wtmp.0

> wtmp; last  (=> shows 3 sessions opened from vm1 as root)

shutdown -fh now

Feel free to contact me if you have any questions.


phoebus <frphoebus>
Sat 29 Mar 2014 04:53:28 PM UTC, comment #9: 

I am wrong: ge.tt only allows for 50 MB, so we need a ftp server somewhere.

Ole Tange <tange>
Group administrator
Sat 29 Mar 2014 04:51:42 PM UTC, comment #8: 

Wonderful. You should not be unhappy that you still have the problem. On the contrary it is very good if you can reproduce the problem between to VMs - especially if you can give me a copy of the two VMs, so I also can experience the issue.

Please give me a copy of the two VMs.

If you do not have a better option you can use http://ge.tt

Ole Tange <tange>
Group administrator
Sat 29 Mar 2014 08:27:20 AM UTC, comment #7: 

I tested agains Centos5.3 (the same you advised). Unfornulately i have the also the issue. VM (into VirtualBox)uses NAT network.

Command used the same: parallel --controlmaster  -S peritmp4 echo ::: 1 2 3


Each time for each session the Connection to master is closed.
I uupdated cose in line 3827 and 3831 tove ssh in verbose mode wth the '-v' parameter.

  3826             my $control_path = $self->control_path_dir()."/ssh-%r@%h:%p"        ;
   3827             $sshcmd = "ssh -v -S ".$control_path;
   3828             $serverlogin = $self->{'string'};
   3829             # OpenSSH_3.6.1p2 gives 'tcgetattr: Invalid argument' with -        tt
   3830             # 2>/dev/null to ignore "process_mux_new_session: tcgetattr:         Invalid argument"
   3831             my $master = "ssh -v -MTS $control_path $serverlogin sleep 1        000000000 2>/dev/null";
   3832             if(not $self->{'control_path'}{$control_path}++) {

Check the output log in atachment.
I very dissapointed and not understand why i can't succes as you.

Thanks for your feedback.

(file #31073)

phoebus <frphoebus>
Fri 28 Mar 2014 11:39:07 AM UTC, comment #6: 

Ok i'll do the test against Centos 5.8 and Centos 6.3.
And get back to you.

phoebus <frphoebus>
Fri 28 Mar 2014 10:56:51 AM UTC, comment #5: 

Yes, that is the correct version. I just successfully tested it on CentOS 5.8.

Can you reproduce it on some virtual machines that I can get?
E.g. http://sourceforge.net/projects/virtualboximage/files/

Ole Tange <tange>
Group administrator
Fri 28 Mar 2014 06:25:25 AM UTC, comment #4: 

I use this parallel version:
http://git.savannah.gnu.org/cgit/parallel.git/snapshot/parallel-06d55fcec3194b2d4c3570581beb75dde4ce57da.tar.gz get from

author Ole Tange <ole@tange.dk> 2014-03-27 02:02:18 (GMT)
committer Ole Tange <ole@tange.dk> 2014-03-27 02:02:18 (GMT)
commit 06d55fcec3194b2d4c3570581beb75dde4ce57da (patch) (unidiff)
tree 710511fd7d0a73072428c82885e758694b433b00
parent d6235b0cc30411903cb8e7220810ec79da4c2cee (diff)
download parallel-06d55fcec3194b2d4c3570581beb75dde4ce57da.tar.gz
Fixed: bug #41964: --controlmaster not seems to reuse OpenSSH connections to the same host.


Could you confirm i get the right version?

phoebus <frphoebus>
Fri 28 Mar 2014 12:47:50 AM UTC, comment #3: 

I could reproduce your earlier error, but not this.

Can you reproduce it on some virtual machines that I can get?
E.g. http://sourceforge.net/projects/virtualboximage/files/

Ole Tange <tange>
Group administrator
Thu 27 Mar 2014 09:56:56 PM UTC, comment #2: 

I tried with the version fixed but i have always the issue.
[root@virtualdev01 .ssh]# parallel --version
GNU parallel 20140326
Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014 Ole Tange and Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
GNU parallel comes with no warranty.

Web site: http://www.gnu.org/software/parallel

When using programs that use GNU Parallel to process data for publication please cite:

O. Tange (2011): GNU Parallel - The Command-Line Power Tool,
;login: The USENIX Magazine, February 2011:42-47.
[root@virtualdev01 .ssh]#


Test done:
[root@virtualdev01 ~]# rm -rf /root/.parallel/tmp
[root@virtualdev01 ~]# parallel --controlmaster  -S peritmp4 echo ::: 1 2 3
When using programs that use GNU Parallel to process data for publication please cite:

  O. Tange (2011): GNU Parallel - The Command-Line Power Tool,
  ;login: The USENIX Magazine, February 2011:42-47.

This helps funding further development; and it won't cost you a cent.

To silence this citation notice run 'parallel --bibtex' once or use '--no-notice'.

1
2
3
[root@virtualdev01 ~]# parallel --controlmaster  -S peritmp4 echo ::: 1 2 3
When using programs that use GNU Parallel to process data for publication please cite:

  O. Tange (2011): GNU Parallel - The Command-Line Power Tool,
  ;login: The USENIX Magazine, February 2011:42-47.

This helps funding further development; and it won't cost you a cent.

To silence this citation notice run 'parallel --bibtex' once or use '--no-notice'.

ControlSocket /root/.parallel/tmp/control_path_dir-Bh1F/ssh-root@peritmp4:22 already exists, disabling multiplexing
1
2
3
[root@virtualdev01 ~]# parallel --controlmaster  -S peritmp4 echo ::: 1 2 3
When using programs that use GNU Parallel to process data for publication please cite:

  O. Tange (2011): GNU Parallel - The Command-Line Power Tool,
  ;login: The USENIX Magazine, February 2011:42-47.

This helps funding further development; and it won't cost you a cent.

To silence this citation notice run 'parallel --bibtex' once or use '--no-notice'.

ControlSocket /root/.parallel/tmp/control_path_dir-8peH/ssh-root@peritmp4:22 already exists, disabling multiplexing
1
2
3
[root@virtualdev01 ~]#

Output:

[root@peritmp4 ~]# last
root     pts/3        virtualdev01     Thu Mar 27 22:54 - 22:54  (00:00)
root     pts/2        virtualdev01     Thu Mar 27 22:54 - 22:54  (00:00)
root     pts/1        virtualdev01     Thu Mar 27 22:54 - 22:54  (00:00)
root     pts/3        virtualdev01     Thu Mar 27 22:54 - 22:54  (00:00)
root     pts/2        virtualdev01     Thu Mar 27 22:54 - 22:54  (00:00)
root     pts/1        virtualdev01     Thu Mar 27 22:54 - 22:54  (00:00)
root     pts/3        virtualdev01     Thu Mar 27 22:54 - 22:54  (00:00)
root     pts/2        virtualdev01     Thu Mar 27 22:54 - 22:54  (00:00)
root     pts/1        virtualdev01     Thu Mar 27 22:54 - 22:54  (00:00)

wtmp begins Thu Mar 27 22:54:36 2014
[root@peritmp4 ~]#



I don't understand why controlmaster doesn't work as expecetd throuhg parallel.

phoebus <frphoebus>
Thu 27 Mar 2014 02:05:34 AM UTC, comment #1: 

Fixed in [06d55fc]

Ole Tange <tange>
Group administrator
Wed 26 Mar 2014 02:58:40 PM UTC, original submission:  

Please find below a test case which show that 3 ssh session are opened. it seems taht contolmaster session is not used.


Environment: OS centos 6.4 64 bits:

[root@virtualdev01 ~]# parallel --version
GNU parallel 20140322
Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014 Ole Tange and Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
GNU parallel comes with no warranty.

Web site: http://www.gnu.org/software/parallel

When using programs that use GNU Parallel to process data for publication please cite:

O. Tange (2011): GNU Parallel - The Command-Line Power Tool,
;login: The USENIX Magazine, February 2011:42-47.
[root@virtualdev01 ~]#

Reproduce the problem:
From box which launch parallel:
[root@virtualdev01 etc]# parallel --controlmaster -S peritmp4 echo ::: 1 2 3
When using programs that use GNU Parallel to process data for publication please cite:

  O. Tange (2011): GNU Parallel - The Command-Line Power Tool,
  ;login: The USENIX Magazine, February 2011:42-47.

This helps funding further development; and it won't cost you a cent.

To silence this citation notice run 'parallel --bibtex' once or use '--no-notice'.

1
2
3
[root@virtualdev01 etc]#


From box which receive parallel request:
root@peritmp4 .ssh]# > /var/log/wtmp
[root@peritmp4 .ssh]# last

wtmp begins Wed Mar 26 15:45:53 2014
[root@peritmp4 .ssh]# last
root     pts/3        virtualdev01     Wed Mar 26 15:46 - 15:46  (00:00)
root     pts/2        virtualdev01     Wed Mar 26 15:46 - 15:46  (00:00)
root     pts/1        virtualdev01     Wed Mar 26 15:46 - 15:46  (00:00)

wtmp begins Wed Mar 26 15:46:30 2014
[root@peritmp4 .ssh]#



Anonymous

 

(Note: upload size limit is set to 16384 kB, after insertion of the required escape characters.)

Attach Files:
   
   
Comment:
   

Attached Files
file #31083:  vm1-vm2.txt added by frphoebus (58KiB - text/plain)

 

Depends on the following items: None found

Items that depend on this one: None found

 

Carbon-Copy List
  • -email is unavailable- added by frphoebus (Posted a comment)
  • -email is unavailable- added by tange (Posted a comment)
  •  

    There are 0 votes so far. Votes easily highlight which items people would like to see resolved in priority, independently of the priority of the item set by tracker managers.

    Only logged-in users can vote.

     

    Follow 7 latest changes.

    Date Changed by Updated Field Previous Value => Replaced by
    2014-05-12 tange Open/ClosedOpen Closed
    2014-03-31 frphoebus Attached File- Added vm1-vm2.txt, #31083
    2014-03-29 frphoebus Attached File- Added centos5-3-controlmaster-log.txt, #31073
    2014-03-28 tange Open/ClosedClosed Open
    2014-03-27 tange StatusNone Fixed
        Assigned toNone tange
        Open/ClosedOpen Closed

    Back to the top

    Powered by Savane 3.13-3230.
    Corresponding source code