/[dms]/dms/src/RemoteJob.py
ViewVC logotype

Diff of /dms/src/RemoteJob.py

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 1.10.2.3 by darktigrou, Sun Mar 27 21:01:07 2005 UTC revision 1.10.2.4 by klyr, Sat Apr 2 17:15:01 2005 UTC
# Line 28  class RemoteJob(threading.Thread): Line 28  class RemoteJob(threading.Thread):
28      """      """
29      # a lock that is shared by all RemoteJob instances      # a lock that is shared by all RemoteJob instances
30      # in order to prevent the creation of same temporary file names      # in order to prevent the creation of same temporary file names
     tmp_file_creation_lock = threading.Lock()  
31      nb_job = 0      nb_job = 0
32    
33      def __init__(self, socket, scheduler_address, scheduer_port):      def __init__(self, socket, scheduler_address, scheduer_port):
# Line 104  class RemoteJob(threading.Thread): Line 103  class RemoteJob(threading.Thread):
103          # send stderr messages          # send stderr messages
104          ProtocolUtils.send_data(self.__client_socket, msg_stderr)          ProtocolUtils.send_data(self.__client_socket, msg_stderr)
105        
106      def send_result_files_back_to_client(\      def send_result_files_back_to_client(self,
107          self,\                                           output_filenames,
108          input_files,\                                           output_files):
         output_files):  
109          """Send the result (output files and output messages) to the client,          """Send the result (output files and output messages) to the client,
110          and replace the temporary input file names in the various messages by          and replace the temporary input file names in the various messages by
111          the original ones."""          the original ones."""
112    
         #clean temporary input files created  
         for tmp_input_file in input_files.values():  
             syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON, "Removing tem file : "\  
                           + tmp_input_file)  
             os.unlink(tmp_input_file)  
113          # FIXME: don't forget to send stdout and stderr          # FIXME: don't forget to send stdout and stderr
114                    
115          # then send output files back to the client          # then send output files back to the client
116          for original_output_file_name in output_files.keys():          for (original_output_file_name, tmp_output_file_name) in output_filenames.items():
117              self.__client_socket.send(Protocol.FILE_COMMAND)              self.__client_socket.send(Protocol.FILE_COMMAND)
118                
119              ProtocolUtils.send_data(self.__client_socket, original_output_file_name)              ProtocolUtils.send_data(self.__client_socket, original_output_file_name)
   
120              # send the file              # send the file
121              tmp_output_file = open(\              # If this not a NamedTemporaryFile
122                  output_files[original_output_file_name], 'rb')              if original_output_file_name not in output_files.keys():
123                    tmp_output_file = open(tmp_output_file_name, 'rb')
124    
125                else:
126                    tmp_output_file = output_files[original_output_file_name]
127    
128              if __debug__:              if __debug__:
129                  syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON, "Sending file : "\                  syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON, "Sending file : %s" %
130                                + output_files[original_output_file_name])                                tmp_output_file_name)
131              ProtocolUtils.send_data(self.__client_socket, tmp_output_file.read())                  
132              tmp_output_file.close()              tmp_output_file.seek(0)
133              os.unlink(output_files[original_output_file_name])              ProtocolUtils.send_data(self.__client_socket, tmp_output_file.read())
134    
135                # If this not a NamedTemporaryFile
136                if original_output_file_name not in output_files.keys():
137                    tmp_output_file.close()
138                    os.unlink(tmp_output_file_name)
139    
140              if __debug__:              if __debug__:
141                  syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON, "File sent : "\                  syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON, "File sent : %s"
142                                + output_files[original_output_file_name])                                % tmp_output_file_name)
143                    
144      def compiler_job(self):      def compiler_job(self):
145          """This method is called when a remote client          """This method is called when a remote client
# Line 156  class RemoteJob(threading.Thread): Line 159  class RemoteJob(threading.Thread):
159          compiler_command_line = ProtocolUtils.recv_data(self.__client_socket)          compiler_command_line = ProtocolUtils.recv_data(self.__client_socket)
160          if __debug__:          if __debug__:
161              syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON,              syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON,
162                            "Compilation command line received : "\                            "Compilation command line received : %s"
163                            + compiler_command_line)                            % compiler_command_line)
164          compiler_command = CompilerCommandFactory.build_compiler_instance\          compiler_command = CompilerCommandFactory.build_compiler_instance\
165                             (compiler_command_line.split())                             (compiler_command_line.split())
166    
167    
168          # get the content of the input files used in the command line we          # get the content of the input files used in the command line we
169          # have just received          # have just received
170            input_temp_filenames = {}
171            # keep a reference on temporary files to prevent it to be garbage collected
172          input_temp_files = {}          input_temp_files = {}
173          output_temp_files = {}          output_temp_files = {}
174            output_temp_filenames = {}
175          command = self.__client_socket.recv(Protocol.COMMAND_TYPE_SIZE)          command = self.__client_socket.recv(Protocol.COMMAND_TYPE_SIZE)
176                                                                                            
177          while command == Protocol.FILE_COMMAND:          while command == Protocol.FILE_COMMAND:
178    
179              file_name = ProtocolUtils.recv_data(self.__client_socket)              file_name = ProtocolUtils.recv_data(self.__client_socket)
180    
181              # FIXME: do we need to create the file inside the              tmp_file = tempfile.NamedTemporaryFile("w+b", -1, FileUtils.get_file_extension(file_name))
182              # critical section ?  
             RemoteJob.tmp_file_creation_lock.acquire()  
             if __debug__:  
                 syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON,\  
                               "Creating temporary file.")  
             tmp_file_name = tempfile.mktemp(\  
                 FileUtils.get_file_extension(file_name))  
             tmp_file = open(tmp_file_name, 'w')  
             RemoteJob.tmp_file_creation_lock.release()  
183              if __debug__:              if __debug__:
184                  syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON,\                  syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON,\
185                                "Temporary file created.")                                "Temporary file created: %s." % tmp_file.name)
186    
187              input_temp_files[file_name] = tmp_file_name              input_temp_filenames[file_name] = tmp_file.name
188                input_temp_files[file_name] = tmp_file
189                
190              tmp_file.write(ProtocolUtils.recv_data(self.__client_socket))              tmp_file.write(ProtocolUtils.recv_data(self.__client_socket))
191              tmp_file.flush()              tmp_file.flush()
192              tmp_file.close()  
193              command = self.__client_socket.recv(Protocol.COMMAND_TYPE_SIZE)              command = self.__client_socket.recv(Protocol.COMMAND_TYPE_SIZE)
                                                   
194    
195          # replace original input files in the command line by the          # replace original input files in the command line by the
196          # temporary ones          # temporary ones
197          compiler_command.replace_input_files(input_temp_files)          compiler_command.replace_input_files(input_temp_filenames)
198          if __debug__:          if __debug__:
199              syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON, \              syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON, \
200                            "New compilation command line :"\                            "New compilation command line : %s"\
201                            + " ".join(compiler_command.get_command_args()))                             % " ".join(compiler_command.get_command_args()))
202    
203    
204          # FIXME We should not use "-o" here, this is compiler dependant          # FIXME We should not use "-o" here, this is compiler dependant
# Line 211  class RemoteJob(threading.Thread): Line 210  class RemoteJob(threading.Thread):
210                  index_output = compiler_command.get_command_args().index("-o") \                  index_output = compiler_command.get_command_args().index("-o") \
211                                 + 1                                 + 1
212                                    
213                  # FIXME: do we need to create the file inside the                  tmp_output_file = tempfile.NamedTemporaryFile()
214                  # critical section ?  
                 RemoteJob.tmp_file_creation_lock.acquire()  
                 tmp_output_file = tempfile.mktemp()  
                 tmp_output_file_hdl = open(tmp_output_file, 'w')  
                 tmp_output_file_hdl.close()  
                 RemoteJob.tmp_file_creation_lock.release()  
215                  # associate the output tmp file with the original one                  # associate the output tmp file with the original one
216                  output_file_name = compiler_command.get_command_args()\                  output_file_name = compiler_command.get_command_args()\
217                                    [index_output]                                    [index_output]
218    
219                    output_temp_filenames[output_file_name] = tmp_output_file.name
220                  output_temp_files[output_file_name] = tmp_output_file                  output_temp_files[output_file_name] = tmp_output_file
221                    
222                  # replace the output file name in the command line                  # replace the output file name in the command line
223                  compiler_command.get_command_args(\                  compiler_command.get_command_args(\
224                      )[index_output] = tmp_output_file                      )[index_output] = tmp_output_file.name
225              except IndexError:              except IndexError:
226                  # if there is no file name after the -o option,                  # if there is no file name after the -o option,
227                  # it means that the command line is wrong, but we                  # it means that the command line is wrong, but we
# Line 236  class RemoteJob(threading.Thread): Line 233  class RemoteJob(threading.Thread):
233              if __debug__:              if __debug__:
234                  syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON, \                  syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON, \
235                                "No -o option in command line.")                                "No -o option in command line.")
236              for original_input_file_name in input_temp_files.keys():              for original_input_file_name in input_temp_filenames.keys():
237                  stop_step = compiler_command.get_stop_step()                  stop_step = compiler_command.get_stop_step()
238    
239                  orig_output_file_name = compiler_command.\                  orig_output_file_name = compiler_command.\
240                                          get_output_file_name_for_step(\                                          get_output_file_name_for_step(\
241                      original_input_file_name,\                      original_input_file_name,\
242                      stop_step)                      stop_step)
243                  output_temp_files[\  
244                    # FIXME: Here there is a problem ... we do not give a temp filename
245                    output_temp_filenames[\
246                      orig_output_file_name] = compiler_command.\                      orig_output_file_name] = compiler_command.\
247                      get_output_file_name_for_step(\                      get_output_file_name_for_step(
248                      input_temp_files[original_input_file_name],\                      input_temp_filenames[original_input_file_name],
249                      stop_step)                      stop_step)
250                                    
251                  if __debug__:                  if __debug__:
252                      syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON, \                      syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON, \
253                                    "File to return to the client : "\                                    "File to return to the client : %s" %
254                                    + output_temp_files[orig_output_file_name])                                    output_temp_filenames[orig_output_file_name])
255                                    
256          # execute the command in a subshell and get the stdout and stderr output          # execute the command in a subshell and get the stdout and stderr output
257          if __debug__:          if __debug__:
# Line 271  class RemoteJob(threading.Thread): Line 271  class RemoteJob(threading.Thread):
271          proc.stdout.close()          proc.stdout.close()
272          exit_code = proc.wait()          exit_code = proc.wait()
273    
274          self.send_output_messages(msg_stdout, msg_stderr, input_temp_files,\          self.send_output_messages(msg_stdout, msg_stderr,
275                                    output_temp_files)                                    input_temp_filenames,
276                                      output_temp_filenames)
277    
278          if os.WIFEXITED(exit_code):          if os.WIFEXITED(exit_code):
279              exit_code = os.WEXITSTATUS(exit_code)              exit_code = os.WEXITSTATUS(exit_code)
# Line 283  class RemoteJob(threading.Thread): Line 284  class RemoteJob(threading.Thread):
284                            "Exit code : " + str(exit_code))                            "Exit code : " + str(exit_code))
285          if exit_code == 0:          if exit_code == 0:
286              # send the result (output files and output messages) to the client              # send the result (output files and output messages) to the client
287              self.send_result_files_back_to_client(input_temp_files,\              self.send_result_files_back_to_client(output_temp_filenames,
288                                                    output_temp_files)                                                    output_temp_files)
289              if __debug__:              if __debug__:
290                  syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON,\                  syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON,\

Legend:
Removed from v.1.10.2.3  
changed lines
  Added in v.1.10.2.4

savannah-hackers-public@gnu.org
ViewVC Help
Powered by ViewVC 1.1.26