28 |
""" |
""" |
29 |
# a lock that is shared by all RemoteJob instances |
# a lock that is shared by all RemoteJob instances |
30 |
# in order to prevent the creation of same temporary file names |
# in order to prevent the creation of same temporary file names |
|
tmp_file_creation_lock = threading.Lock() |
|
31 |
nb_job = 0 |
nb_job = 0 |
32 |
|
|
33 |
def __init__(self, socket, scheduler_address, scheduer_port): |
def __init__(self, socket, scheduler_address, scheduer_port): |
103 |
# send stderr messages |
# send stderr messages |
104 |
ProtocolUtils.send_data(self.__client_socket, msg_stderr) |
ProtocolUtils.send_data(self.__client_socket, msg_stderr) |
105 |
|
|
106 |
def send_result_files_back_to_client(\ |
def send_result_files_back_to_client(self, |
107 |
self,\ |
output_filenames, |
108 |
input_files,\ |
output_files): |
|
output_files): |
|
109 |
"""Send the result (output files and output messages) to the client, |
"""Send the result (output files and output messages) to the client, |
110 |
and replace the temporary input file names in the various messages by |
and replace the temporary input file names in the various messages by |
111 |
the original ones.""" |
the original ones.""" |
112 |
|
|
|
#clean temporary input files created |
|
|
for tmp_input_file in input_files.values(): |
|
|
syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON, "Removing tem file : "\ |
|
|
+ tmp_input_file) |
|
|
os.unlink(tmp_input_file) |
|
113 |
# FIXME: don't forget to send stdout and stderr |
# FIXME: don't forget to send stdout and stderr |
114 |
|
|
115 |
# then send output files back to the client |
# then send output files back to the client |
116 |
for original_output_file_name in output_files.keys(): |
for (original_output_file_name, tmp_output_file_name) in output_filenames.items(): |
117 |
self.__client_socket.send(Protocol.FILE_COMMAND) |
self.__client_socket.send(Protocol.FILE_COMMAND) |
118 |
|
|
119 |
ProtocolUtils.send_data(self.__client_socket, original_output_file_name) |
ProtocolUtils.send_data(self.__client_socket, original_output_file_name) |
|
|
|
120 |
# send the file |
# send the file |
121 |
tmp_output_file = open(\ |
# If this not a NamedTemporaryFile |
122 |
output_files[original_output_file_name], 'rb') |
if original_output_file_name not in output_files.keys(): |
123 |
|
tmp_output_file = open(tmp_output_file_name, 'rb') |
124 |
|
|
125 |
|
else: |
126 |
|
tmp_output_file = output_files[original_output_file_name] |
127 |
|
|
128 |
if __debug__: |
if __debug__: |
129 |
syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON, "Sending file : "\ |
syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON, "Sending file : %s" % |
130 |
+ output_files[original_output_file_name]) |
tmp_output_file_name) |
131 |
ProtocolUtils.send_data(self.__client_socket, tmp_output_file.read()) |
|
132 |
tmp_output_file.close() |
tmp_output_file.seek(0) |
133 |
os.unlink(output_files[original_output_file_name]) |
ProtocolUtils.send_data(self.__client_socket, tmp_output_file.read()) |
134 |
|
|
135 |
|
# If this not a NamedTemporaryFile |
136 |
|
if original_output_file_name not in output_files.keys(): |
137 |
|
tmp_output_file.close() |
138 |
|
os.unlink(tmp_output_file_name) |
139 |
|
|
140 |
if __debug__: |
if __debug__: |
141 |
syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON, "File sent : "\ |
syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON, "File sent : %s" |
142 |
+ output_files[original_output_file_name]) |
% tmp_output_file_name) |
143 |
|
|
144 |
def compiler_job(self): |
def compiler_job(self): |
145 |
"""This method is called when a remote client |
"""This method is called when a remote client |
159 |
compiler_command_line = ProtocolUtils.recv_data(self.__client_socket) |
compiler_command_line = ProtocolUtils.recv_data(self.__client_socket) |
160 |
if __debug__: |
if __debug__: |
161 |
syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON, |
syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON, |
162 |
"Compilation command line received : "\ |
"Compilation command line received : %s" |
163 |
+ compiler_command_line) |
% compiler_command_line) |
164 |
compiler_command = CompilerCommandFactory.build_compiler_instance\ |
compiler_command = CompilerCommandFactory.build_compiler_instance\ |
165 |
(compiler_command_line.split()) |
(compiler_command_line.split()) |
166 |
|
|
167 |
|
|
168 |
# get the content of the input files used in the command line we |
# get the content of the input files used in the command line we |
169 |
# have just received |
# have just received |
170 |
|
input_temp_filenames = {} |
171 |
|
# keep a reference on temporary files to prevent it to be garbage collected |
172 |
input_temp_files = {} |
input_temp_files = {} |
173 |
output_temp_files = {} |
output_temp_files = {} |
174 |
|
output_temp_filenames = {} |
175 |
command = self.__client_socket.recv(Protocol.COMMAND_TYPE_SIZE) |
command = self.__client_socket.recv(Protocol.COMMAND_TYPE_SIZE) |
176 |
|
|
177 |
while command == Protocol.FILE_COMMAND: |
while command == Protocol.FILE_COMMAND: |
178 |
|
|
179 |
file_name = ProtocolUtils.recv_data(self.__client_socket) |
file_name = ProtocolUtils.recv_data(self.__client_socket) |
180 |
|
|
181 |
# FIXME: do we need to create the file inside the |
tmp_file = tempfile.NamedTemporaryFile("w+b", -1, FileUtils.get_file_extension(file_name)) |
182 |
# critical section ? |
|
|
RemoteJob.tmp_file_creation_lock.acquire() |
|
|
if __debug__: |
|
|
syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON,\ |
|
|
"Creating temporary file.") |
|
|
tmp_file_name = tempfile.mktemp(\ |
|
|
FileUtils.get_file_extension(file_name)) |
|
|
tmp_file = open(tmp_file_name, 'w') |
|
|
RemoteJob.tmp_file_creation_lock.release() |
|
183 |
if __debug__: |
if __debug__: |
184 |
syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON,\ |
syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON,\ |
185 |
"Temporary file created.") |
"Temporary file created: %s." % tmp_file.name) |
186 |
|
|
187 |
input_temp_files[file_name] = tmp_file_name |
input_temp_filenames[file_name] = tmp_file.name |
188 |
|
input_temp_files[file_name] = tmp_file |
189 |
|
|
190 |
tmp_file.write(ProtocolUtils.recv_data(self.__client_socket)) |
tmp_file.write(ProtocolUtils.recv_data(self.__client_socket)) |
191 |
tmp_file.flush() |
tmp_file.flush() |
192 |
tmp_file.close() |
|
193 |
command = self.__client_socket.recv(Protocol.COMMAND_TYPE_SIZE) |
command = self.__client_socket.recv(Protocol.COMMAND_TYPE_SIZE) |
|
|
|
194 |
|
|
195 |
# replace original input files in the command line by the |
# replace original input files in the command line by the |
196 |
# temporary ones |
# temporary ones |
197 |
compiler_command.replace_input_files(input_temp_files) |
compiler_command.replace_input_files(input_temp_filenames) |
198 |
if __debug__: |
if __debug__: |
199 |
syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON, \ |
syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON, \ |
200 |
"New compilation command line :"\ |
"New compilation command line : %s"\ |
201 |
+ " ".join(compiler_command.get_command_args())) |
% " ".join(compiler_command.get_command_args())) |
202 |
|
|
203 |
|
|
204 |
# FIXME We should not use "-o" here, this is compiler dependant |
# FIXME We should not use "-o" here, this is compiler dependant |
210 |
index_output = compiler_command.get_command_args().index("-o") \ |
index_output = compiler_command.get_command_args().index("-o") \ |
211 |
+ 1 |
+ 1 |
212 |
|
|
213 |
# FIXME: do we need to create the file inside the |
tmp_output_file = tempfile.NamedTemporaryFile() |
214 |
# critical section ? |
|
|
RemoteJob.tmp_file_creation_lock.acquire() |
|
|
tmp_output_file = tempfile.mktemp() |
|
|
tmp_output_file_hdl = open(tmp_output_file, 'w') |
|
|
tmp_output_file_hdl.close() |
|
|
RemoteJob.tmp_file_creation_lock.release() |
|
215 |
# associate the output tmp file with the original one |
# associate the output tmp file with the original one |
216 |
output_file_name = compiler_command.get_command_args()\ |
output_file_name = compiler_command.get_command_args()\ |
217 |
[index_output] |
[index_output] |
218 |
|
|
219 |
|
output_temp_filenames[output_file_name] = tmp_output_file.name |
220 |
output_temp_files[output_file_name] = tmp_output_file |
output_temp_files[output_file_name] = tmp_output_file |
221 |
|
|
222 |
# replace the output file name in the command line |
# replace the output file name in the command line |
223 |
compiler_command.get_command_args(\ |
compiler_command.get_command_args(\ |
224 |
)[index_output] = tmp_output_file |
)[index_output] = tmp_output_file.name |
225 |
except IndexError: |
except IndexError: |
226 |
# if there is no file name after the -o option, |
# if there is no file name after the -o option, |
227 |
# it means that the command line is wrong, but we |
# it means that the command line is wrong, but we |
233 |
if __debug__: |
if __debug__: |
234 |
syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON, \ |
syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON, \ |
235 |
"No -o option in command line.") |
"No -o option in command line.") |
236 |
for original_input_file_name in input_temp_files.keys(): |
for original_input_file_name in input_temp_filenames.keys(): |
237 |
stop_step = compiler_command.get_stop_step() |
stop_step = compiler_command.get_stop_step() |
238 |
|
|
239 |
orig_output_file_name = compiler_command.\ |
orig_output_file_name = compiler_command.\ |
240 |
get_output_file_name_for_step(\ |
get_output_file_name_for_step(\ |
241 |
original_input_file_name,\ |
original_input_file_name,\ |
242 |
stop_step) |
stop_step) |
243 |
output_temp_files[\ |
|
244 |
|
# FIXME: Here there is a problem ... we do not give a temp filename |
245 |
|
output_temp_filenames[\ |
246 |
orig_output_file_name] = compiler_command.\ |
orig_output_file_name] = compiler_command.\ |
247 |
get_output_file_name_for_step(\ |
get_output_file_name_for_step( |
248 |
input_temp_files[original_input_file_name],\ |
input_temp_filenames[original_input_file_name], |
249 |
stop_step) |
stop_step) |
250 |
|
|
251 |
if __debug__: |
if __debug__: |
252 |
syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON, \ |
syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON, \ |
253 |
"File to return to the client : "\ |
"File to return to the client : %s" % |
254 |
+ output_temp_files[orig_output_file_name]) |
output_temp_filenames[orig_output_file_name]) |
255 |
|
|
256 |
# execute the command in a subshell and get the stdout and stderr output |
# execute the command in a subshell and get the stdout and stderr output |
257 |
if __debug__: |
if __debug__: |
271 |
proc.stdout.close() |
proc.stdout.close() |
272 |
exit_code = proc.wait() |
exit_code = proc.wait() |
273 |
|
|
274 |
self.send_output_messages(msg_stdout, msg_stderr, input_temp_files,\ |
self.send_output_messages(msg_stdout, msg_stderr, |
275 |
output_temp_files) |
input_temp_filenames, |
276 |
|
output_temp_filenames) |
277 |
|
|
278 |
if os.WIFEXITED(exit_code): |
if os.WIFEXITED(exit_code): |
279 |
exit_code = os.WEXITSTATUS(exit_code) |
exit_code = os.WEXITSTATUS(exit_code) |
284 |
"Exit code : " + str(exit_code)) |
"Exit code : " + str(exit_code)) |
285 |
if exit_code == 0: |
if exit_code == 0: |
286 |
# send the result (output files and output messages) to the client |
# send the result (output files and output messages) to the client |
287 |
self.send_result_files_back_to_client(input_temp_files,\ |
self.send_result_files_back_to_client(output_temp_filenames, |
288 |
output_temp_files) |
output_temp_files) |
289 |
if __debug__: |
if __debug__: |
290 |
syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON,\ |
syslog.syslog(syslog.LOG_DEBUG | syslog.LOG_DAEMON,\ |