33 |
open CommonFile |
open CommonFile |
34 |
open CommonClient |
open CommonClient |
35 |
open CommonComplexOptions |
open CommonComplexOptions |
36 |
open CommonOptions |
open CommonOptions |
37 |
open CommonResult |
open CommonResult |
38 |
open CommonTypes |
open CommonTypes |
39 |
|
|
40 |
open GuiTypes |
open GuiTypes |
41 |
open GuiProto |
open GuiProto |
42 |
|
|
43 |
open DonkeyTypes |
open DonkeyTypes |
44 |
open DonkeyMftp |
open DonkeyMftp |
45 |
open DonkeyProtoCom |
open DonkeyProtoCom |
46 |
open DonkeySources |
open DonkeySources |
53 |
| History |
| History |
54 |
| SizeLRU |
| SizeLRU |
55 |
| ExtLRU |
| ExtLRU |
56 |
|
|
57 |
(* This means that we have just received the fact that this client |
(* prints a new logline with date, module and starts newline *) |
58 |
|
let lprintf_nl () = |
59 |
|
lprintf "%s[EDK] " |
60 |
|
(log_time ()); lprintf_nl2 |
61 |
|
|
62 |
|
(* prints a new logline with date, module and does not start newline *) |
63 |
|
let lprintf_n () = |
64 |
|
lprintf "%s[EDK] " |
65 |
|
(log_time ()); lprintf |
66 |
|
|
67 |
|
(* This means that we have just received the fact that this client |
68 |
has this file. *) |
has this file. *) |
69 |
|
|
70 |
type file_info = { |
type file_info = { |
72 |
fi_ext : string; |
fi_ext : string; |
73 |
mutable fi_time : int; |
mutable fi_time : int; |
74 |
} |
} |
75 |
|
|
76 |
type neighbour = { |
type neighbour = { |
77 |
mutable nb_files : (int * file_info) list; |
mutable nb_files : (int * file_info) list; |
78 |
mutable nb_sort : int; |
mutable nb_sort : int; |
79 |
mutable nb_client : client; |
mutable nb_client : client; |
80 |
} |
} |
81 |
|
|
82 |
let neighbours = Hashtbl.create 131 |
let neighbours = Hashtbl.create 131 |
83 |
|
|
84 |
let file_info file time = |
let file_info file time = |
89 |
fi_ext = String.lowercase ( |
fi_ext = String.lowercase ( |
90 |
Filename2.last_extension (file_best_name file)); |
Filename2.last_extension (file_best_name file)); |
91 |
} |
} |
92 |
|
|
93 |
exception Continue |
exception Continue |
94 |
|
|
95 |
let ngood_propositions = Array.create 4 zero |
let ngood_propositions = Array.create 4 zero |
96 |
let nbad_propositions = Array.create 4 zero |
let nbad_propositions = Array.create 4 zero |
97 |
|
|
98 |
let incr_propositions kind result = |
let incr_propositions kind result = |
99 |
let propositions = match result with |
let propositions = match result with |
100 |
File_not_found -> nbad_propositions |
File_not_found -> nbad_propositions |
101 |
| File_found -> ngood_propositions |
| File_found -> ngood_propositions |
110 |
propositions.(kind) <- propositions.(kind) ++ one |
propositions.(kind) <- propositions.(kind) ++ one |
111 |
|
|
112 |
let propositions = Fifo.create () |
let propositions = Fifo.create () |
113 |
|
|
114 |
let rec compute_stats () = |
let rec compute_stats () = |
115 |
if not (Fifo.empty propositions) then |
if not (Fifo.empty propositions) then |
116 |
let (s, file, kind, time) = Fifo.head propositions in |
let (s, file, kind, time) = Fifo.head propositions in |
120 |
| File_new_source -> |
| File_new_source -> |
121 |
if time + 1800 < last_time () then begin |
if time + 1800 < last_time () then begin |
122 |
if !verbose_hidden_errors then |
if !verbose_hidden_errors then |
123 |
lprintf "DonkeyNeighbours: WARNING, source was not tested\n"; |
lprintf_nl () "WARNING, source was not tested"; |
124 |
raise Continue |
raise Continue |
125 |
end |
end |
126 |
| File_expected -> |
| File_expected -> |
130 |
end |
end |
131 |
| File_possible -> |
| File_possible -> |
132 |
if !verbose_hidden_errors then |
if !verbose_hidden_errors then |
133 |
lprintf "DonkeyNeighbours: WARNING, source was unknown\n"; |
lprintf_nl () "WARNING, source was unknown"; |
134 |
if time + 1800 < last_time () then raise Continue |
if time + 1800 < last_time () then raise Continue |
135 |
| File_not_found -> |
| File_not_found -> |
136 |
incr_propositions kind File_not_found; |
incr_propositions kind File_not_found; |
142 |
raise Continue |
raise Continue |
143 |
| File_unknown -> assert false |
| File_unknown -> assert false |
144 |
with |
with |
145 |
| Not_found -> |
| Not_found -> |
146 |
(* For some reason, the request was forgotten. Forget it... *) |
(* For some reason, the request was forgotten. Forget it... *) |
147 |
if !verbose_hidden_errors then |
if !verbose_hidden_errors then |
148 |
lprintf "DonkeyNeighbours: ERROR, request was forgotten\n"; |
lprintf_nl () "ERROR, request was forgotten"; |
149 |
let _ = Fifo.take propositions in |
let _ = Fifo.take propositions in |
150 |
compute_stats () |
compute_stats () |
151 |
| Continue -> |
| Continue -> |
152 |
let _ = Fifo.take propositions in |
let _ = Fifo.take propositions in |
153 |
compute_stats () |
compute_stats () |
154 |
|
|
155 |
let propose_source file c kind = |
let propose_source file c kind = |
156 |
try |
try |
157 |
List.iter (fun r -> |
List.iter (fun r -> |
158 |
if r.request_file == file.file_sources then raise Exit |
if r.request_file == file.file_sources then raise Exit |
163 |
let s = DonkeySources.find_source_by_uid c.client_kind in |
let s = DonkeySources.find_source_by_uid c.client_kind in |
164 |
DonkeySources.set_request_result s file.file_sources File_new_source; |
DonkeySources.set_request_result s file.file_sources File_new_source; |
165 |
Fifo.put propositions (s, file, kind, last_time ()) |
Fifo.put propositions (s, file, kind, last_time ()) |
166 |
| _ -> |
| _ -> |
167 |
if !verbose_hidden_errors then |
if !verbose_hidden_errors then |
168 |
lprintf "DonkeyNeighbours: ERROR, proposed client is indirect\n"; |
lprintf_nl () "ERROR, proposed client is indirect"; |
169 |
raise Exit |
raise Exit |
170 |
|
|
171 |
with Exit -> |
with Exit -> |
172 |
(* This client might already have queried this source *) |
(* This client might already have queried this source *) |
173 |
() |
() |
174 |
|
|
175 |
let new_neighbour c file = |
let new_neighbour c file = |
176 |
compute_stats (); |
compute_stats (); |
177 |
match c.client_kind with |
match c.client_kind with |
178 |
Indirect_address _ -> () |
Indirect_address _ -> () |
179 |
| _ -> |
| _ -> |
180 |
|
|
181 |
let client_num = client_num c in |
let client_num = client_num c in |
182 |
let file_num = file_num file in |
let file_num = file_num file in |
183 |
let time = last_time () in |
let time = last_time () in |
184 |
|
|
185 |
try |
try |
186 |
let nb = Hashtbl.find neighbours client_num in |
let nb = Hashtbl.find neighbours client_num in |
187 |
try |
try |
196 |
nb_sort = min_int; |
nb_sort = min_int; |
197 |
nb_client = c; |
nb_client = c; |
198 |
} |
} |
199 |
|
|
200 |
(* |
(* |
201 |
We should try to sort files by "proximity", and then try to move sources |
We should try to sort files by "proximity", and then try to move sources |
202 |
from one to another, maybe depending on the popularity of the file. |
from one to another, maybe depending on the popularity of the file. |
203 |
Before adding a client as a source for a file, we should check that |
Before adding a client as a source for a file, we should check that |
208 |
* Propose the 5 best History sources to all current files |
* Propose the 5 best History sources to all current files |
209 |
* For every file, propose 5 best LRU sources by extension, and 5 best LRU |
* For every file, propose 5 best LRU sources by extension, and 5 best LRU |
210 |
sources by size |
sources by size |
|
|
|
211 |
*) |
*) |
212 |
|
|
213 |
let recover_downloads current_files = |
let recover_downloads current_files = |
214 |
|
|
215 |
(* Global LRU *) |
(* Global LRU *) |
216 |
let list = ref [] in |
let list = ref [] in |
228 |
let s2 = nb2.nb_sort in |
let s2 = nb2.nb_sort in |
229 |
if s1 > s2 then -1 else |
if s1 > s2 then -1 else |
230 |
if s2 > s1 then 1 else 0) !list in |
if s2 > s1 then 1 else 0) !list in |
231 |
|
|
232 |
let keep_glru,_ = List2.cut 333 glru in |
let keep_glru,_ = List2.cut 333 glru in |
233 |
let best_glru,_ = List2.cut 5 glru in |
let best_glru,_ = List2.cut 5 glru in |
234 |
(* For each file, propose the 5 first elements of the GLRU as a potential |
(* For each file, propose the 5 first elements of the GLRU as a potential |
250 |
let s2 = nb2.nb_sort in |
let s2 = nb2.nb_sort in |
251 |
if s1 > s2 then -1 else |
if s1 > s2 then -1 else |
252 |
if s2 > s1 then 1 else 0) !list in |
if s2 > s1 then 1 else 0) !list in |
253 |
|
|
254 |
let keep_history,_ = List2.cut 333 history in |
let keep_history,_ = List2.cut 333 history in |
255 |
let best_history,_ = List2.cut 5 history in |
let best_history,_ = List2.cut 5 history in |
256 |
(* For each file, propose the 5 first elements of the GLRU as a potential |
(* For each file, propose the 5 first elements of the GLRU as a potential |
271 |
[] -> assert false |
[] -> assert false |
272 |
| nb2 :: _ -> |
| nb2 :: _ -> |
273 |
if nb2 != nb then list := nb :: !list |
if nb2 != nb then list := nb :: !list |
274 |
with Not_found -> |
with Not_found -> |
275 |
sizes_list := (fi.fi_size, ref [nb]) :: !sizes_list |
sizes_list := (fi.fi_size, ref [nb]) :: !sizes_list |
276 |
); |
); |
277 |
|
|
278 |
(try |
(try |
279 |
let list = List.assq fi.fi_ext !extensions_list in |
let list = List.assq fi.fi_ext !extensions_list in |
280 |
match !list with |
match !list with |
281 |
[] -> assert false |
[] -> assert false |
282 |
| nb2 :: _ -> |
| nb2 :: _ -> |
283 |
if nb2 != nb then list := nb :: !list |
if nb2 != nb then list := nb :: !list |
284 |
with Not_found -> |
with Not_found -> |
285 |
extensions_list := (fi.fi_ext, ref [nb]) :: !extensions_list |
extensions_list := (fi.fi_ext, ref [nb]) :: !extensions_list |
286 |
) |
) |
287 |
|
|
288 |
) nb.nb_files |
) nb.nb_files |
289 |
) neighbours; |
) neighbours; |
290 |
|
|
291 |
List.iter (fun (ext, list) -> |
List.iter (fun (ext, list) -> |
292 |
List.iter (fun nb -> |
List.iter (fun nb -> |
293 |
nb.nb_sort <- min_int; |
nb.nb_sort <- min_int; |
301 |
let s2 = nb2.nb_sort in |
let s2 = nb2.nb_sort in |
302 |
if s1 > s2 then -1 else |
if s1 > s2 then -1 else |
303 |
if s2 > s1 then 1 else 0) !list |
if s2 > s1 then 1 else 0) !list |
304 |
|
|
|
|
|
305 |
) !extensions_list; |
) !extensions_list; |
306 |
|
|
307 |
List.iter (fun (size, list) -> |
List.iter (fun (size, list) -> |
308 |
List.iter (fun nb -> |
List.iter (fun nb -> |
309 |
nb.nb_sort <- min_int; |
nb.nb_sort <- min_int; |
318 |
if s1 > s2 then -1 else |
if s1 > s2 then -1 else |
319 |
if s2 > s1 then 1 else 0) !list |
if s2 > s1 then 1 else 0) !list |
320 |
) !sizes_list; |
) !sizes_list; |
321 |
|
|
|
|
|
322 |
List.iter (fun file -> |
List.iter (fun file -> |
323 |
let fi_size = int_of_float (log ( |
let fi_size = int_of_float (log ( |
324 |
Int64.to_float (file_size file ++ one))) in |
Int64.to_float (file_size file ++ one))) in |
325 |
let fi_ext = String.lowercase ( |
let fi_ext = String.lowercase ( |
326 |
Filename2.last_extension (file_best_name file)) in |
Filename2.last_extension (file_best_name file)) in |
327 |
|
|
328 |
(try |
(try |
329 |
let list = List.assoc fi_size !sizes_list in |
let list = List.assoc fi_size !sizes_list in |
330 |
let best_slru, _ = List2.cut 5 !list in |
let best_slru, _ = List2.cut 5 !list in |
331 |
List.iter (fun nb -> |
List.iter (fun nb -> |
332 |
propose_source file nb.nb_client SizeLRU) best_slru |
propose_source file nb.nb_client SizeLRU) best_slru |
333 |
with _ -> ()); |
with _ -> ()); |
334 |
|
|
335 |
(try |
(try |
336 |
let list = List.assoc fi_ext !extensions_list in |
let list = List.assoc fi_ext !extensions_list in |
337 |
let best_slru, _ = List2.cut 5 !list in |
let best_slru, _ = List2.cut 5 !list in |
338 |
List.iter (fun nb -> |
List.iter (fun nb -> |
339 |
propose_source file nb.nb_client ExtLRU) best_slru |
propose_source file nb.nb_client ExtLRU) best_slru |
340 |
with _ -> ()); |
with _ -> ()); |
|
|
|
341 |
) current_files; |
) current_files; |
342 |
|
|
343 |
|
|
347 |
list_list := !list :: !list_list) !extensions_list; |
list_list := !list :: !list_list) !extensions_list; |
348 |
List.iter (fun (_, list) -> |
List.iter (fun (_, list) -> |
349 |
list_list := !list :: !list_list) !sizes_list; |
list_list := !list :: !list_list) !sizes_list; |
350 |
|
|
351 |
let rec iter rem todo_list done_list keep = |
let rec iter rem todo_list done_list keep = |
352 |
if rem = 0 then keep else |
if rem = 0 then keep else |
353 |
match todo_list with |
match todo_list with |
367 |
i.e. the table can only retain at most 1 MB of data after every call |
i.e. the table can only retain at most 1 MB of data after every call |
368 |
to this function. *) |
to this function. *) |
369 |
Hashtbl.clear neighbours; |
Hashtbl.clear neighbours; |
370 |
|
|
371 |
List.iter (fun list -> |
List.iter (fun list -> |
372 |
List.iter (fun nb -> |
List.iter (fun nb -> |
373 |
let cnum = client_num nb.nb_client in |
let cnum = client_num nb.nb_client in |
374 |
if not (Hashtbl.mem neighbours cnum) then |
if not (Hashtbl.mem neighbours cnum) then |
375 |
Hashtbl.add neighbours cnum nb |
Hashtbl.add neighbours cnum nb |
376 |
) list) |
) list) |
377 |
[keep_glru; keep_history; keep_slru] |
[keep_glru; keep_history; keep_slru] |
378 |
|
|
|
|
|
|
|
|
379 |
open LittleEndian |
open LittleEndian |
380 |
|
|
381 |
let _ = |
let _ = |
382 |
CommonWeb.add_redirector_info "DKNB" (fun buf -> |
CommonWeb.add_redirector_info "DKNB" (fun buf -> |
383 |
compute_stats (); |
compute_stats (); |
384 |
let len = Array.length ngood_propositions in |
let len = Array.length ngood_propositions in |
385 |
buf_int buf len; |
buf_int buf len; |
386 |
if !verbose_redirector then lprintf "DonkeyNeighbours stats:\n"; |
if !verbose_redirector then lprintf_n () "Stats:"; |
387 |
for i = 0 to len - 1 do |
for i = 0 to len - 1 do |
388 |
if !verbose_redirector then lprintf " Good[%d] = %Ld\n" i ngood_propositions.(i); |
if !verbose_redirector then lprintf_nl2 " Good[%d] = %Ld\n" i ngood_propositions.(i); |
389 |
if !verbose_redirector then lprintf " Bad[%d] = %Ld\n" i nbad_propositions.(i); |
if !verbose_redirector then lprintf_nl2 " Bad[%d] = %Ld\n" i nbad_propositions.(i); |
390 |
buf_int64 buf ngood_propositions.(i); |
buf_int64 buf ngood_propositions.(i); |
391 |
buf_int64 buf nbad_propositions.(i) |
buf_int64 buf nbad_propositions.(i) |
392 |
done; |
done; |
393 |
let len = Fifo.length propositions in |
let len = Fifo.length propositions in |
394 |
if !verbose_redirector then lprintf " Waiting propositions: %d\n" len; |
if !verbose_redirector then lprintf_nl2 " Waiting propositions: %d\n" len; |
395 |
buf_int buf len; |
buf_int buf len; |
396 |
let len = ref 0 in |
let len = ref 0 in |
397 |
Hashtbl.iter (fun _ _ -> incr len) neighbours; |
Hashtbl.iter (fun _ _ -> incr len) neighbours; |
398 |
if !verbose_redirector then lprintf " Total neighbours: %d\n" !len; |
if !verbose_redirector then lprintf_nl2 " Total neighbours: %d\n" !len; |
399 |
buf_int buf !len |
buf_int buf !len |
400 |
) |
) |