/[mldonkey]/mldonkey/src/networks/donkey/donkeyNeighbours.ml
ViewVC logotype

Diff of /mldonkey/src/networks/donkey/donkeyNeighbours.ml

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 1.6 by spiralvoice, Sun Apr 24 19:44:16 2005 UTC revision 1.7 by spiralvoice, Mon Aug 8 16:47:31 2005 UTC
# Line 33  open CommonGlobals Line 33  open CommonGlobals
33  open CommonFile  open CommonFile
34  open CommonClient  open CommonClient
35  open CommonComplexOptions  open CommonComplexOptions
36  open CommonOptions    open CommonOptions
37  open CommonResult  open CommonResult
38  open CommonTypes  open CommonTypes
39    
40  open GuiTypes  open GuiTypes
41  open GuiProto    open GuiProto
42    
43  open DonkeyTypes    open DonkeyTypes
44  open DonkeyMftp  open DonkeyMftp
45  open DonkeyProtoCom  open DonkeyProtoCom
46  open DonkeySources  open DonkeySources
# Line 53  type proposal = Line 53  type proposal =
53  | History  | History
54  | SizeLRU  | SizeLRU
55  | ExtLRU  | ExtLRU
56      
57  (* This means that we have just received the fact that this client  (* prints a new logline with date, module and starts newline *)
58    let lprintf_nl () =
59      lprintf "%s[EDK] "
60        (log_time ()); lprintf_nl2
61    
62    (* prints a new logline with date, module and does not start newline *)
63    let lprintf_n () =
64      lprintf "%s[EDK] "
65        (log_time ()); lprintf
66    
67    (* This means that we have just received the fact that this client
68  has this file. *)  has this file. *)
69    
70  type file_info = {  type file_info = {
# Line 62  type file_info = { Line 72  type file_info = {
72      fi_ext : string;      fi_ext : string;
73      mutable fi_time : int;      mutable fi_time : int;
74    }    }
75      
76  type neighbour = {  type neighbour = {
77      mutable nb_files : (int * file_info) list;      mutable nb_files : (int * file_info) list;
78      mutable nb_sort : int;      mutable nb_sort : int;
79      mutable nb_client : client;      mutable nb_client : client;
80    }    }
81      
82  let neighbours = Hashtbl.create 131  let neighbours = Hashtbl.create 131
83    
84  let file_info file time =  let file_info file time =
# Line 79  let file_info file time = Line 89  let file_info file time =
89      fi_ext = String.lowercase (      fi_ext = String.lowercase (
90        Filename2.last_extension (file_best_name file));        Filename2.last_extension (file_best_name file));
91    }    }
92      
93  exception Continue    exception Continue
94      
95  let ngood_propositions = Array.create 4 zero  let ngood_propositions = Array.create 4 zero
96  let nbad_propositions = Array.create 4 zero  let nbad_propositions = Array.create 4 zero
97      
98  let incr_propositions kind result =  let incr_propositions kind result =
99    let propositions = match result with    let propositions = match result with
100        File_not_found -> nbad_propositions        File_not_found -> nbad_propositions
101      | File_found -> ngood_propositions      | File_found -> ngood_propositions
# Line 100  let incr_propositions kind result = Line 110  let incr_propositions kind result =
110    propositions.(kind) <- propositions.(kind) ++ one    propositions.(kind) <- propositions.(kind) ++ one
111    
112  let propositions = Fifo.create ()  let propositions = Fifo.create ()
113      
114  let rec compute_stats () =  let rec compute_stats () =
115    if not (Fifo.empty propositions) then    if not (Fifo.empty propositions) then
116      let (s, file, kind, time) = Fifo.head propositions in      let (s, file, kind, time) = Fifo.head propositions in
# Line 110  let rec compute_stats () = Line 120  let rec compute_stats () =
120        | File_new_source ->        | File_new_source ->
121            if time + 1800 < last_time () then begin            if time + 1800 < last_time () then begin
122                if !verbose_hidden_errors then                if !verbose_hidden_errors then
123                  lprintf "DonkeyNeighbours: WARNING, source was not tested\n";                  lprintf_nl () "WARNING, source was not tested";
124                raise Continue                raise Continue
125              end              end
126        | File_expected  ->        | File_expected  ->
# Line 120  let rec compute_stats () = Line 130  let rec compute_stats () =
130              end              end
131        | File_possible ->        | File_possible ->
132            if !verbose_hidden_errors then            if !verbose_hidden_errors then
133              lprintf "DonkeyNeighbours: WARNING, source was unknown\n";              lprintf_nl () "WARNING, source was unknown";
134            if time + 1800 < last_time () then raise Continue            if time + 1800 < last_time () then raise Continue
135        | File_not_found ->        | File_not_found ->
136            incr_propositions kind File_not_found;            incr_propositions kind File_not_found;
# Line 132  let rec compute_stats () = Line 142  let rec compute_stats () =
142            raise Continue            raise Continue
143        | File_unknown -> assert false        | File_unknown -> assert false
144      with      with
145      | Not_found ->      | Not_found ->
146  (* For some reason, the request was forgotten. Forget it... *)  (* For some reason, the request was forgotten. Forget it... *)
147          if !verbose_hidden_errors then          if !verbose_hidden_errors then
148            lprintf "DonkeyNeighbours: ERROR, request was forgotten\n";            lprintf_nl () "ERROR, request was forgotten";
149          let _ = Fifo.take propositions in          let _ = Fifo.take propositions in
150          compute_stats ()          compute_stats ()
151      | Continue ->              | Continue ->
152          let _ = Fifo.take propositions in          let _ = Fifo.take propositions in
153          compute_stats ()          compute_stats ()
154              
155  let propose_source file c kind =  let propose_source file c kind =
156    try    try
157      List.iter (fun r ->      List.iter (fun r ->
158          if r.request_file == file.file_sources then raise Exit          if r.request_file == file.file_sources then raise Exit
# Line 153  let propose_source file c kind = Line 163  let propose_source file c kind =
163          let s = DonkeySources.find_source_by_uid c.client_kind in          let s = DonkeySources.find_source_by_uid c.client_kind in
164          DonkeySources.set_request_result s file.file_sources File_new_source;          DonkeySources.set_request_result s file.file_sources File_new_source;
165          Fifo.put propositions (s, file, kind, last_time ())          Fifo.put propositions (s, file, kind, last_time ())
166      | _ ->      | _ ->
167          if !verbose_hidden_errors then          if !verbose_hidden_errors then
168            lprintf "DonkeyNeighbours: ERROR, proposed client is indirect\n";            lprintf_nl () "ERROR, proposed client is indirect";
169          raise Exit          raise Exit
170            
171    with Exit ->    with Exit ->
172  (* This client might already have queried this source *)  (* This client might already have queried this source *)
173        ()        ()
174      
175  let new_neighbour c file =  let new_neighbour c file =
176    compute_stats ();      compute_stats ();
177    match c.client_kind with    match c.client_kind with
178      Indirect_address _ -> ()      Indirect_address _ -> ()
179    | _ ->    | _ ->
180          
181        let client_num = client_num c in        let client_num = client_num c in
182        let file_num = file_num file in        let file_num = file_num file in
183        let time = last_time () in        let time = last_time () in
184          
185        try        try
186          let nb = Hashtbl.find neighbours client_num in          let nb = Hashtbl.find neighbours client_num in
187          try          try
# Line 186  let new_neighbour c file = Line 196  let new_neighbour c file =
196              nb_sort = min_int;              nb_sort = min_int;
197              nb_client = c;              nb_client = c;
198            }            }
199          
200  (*  (*
201    We should try to sort files by "proximity", and then try to move sources    We should try to sort files by "proximity", and then try to move sources
202  from one to another, maybe depending on the popularity of the file.  from one to another, maybe depending on the popularity of the file.
203  Before adding a client as a source for a file, we should check that  Before adding a client as a source for a file, we should check that
# Line 198  Currently, the function does: Line 208  Currently, the function does:
208  * Propose the 5 best History sources to all current files  * Propose the 5 best History sources to all current files
209  * For every file, propose 5 best LRU sources by extension, and 5 best LRU  * For every file, propose 5 best LRU sources by extension, and 5 best LRU
210      sources by size      sources by size
     
211    *)    *)
212    
213  let recover_downloads current_files =  let recover_downloads current_files =
214    
215  (* Global LRU *)  (* Global LRU *)
216    let list = ref [] in    let list = ref [] in
# Line 219  let recover_downloads current_files = Line 228  let recover_downloads current_files =
228          let s2 = nb2.nb_sort in          let s2 = nb2.nb_sort in
229          if s1 > s2 then -1 else          if s1 > s2 then -1 else
230          if s2 > s1 then 1 else 0) !list in          if s2 > s1 then 1 else 0) !list in
231      
232    let keep_glru,_ = List2.cut 333 glru in    let keep_glru,_ = List2.cut 333 glru in
233    let best_glru,_ = List2.cut 5 glru in    let best_glru,_ = List2.cut 5 glru in
234  (* For each file, propose the 5 first elements of the GLRU as a potential  (* For each file, propose the 5 first elements of the GLRU as a potential
# Line 241  source. *) Line 250  source. *)
250          let s2 = nb2.nb_sort in          let s2 = nb2.nb_sort in
251          if s1 > s2 then -1 else          if s1 > s2 then -1 else
252          if s2 > s1 then 1 else 0) !list in          if s2 > s1 then 1 else 0) !list in
253      
254    let keep_history,_ = List2.cut 333 history in    let keep_history,_ = List2.cut 333 history in
255    let best_history,_ = List2.cut 5 history in    let best_history,_ = List2.cut 5 history in
256  (* For each file, propose the 5 first elements of the GLRU as a potential  (* For each file, propose the 5 first elements of the GLRU as a potential
# Line 262  source. *) Line 271  source. *)
271                  [] -> assert false                  [] -> assert false
272                | nb2 :: _ ->                | nb2 :: _ ->
273                    if nb2 != nb then list := nb :: !list                    if nb2 != nb then list := nb :: !list
274              with Not_found ->              with Not_found ->
275                  sizes_list := (fi.fi_size, ref [nb]) :: !sizes_list                  sizes_list := (fi.fi_size, ref [nb]) :: !sizes_list
276            );            );
277              
278            (try            (try
279                let list = List.assq fi.fi_ext !extensions_list in                let list = List.assq fi.fi_ext !extensions_list in
280                match !list with                match !list with
281                  [] -> assert false                  [] -> assert false
282                | nb2 :: _ ->                | nb2 :: _ ->
283                    if nb2 != nb then list := nb :: !list                    if nb2 != nb then list := nb :: !list
284              with Not_found ->              with Not_found ->
285                  extensions_list := (fi.fi_ext, ref [nb]) :: !extensions_list                  extensions_list := (fi.fi_ext, ref [nb]) :: !extensions_list
286            )            )
287          
288        ) nb.nb_files        ) nb.nb_files
289    ) neighbours;    ) neighbours;
290      
291    List.iter (fun (ext, list) ->    List.iter (fun (ext, list) ->
292        List.iter (fun nb ->        List.iter (fun nb ->
293            nb.nb_sort <- min_int;            nb.nb_sort <- min_int;
# Line 292  source. *) Line 301  source. *)
301            let s2 = nb2.nb_sort in            let s2 = nb2.nb_sort in
302            if s1 > s2 then -1 else            if s1 > s2 then -1 else
303            if s2 > s1 then 1 else 0) !list            if s2 > s1 then 1 else 0) !list
304      
     
305    ) !extensions_list;    ) !extensions_list;
306      
307    List.iter (fun (size, list) ->    List.iter (fun (size, list) ->
308        List.iter (fun nb ->        List.iter (fun nb ->
309            nb.nb_sort <- min_int;            nb.nb_sort <- min_int;
# Line 310  source. *) Line 318  source. *)
318            if s1 > s2 then -1 else            if s1 > s2 then -1 else
319            if s2 > s1 then 1 else 0) !list            if s2 > s1 then 1 else 0) !list
320    ) !sizes_list;    ) !sizes_list;
321      
     
322    List.iter (fun file ->    List.iter (fun file ->
323        let fi_size = int_of_float (log (        let fi_size = int_of_float (log (
324              Int64.to_float (file_size file ++ one))) in              Int64.to_float (file_size file ++ one))) in
325        let fi_ext = String.lowercase (        let fi_ext = String.lowercase (
326            Filename2.last_extension (file_best_name file)) in            Filename2.last_extension (file_best_name file)) in
327          
328        (try        (try
329            let list = List.assoc fi_size !sizes_list in            let list = List.assoc fi_size !sizes_list in
330            let best_slru, _ = List2.cut 5 !list in            let best_slru, _ = List2.cut 5 !list in
331            List.iter (fun nb ->            List.iter (fun nb ->
332                propose_source file nb.nb_client SizeLRU) best_slru                propose_source file nb.nb_client SizeLRU) best_slru
333          with _ -> ());          with _ -> ());
334          
335        (try        (try
336            let list = List.assoc fi_ext !extensions_list in            let list = List.assoc fi_ext !extensions_list in
337            let best_slru, _ = List2.cut 5 !list in            let best_slru, _ = List2.cut 5 !list in
338            List.iter (fun nb ->            List.iter (fun nb ->
339                propose_source file nb.nb_client ExtLRU) best_slru                propose_source file nb.nb_client ExtLRU) best_slru
340          with _ -> ());          with _ -> ());
     
341    ) current_files;    ) current_files;
342    
343    
# Line 341  source. *) Line 347  source. *)
347        list_list := !list :: !list_list) !extensions_list;        list_list := !list :: !list_list) !extensions_list;
348    List.iter (fun (_, list) ->    List.iter (fun (_, list) ->
349        list_list := !list :: !list_list) !sizes_list;        list_list := !list :: !list_list) !sizes_list;
350      
351    let rec iter rem todo_list done_list keep =    let rec iter rem todo_list done_list keep =
352      if rem = 0 then keep else      if rem = 0 then keep else
353      match todo_list with      match todo_list with
# Line 361  source. *) Line 367  source. *)
367  i.e. the table can only retain at most 1 MB of data after every call  i.e. the table can only retain at most 1 MB of data after every call
368    to this function. *)    to this function. *)
369    Hashtbl.clear neighbours;    Hashtbl.clear neighbours;
370      
371    List.iter (fun list ->    List.iter (fun list ->
372        List.iter (fun nb ->        List.iter (fun nb ->
373            let cnum = client_num nb.nb_client in            let cnum = client_num nb.nb_client in
374            if not (Hashtbl.mem neighbours cnum) then            if not (Hashtbl.mem neighbours cnum) then
375              Hashtbl.add neighbours cnum nb              Hashtbl.add neighbours cnum nb
376        ) list)        ) list)
377    [keep_glru; keep_history; keep_slru]    [keep_glru; keep_history; keep_slru]
378    
           
     
379  open LittleEndian  open LittleEndian
380      
381  let _ =  let _ =
382    CommonWeb.add_redirector_info "DKNB" (fun buf ->    CommonWeb.add_redirector_info "DKNB" (fun buf ->
383        compute_stats ();        compute_stats ();
384        let len = Array.length ngood_propositions in        let len = Array.length ngood_propositions in
385        buf_int buf len;        buf_int buf len;
386        if !verbose_redirector then lprintf "DonkeyNeighbours stats:\n";        if !verbose_redirector then lprintf_n () "Stats:";
387        for i = 0 to len - 1 do        for i = 0 to len - 1 do
388          if !verbose_redirector then lprintf "   Good[%d] = %Ld\n" i ngood_propositions.(i);          if !verbose_redirector then lprintf_nl2 "   Good[%d] = %Ld\n" i ngood_propositions.(i);
389          if !verbose_redirector then lprintf "   Bad[%d] = %Ld\n" i nbad_propositions.(i);          if !verbose_redirector then lprintf_nl2 "   Bad[%d] = %Ld\n" i nbad_propositions.(i);
390          buf_int64 buf ngood_propositions.(i);          buf_int64 buf ngood_propositions.(i);
391          buf_int64 buf nbad_propositions.(i)          buf_int64 buf nbad_propositions.(i)
392        done;        done;
393        let len = Fifo.length propositions in        let len = Fifo.length propositions in
394        if !verbose_redirector then lprintf "   Waiting propositions: %d\n" len;        if !verbose_redirector then lprintf_nl2 "   Waiting propositions: %d\n" len;
395        buf_int buf len;        buf_int buf len;
396        let len = ref 0 in        let len = ref 0 in
397        Hashtbl.iter (fun _ _ -> incr len) neighbours;        Hashtbl.iter (fun _ _ -> incr len) neighbours;
398        if !verbose_redirector then lprintf "   Total neighbours: %d\n" !len;        if !verbose_redirector then lprintf_nl2 "   Total neighbours: %d\n" !len;
399        buf_int buf !len        buf_int buf !len
400    )    )

Legend:
Removed from v.1.6  
changed lines
  Added in v.1.7

savannah-hackers-public@gnu.org
ViewVC Help
Powered by ViewVC 1.1.26