38 |
(* let debug_all = true *) |
(* let debug_all = true *) |
39 |
let exit_on_error = ref false |
let exit_on_error = ref false |
40 |
|
|
41 |
|
(* prints a new logline with date, module and starts newline *) |
42 |
|
let lprintf_nl () = |
43 |
|
lprintf "%s[cSw2] " |
44 |
|
(log_time ()); lprintf_nl2 |
45 |
|
|
46 |
|
(* prints a new logline with date, module and does not start newline *) |
47 |
|
let lprintf_n () = |
48 |
|
lprintf "%s[cSw2] " |
49 |
|
(log_time ()); lprintf |
50 |
|
|
51 |
open CommonTypes |
open CommonTypes |
52 |
|
|
53 |
module Make(M: CommonEnv) = struct |
module Make(M: CommonEnv) = struct |
231 |
(*************************************************************************) |
(*************************************************************************) |
232 |
|
|
233 |
let print_uploader up = |
let print_uploader up = |
234 |
lprintf " interesting complete_blocks: %d\n " up.up_ncomplete; |
lprintf_n () " interesting complete_blocks: %d\n " up.up_ncomplete; |
235 |
Array.iter (fun i -> lprintf " %d " i) up.up_complete_blocks; |
Array.iter (fun i -> lprintf " %d " i) up.up_complete_blocks; |
236 |
lprintf " interesting partial_blocks: %d\n " up.up_npartial; |
lprint_newline (); |
237 |
|
lprintf_n () " interesting partial_blocks: %d\n " up.up_npartial; |
238 |
Array.iter (fun (i, begin_pos, end_pos) -> |
Array.iter (fun (i, begin_pos, end_pos) -> |
239 |
lprintf " %d[%s...%s] " i |
lprintf " %d[%s...%s] " i |
240 |
(Int64.to_string begin_pos) |
(Int64.to_string begin_pos) |
241 |
(Int64.to_string end_pos)) up.up_partial_blocks |
(Int64.to_string end_pos)) up.up_partial_blocks; |
242 |
|
lprint_newline () |
243 |
|
|
244 |
(*************************************************************************) |
(*************************************************************************) |
245 |
(* *) |
(* *) |
719 |
in |
in |
720 |
let i = iter 0 (Array.length b - 1) in |
let i = iter 0 (Array.length b - 1) in |
721 |
if debug_all then |
if debug_all then |
722 |
lprintf "%Ld is block %d [%Ld-%Ld]\n" chunk_pos i |
lprintf_nl () "%Ld is block %d [%Ld-%Ld]" chunk_pos i |
723 |
(compute_block_begin s i) (compute_block_end s i); |
(compute_block_begin s i) (compute_block_end s i); |
724 |
i |
i |
725 |
|
|
751 |
let current_end = min block_end chunk_end in |
let current_end = min block_end chunk_end in |
752 |
|
|
753 |
if debug_all then |
if debug_all then |
754 |
lprintf "Apply: %d %s-%s %s-%s\n" |
lprintf_nl () "Apply: %d %s-%s %s-%s" |
755 |
i |
i |
756 |
(Int64.to_string block_begin) |
(Int64.to_string block_begin) |
757 |
(Int64.to_string block_end) |
(Int64.to_string block_end) |
776 |
(*************************************************************************) |
(*************************************************************************) |
777 |
|
|
778 |
let print_s str s = |
let print_s str s = |
779 |
lprintf "Ranges after %s:\n" str; |
lprintf_nl () "Ranges after %s:" str; |
780 |
|
|
781 |
let rec iter r = |
let rec iter r = |
782 |
lprintf " %s(%s)-%s(%d)" |
lprintf_n () " %s(%s)-%s(%d)" |
783 |
(Int64.to_string r.range_begin) |
(Int64.to_string r.range_begin) |
784 |
(Int64.to_string r.range_current_begin) |
(Int64.to_string r.range_current_begin) |
785 |
(Int64.to_string r.range_end) r.range_nuploading; |
(Int64.to_string r.range_end) r.range_nuploading; |
786 |
match r.range_next with |
match r.range_next with |
787 |
None -> lprintf "\n" |
None -> lprint_newline() |
788 |
| Some r -> iter r |
| Some r -> iter r |
789 |
in |
in |
790 |
|
|
791 |
Array.iteri (fun i b -> |
Array.iteri (fun i b -> |
792 |
lprintf " %d: " i; |
lprintf_n () " %d: " i; |
793 |
let block_begin = compute_block_begin s i in |
let block_begin = compute_block_begin s i in |
794 |
let block_end = compute_block_end s i in |
let block_end = compute_block_end s i in |
795 |
lprintf "%Ld - %Ld [%Ld] %c " block_begin block_end |
lprintf "%Ld - %Ld [%Ld] %c " block_begin block_end |
807 |
(Int64.to_string b.block_begin) |
(Int64.to_string b.block_begin) |
808 |
(Int64.to_string b.block_end); |
(Int64.to_string b.block_end); |
809 |
iter b.block_ranges |
iter b.block_ranges |
810 |
| EmptyBlock -> lprintf "_\n" |
| EmptyBlock -> lprintf_nl2 "_" |
811 |
| CompleteBlock -> lprintf "C\n" |
| CompleteBlock -> lprintf_nl2 "C" |
812 |
| VerifiedBlock -> lprintf "V\n" |
| VerifiedBlock -> lprintf_nl2 "V" |
813 |
) s.s_blocks; |
) s.s_blocks; |
814 |
|
|
815 |
lprintf "Files:\n"; |
lprintf_nl () "Files:"; |
816 |
List.iter (fun t -> |
List.iter (fun t -> |
817 |
lprintf " File num: %d\n" (file_num t.t_file); |
lprintf_nl () " File num: %d" (file_num t.t_file); |
818 |
lprintf " %s\n" (if t.t_primary then "primary" else "secondary"); |
lprintf_nl () " %s" (if t.t_primary then "primary" else "secondary"); |
819 |
lprintf " Downloaded: %Ld\n" (file_downloaded t.t_file); |
lprintf_nl () " Downloaded: %Ld" (file_downloaded t.t_file); |
820 |
lprintf " Bitmap: %s\n" t.t_converted_verified_bitmap |
lprintf_nl () " Bitmap: %s" t.t_converted_verified_bitmap |
821 |
) s.s_networks |
) s.s_networks |
822 |
|
|
823 |
(*************************************************************************) |
(*************************************************************************) |
842 |
(*************************************************************************) |
(*************************************************************************) |
843 |
|
|
844 |
let print_block b = |
let print_block b = |
845 |
lprintf "Block %d: %s-%s" |
lprintf_n () "Block %d: %s-%s" |
846 |
b.block_num |
b.block_num |
847 |
(Int64.to_string b.block_begin) |
(Int64.to_string b.block_begin) |
848 |
(Int64.to_string b.block_end); |
(Int64.to_string b.block_end); |
849 |
lprintf "\n"; |
lprint_newline (); |
850 |
lprintf " ranges:\n"; |
lprintf_nl () " ranges:"; |
851 |
let rec iter_range r = |
let rec iter_range r = |
852 |
lprintf " %Ld-%Ld\n" r.range_current_begin r.range_end; |
lprintf_nl () " %Ld-%Ld" r.range_current_begin r.range_end; |
853 |
match r.range_next with |
match r.range_next with |
854 |
None -> () |
None -> () |
855 |
| Some rr -> iter_range rr |
| Some rr -> iter_range rr |
856 |
in |
in |
857 |
iter_range b.block_ranges; |
iter_range b.block_ranges; |
858 |
lprintf "\n" |
lprint_newline () |
859 |
|
|
860 |
(*************************************************************************) |
(*************************************************************************) |
861 |
(* *) |
(* *) |
876 |
| _ -> () |
| _ -> () |
877 |
end; |
end; |
878 |
if file_downloaded t.t_file < zero then |
if file_downloaded t.t_file < zero then |
879 |
lprintf "ERROR: file_downloaded < zero !!!\n"; |
lprintf_nl () "ERROR: file_downloaded < zero!"; |
880 |
|
|
881 |
| _ -> () |
| _ -> () |
882 |
|
|
1103 |
end else begin |
end else begin |
1104 |
|
|
1105 |
if !verbose_swarming then |
if !verbose_swarming then |
1106 |
lprintf "VERIFICATION OF BLOC %d OF %s FAILED\n" |
lprintf_nl () "VERIFICATION OF BLOCK %d OF %s FAILED" |
1107 |
i (file_best_name t.t_file); |
i (file_best_name t.t_file); |
1108 |
t.t_ncomplete_blocks <- t.t_ncomplete_blocks - 1; |
t.t_ncomplete_blocks <- t.t_ncomplete_blocks - 1; |
1109 |
|
|
1111 |
s.s_verified_bitmap.[i] = '2' |
s.s_verified_bitmap.[i] = '2' |
1112 |
) t.t_t2s_blocks.(i) then begin |
) t.t_t2s_blocks.(i) then begin |
1113 |
|
|
1114 |
lprintf "Verification of complete block %d of %s FAILED, redownloading it.\n" |
lprintf_nl () "Verification of complete block %d of %s FAILED, redownloading it." |
1115 |
i (file_best_name t.t_file); |
i (file_best_name t.t_file); |
1116 |
|
|
1117 |
t.t_converted_verified_bitmap.[i] <- '0'; |
t.t_converted_verified_bitmap.[i] <- '0'; |
1133 |
end else begin |
end else begin |
1134 |
let nsub = ref 0 in |
let nsub = ref 0 in |
1135 |
|
|
1136 |
lprintf " Swarmer was incomplete: "; |
lprintf_n () " Swarmer was incomplete: "; |
1137 |
List.iter (fun i -> |
List.iter (fun i -> |
1138 |
lprintf "%c" s.s_verified_bitmap.[i]; |
lprintf "%c" s.s_verified_bitmap.[i]; |
1139 |
if s.s_verified_bitmap.[i] = '2' then incr nsub; |
if s.s_verified_bitmap.[i] = '2' then incr nsub; |
1140 |
) t.t_t2s_blocks.(i); |
) t.t_t2s_blocks.(i); |
1141 |
lprintf " = %d/%d\n" !nsub (List.length t.t_t2s_blocks.(i)); |
lprintf_nl2 " = %d/%d" !nsub (List.length t.t_t2s_blocks.(i)); |
1142 |
|
|
1143 |
t.t_converted_verified_bitmap.[i] <- '1' |
t.t_converted_verified_bitmap.[i] <- '1' |
1144 |
end; |
end; |
1168 |
|
|
1169 |
end else begin |
end else begin |
1170 |
|
|
1171 |
lprintf "VERIFICATION OF BLOCS OF %s FAILED\n" |
lprintf_nl () "VERIFICATION OF BLOCKS OF %s FAILED\n" |
1172 |
(file_best_name t.t_file); |
(file_best_name t.t_file); |
1173 |
|
|
1174 |
for i = 0 to nblocks - 1 do |
for i = 0 to nblocks - 1 do |
1201 |
end else begin |
end else begin |
1202 |
let nsub = ref 0 in |
let nsub = ref 0 in |
1203 |
|
|
1204 |
lprintf " Swarmer was incomplete: "; |
lprintf_n () " Swarmer was incomplete: "; |
1205 |
List.iter (fun i -> |
List.iter (fun i -> |
1206 |
lprintf "%c" s.s_verified_bitmap.[i]; |
lprintf "%c" s.s_verified_bitmap.[i]; |
1207 |
if s.s_verified_bitmap.[i] = '2' then incr nsub; |
if s.s_verified_bitmap.[i] = '2' then incr nsub; |
1208 |
) t.t_t2s_blocks.(i); |
) t.t_t2s_blocks.(i); |
1209 |
lprintf " = %d/%d\n" !nsub (List.length t.t_t2s_blocks.(i)); |
lprintf_nl2 " = %d/%d" !nsub (List.length t.t_t2s_blocks.(i)); |
1210 |
|
|
1211 |
t.t_converted_verified_bitmap.[i] <- '1' |
t.t_converted_verified_bitmap.[i] <- '1' |
1212 |
end; |
end; |
1367 |
s.s_blocks.(i) <- PartialBlock b; |
s.s_blocks.(i) <- PartialBlock b; |
1368 |
if s.s_verified_bitmap.[i] < '1' then |
if s.s_verified_bitmap.[i] < '1' then |
1369 |
set_bitmap_1 s i; |
set_bitmap_1 s i; |
1370 |
if debug_all then lprintf "NB[%s]" s.s_verified_bitmap; |
if debug_all then lprintf_nl () "NB[%s]" s.s_verified_bitmap; |
1371 |
b |
b |
1372 |
|
|
1373 |
|
|
1472 |
(* lprintf "iter range %Ld-%Ld\n" r.range_begin r.range_end; *) |
(* lprintf "iter range %Ld-%Ld\n" r.range_begin r.range_end; *) |
1473 |
(try range_received None r chunk_begin chunk_end |
(try range_received None r chunk_begin chunk_end |
1474 |
with e -> |
with e -> |
1475 |
lprintf "EXCEPTION IN range_received: %s\n" |
lprintf_nl () "EXCEPTION IN range_received: %s" |
1476 |
(Printexc2.to_string e); |
(Printexc2.to_string e); |
1477 |
exit 2); |
exit 2); |
1478 |
match range_next with |
match range_next with |
1653 |
up.up_ncomplete <- Array.length complete_blocks; |
up.up_ncomplete <- Array.length complete_blocks; |
1654 |
|
|
1655 |
if Array.length partial_blocks > 0 then |
if Array.length partial_blocks > 0 then |
1656 |
lprintf "WARNING: partial_blocks = %d\n" (Array.length partial_blocks); |
lprintf_nl () "WARNING: partial_blocks = %d" (Array.length partial_blocks); |
1657 |
up.up_partial_blocks <- partial_blocks; |
up.up_partial_blocks <- partial_blocks; |
1658 |
up.up_npartial <- Array.length partial_blocks; |
up.up_npartial <- Array.length partial_blocks; |
1659 |
|
|
1773 |
else |
else |
1774 |
lprintf "%d" s.s_nuploading.(i) |
lprintf "%d" s.s_nuploading.(i) |
1775 |
done; |
done; |
1776 |
lprintf "\n"; |
lprint_newline (); |
1777 |
for i = 0 to nblocks - 1 do |
for i = 0 to nblocks - 1 do |
1778 |
|
|
1779 |
match s.s_blocks.(i) with |
match s.s_blocks.(i) with |
1794 |
lprintf " }"; |
lprintf " }"; |
1795 |
|
|
1796 |
done; |
done; |
1797 |
lprintf "\n" |
lprint_newline () |
1798 |
|
|
1799 |
(*************************************************************************) |
(*************************************************************************) |
1800 |
(* *) |
(* *) |
1819 |
| PartialBlock b -> |
| PartialBlock b -> |
1820 |
b, b.block_begin, b.block_end |
b, b.block_begin, b.block_end |
1821 |
| VerifiedBlock -> |
| VerifiedBlock -> |
1822 |
lprintf "ERROR: verified block in permute_and_return %d\n" b; |
lprintf_nl () "ERROR: verified block in permute_and_return %d\n" b; |
1823 |
assert false |
assert false |
1824 |
| CompleteBlock -> |
| CompleteBlock -> |
1825 |
lprintf "ERROR: complete block in permute_and_return %d\n" b; |
lprintf_nl () "ERROR: complete block in permute_and_return %d\n" b; |
1826 |
assert false |
assert false |
1827 |
|
|
1828 |
(*************************************************************************) |
(*************************************************************************) |
1911 |
|
|
1912 |
let random_int max = |
let random_int max = |
1913 |
let x = Random.int max in |
let x = Random.int max in |
1914 |
if debug_all then lprintf "(Random %d -> %d)" max x; |
if debug_all then lprintf_nl () "(Random %d -> %d)" max x; |
1915 |
x |
x |
1916 |
|
|
1917 |
let select_block up = |
let select_block up = |
1949 |
(************* Try to download the movie index and the first minute to |
(************* Try to download the movie index and the first minute to |
1950 |
allow preview of the file as soon as possible *) |
allow preview of the file as soon as possible *) |
1951 |
|
|
1952 |
if debug_all then lprintf "{First}"; |
if debug_all then lprintf_nl () "{First}"; |
1953 |
|
|
1954 |
let download_first n b = |
let download_first n b = |
1955 |
(* lprintf "download_first %d\n" n; *) |
(* lprintf "download_first %d\n" n; *) |
1980 |
(************* If the file can be verified, and we don't have a lot of blocks |
(************* If the file can be verified, and we don't have a lot of blocks |
1981 |
yet, try to download the partial ones as soon as possible *) |
yet, try to download the partial ones as soon as possible *) |
1982 |
|
|
1983 |
if debug_all then lprintf "{PartialBlock}"; |
if debug_all then lprintf_nl () "{PartialBlock}"; |
1984 |
|
|
1985 |
let download_partial max_uploaders = |
let download_partial max_uploaders = |
1986 |
let partial_block = ref (-1) in |
let partial_block = ref (-1) in |
2011 |
(************* Download partial chunks from the verification point of view *) |
(************* Download partial chunks from the verification point of view *) |
2012 |
|
|
2013 |
if List.length s.s_networks > 1 then begin |
if List.length s.s_networks > 1 then begin |
2014 |
if debug_all then lprintf "{PartialChunk}"; |
if debug_all then lprintf_n () "{PartialChunk}"; |
2015 |
|
|
2016 |
let my_t = if t.t_verifier <> NoVerification then t |
let my_t = if t.t_verifier <> NoVerification then t |
2017 |
else match s.s_networks with t :: _ -> t | _ -> t in |
else match s.s_networks with t :: _ -> t | _ -> t in |
2031 |
(* TODO remove this *) |
(* TODO remove this *) |
2032 |
let b = should_download_block s n in |
let b = should_download_block s n in |
2033 |
|
|
2034 |
if !verbose_swarming then begin |
if !verbose_swarming then |
2035 |
lprintf " test %d %c %d %b %d\n" |
lprintf_nl2 " test %d %c %d %b %d" |
2036 |
n s.s_verified_bitmap.[n] s.s_nuploading.(n) |
n s.s_verified_bitmap.[n] s.s_nuploading.(n) |
2037 |
b nbs; |
b nbs; |
|
end; |
|
2038 |
|
|
2039 |
if s.s_verified_bitmap.[n] < '2' && |
if s.s_verified_bitmap.[n] < '2' && |
2040 |
s.s_nuploading.(n) < max_uploaders && |
s.s_nuploading.(n) < max_uploaders && |
2049 |
done; |
done; |
2050 |
if !partial_block <> -1 then begin |
if !partial_block <> -1 then begin |
2051 |
if !verbose_swarming then |
if !verbose_swarming then |
2052 |
lprintf "PartialChunk won for %d waiting blocks" |
lprintf_n () "PartialChunk won for %d waiting blocks" |
2053 |
!partial_remaining; |
!partial_remaining; |
2054 |
raise (BlockFound !partial_block) |
raise (BlockFound !partial_block) |
2055 |
end |
end |
2130 |
(* print_s "NO BLOCK FOUND" s; *) |
(* print_s "NO BLOCK FOUND" s; *) |
2131 |
raise Not_found |
raise Not_found |
2132 |
|
|
2133 |
(*************************************************************************) |
(*************************************************************************) |
2134 |
(* *) |
(* *) |
2135 |
(* find_block *) |
(* find_block *) |
2136 |
(* *) |
(* *) |
2171 |
if debug_all then lprintf " = %d \n" num; |
if debug_all then lprintf " = %d \n" num; |
2172 |
b |
b |
2173 |
with e -> |
with e -> |
2174 |
if debug_all then lprintf "Exception %s\n" (Printexc2.to_string e); |
if debug_all then lprintf_nl () "Exception %s" (Printexc2.to_string e); |
2175 |
raise e |
raise e |
2176 |
|
|
2177 |
(*************************************************************************) |
(*************************************************************************) |
2266 |
up.up_ranges <- up.up_ranges @ [key]; |
up.up_ranges <- up.up_ranges @ [key]; |
2267 |
r.range_nuploading <- r.range_nuploading + 1; |
r.range_nuploading <- r.range_nuploading + 1; |
2268 |
if r.range_current_begin = r.range_end then |
if r.range_current_begin = r.range_end then |
2269 |
lprintf "CS error: range is empty error\n"; |
lprintf_nl () "error: range is empty error"; |
2270 |
key |
key |
2271 |
end else |
end else |
2272 |
match r.range_next with |
match r.range_next with |
2305 |
let file_end = file_begin ++ (Int64.of_int string_len) in |
let file_end = file_begin ++ (Int64.of_int string_len) in |
2306 |
|
|
2307 |
if !verbose_swarming then |
if !verbose_swarming then |
2308 |
lprintf "received on %s-%s\n" |
lprintf_nl () "received on %s-%s" |
2309 |
(Int64.to_string file_begin) |
(Int64.to_string file_begin) |
2310 |
(Int64.to_string file_end); |
(Int64.to_string file_end); |
2311 |
|
|
2338 |
string_len < string_length then begin |
string_len < string_length then begin |
2339 |
if !verbose_hidden_errors then |
if !verbose_hidden_errors then |
2340 |
begin |
begin |
2341 |
lprintf "[ERROR CommonSwarming]: BAD WRITE\n"; |
lprintf_nl () "BAD WRITE"; |
2342 |
lprintf " for range %Ld-%Ld (string_pos %d)\n" |
lprintf_nl () " for range %Ld-%Ld (string_pos %d)" |
2343 |
r.range_begin r.range_end string_pos; |
r.range_begin r.range_end string_pos; |
2344 |
|
|
2345 |
lprintf " received: file_pos:%Ld string:%d %d\n" |
lprintf_nl () " received: file_pos:%Ld string:%d %d" |
2346 |
file_begin string_begin string_len; |
file_begin string_begin string_len; |
2347 |
lprintf " ranges:\n"; |
lprintf_nl () " ranges:"; |
2348 |
List.iter (fun (_,_,r) -> |
List.iter (fun (_,_,r) -> |
2349 |
lprintf " range: %Ld-[%Ld]-%Ld" |
lprintf_n () " range: %Ld-[%Ld]-%Ld" |
2350 |
r.range_begin r.range_current_begin |
r.range_begin r.range_current_begin |
2351 |
r.range_end; |
r.range_end; |
2352 |
(match r.range_next with |
(match r.range_next with |
2357 |
None -> () |
None -> () |
2358 |
| Some rr -> |
| Some rr -> |
2359 |
lprintf " prev: %Ld" rr.range_begin); |
lprintf " prev: %Ld" rr.range_begin); |
2360 |
lprintf "\n"; |
lprint_newline (); |
2361 |
let b = r.range_block in |
let b = r.range_block in |
2362 |
lprintf " block: %d[%c] %Ld-%Ld [%s]" |
lprintf_n () " block: %d[%c] %Ld-%Ld [%s]" |
2363 |
b.block_num |
b.block_num |
2364 |
s.s_verified_bitmap.[b.block_num] |
s.s_verified_bitmap.[b.block_num] |
2365 |
b.block_begin b.block_end |
b.block_begin b.block_end |
2373 |
lprintf " first range: %Ld(%Ld)" |
lprintf " first range: %Ld(%Ld)" |
2374 |
br.range_begin |
br.range_begin |
2375 |
(br.range_end -- br.range_current_begin); |
(br.range_end -- br.range_current_begin); |
2376 |
lprintf "\n"; |
lprint_newline (); |
2377 |
) up.up_ranges; |
) up.up_ranges; |
2378 |
end; |
end; |
2379 |
if !exit_on_error then exit 2 |
if !exit_on_error then exit 2 |
2408 |
|
|
2409 |
let rec iter_block_out i block_begin list = |
let rec iter_block_out i block_begin list = |
2410 |
if debug_present_chunks then |
if debug_present_chunks then |
2411 |
lprintf "iter_block_out %d bb: %s\n" |
lprintf_nl () "iter_block_out %d bb: %s" |
2412 |
i |
i |
2413 |
(Int64.to_string block_begin); |
(Int64.to_string block_begin); |
2414 |
|
|
2425 |
|
|
2426 |
and iter_block_in i block_begin chunk_begin list = |
and iter_block_in i block_begin chunk_begin list = |
2427 |
if debug_present_chunks then |
if debug_present_chunks then |
2428 |
lprintf "iter_block_in %d bb: %s cb:%s \n" |
lprintf_nl () "iter_block_in %d bb: %s cb:%s" |
2429 |
i |
i |
2430 |
(Int64.to_string block_begin) |
(Int64.to_string block_begin) |
2431 |
(Int64.to_string chunk_begin) |
(Int64.to_string chunk_begin) |
2447 |
|
|
2448 |
and iter_range_out i block_end block_begin r list = |
and iter_range_out i block_end block_begin r list = |
2449 |
if debug_present_chunks then |
if debug_present_chunks then |
2450 |
lprintf "iter_range_out %d nb: %s bb:%s\n" |
lprintf_nl () "iter_range_out %d nb: %s bb:%s" |
2451 |
i |
i |
2452 |
(Int64.to_string block_end) |
(Int64.to_string block_end) |
2453 |
(Int64.to_string block_begin) |
(Int64.to_string block_begin) |
2482 |
|
|
2483 |
and iter_range_in i block_end chunk_begin chunk_end r list = |
and iter_range_in i block_end chunk_begin chunk_end r list = |
2484 |
if debug_present_chunks then |
if debug_present_chunks then |
2485 |
lprintf "iter_range_in %d bn: %s cb:%s ce: %s\n" |
lprintf_nl () "iter_range_in %d bn: %s cb:%s ce: %s" |
2486 |
i |
i |
2487 |
(Int64.to_string block_end) |
(Int64.to_string block_end) |
2488 |
(Int64.to_string chunk_begin) |
(Int64.to_string chunk_begin) |
2634 |
() |
() |
2635 |
) t.t_t2s_blocks.(i); |
) t.t_t2s_blocks.(i); |
2636 |
if t.t_converted_verified_bitmap.[i] <> '3' then |
if t.t_converted_verified_bitmap.[i] <> '3' then |
2637 |
lprintf "FIELD AS BEEN CLEARED\n" |
lprintf_nl () "FIELD AS BEEN CLEARED" |
2638 |
| _ -> () |
| _ -> () |
2639 |
done |
done |
2640 |
|
|
2803 |
(get_value "file_all_chunks" value_to_string) |
(get_value "file_all_chunks" value_to_string) |
2804 |
|
|
2805 |
with e -> |
with e -> |
2806 |
lprintf "Exception %s while loading bitmap\n" |
lprintf_nl () "Exception %s while loading bitmap" |
2807 |
(Printexc2.to_string e); |
(Printexc2.to_string e); |
2808 |
); |
); |
2809 |
|
|
2813 |
*) |
*) |
2814 |
|
|
2815 |
if primary then begin |
if primary then begin |
2816 |
if !verbose then lprintf "Loading present...\n"; |
if !verbose then lprintf_nl () "Loading present..."; |
2817 |
let present = try |
let present = try |
2818 |
let present = |
let present = |
2819 |
(get_value "file_present_chunks" |
(get_value "file_present_chunks" |
2822 |
set_present t present; |
set_present t present; |
2823 |
present |
present |
2824 |
with e -> |
with e -> |
2825 |
lprintf "Exception %s while set present\n" |
lprintf_nl () "Exception %s while set present" |
2826 |
(Printexc2.to_string e); |
(Printexc2.to_string e); |
2827 |
[] |
[] |
2828 |
in |
in |
2829 |
if !verbose then lprintf "Downloaded after present %Ld\n" (downloaded t); |
if !verbose then lprintf_nl () "Downloaded after present %Ld" (downloaded t); |
2830 |
|
|
2831 |
if !verbose then lprintf "Loading absent...\n"; |
if !verbose then lprintf_nl () "Loading absent..."; |
2832 |
(try |
(try |
2833 |
set_absent t |
set_absent t |
2834 |
(get_value "file_absent_chunks" |
(get_value "file_absent_chunks" |
2835 |
(value_to_list value_to_int64_pair)); |
(value_to_list value_to_int64_pair)); |
2836 |
with e -> |
with e -> |
2837 |
if !verbose_hidden_errors then lprintf "Exception %s while set absent\n" |
if !verbose_hidden_errors then lprintf_nl () "Exception %s while set absent" |
2838 |
(Printexc2.to_string e); |
(Printexc2.to_string e); |
2839 |
); |
); |
2840 |
if !verbose then lprintf "Downloaded after absent %Ld\n" (downloaded t); |
if !verbose then lprintf_nl () "Downloaded after absent %Ld" (downloaded t); |
2841 |
(try |
(try |
2842 |
let d = get_value "file_downloaded" value_to_int64 in |
let d = get_value "file_downloaded" value_to_int64 in |
2843 |
|
|
2844 |
if d <> downloaded t && !verbose_hidden_errors then begin |
if d <> downloaded t && !verbose_hidden_errors then begin |
2845 |
lprintf "ERROR: CommonSwarming: stored downloaded value not restored !!! (%Ld/%Ld)\n" (downloaded t) d; |
lprintf_nl () "ERROR: stored downloaded value not restored !!! (%Ld/%Ld)" (downloaded t) d; |
2846 |
lprintf "ERROR: CommonSwarming: present:\n"; |
lprintf_nl () "ERROR: present:"; |
2847 |
List.iter (fun (x,y) -> |
List.iter (fun (x,y) -> |
2848 |
lprintf " (%Ld,%Ld);\n" x y |
lprintf_nl () " (%Ld,%Ld);" x y |
2849 |
) present; |
) present; |
2850 |
|
|
2851 |
let p = present_chunks t in |
let p = present_chunks t in |
2852 |
lprintf "ERROR: CommonSwarming: present now:\n"; |
lprintf_nl () "ERROR: present now:"; |
2853 |
|
|
2854 |
let total = ref zero in |
let total = ref zero in |
2855 |
List.iter (fun (x,y) -> |
List.iter (fun (x,y) -> |
2856 |
lprintf " (%Ld,%Ld);\n" x y; |
lprintf_nl () " (%Ld,%Ld);" x y; |
2857 |
total := !total ++ (y -- x); |
total := !total ++ (y -- x); |
2858 |
) p; |
) p; |
2859 |
|
|
2860 |
lprintf "ERROR: total %Ld\n" !total; |
lprintf_nl () "ERROR: total %Ld" !total; |
2861 |
if p = present then begin |
if p = present then begin |
2862 |
lprintf "ERROR: both appear to be the same !!!\n"; |
lprintf_nl () "ERROR: both appear to be the same!"; |
2863 |
end; |
end; |
2864 |
|
|
2865 |
|
|
2974 |
let t2 = match s2.s_networks with |
let t2 = match s2.s_networks with |
2975 |
[t] -> t |
[t] -> t |
2976 |
| list -> |
| list -> |
2977 |
lprintf "s_networks: %d files\n" (List.length list); |
lprintf_nl () "s_networks: %d files" (List.length list); |
2978 |
failwith "Second file is already merged with other files" |
failwith "Second file is already merged with other files" |
2979 |
in |
in |
2980 |
|
|
3105 |
done; |
done; |
3106 |
) tail |
) tail |
3107 |
with e -> |
with e -> |
3108 |
print_s "ERROR"s; |
print_s "ERROR" s; |
3109 |
raise e |
raise e |
3110 |
|
|
3111 |
(*************************************************************************) |
(*************************************************************************) |
3210 |
if bitmap.[i] <> '3' then raise Not_found; |
if bitmap.[i] <> '3' then raise Not_found; |
3211 |
done; |
done; |
3212 |
if file_size file <> downloaded t then |
if file_size file <> downloaded t then |
3213 |
lprintf "Downloaded size differs after complete verification\n"; |
lprintf_nl () "Downloaded size differs after complete verification"; |
3214 |
true |
true |
3215 |
with _ -> false |
with _ -> false |
3216 |
|
|
3272 |
let len = Int64.to_int len in |
let len = Int64.to_int len in |
3273 |
let s = String.create len in |
let s = String.create len in |
3274 |
let ss = String.create len in |
let ss = String.create len in |
3275 |
lprintf "file_verify: READ %Ld %d\n" fpos len; |
lprintf_nl () "file_verify: READ %Ld %d" fpos len; |
3276 |
Unix32.flush_fd file.file_fd; |
Unix32.flush_fd file.file_fd; |
3277 |
Unix32.read file.file_fd fpos ss 0 len; |
Unix32.read file.file_fd fpos ss 0 len; |
3278 |
Unix32.read file_check_fd fpos s 0 len; |
Unix32.read file_check_fd fpos s 0 len; |
3279 |
let result = (ss = s) in |
let result = (ss = s) in |
3280 |
if not result then begin |
if not result then begin |
3281 |
lprintf "ERROR in file_verify %d %Ld-%Ld\n" file.file_num fpos epos; |
lprintf_nl () "ERROR in file_verify %d %Ld-%Ld" file.file_num fpos epos; |
3282 |
(* lprintf "[%s]\n" (String.escaped ss); |
(* lprintf "[%s]\n" (String.escaped ss); |
3283 |
lprintf "[%s]\n" (String.escaped s); *) |
lprintf "[%s]\n" (String.escaped s); *) |
3284 |
(* if !exit_on_error then exit 2; *) |
(* if !exit_on_error then exit 2; *) |
3288 |
let file_mtime file = 0. |
let file_mtime file = 0. |
3289 |
|
|
3290 |
let file_write f fpos s spos len = |
let file_write f fpos s spos len = |
3291 |
lprintf "file_write: WRITE %d\n" len; |
lprintf_nl () "file_write: WRITE %d" len; |
3292 |
Unix32.write f.file_fd fpos s spos len; |
Unix32.write f.file_fd fpos s spos len; |
3293 |
let ss = String.create len in |
let ss = String.create len in |
3294 |
lprintf "file_write: READ %Ld %d\n" fpos len; |
lprintf_nl () "file_write: READ %Ld %d" fpos len; |
3295 |
Unix32.read file_check_fd fpos ss 0 len; |
Unix32.read file_check_fd fpos ss 0 len; |
3296 |
if ss <> (String.sub s spos len) then |
if ss <> (String.sub s spos len) then |
3297 |
lprintf "Writting bad content !\n" |
lprintf_nl () "Writting bad content!" |
3298 |
|
|
3299 |
let file_copy _ _ _ _ _ = () |
let file_copy _ _ _ _ _ = () |
3300 |
let file_state file = FileDownloading |
let file_state file = FileDownloading |
3336 |
in |
in |
3337 |
|
|
3338 |
let files = Array.init nfiles (fun i -> |
let files = Array.init nfiles (fun i -> |
3339 |
lprintf "Creating swarmer %d\n" i; |
lprintf_nl () "Creating swarmer %d" i; |
3340 |
|
|
3341 |
let block_size = block_sizes.(i) in |
let block_size = block_sizes.(i) in |
3342 |
let nchunks = Int64.to_int (Int64.div |
let nchunks = Int64.to_int (Int64.div |