The parallelization shunt is a piece of code that I've been copypasting in the collection modules (List, Array, etc.) It gets you the parallel combinators as long as you have a compatible module signature (I'm using copypaste as a ghetto functor.) The functions you need for all the combinators are: splitInto, foldl, foldl1, foldr, foldr1, iter, map, filter, concat, groupsOf, init, length, and, for pzipWith, either unsafe_get or zipWith, depending on whether you have an array-like or a list-like data structure:
let par_mapReduce ?process_count ~combine ~process l =
let process_count = process_count |? !global_process_count in
splitInto process_count l |> par_map ~process_count process |> combine
let pmapReduce combine process = par_mapReduce ~combine ~process
let pfoldl r f init = pmapReduce (List.foldl1 r) (foldl f init)
let pfoldl1 f = pmapReduce (List.foldl1 f) (foldl1 f)
let pfoldr r f init = pmapReduce (List.foldr1 r) (foldr f init)
let pfoldr1 f = pmapReduce (List.foldr1 f) (foldr1 f)
let piter f = pmapReduce ignore (iter f)
let pmap f = pmapReduce concat (map f)
let pfilter f = pmapReduce concat (filter f)
let pfoldlSeqN ?process_count n r f init l =
List.foldl (fun acc il -> r acc (pfoldl ?process_count r f init il))
init (groupsOf n l)
let piterSeqN ?process_count n r f l =
List.iter (fun l -> iter r (pmap ?process_count f l)) (groupsOf n l)
let pinit ?process_count f l =
let process_count = process_count |? !global_process_count in
let plen = int (ceil (float l /. float process_count)) in
let process i =
let start = plen * i in
let len = min plen (l - start) in
init (fun j -> f (start + j)) len in
concat (par_map ~process_count process (0--(process_count-1)))
(* for array-likes *)
let pzipWith ?process_count f a b =
let process_count = process_count |? !global_process_count in
let len = min (length a) (length b) in
pinit ~process_count (fun i ->
f (unsafe_get a i) (unsafe_get b i)
) len
(* for list-likes *)
let pzipWith ?process_count f a b =
let process_count = process_count |? !global_process_count in
let len = min (len a) (len b) in
let plen = int (ceil (float len /. float process_count)) in
let aspl = groupsOf plen a in
let bspl = groupsOf plen b in
concat (par_map ~process_count (uncurry (zipWith f)) (zip aspl bspl))
No comments:
Post a Comment