typestar

par_wordcount.rs in Rust

Word frequencies over a corpus, serial and parallel, with the timings compared.

use std::collections::HashMap;
use std::time::Instant;

use rayon::prelude::*;

fn corpus(copies: usize) -> Vec<String> {
    let base = [
        "the quick brown fox jumps over the lazy dog",
        "practice makes the difference between typing and thinking",
        "ownership borrowing lifetimes traits iterators errors",
    ];
    base.iter()
        .cycle()
        .take(base.len() * copies)
        .map(|line| line.to_string())
        .collect()
}

fn serial_counts(lines: &[String]) -> HashMap<&str, u32> {
    let mut counts = HashMap::new();
    for line in lines {
        for word in line.split_whitespace() {
            *counts.entry(word).or_insert(0) += 1;
        }
    }
    counts
}

fn parallel_counts(lines: &[String]) -> HashMap<&str, u32> {
    lines
        .par_iter()
        .fold(HashMap::new, |mut acc: HashMap<&str, u32>, line| {
            for word in line.split_whitespace() {
                *acc.entry(word).or_insert(0) += 1;
            }
            acc
        })
        .reduce(HashMap::new, |mut a, b| {
            for (word, n) in b {
                *a.entry(word).or_insert(0) += n;
            }
            a
        })
}

fn top(counts: &HashMap<&str, u32>, n: usize) -> Vec<(String, u32)> {
    let mut pairs: Vec<(String, u32)> =
        counts.iter().map(|(w, c)| (w.to_string(), *c)).collect();
    pairs.sort_by(|a, b| b.1.cmp(&a.1).then(a.0.cmp(&b.0)));
    pairs.into_iter().take(n).collect()
}

fn main() {
    let lines = corpus(20_000);
    println!("{} lines, {} threads", lines.len(),
             rayon::current_num_threads());

    let started = Instant::now();
    let one = serial_counts(&lines);
    let serial_time = started.elapsed();

    let started = Instant::now();
    let many = parallel_counts(&lines);
    let parallel_time = started.elapsed();

    assert_eq!(one, many);
    println!("serial   {:?}", serial_time);
    println!("parallel {:?}", parallel_time);
    println!(
        "speedup  {:.2}x",
        serial_time.as_secs_f64() / parallel_time.as_secs_f64()
    );

    for (word, count) in top(&many, 5) {
        println!("{count:>7}  {word}");
    }
}

How it works

  1. The parallel path folds one map per thread, then merges them.
  2. Both paths produce the same table, which the code asserts.
  3. The timings print side by side so the win is visible.

Keywords and builtins used here

The run, in numbers

Lines
77
Characters to type
1770
Tokens
548
Three-star pace
115 tpm

At the three-star pace of 115 tokens a minute, this run takes about 286 seconds.

Type this snippet

Step 1 of 1 in Encore, step 9 of 9 in Data parallelism with rayon.

← Previous