Merge pull request #2619 from dathere/replace-ahash-with-foldhash

refactor: replace ahash with faster foldhash

Author: jqnatividad

Cargo.lock

@@ -70,7 +70,6 @@ inherits = "release"
 panic    = "abort"
 
 [dependencies]
-ahash = "0.8"
 arboard = { version = "3.4.1", default-features = false, optional = true }
 atoi_simd = "0.16"
 base62 = { version = "2.2", optional = true }
@@ -110,6 +109,7 @@ eudex = { version = "0.1", optional = true }
 ext-sort = { version = "0.1", default-features = false }
 fast-float2 = "0.2"
 flate2 = { version = "1", optional = true }
+foldhash = "0.1"
 file-format = { version = "0.26", features = ["reader"] }
 filetime = "0.2"
 flexi_logger = { version = "0.29", features = [

Cargo.toml

@@ -192,9 +192,9 @@ impl Args {
     // this algorithm is largely inspired by https://github.com/vi/csvcatrow by @vi
     // https://github.com/dathere/qsv/issues/527
     fn cat_rowskey(&self) -> CliResult<()> {
-        // ahash is a faster hasher than the default one used by IndexSet and IndexMap
-        type AhashIndexSet<T> = IndexSet<T, ahash::RandomState>;
-        type AhashIndexMap<T, T2> = IndexMap<T, T2, ahash::RandomState>;
+        // foldhash is a faster hasher than the default one used by IndexSet and IndexMap
+        type FhashIndexSet<T> = IndexSet<T, foldhash::fast::RandomState>;
+        type FhashIndexMap<T, T2> = IndexMap<T, T2, foldhash::fast::RandomState>;
 
         let Ok(group_kind) = GroupKind::from_str(&self.flag_group) else {
             return fail_incorrectusage_clierror!(
@@ -204,7 +204,7 @@ impl Args {
             );
         };
 
-        let mut columns_global: AhashIndexSet<Box<[u8]>> = AhashIndexSet::default();
+        let mut columns_global: FhashIndexSet<Box<[u8]>> = FhashIndexSet::default();
 
         if group_kind != GroupKind::None {
             columns_global.insert(self.flag_group_name.as_bytes().to_vec().into_boxed_slice());
@@ -273,7 +273,7 @@ impl Args {
         let mut conf_path;
         let mut rdr;
         let mut header: &csv::ByteRecord;
-        let mut columns_of_this_file: AhashIndexMap<Box<[u8]>, usize> = AhashIndexMap::default();
+        let mut columns_of_this_file: FhashIndexMap<Box<[u8]>, usize> = FhashIndexMap::default();
         columns_of_this_file.reserve(num_columns_global);
         let mut row: csv::ByteRecord = csv::ByteRecord::with_capacity(500, num_columns_global);
 

src/cmd/cat.rs

@@ -55,8 +55,8 @@ Common options:
 
 use std::{collections::hash_map::Entry, fs, io, str};
 
-use ahash::AHashMap;
 use byteorder::{BigEndian, WriteBytesExt};
+use foldhash::{HashMap, HashMapExt};
 use serde::Deserialize;
 
 use crate::{
@@ -185,14 +185,14 @@ impl Args {
 #[allow(dead_code)]
 struct ValueIndex<R> {
     // This maps tuples of values to corresponding rows.
-    values:   AHashMap<Vec<ByteString>, Vec<usize>>,
+    values:   HashMap<Vec<ByteString>, Vec<usize>>,
     idx:      Indexed<R, io::Cursor<Vec<u8>>>,
     num_rows: usize,
 }
 
 impl<R: io::Read + io::Seek> ValueIndex<R> {
     fn new(mut rdr: csv::Reader<R>, sel: &Selection, casei: bool) -> CliResult<ValueIndex<R>> {
-        let mut val_idx = AHashMap::with_capacity(10000);
+        let mut val_idx = HashMap::with_capacity(10000);
         let mut row_idx = io::Cursor::new(Vec::with_capacity(8 * 10000));
         let (mut rowi, mut count) = (0_usize, 0_usize);
 

src/cmd/exclude.rs

@@ -53,7 +53,7 @@ Common options:
 
 use std::{io, iter, ops};
 
-use ahash::AHashMap;
+use foldhash::{HashMap, HashMapExt};
 use serde::Deserialize;
 
 use crate::{
@@ -144,8 +144,8 @@ impl ops::Deref for ByteRecord {
 }
 
 type GroupKey = Option<ByteRecord>;
-type GroupBuffer = AHashMap<GroupKey, Vec<ByteRecord>>;
-type Grouper = AHashMap<GroupKey, GroupValues>;
+type GroupBuffer = HashMap<GroupKey, Vec<ByteRecord>>;
+type Grouper = HashMap<GroupKey, GroupValues>;
 type GroupKeySelection = Option<Selection>;
 
 trait GroupKeyConstructor {
@@ -163,14 +163,14 @@ impl GroupKeyConstructor for GroupKeySelection {
 
 #[derive(Debug)]
 struct GroupValues {
-    map:     AHashMap<usize, ByteString>,
+    map:     HashMap<usize, ByteString>,
     default: Option<ByteString>,
 }
 
 impl GroupValues {
     fn new(default: Option<ByteString>) -> Self {
         Self {
-            map: AHashMap::new(),
+            map: HashMap::new(),
             default,
         }
     }

src/cmd/fill.rs

@@ -368,9 +368,9 @@ use std::{
     path::{Path, PathBuf},
 };
 
-use ahash::RandomState;
 use cached::{SizedCache, proc_macro::cached};
 use dynfmt2::Format;
+use foldhash::fast::RandomState;
 use geosuggest_core::{
     CitiesRecord, CountryRecord, Engine,
     storage::{self, IndexStorage},

src/cmd/geocode.rs

@@ -92,8 +92,8 @@ Common options:
 
 use std::{collections::hash_map::Entry, fmt, io, iter::repeat_n, mem::swap, str};
 
-use ahash::AHashMap;
 use byteorder::{BigEndian, WriteBytesExt};
+use foldhash::{HashMap, HashMapExt};
 use serde::Deserialize;
 
 use crate::{
@@ -463,7 +463,7 @@ impl Args {
 
 struct ValueIndex<R> {
     // This maps tuples of values to corresponding rows.
-    values:   AHashMap<Vec<ByteString>, Vec<usize>>,
+    values:   HashMap<Vec<ByteString>, Vec<usize>>,
     idx:      Indexed<R, io::Cursor<Vec<u8>>>,
     num_rows: usize,
 }
@@ -503,7 +503,7 @@ impl<R: io::Read + io::Seek> ValueIndex<R> {
         zerosi: bool,
         nulls: bool,
     ) -> CliResult<ValueIndex<R>> {
-        let mut val_idx = AHashMap::with_capacity(20_000);
+        let mut val_idx = HashMap::with_capacity(20_000);
         let mut row_idx = io::Cursor::new(Vec::with_capacity(8 * 20_000));
         let (mut rowi, mut count) = (0_usize, 0_usize);
 

src/cmd/join.rs

@@ -62,7 +62,7 @@ use std::{
     path::Path,
 };
 
-use ahash::AHashMap;
+use foldhash::{HashMap, HashMapExt};
 use regex::Regex;
 use serde::Deserialize;
 
@@ -124,7 +124,7 @@ impl Args {
         let key_col = self.key_column(&rconfig, &headers)?;
         let mut r#gen = WriterGenerator::new(self.flag_filename.clone());
 
-        let mut writers: AHashMap<Vec<u8>, BoxedWriter> = AHashMap::new();
+        let mut writers: HashMap<Vec<u8>, BoxedWriter> = HashMap::new();
         let mut row = csv::ByteRecord::new();
         while rdr.read_byte_record(&mut row)? {
             // Decide what file to put this in.

src/cmd/partition.rs

@@ -66,8 +66,8 @@ Common options:
                             Must be a single character. (default: ,)
 "#;
 
-use ahash::AHashMap;
 use dynfmt2::Format;
+use foldhash::{HashMap, HashMapExt};
 use serde::Deserialize;
 
 use crate::{
@@ -90,8 +90,8 @@ struct Args {
     flag_delimiter:  Option<Delimiter>,
 }
 
-type Values = AHashMap<String, String>;
-type ValuesNum = AHashMap<String, u64>;
+type Values = HashMap<String, String>;
+type ValuesNum = HashMap<String, u64>;
 
 pub fn run(argv: &[&str]) -> CliResult<()> {
     let args: Args = util::get_args(USAGE, argv)?;

src/cmd/pseudo.rs

@@ -101,7 +101,7 @@ Common options:
 
 use std::collections::HashMap;
 
-use ahash::RandomState;
+use foldhash::fast::RandomState;
 use serde::{Deserialize, Serialize};
 
 use crate::{

src/cmd/safenames.rs

@@ -80,8 +80,8 @@ Common options:
 
 use std::{fs::File, io::Write, path::Path};
 
-use ahash::{AHashMap, AHashSet};
 use csv::ByteRecord;
+use foldhash::{HashMap, HashMapExt, HashSet};
 use grex::RegExpBuilder;
 use itertools::Itertools;
 use log::{debug, error, info, warn};
@@ -486,7 +486,7 @@ fn build_low_cardinality_column_selector_arg(
 fn get_unique_values(
     args: &util::SchemaArgs,
     column_select_arg: &str,
-) -> CliResult<AHashMap<String, Vec<String>>> {
+) -> CliResult<HashMap<String, Vec<String>>> {
     // prepare arg for invoking cmd::frequency
     let freq_args = crate::cmd::frequency::Args {
         arg_input:            args.arg_input.clone(),
@@ -530,8 +530,8 @@ fn get_unique_values(
 fn construct_map_of_unique_values(
     freq_csv_fields: &ByteRecord,
     frequency_tables: &[Frequencies<Vec<u8>>],
-) -> CliResult<AHashMap<String, Vec<String>>> {
-    let mut unique_values_map: AHashMap<String, Vec<String>> = AHashMap::new();
+) -> CliResult<HashMap<String, Vec<String>>> {
+    let mut unique_values_map: HashMap<String, Vec<String>> = HashMap::new();
     let mut unique_values = Vec::with_capacity(freq_csv_fields.len());
     // iterate through fields and gather unique values for each field
     for (i, header_byte_slice) in freq_csv_fields.iter().enumerate() {
@@ -592,7 +592,7 @@ fn get_required_fields(properties_map: &Map<String, Value>) -> Vec<Value> {
 fn generate_string_patterns(
     args: &util::SchemaArgs,
     properties_map: &Map<String, Value>,
-) -> CliResult<AHashMap<String, String>> {
+) -> CliResult<HashMap<String, String>> {
     let rconfig = Config::new(args.arg_input.as_ref())
         .delimiter(args.flag_delimiter)
         .no_headers(args.flag_no_headers)
@@ -603,7 +603,7 @@ fn generate_string_patterns(
     let headers = rdr.byte_headers()?.clone();
     let sel = rconfig.selection(&headers)?;
 
-    let mut pattern_map: AHashMap<String, String> = AHashMap::new();
+    let mut pattern_map: HashMap<String, String> = HashMap::new();
 
     // return empty pattern map when:
     //  * no columns are selected
@@ -615,7 +615,7 @@ fn generate_string_patterns(
     }
 
     // Map each Header to its unique Set of values
-    let mut unique_values_map: AHashMap<String, AHashSet<String>> = AHashMap::new();
+    let mut unique_values_map: HashMap<String, HashSet<String>> = HashMap::new();
 
     #[allow(unused_assignments)]
     let mut record = csv::ByteRecord::new();

src/cmd/schema.rs

@@ -128,7 +128,7 @@ use std::{
     },
 };
 
-use ahash::{HashMap, HashMapExt};
+use foldhash::{HashMap, HashMapExt};
 #[cfg(any(feature = "feature_capable", feature = "lite"))]
 use indicatif::{ProgressBar, ProgressDrawTarget};
 use minijinja::{Environment, Value, value::ValueKind};
@@ -852,7 +852,7 @@ fn register_lookup(
     let row_len = lookup_table.headers.len();
     for record in rdr.records().flatten() {
         let mut row_data: HashMap<String, String> =
-            HashMap::with_capacity_and_hasher(row_len, ahash::RandomState::new());
+            HashMap::with_capacity_and_hasher(row_len, foldhash::fast::RandomState::default());
 
         // Store all fields for this row
         for (header, value) in lookup_table.headers.iter().zip(record.iter()) {

src/cmd/template.rs

@@ -206,8 +206,8 @@ use std::{
     },
 };
 
-use ahash::{HashSet, HashSetExt};
 use csv::ByteRecord;
+use foldhash::{HashSet, HashSetExt};
 use indicatif::HumanCount;
 #[cfg(any(feature = "feature_capable", feature = "lite"))]
 use indicatif::{ProgressBar, ProgressDrawTarget};