// codesnip-guard: DaryHeap
#[cfg_attr(any(),rustfmt::skip)]pub use self::dary_heap::{DaryHeapI32,DaryHeapI64,DaryHeapI128,DaryHeapU32,DaryHeapU64,DaryHeapU128};#[cfg_attr(any(),rustfmt::skip)]mod dary_heap{#[cfg(target_arch="x86_64")]use super::simd;use super::{SimdBackend,simd_backend};#[repr(C,align(64))]#[derive(Clone,Debug)]struct HeapBlock<T,const D:usize>([T;D]);impl<T:Copy,const D:usize>HeapBlock<T,D>{#[inline(always)]fn filled(value:T)->Self{Self([value;D])}#[inline(always)]fn get(&self,index:usize)->T{unsafe{*self.0 .get_unchecked(index)}}#[inline(always)]fn set(&mut self,index:usize,value:T){unsafe{*self.0 .get_unchecked_mut(index)=value;}}}#[repr(C,align(64))]#[derive(Clone,Debug)]struct U128HeapBlock{low:[u64;4],high:[u64;4]}impl U128HeapBlock{#[inline(always)]fn filled(value:u128)->Self{Self{low:[value as u64;4],high:[(value>>64)as u64;4]}}#[inline(always)]fn get(&self,index:usize)->u128{unsafe{(*self.high.get_unchecked(index)as u128)<<64|*self.low.get_unchecked(index)as u128}}#[inline(always)]fn set(&mut self,index:usize,value:u128){unsafe{*self.low.get_unchecked_mut(index)=value as u64;*self.high.get_unchecked_mut(index)=(value>>64)as u64;}}}#[inline(always)]fn max_index<T:Copy+Ord,const D:usize>(values:&[T;D])->usize{let mut maximum=values[0];let mut result=0;for(index,&value)in values.iter().enumerate().skip(1){if value>maximum{maximum=value;result=index;}}result}#[inline(always)]fn max_index_u128(values:&U128HeapBlock)->usize{let mut result=0;for index in 1..4{if values.high[index]>values.high[result]||(values.high[index]==values.high[result]&&values.low[index]>values.low[result]){result=index;}}result}macro_rules!define_dary_heap{($name:ident,$doc:literal,$value:ty,$storage:ty,$branch:expr,$block:ty,encode=$encode:expr,decode=$decode:expr$(,$field:ident:$field_type:ty=$field_value:expr)*$(,)?)=>{#[doc=$doc]#[derive(Clone,Debug)]pub struct$name{root:$storage,blocks:Vec<$block>,len:usize,$(#[cfg(target_arch="x86_64")]$field:$field_type,)*}impl$name{pub fn new()->Self{Self::with_capacity(0)}pub fn with_capacity(capacity:usize)->Self{Self::empty(capacity$(,$field_value)*)}#[inline]pub fn len(&self)->usize{self.len}#[inline]pub fn is_empty(&self)->bool{self.len==0}#[inline]pub fn peek(&self)->Option<$value>{(self.len!=0).then(||Self::decode(self.root))}pub fn push(&mut self,value:$value){let value=Self::encode(value);if self.len==0{self.root=value;self.len=1;return;}let mut hole=self.len;let block=(hole-1)/$branch;if block==self.blocks.len(){self.blocks.push(<$block>::filled(<$storage>::MIN));}self.len+=1;while hole!=0{let parent=(hole-1)/$branch;let parent_key=self.key(parent);if parent_key>=value{break;}self.set_key(hole,parent_key);hole=parent;}self.set_key(hole,value);}pub fn pop(&mut self)->Option<$value>{if self.len==0{return None;}let result=self.root;if self.len==1{self.root=<$storage>::MIN;self.len=0;return Some(Self::decode(result));}let last=self.len-1;let value=self.key(last);self.set_key(last,<$storage>::MIN);self.len=last;self.sift_down_after_pop(0,value);Some(Self::decode(result))}#[doc=" Unconditionally replaces the greatest value, or inserts into an empty heap."]pub fn replace(&mut self,value:$value)->Option<$value>{if self.len==0{self.push(value);return None;}let result=self.root;self.sift_down(0,Self::encode(value));Some(Self::decode(result))}pub fn clear(&mut self){self.root=<$storage>::MIN;self.blocks.clear();self.len=0;}pub fn into_sorted_vec(mut self)->Vec<$value>{let mut values=Vec::with_capacity(self.len);while let Some(value)=self.pop(){values.push(value);}values.reverse();values}fn empty(capacity:usize$(,$field:$field_type)*)->Self{$({let _=&$field;})*Self{root:<$storage>::MIN,blocks:Vec::with_capacity(capacity.saturating_sub(1).div_ceil($branch)),len:0,$(#[cfg(target_arch="x86_64")]$field,)*}}fn build(values:Vec<$value>$(,$field:$field_type)*)->Self{let len=values.len();let mut heap=Self::empty(len$(,$field)*);heap.len=len;if let Some((&root,values))=values.split_first(){heap.root=Self::encode(root);heap.blocks.resize(len.saturating_sub(1).div_ceil($branch),<$block>::filled(<$storage>::MIN),);for(index,&value)in values.iter().enumerate(){heap.blocks[index/$branch].set(index%$branch,Self::encode(value));}heap.heapify();}heap}#[inline(always)]fn encode(value:$value)->$storage{($encode)(value)}#[inline(always)]fn decode(value:$storage)->$value{($decode)(value)}#[inline(always)]fn key(&self,index:usize)->$storage{if index==0{self.root}else{unsafe{self.blocks.get_unchecked((index-1)/$branch)}.get((index-1)%$branch)}}#[inline(always)]fn set_key(&mut self,index:usize,value:$storage){if index==0{self.root=value;}else{unsafe{self.blocks.get_unchecked_mut((index-1)/$branch)}.set((index-1)%$branch,value);}}#[inline(always)]fn sift_down_by<F>(&mut self,mut hole:usize,value:$storage,mut max_index:F)where F:FnMut(&$block)->usize,{if self.len<=1{self.set_key(hole,value);return;}let last_parent=(self.len-2)/$branch;while hole<=last_parent{let block=unsafe{self.blocks.get_unchecked(hole)};let lane=max_index(block);let child_key=block.get(lane);if child_key<=value{break;}self.set_key(hole,child_key);hole=hole*$branch+lane+1;}self.set_key(hole,value);}fn heapify(&mut self){if self.len<=1{return;}for parent in(0..=(self.len-2)/$branch).rev(){let value=self.key(parent);self.sift_down(parent,value);}}}impl Default for$name{fn default()->Self{Self::new()}}impl From<Vec<$value>>for$name{fn from(values:Vec<$value>)->Self{Self::build(values$(,$field_value)*)}}impl Extend<$value>for$name{fn extend<I>(&mut self,iter:I)where I:IntoIterator<Item=$value>,{for value in iter{self.push(value);}}}impl FromIterator<$value>for$name{fn from_iter<I>(iter:I)->Self where I:IntoIterator<Item=$value>,{let values:Vec<_> =iter.into_iter().collect();Self::from(values)}}};}define_dary_heap!(DaryHeapU32,"A cache-line-oriented 16-ary max-heap for medium-to-large 32-bit heaps. `BinaryHeap` can be faster for small heaps and monotone replacements.",u32,u32,16,HeapBlock<u32,16>,encode=|value|value,decode=|value|value,backend:SimdBackend=simd_backend(),);define_dary_heap!(DaryHeapI32,"A cache-line-oriented 16-ary max-heap for medium-to-large 32-bit heaps. `BinaryHeap` can be faster for small heaps and monotone replacements.",i32,u32,16,HeapBlock<u32,16>,encode=|value:i32|value as u32^(1<<31),decode=|value:u32|(value^(1<<31))as i32,backend:SimdBackend=simd_backend(),);define_dary_heap!(DaryHeapU64,"A cache-line-oriented 8-ary max-heap for large 64-bit heaps. `BinaryHeap` can be faster for small heaps and monotone replacements.",u64,u64,8,HeapBlock<u64,8>,encode=|value|value,decode=|value|value,backend:SimdBackend=simd_backend(),);define_dary_heap!(DaryHeapI64,"A cache-line-oriented 8-ary max-heap for large 64-bit heaps. `BinaryHeap` can be faster for small heaps and monotone replacements.",i64,u64,8,HeapBlock<u64,8>,encode=|value:i64|value as u64^(1<<63),decode=|value:u64|(value^(1<<63))as i64,backend:SimdBackend=simd_backend(),);define_dary_heap!(DaryHeapU128,"A cache-line-oriented 4-ary max-heap for large full-width 128-bit heaps. `BinaryHeap` can be faster for small heaps, monotone replacements, and heavily repeated keys.",u128,u128,4,U128HeapBlock,encode=|value|value,decode=|value|value,backend:SimdBackend=simd_backend(),);define_dary_heap!(DaryHeapI128,"A cache-line-oriented 4-ary max-heap for large full-width 128-bit heaps. `BinaryHeap` can be faster for small heaps, monotone replacements, and heavily repeated keys.",i128,u128,4,U128HeapBlock,encode=|value:i128|value as u128^(1<<127),decode=|value:u128|(value^(1<<127))as i128,backend:SimdBackend=simd_backend(),);macro_rules!impl_simd_heap{($name:ident,$value:ty,$branch:expr,$max_avx2:ident,$max_avx512:ident)=>{impl$name{#[inline(always)]fn sift_down_scalar(&mut self,hole:usize,value:$value){self.sift_down_by(hole,value,|block|max_index(&block.0))}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2")]unsafe fn sift_down_avx2(&mut self,hole:usize,value:$value){self.sift_down_by(hole,value,|block|unsafe{simd::$max_avx2(&block.0)})}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx512f")]unsafe fn sift_down_avx512(&mut self,hole:usize,value:$value){self.sift_down_by(hole,value,|block|unsafe{simd::$max_avx512(&block.0)})}#[inline]fn sift_down(&mut self,hole:usize,value:$value){#[cfg(target_arch="x86_64")]match self.backend{SimdBackend::Scalar=>self.sift_down_scalar(hole,value),SimdBackend::Avx2=>unsafe{self.sift_down_avx2(hole,value)},SimdBackend::Avx512=>unsafe{self.sift_down_avx512(hole,value)},}#[cfg(not(target_arch="x86_64"))]self.sift_down_scalar(hole,value);}#[inline(always)]fn sift_down_after_pop(&mut self,hole:usize,value:$value){self.sift_down(hole,value);}}};}impl_simd_heap!(DaryHeapU32,u32,16,max_index_u32x16_avx2,max_index_u32x16_avx512);impl_simd_heap!(DaryHeapI32,u32,16,max_index_u32x16_avx2,max_index_u32x16_avx512);impl_simd_heap!(DaryHeapU64,u64,8,max_index_u64x8_avx2,max_index_u64x8_avx512);impl_simd_heap!(DaryHeapI64,u64,8,max_index_u64x8_avx2,max_index_u64x8_avx512);macro_rules!impl_u128_heap{($name:ident)=>{impl$name{#[inline(always)]fn sift_down_scalar(&mut self,hole:usize,value:u128){self.sift_down_by(hole,value,max_index_u128)}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2")]unsafe fn sift_down_avx2(&mut self,hole:usize,value:u128){self.sift_down_by(hole,value,|block|unsafe{simd::max_index_u128x4_avx2(&block.low,&block.high)})}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2,avx512f,avx512vl")]unsafe fn sift_down_avx512(&mut self,hole:usize,value:u128){self.sift_down_by(hole,value,|block|unsafe{simd::max_index_u128x4_avx512(&block.low,&block.high)})}#[inline]fn sift_down(&mut self,hole:usize,value:u128){#[cfg(target_arch="x86_64")]match self.backend{SimdBackend::Scalar=>self.sift_down_scalar(hole,value),SimdBackend::Avx2=>unsafe{self.sift_down_avx2(hole,value)},SimdBackend::Avx512=>unsafe{self.sift_down_avx512(hole,value)},}#[cfg(not(target_arch="x86_64"))]self.sift_down_scalar(hole,value);}#[inline]fn sift_down_after_pop(&mut self,hole:usize,value:u128){#[cfg(target_arch="x86_64")]match self.backend{SimdBackend::Scalar=>self.sift_down_scalar(hole,value),SimdBackend::Avx2 if self.len<1<<18=>self.sift_down_scalar(hole,value),SimdBackend::Avx2=>unsafe{self.sift_down_avx2(hole,value)},SimdBackend::Avx512 if self.len<1<<15=>self.sift_down_scalar(hole,value),SimdBackend::Avx512=>unsafe{self.sift_down_avx512(hole,value)},}#[cfg(not(target_arch="x86_64"))]self.sift_down_scalar(hole,value);}}};}impl_u128_heap!(DaryHeapU128);impl_u128_heap!(DaryHeapI128);}
// codesnip-guard: _simd
#[cfg_attr(any(),rustfmt::skip)]pub mod simd{#![allow(unsafe_op_in_unsafe_fn)]#[cfg(target_arch="x86_64")]use std::arch::x86_64::*;#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2")]#[inline]pub unsafe fn first_ge_u16x32_avx2(values:&[u16;32],key:u16)->usize{let sign=_mm256_set1_epi16(i16::MIN);let key=_mm256_xor_si256(_mm256_set1_epi16(key as i16),sign);let low=_mm256_xor_si256(_mm256_loadu_si256(values.as_ptr().cast()),sign);let high=_mm256_xor_si256(_mm256_loadu_si256(values.as_ptr().add(16).cast()),sign);let low=_mm256_movemask_epi8(_mm256_cmpgt_epi16(key,low))as u32 as u64;let high=_mm256_movemask_epi8(_mm256_cmpgt_epi16(key,high))as u32 as u64;(!(low|high<<32)).trailing_zeros()as usize/2}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2")]#[inline]pub unsafe fn first_gt_u16x32_avx2(values:&[u16;32],key:u16)->usize{let sign=_mm256_set1_epi16(i16::MIN);let key=_mm256_xor_si256(_mm256_set1_epi16(key as i16),sign);let low=_mm256_xor_si256(_mm256_loadu_si256(values.as_ptr().cast()),sign);let high=_mm256_xor_si256(_mm256_loadu_si256(values.as_ptr().add(16).cast()),sign);let low=_mm256_movemask_epi8(_mm256_cmpgt_epi16(low,key))as u32 as u64;let high=_mm256_movemask_epi8(_mm256_cmpgt_epi16(high,key))as u32 as u64;(low|high<<32).trailing_zeros()as usize/2}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2")]#[inline]pub unsafe fn first_ge_u32x16_avx2(values:&[u32;16],key:u32)->usize{let sign=_mm256_set1_epi32(i32::MIN);let key=_mm256_xor_si256(_mm256_set1_epi32(key as i32),sign);let low=_mm256_xor_si256(_mm256_loadu_si256(values.as_ptr().cast()),sign);let high=_mm256_xor_si256(_mm256_loadu_si256(values.as_ptr().add(8).cast()),sign);let low=_mm256_movemask_ps(_mm256_castsi256_ps(_mm256_cmpgt_epi32(key,low)));let high=_mm256_movemask_ps(_mm256_castsi256_ps(_mm256_cmpgt_epi32(key,high)));let mask=((!low as u32)&0xff)|(((!high as u32)&0xff)<<8);(mask as u16).trailing_zeros()as usize}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2")]#[inline]pub unsafe fn first_gt_u32x16_avx2(values:&[u32;16],key:u32)->usize{let sign=_mm256_set1_epi32(i32::MIN);let key=_mm256_xor_si256(_mm256_set1_epi32(key as i32),sign);let low=_mm256_xor_si256(_mm256_loadu_si256(values.as_ptr().cast()),sign);let high=_mm256_xor_si256(_mm256_loadu_si256(values.as_ptr().add(8).cast()),sign);let low=_mm256_movemask_ps(_mm256_castsi256_ps(_mm256_cmpgt_epi32(low,key)));let high=_mm256_movemask_ps(_mm256_castsi256_ps(_mm256_cmpgt_epi32(high,key)));let mask=low as u32|((high as u32)<<8);(mask as u16).trailing_zeros()as usize}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2")]#[inline]pub unsafe fn first_ge_u64x8_avx2(values:&[u64;8],key:u64)->usize{let sign=_mm256_set1_epi64x(i64::MIN);let key=_mm256_xor_si256(_mm256_set1_epi64x(key as i64),sign);let low=_mm256_xor_si256(_mm256_loadu_si256(values.as_ptr().cast()),sign);let high=_mm256_xor_si256(_mm256_loadu_si256(values.as_ptr().add(4).cast()),sign);let low=_mm256_movemask_pd(_mm256_castsi256_pd(_mm256_cmpgt_epi64(key,low)));let high=_mm256_movemask_pd(_mm256_castsi256_pd(_mm256_cmpgt_epi64(key,high)));let mask=((!low as u32)&0x0f)|(((!high as u32)&0x0f)<<4);(mask as u8).trailing_zeros()as usize}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2")]#[inline]pub unsafe fn first_gt_u64x8_avx2(values:&[u64;8],key:u64)->usize{let sign=_mm256_set1_epi64x(i64::MIN);let key=_mm256_xor_si256(_mm256_set1_epi64x(key as i64),sign);let low=_mm256_xor_si256(_mm256_loadu_si256(values.as_ptr().cast()),sign);let high=_mm256_xor_si256(_mm256_loadu_si256(values.as_ptr().add(4).cast()),sign);let low=_mm256_movemask_pd(_mm256_castsi256_pd(_mm256_cmpgt_epi64(low,key)));let high=_mm256_movemask_pd(_mm256_castsi256_pd(_mm256_cmpgt_epi64(high,key)));let mask=low as u32|((high as u32)<<4);(mask as u8).trailing_zeros()as usize}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx512f,avx512bw")]#[inline]pub unsafe fn first_ge_u16x32_avx512(values:&[u16;32],key:u16)->usize{let values=_mm512_loadu_si512(values.as_ptr().cast());let key=_mm512_set1_epi16(key as i16);(!_mm512_cmplt_epu16_mask(values,key)).trailing_zeros()as usize}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx512f,avx512bw")]#[inline]pub unsafe fn first_gt_u16x32_avx512(values:&[u16;32],key:u16)->usize{let values=_mm512_loadu_si512(values.as_ptr().cast());let key=_mm512_set1_epi16(key as i16);_mm512_cmpgt_epu16_mask(values,key).trailing_zeros()as usize}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx512f")]#[inline]pub unsafe fn first_ge_u32x16_avx512(values:&[u32;16],key:u32)->usize{let values=_mm512_loadu_si512(values.as_ptr().cast());let key=_mm512_set1_epi32(key as i32);(!_mm512_cmplt_epu32_mask(values,key)).trailing_zeros()as usize}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx512f")]#[inline]pub unsafe fn first_gt_u32x16_avx512(values:&[u32;16],key:u32)->usize{let values=_mm512_loadu_si512(values.as_ptr().cast());let key=_mm512_set1_epi32(key as i32);_mm512_cmpgt_epu32_mask(values,key).trailing_zeros()as usize}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx512f")]#[inline]pub unsafe fn first_ge_u64x8_avx512(values:&[u64;8],key:u64)->usize{let values=_mm512_loadu_si512(values.as_ptr().cast());let key=_mm512_set1_epi64(key as i64);(!_mm512_cmplt_epu64_mask(values,key)).trailing_zeros()as usize}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx512f")]#[inline]pub unsafe fn first_gt_u64x8_avx512(values:&[u64;8],key:u64)->usize{let values=_mm512_loadu_si512(values.as_ptr().cast());let key=_mm512_set1_epi64(key as i64);_mm512_cmpgt_epu64_mask(values,key).trailing_zeros()as usize}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2")]#[inline]pub unsafe fn add_suffix_u32x16_avx2(values:&mut[u32;16],index:usize,delta:u32){let index=_mm256_set1_epi32(index as i32-1);let delta=_mm256_set1_epi32(delta as i32);let low_mask=_mm256_cmpgt_epi32(_mm256_setr_epi32(0,1,2,3,4,5,6,7),index);let high_mask=_mm256_cmpgt_epi32(_mm256_setr_epi32(8,9,10,11,12,13,14,15),index);let low=_mm256_add_epi32(_mm256_loadu_si256(values.as_ptr().cast()),_mm256_and_si256(delta,low_mask));let high=_mm256_add_epi32(_mm256_loadu_si256(values.as_ptr().add(8).cast()),_mm256_and_si256(delta,high_mask));_mm256_storeu_si256(values.as_mut_ptr().cast(),low);_mm256_storeu_si256(values.as_mut_ptr().add(8).cast(),high);}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2")]#[inline]pub unsafe fn add_suffix_u64x8_avx2(values:&mut[u64;8],index:usize,delta:u64){let index=_mm256_set1_epi64x(index as i64-1);let delta=_mm256_set1_epi64x(delta as i64);let low_mask=_mm256_cmpgt_epi64(_mm256_setr_epi64x(0,1,2,3),index);let high_mask=_mm256_cmpgt_epi64(_mm256_setr_epi64x(4,5,6,7),index);let low=_mm256_add_epi64(_mm256_loadu_si256(values.as_ptr().cast()),_mm256_and_si256(delta,low_mask));let high=_mm256_add_epi64(_mm256_loadu_si256(values.as_ptr().add(4).cast()),_mm256_and_si256(delta,high_mask));_mm256_storeu_si256(values.as_mut_ptr().cast(),low);_mm256_storeu_si256(values.as_mut_ptr().add(4).cast(),high);}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx512f")]#[inline]pub unsafe fn add_suffix_u32x16_avx512(values:&mut[u32;16],index:usize,delta:u32){let values_vector=_mm512_loadu_si512(values.as_ptr().cast());let values_vector=_mm512_mask_add_epi32(values_vector,u16::MAX<<index,values_vector,_mm512_set1_epi32(delta as i32));_mm512_storeu_si512(values.as_mut_ptr().cast(),values_vector);}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx512f")]#[inline]pub unsafe fn add_suffix_u64x8_avx512(values:&mut[u64;8],index:usize,delta:u64){let values_vector=_mm512_loadu_si512(values.as_ptr().cast());let values_vector=_mm512_mask_add_epi64(values_vector,u8::MAX<<index,values_vector,_mm512_set1_epi64(delta as i64));_mm512_storeu_si512(values.as_mut_ptr().cast(),values_vector);}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2")]#[inline]pub unsafe fn max_index_u32x16_avx2(values:&[u32;16])->usize{let low=_mm256_loadu_si256(values.as_ptr().cast());let high=_mm256_loadu_si256(values.as_ptr().add(8).cast());let mut maximum=_mm256_max_epu32(low,high);maximum=_mm256_max_epu32(maximum,_mm256_permute2x128_si256::<0x01>(maximum,maximum));maximum=_mm256_max_epu32(maximum,_mm256_shuffle_epi32::<0x4e>(maximum));maximum=_mm256_max_epu32(maximum,_mm256_shuffle_epi32::<0xb1>(maximum));let maximum=_mm256_set1_epi32(_mm256_extract_epi32::<0>(maximum));let low=_mm256_movemask_ps(_mm256_castsi256_ps(_mm256_cmpeq_epi32(low,maximum)));let high=_mm256_movemask_ps(_mm256_castsi256_ps(_mm256_cmpeq_epi32(high,maximum)));((low as u32|((high as u32)<<8))as u16).trailing_zeros()as usize}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx512f")]#[inline]pub unsafe fn max_index_u32x16_avx512(values:&[u32;16])->usize{let values=_mm512_loadu_si512(values.as_ptr().cast());let maximum=_mm512_set1_epi32(_mm512_reduce_max_epu32(values)as i32);_mm512_cmpeq_epi32_mask(values,maximum).trailing_zeros()as usize}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2")]#[inline]unsafe fn max_epu64(left:__m256i,right:__m256i)->__m256i{let sign=_mm256_set1_epi64x(i64::MIN);let greater=_mm256_cmpgt_epi64(_mm256_xor_si256(left,sign),_mm256_xor_si256(right,sign));_mm256_blendv_epi8(right,left,greater)}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2")]#[inline]pub unsafe fn max_index_u64x8_avx2(values:&[u64;8])->usize{let sign=_mm256_set1_epi64x(i64::MIN);let low=_mm256_xor_si256(_mm256_loadu_si256(values.as_ptr().cast()),sign);let high=_mm256_xor_si256(_mm256_loadu_si256(values.as_ptr().add(4).cast()),sign);let mut maximum=max_i64x4(low,high);maximum=max_i64x4(maximum,_mm256_permute4x64_epi64::<0x4e>(maximum));maximum=max_i64x4(maximum,_mm256_permute4x64_epi64::<0xb1>(maximum));let low=_mm256_movemask_pd(_mm256_castsi256_pd(_mm256_cmpeq_epi64(low,maximum)));let high=_mm256_movemask_pd(_mm256_castsi256_pd(_mm256_cmpeq_epi64(high,maximum)));((low as u32|((high as u32)<<4))as u8).trailing_zeros()as usize}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx512f")]#[inline]pub unsafe fn max_index_u64x8_avx512(values:&[u64;8])->usize{let values=_mm512_loadu_si512(values.as_ptr().cast());let maximum=_mm512_set1_epi64(_mm512_reduce_max_epu64(values)as i64);_mm512_cmpeq_epi64_mask(values,maximum).trailing_zeros()as usize}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2")]#[inline]pub unsafe fn max_index_u128x4_avx2(low:&[u64;4],high:&[u64;4])->usize{let low=_mm256_loadu_si256(low.as_ptr().cast());let high=_mm256_loadu_si256(high.as_ptr().cast());let mut maximum_high=high;maximum_high=max_epu64(maximum_high,_mm256_permute4x64_epi64::<0x4e>(maximum_high));maximum_high=max_epu64(maximum_high,_mm256_permute4x64_epi64::<0xb1>(maximum_high));let high_equal=_mm256_cmpeq_epi64(high,maximum_high);let high_mask=_mm256_movemask_pd(_mm256_castsi256_pd(high_equal))as u32&15;if high_mask.is_power_of_two(){return high_mask.trailing_zeros()as usize;}let mut maximum_low=_mm256_and_si256(low,high_equal);maximum_low=max_epu64(maximum_low,_mm256_permute4x64_epi64::<0x4e>(maximum_low));maximum_low=max_epu64(maximum_low,_mm256_permute4x64_epi64::<0xb1>(maximum_low));let both=_mm256_and_si256(high_equal,_mm256_cmpeq_epi64(low,maximum_low));(_mm256_movemask_pd(_mm256_castsi256_pd(both))as u32&15).trailing_zeros()as usize}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2,avx512f,avx512vl")]#[inline]pub unsafe fn max_index_u128x4_avx512(low:&[u64;4],high:&[u64;4])->usize{let low=_mm256_loadu_si256(low.as_ptr().cast());let high=_mm256_loadu_si256(high.as_ptr().cast());let mut maximum_high=high;maximum_high=_mm256_max_epu64(maximum_high,_mm256_permute4x64_epi64::<0x4e>(maximum_high));maximum_high=_mm256_max_epu64(maximum_high,_mm256_permute4x64_epi64::<0xb1>(maximum_high));let high_equal=_mm256_cmpeq_epi64(high,maximum_high);let high_mask=_mm256_movemask_pd(_mm256_castsi256_pd(high_equal))as u32&15;if high_mask.is_power_of_two(){return high_mask.trailing_zeros()as usize;}let mut maximum_low=_mm256_and_si256(low,high_equal);maximum_low=_mm256_max_epu64(maximum_low,_mm256_permute4x64_epi64::<0x4e>(maximum_low));maximum_low=_mm256_max_epu64(maximum_low,_mm256_permute4x64_epi64::<0xb1>(maximum_low));let both=_mm256_and_si256(high_equal,_mm256_cmpeq_epi64(low,maximum_low));(_mm256_movemask_pd(_mm256_castsi256_pd(both))as u32&15).trailing_zeros()as usize}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2")]#[inline]unsafe fn reduce_min_i32x8(mut values:__m256i)->i32{values=_mm256_min_epi32(values,_mm256_permute2x128_si256::<0x01>(values,values));values=_mm256_min_epi32(values,_mm256_shuffle_epi32::<0x4e>(values));values=_mm256_min_epi32(values,_mm256_shuffle_epi32::<0xb1>(values));_mm256_extract_epi32::<0>(values)}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2")]#[inline]unsafe fn reduce_max_i32x8(mut values:__m256i)->i32{values=_mm256_max_epi32(values,_mm256_permute2x128_si256::<0x01>(values,values));values=_mm256_max_epi32(values,_mm256_shuffle_epi32::<0x4e>(values));values=_mm256_max_epi32(values,_mm256_shuffle_epi32::<0xb1>(values));_mm256_extract_epi32::<0>(values)}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2")]#[inline]unsafe fn reduce_sum_i32x8(mut values:__m256i)->i32{values=_mm256_add_epi32(values,_mm256_permute2x128_si256::<0x01>(values,values));values=_mm256_add_epi32(values,_mm256_shuffle_epi32::<0x4e>(values));values=_mm256_add_epi32(values,_mm256_shuffle_epi32::<0xb1>(values));_mm256_extract_epi32::<0>(values)}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2")]#[inline]pub unsafe fn minimum_i32x16_avx2(values:&[i32;16])->i32{let low=_mm256_loadu_si256(values.as_ptr().cast());let high=_mm256_loadu_si256(values.as_ptr().add(8).cast());reduce_min_i32x8(_mm256_min_epi32(low,high))}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2")]#[inline]pub unsafe fn maximum_i32x16_avx2(values:&[i32;16])->i32{let low=_mm256_loadu_si256(values.as_ptr().cast());let high=_mm256_loadu_si256(values.as_ptr().add(8).cast());reduce_max_i32x8(_mm256_max_epi32(low,high))}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2")]#[inline]pub unsafe fn sum_i32x16_avx2(values:&[i32;16])->i32{let low=_mm256_loadu_si256(values.as_ptr().cast());let high=_mm256_loadu_si256(values.as_ptr().add(8).cast());reduce_sum_i32x8(_mm256_add_epi32(low,high))}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2")]#[inline]unsafe fn range_mask_i32x8(start:usize,end:usize,offset:i32)->__m256i{let lanes=_mm256_setr_epi32(offset,offset+1,offset+2,offset+3,offset+4,offset+5,offset+6,offset+7);_mm256_and_si256(_mm256_cmpgt_epi32(lanes,_mm256_set1_epi32(start as i32-1)),_mm256_cmpgt_epi32(_mm256_set1_epi32(end as i32),lanes))}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2")]#[inline]pub unsafe fn minimum_range_i32x16_avx2(values:&[i32;16],start:usize,end:usize)->i32{let unit=_mm256_set1_epi32(i32::MAX);let low=_mm256_blendv_epi8(unit,_mm256_loadu_si256(values.as_ptr().cast()),range_mask_i32x8(start,end,0));let high=_mm256_blendv_epi8(unit,_mm256_loadu_si256(values.as_ptr().add(8).cast()),range_mask_i32x8(start,end,8));reduce_min_i32x8(_mm256_min_epi32(low,high))}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2")]#[inline]pub unsafe fn maximum_range_i32x16_avx2(values:&[i32;16],start:usize,end:usize)->i32{let unit=_mm256_set1_epi32(i32::MIN);let low=_mm256_blendv_epi8(unit,_mm256_loadu_si256(values.as_ptr().cast()),range_mask_i32x8(start,end,0));let high=_mm256_blendv_epi8(unit,_mm256_loadu_si256(values.as_ptr().add(8).cast()),range_mask_i32x8(start,end,8));reduce_max_i32x8(_mm256_max_epi32(low,high))}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2")]#[inline]pub unsafe fn sum_range_i32x16_avx2(values:&[i32;16],start:usize,end:usize)->i32{let low=_mm256_and_si256(_mm256_loadu_si256(values.as_ptr().cast()),range_mask_i32x8(start,end,0));let high=_mm256_and_si256(_mm256_loadu_si256(values.as_ptr().add(8).cast()),range_mask_i32x8(start,end,8));reduce_sum_i32x8(_mm256_add_epi32(low,high))}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx512f")]#[inline]pub unsafe fn minimum_i32x16_avx512(values:&[i32;16])->i32{_mm512_reduce_min_epi32(_mm512_loadu_si512(values.as_ptr().cast()))}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx512f")]#[inline]pub unsafe fn maximum_i32x16_avx512(values:&[i32;16])->i32{_mm512_reduce_max_epi32(_mm512_loadu_si512(values.as_ptr().cast()))}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx512f")]#[inline]pub unsafe fn sum_i32x16_avx512(values:&[i32;16])->i32{_mm512_reduce_add_epi32(_mm512_loadu_si512(values.as_ptr().cast()))}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx512f")]#[inline]pub unsafe fn minimum_range_i32x16_avx512(values:&[i32;16],start:usize,end:usize)->i32{let mask=(u16::MAX<<start)&(u16::MAX>>(16-end));_mm512_mask_reduce_min_epi32(mask,_mm512_loadu_si512(values.as_ptr().cast()))}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx512f")]#[inline]pub unsafe fn maximum_range_i32x16_avx512(values:&[i32;16],start:usize,end:usize)->i32{let mask=(u16::MAX<<start)&(u16::MAX>>(16-end));_mm512_mask_reduce_max_epi32(mask,_mm512_loadu_si512(values.as_ptr().cast()))}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx512f")]#[inline]pub unsafe fn sum_range_i32x16_avx512(values:&[i32;16],start:usize,end:usize)->i32{let mask=(u16::MAX<<start)&(u16::MAX>>(16-end));_mm512_mask_reduce_add_epi32(mask,_mm512_loadu_si512(values.as_ptr().cast()))}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2")]#[inline]unsafe fn min_i64x4(left:__m256i,right:__m256i)->__m256i{_mm256_blendv_epi8(left,right,_mm256_cmpgt_epi64(left,right))}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2")]#[inline]unsafe fn max_i64x4(left:__m256i,right:__m256i)->__m256i{_mm256_blendv_epi8(right,left,_mm256_cmpgt_epi64(left,right))}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2")]#[inline]unsafe fn reduce_min_i64x4(mut values:__m256i)->i64{values=min_i64x4(values,_mm256_permute4x64_epi64::<0x4e>(values));values=min_i64x4(values,_mm256_permute4x64_epi64::<0xb1>(values));_mm256_extract_epi64::<0>(values)}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2")]#[inline]unsafe fn reduce_max_i64x4(mut values:__m256i)->i64{values=max_i64x4(values,_mm256_permute4x64_epi64::<0x4e>(values));values=max_i64x4(values,_mm256_permute4x64_epi64::<0xb1>(values));_mm256_extract_epi64::<0>(values)}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2")]#[inline]unsafe fn reduce_sum_i64x4(mut values:__m256i)->i64{values=_mm256_add_epi64(values,_mm256_permute4x64_epi64::<0x4e>(values));values=_mm256_add_epi64(values,_mm256_permute4x64_epi64::<0xb1>(values));_mm256_extract_epi64::<0>(values)}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2")]#[inline]pub unsafe fn minimum_i64x8_avx2(values:&[i64;8])->i64{let low=_mm256_loadu_si256(values.as_ptr().cast());let high=_mm256_loadu_si256(values.as_ptr().add(4).cast());reduce_min_i64x4(min_i64x4(low,high))}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2")]#[inline]pub unsafe fn maximum_i64x8_avx2(values:&[i64;8])->i64{let low=_mm256_loadu_si256(values.as_ptr().cast());let high=_mm256_loadu_si256(values.as_ptr().add(4).cast());reduce_max_i64x4(max_i64x4(low,high))}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2")]#[inline]pub unsafe fn sum_i64x8_avx2(values:&[i64;8])->i64{let low=_mm256_loadu_si256(values.as_ptr().cast());let high=_mm256_loadu_si256(values.as_ptr().add(4).cast());reduce_sum_i64x4(_mm256_add_epi64(low,high))}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2")]#[inline]unsafe fn range_mask_i64x4(start:usize,end:usize,offset:i64)->__m256i{let lanes=_mm256_setr_epi64x(offset,offset+1,offset+2,offset+3);_mm256_and_si256(_mm256_cmpgt_epi64(lanes,_mm256_set1_epi64x(start as i64-1)),_mm256_cmpgt_epi64(_mm256_set1_epi64x(end as i64),lanes))}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2")]#[inline]pub unsafe fn minimum_range_i64x8_avx2(values:&[i64;8],start:usize,end:usize)->i64{let unit=_mm256_set1_epi64x(i64::MAX);let low=_mm256_blendv_epi8(unit,_mm256_loadu_si256(values.as_ptr().cast()),range_mask_i64x4(start,end,0));let high=_mm256_blendv_epi8(unit,_mm256_loadu_si256(values.as_ptr().add(4).cast()),range_mask_i64x4(start,end,4));reduce_min_i64x4(min_i64x4(low,high))}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2")]#[inline]pub unsafe fn maximum_range_i64x8_avx2(values:&[i64;8],start:usize,end:usize)->i64{let unit=_mm256_set1_epi64x(i64::MIN);let low=_mm256_blendv_epi8(unit,_mm256_loadu_si256(values.as_ptr().cast()),range_mask_i64x4(start,end,0));let high=_mm256_blendv_epi8(unit,_mm256_loadu_si256(values.as_ptr().add(4).cast()),range_mask_i64x4(start,end,4));reduce_max_i64x4(max_i64x4(low,high))}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2")]#[inline]pub unsafe fn sum_range_i64x8_avx2(values:&[i64;8],start:usize,end:usize)->i64{let low=_mm256_and_si256(_mm256_loadu_si256(values.as_ptr().cast()),range_mask_i64x4(start,end,0));let high=_mm256_and_si256(_mm256_loadu_si256(values.as_ptr().add(4).cast()),range_mask_i64x4(start,end,4));reduce_sum_i64x4(_mm256_add_epi64(low,high))}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx512f")]#[inline]pub unsafe fn minimum_i64x8_avx512(values:&[i64;8])->i64{_mm512_reduce_min_epi64(_mm512_loadu_si512(values.as_ptr().cast()))}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx512f")]#[inline]pub unsafe fn maximum_i64x8_avx512(values:&[i64;8])->i64{_mm512_reduce_max_epi64(_mm512_loadu_si512(values.as_ptr().cast()))}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx512f")]#[inline]pub unsafe fn sum_i64x8_avx512(values:&[i64;8])->i64{_mm512_reduce_add_epi64(_mm512_loadu_si512(values.as_ptr().cast()))}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx512f")]#[inline]pub unsafe fn minimum_range_i64x8_avx512(values:&[i64;8],start:usize,end:usize)->i64{let mask=(u8::MAX<<start)&(u8::MAX>>(8-end));_mm512_mask_reduce_min_epi64(mask,_mm512_loadu_si512(values.as_ptr().cast()))}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx512f")]#[inline]pub unsafe fn maximum_range_i64x8_avx512(values:&[i64;8],start:usize,end:usize)->i64{let mask=(u8::MAX<<start)&(u8::MAX>>(8-end));_mm512_mask_reduce_max_epi64(mask,_mm512_loadu_si512(values.as_ptr().cast()))}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx512f")]#[inline]pub unsafe fn sum_range_i64x8_avx512(values:&[i64;8],start:usize,end:usize)->i64{let mask=(u8::MAX<<start)&(u8::MAX>>(8-end));_mm512_mask_reduce_add_epi64(mask,_mm512_loadu_si512(values.as_ptr().cast()))}}
// codesnip-guard: avx_helper
#[cfg_attr(any(),rustfmt::skip)]pub use self::avx_helper::{SimdBackend,avx512_enabled,avx512_supported,disable_avx512,enable_avx512,simd_backend};#[cfg_attr(any(),rustfmt::skip)]mod avx_helper{use std::sync::atomic::{AtomicBool,Ordering};#[derive(Copy,Clone,Debug,Eq,PartialEq)]pub enum SimdBackend{Scalar,Avx2,Avx512}static AVX512_ENABLED:AtomicBool=AtomicBool::new(true);#[inline]pub fn disable_avx512(){AVX512_ENABLED.store(false,Ordering::Relaxed);}#[inline]pub fn enable_avx512(){AVX512_ENABLED.store(true,Ordering::Relaxed);}#[inline]pub fn avx512_enabled()->bool{AVX512_ENABLED.load(Ordering::Relaxed)}#[inline]pub fn avx512_supported()->bool{#[cfg(any(target_arch="x86",target_arch="x86_64"))]return is_x86_feature_detected!("avx512f")&&is_x86_feature_detected!("avx512dq")&&is_x86_feature_detected!("avx512cd")&&is_x86_feature_detected!("avx512bw")&&is_x86_feature_detected!("avx512vl");#[cfg(not(any(target_arch="x86",target_arch="x86_64")))]false}#[inline]pub fn simd_backend()->SimdBackend{#[cfg(any(target_arch="x86",target_arch="x86_64"))]{if avx512_enabled()&&avx512_supported(){return SimdBackend::Avx512;}if is_x86_feature_detected!("avx2"){return SimdBackend::Avx2;}}SimdBackend::Scalar}#[macro_export]macro_rules!avx_helper{(@dispatch$backend:path,$kind:ident;$avx512:expr,$avx2:expr,$scalar:expr)=>{{#[cfg(target_arch="x86_64")]{match$backend(){$kind::Avx512=>$avx512,$kind::Avx2=>$avx2,$kind::Scalar=>$scalar,}}#[cfg(not(target_arch="x86_64"))]$scalar}};(@dispatch_avx2_fma$avx2:expr,$scalar:expr)=>{{#[cfg(target_arch="x86_64")]{if is_x86_feature_detected!("avx2")&&is_x86_feature_detected!("fma"){$avx2}else{$scalar}}#[cfg(not(target_arch="x86_64"))]$scalar}};(@avx512$(#[$meta:meta])*$vis:vis fn$name:ident$(<$($T:ident),+>)?($($i:ident:$t:ty),*)->$ret:ty where[$($clauses:tt)*]$body:block)=>{$(#[$meta])*$vis fn$name$(<$($T)*>)?($($i:$t),*)->$ret where$($clauses)*{if$crate::avx512_supported(){$crate::avx_helper!(@def_avx512 fn avx512$(<$($T)*>)?($($i:$t),*)->$ret where[$($clauses)*]$body);unsafe{avx512$(::<$($T),*>)?($($i),*)}}else if is_x86_feature_detected!("avx2"){$crate::avx_helper!(@def_avx2 fn avx2$(<$($T)*>)?($($i:$t),*)->$ret where[$($clauses)*]$body);unsafe{avx2$(::<$($T),*>)?($($i),*)}}else{$body}}};(@avx2$(#[$meta:meta])*$vis:vis fn$name:ident$(<$($T:ident),+>)?($($i:ident:$t:ty),*)->$ret:ty where[$($clauses:tt)*]$body:block)=>{$(#[$meta])*$vis fn$name$(<$($T)*>)?($($i:$t),*)->$ret where$($clauses)*{if is_x86_feature_detected!("avx2"){$crate::avx_helper!(@def_avx2 fn avx2$(<$($T)*>)?($($i:$t),*)->$ret where[$($clauses)*]$body);unsafe{avx2$(::<$($T),*>)?($($i),*)}}else{$body}}};(@def_avx512 fn$name:ident$(<$($T:ident),+>)?($($args:tt)*)->$ret:ty where[$($clauses:tt)*]$body:block)=>{#[target_feature(enable="avx512f,avx512dq,avx512cd,avx512bw,avx512vl")]unsafe fn$name$(<$($T)*>)?($($args)*)->$ret where$($clauses)*$body};(@def_avx2 fn$name:ident$(<$($T:ident),+>)?($($args:tt)*)->$ret:ty where[$($clauses:tt)*]$body:block)=>{#[target_feature(enable="avx2")]unsafe fn$name$(<$($T)*>)?($($args)*)->$ret where$($clauses)*$body};(@$tag:ident$(#[$meta:meta])*$vis:vis fn$name:ident$(<$($T:ident),+>)?($($args:tt)*)->$ret:ty$body:block)=>{$crate::avx_helper!(@$tag$(#[$meta])*$vis fn$name$(<$($T)*>)?($($args)*)->$ret where[]$body);};(@$tag:ident$(#[$meta:meta])*$vis:vis fn$name:ident$(<$($T:ident),+>)?($($args:tt)*)$($t:tt)*)=>{$crate::avx_helper!(@$tag$(#[$meta])*$vis fn$name$(<$($T)*>)?($($args)*)->()$($t)*);};($($t:tt)*)=>{::std::compile_error!($($t)*);}}}